From 379c3bab27e9dc744c1e460260811178997ea740 Mon Sep 17 00:00:00 2001 From: Ryan Angilly Date: Mon, 27 Jul 2026 10:11:15 -0600 Subject: [PATCH 1/5] feat: author and evaluate insight suite metrics Reissues NVIDIA-dev/NeMo-Optimizer#91 in the nemo-platform monorepo. Signed-off-by: Ryan Angilly --- .../nemo_experimentalist_plugin/entities.py | 12 + .../eval_author/README.md | 19 +- .../eval_author/agent.py | 113 +++--- .../eval_author/materialization.py | 59 +-- .../eval_author/models.py | 8 +- .../experimentalist/components/loop.py | 81 +++- .../experimentalist/experiment_mirror.py | 15 +- .../test_eval_author_repair_e2e.py | 375 ++++++++++++++++++ .../test_loop_insight_suite.py | 263 ++++++++++++ .../tests/test_eval_author_agent.py | 172 ++++---- .../tests/test_eval_author_materialization.py | 159 +------- .../tests/test_experiment_mirror.py | 29 +- 12 files changed, 920 insertions(+), 385 deletions(-) create mode 100644 plugins/nemo-experimentalist/tests/experimentalist/test_eval_author_repair_e2e.py create mode 100644 plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py index 99f859c5bd..e6e7070fe1 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py @@ -149,6 +149,14 @@ class Candidate(NemoEntity, entity_type="candidate"): default=None, description="Validation split trial results from the last evaluation run.", ) + insight_reward: dict[str, float] | None = Field( + default=None, + description="Multi-dimensional reward on the materialized Insight suite.", + ) + insight_reward_details: Sequence[TrialResult] | None = Field( + default=None, + description="Insight-suite trial results from the last evaluation run.", + ) validation_trajectory_reward: dict[str, float] | None = Field( default=None, description="Validation trajectory reward: aggregate + per-node scores.", @@ -180,6 +188,9 @@ def __repr__(self) -> str: if self.validation_reward: scores = ", ".join(f"{k}={v:.3f}" for k, v in self.validation_reward.items()) parts.append(f", validation_reward={{{scores}}}") + if self.insight_reward: + scores = ", ".join(f"{k}={v:.3f}" for k, v in self.insight_reward.items()) + parts.append(f", insight_reward={{{scores}}}") if self.killed_round is not None: parts.append(f", killed_round={self.killed_round}") parts.append(")") @@ -191,6 +202,7 @@ def slim(self) -> "Candidate": update={ "train_reward_details": None, "validation_reward_details": None, + "insight_reward_details": None, "validation_trajectory_reward_details": None, } ) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md index f3b1cb8538..810d0ae0ea 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md @@ -16,7 +16,7 @@ runs the top-level Eval Author before beginning optimization. ## Current Files - `agent.py` defines the canonical `EvalAuthor` agent. -- `materialization.py` stages, validates, and publishes Insight suites. +- `materialization.py` stages, validates, and persists Insight suites locally. - `models.py` defines the lightweight `EvalAuthorConfig` and `EvalAuthorResult` models. - `run.py` defines `run_eval_author(...)`, a reusable orchestration function for Python callers. @@ -51,21 +51,20 @@ eval-and-optimize/eval_author//insight-suite/ Each Eval Author invocation fills a fresh candidate suite from the current template and traces. The complete suite is Harbor-validated locally, promoted to the experiment-local working copy with backup-and-restore failure handling, and -uploaded to a newly created NeMo Platform Fileset. Eval Author verifies the remote -file inventory before returning. -An incomplete Fileset is deleted if upload or verification fails; Filesets from -earlier successful invocations are never modified or reused. +analyzed for the Insight's root cause. Eval Author then adds normalized +Insight-specific verifier metric keys to every materialized task while +preserving the template's existing task metrics. The metric authoring step is +scoped to the Insight suite; the user's train and validation datasets remain +unchanged. The authored verifiers must pass static Harbor validation before the +local suite is returned to the optimization loop. Task-template inputs may be local paths, `file://` URIs, or NeMo Platform `fileset:///` references. Fileset-backed templates are downloaded into the experiment-local staging directory before Harbor parses them. The staged template is refreshed on every invocation rather than reused. -`EvalAuthorResult.insight_suite` contains a durable `DatasetRef` whose URI uses the -`fileset:///` form. Downstream agents may store and pass that -reference without understanding its storage. A component that needs Harbor's -local filesystem layout must hydrate the Fileset into its own working directory -before calling `HarborDataset.from_path`. +`EvalAuthorResult.insight_suite` contains the experiment-local materialized +`Dataset` for immediate evaluation by the optimization loop. ## Intended Invocation diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py index 1fafcf92ce..6c13623ec2 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py @@ -9,7 +9,6 @@ import asyncio import logging -from dataclasses import dataclass from pathlib import Path from typing import Any @@ -49,37 +48,6 @@ logger = logging.getLogger(__name__) -@dataclass(frozen=True) -class EvalAuthorDatasetValidationFailure: - """Validation failure for one Eval Author dataset split.""" - - split: str - error: DatasetValidationError - - -class EvalAuthorDatasetValidationError(DatasetValidationError): - """Validation failures from datasets returned by the Eval Author.""" - - def __init__(self, failures: list[EvalAuthorDatasetValidationFailure]) -> None: - self.failures = tuple(failures) - details = "\n".join(f"{failure.split} dataset:\n{failure.error}" for failure in failures) - super().__init__(f"Eval Author dataset validation failed:\n{details}") - - -async def _validate_eval_author_result(result: EvalAuthorResult) -> None: - """Validate both Eval Author output splits and aggregate authoring failures.""" - failures: list[EvalAuthorDatasetValidationFailure] = [] - for split, dataset in (("train", result.train_dataset), ("validation", result.validation_dataset)): - try: - await dataset.validate() - except DatasetValidationError as exc: - failures.append(EvalAuthorDatasetValidationFailure(split=split, error=exc)) - - if failures: - first_failure = failures[0] - raise EvalAuthorDatasetValidationError(failures) from first_failure.error - - class EvalAuthor(Agent, llm=get_smart_model()): """Insights are failure modes of an agent in production. @@ -131,47 +99,57 @@ async def discover_runner(self, dataset: Dataset) -> str: ... @strategy(CodeActStrategy(config=CodeActConfig(max_iterations=60, cell_timeout=3600.0))) - async def augment_dataset( + async def author_insight_metrics( self, insight: Insight, diagnostics: list[tuple[str, Diagnostic]], - train_dataset: Dataset, - validation_dataset: Dataset, + insight_suite: Dataset, runner_conventions: str, validation_feedback: str | None = None, - ) -> EvalAuthorResult: - """Augment existing dataset tasks with evaluation metrics that capture the insight. + ) -> str: + """Author verifier metrics for the materialized tasks that capture the insight. Args: insight: The insight whose failure mode the tasks should detect. diagnostics: Per-trace ``(trace_ref, Diagnostic)`` pairs for concrete evidence. - train_dataset: The train dataset to augment for optimization feedback. - validation_dataset: The validation dataset to augment for scoring. + insight_suite: The materialized tasks recreated from the Insight's production traces. runner_conventions: Summary of how this dataset's runner works (from ``discover_runner``). Use this as the authoritative reference for what artifacts exist at evaluation runtime, how tasks are structured, and how to add metrics. validation_feedback: Actionable failures from mandatory validation of - the previous augmentation attempt. If provided, repair every reported + the previous metric-authoring attempt. If provided, repair every reported file before returning. Refer to ``self.context["dataset_documentation"]`` for the dataset-specific API and metric authoring conventions (file layout, how to add/remove/modify a metric). - **Scope: every task in both datasets** + **Scope: new grades on the materialized Insight tasks** + + Add at least one new Insight-specific metric key to every task in ``insight_suite``. + Use the same new metric key set and shared scoring semantics across the entire + suite. Preserve every existing verifier metric, including the task's ordinary + ``reward`` or ``score``; append the Insight signal instead of replacing the + task's original notion of success. - Add the metric to every task in ``train_dataset`` and ``validation_dataset``. - A metric is only useful as a suite-wide signal, not a per-sample patch. + Only edit verifier files in the materialized Insight suite. Do not modify the + user's train or validation datasets, and do not change task instructions, + environments, solutions, or other agent-visible inputs. This work adds new + grades to the new rows; it does not add new agent output to old benchmark rows. + + Name each new metric after the root-cause behavior, not a trace id or surface + symptom. Measure the current Harbor run from runtime artifacts such as OTLP + traces or agent outputs. Do not hard-code scores for the production traces that + motivated the Insight. **Validate while authoring** - After every verifier edit, call ``await train_dataset.validate()`` and - ``await validation_dataset.validate()``. These validation tools perform + After every verifier edit, call ``await insight_suite.validate()``. This performs evaluator-specific static checks without launching trials or executing verifier - code. If either raises ``DatasetValidationError``, use its task, path, and source - location diagnostics to repair the files, then call the tools again. Do not return - until both datasets pass validation. If ``validation_feedback`` is provided, it - means the previous result failed the mandatory validation performed by the caller; - fix all reported failures and revalidate both datasets. + code. If it raises ``DatasetValidationError``, use its task, path, and source + location diagnostics to repair the files, then call it again. Do not return until + the suite passes validation. If ``validation_feedback`` is provided, the caller's + mandatory validation found errors in the previous attempt; fix every reported + failure and revalidate the suite. **Metric quality** @@ -197,8 +175,9 @@ async def augment_dataset( objects for Y, so X is missing from its context. Measure whether the agent retrieves all required objects, not merely whether X appears in the final answer. - Return an ``EvalAuthorResult(train_dataset=..., validation_dataset=..., summary=...)`` - with the same dataset objects and a summary of what was added. + Return a concise summary naming the new metric key(s), what they measure, and + which runtime evidence they score. The caller retains the materialized suite and + the user's unchanged train and validation datasets. """ # noqa: D413 ... @@ -291,8 +270,8 @@ async def _run( insight: The Insight to investigate with relevant traces. agent_path: Agent root, relative to ``experiment_dir`` or absolute. task_template: Parsed evaluator task containing explicit placeholders. - train_dataset: The train dataset to augment. - validation_dataset: The validation dataset to augment. + train_dataset: The train dataset, returned unchanged. + validation_dataset: The validation dataset, returned unchanged. client: Existing NeMo Platform client used for Intake requests. """ resolved_agent = self.experiment_dir / agent_path @@ -358,39 +337,41 @@ async def _run( diagnostics.append((ref, result)) analysis_statuses[task.id] = ("completed", None) insight_suite.record_analysis(analysis_statuses) - insight_suite_ref = await insight_suite.publish_fileset(client, insight.workspace) - self.context["dataset_documentation"] = doc(type(train_dataset), inline_depth=1) - runner_conventions = await self.discover_runner(train_dataset) - result = await self.augment_dataset( + self.context["dataset_documentation"] = doc(type(materialized_dataset), inline_depth=1) + runner_conventions = await self.discover_runner(materialized_dataset) + summary = await self.author_insight_metrics( insight, diagnostics, - train_dataset, - validation_dataset, + materialized_dataset, runner_conventions, ) for repair_attempt in range(self._config.max_validation_repair_attempts + 1): try: - await _validate_eval_author_result(result) + await materialized_dataset.validate() except DatasetValidationError as exc: if repair_attempt >= self._config.max_validation_repair_attempts: raise logger.warning( - "Eval Author dataset validation failed; requesting repair attempt %d/%d: %s", + "Eval Author Insight metric validation failed; requesting repair attempt %d/%d: %s", repair_attempt + 1, self._config.max_validation_repair_attempts, exc, ) - result = await self.augment_dataset( + summary = await self.author_insight_metrics( insight, diagnostics, - result.train_dataset, - result.validation_dataset, + materialized_dataset, runner_conventions, validation_feedback=str(exc), ) else: - return result.model_copy(update={"insight_suite": insight_suite_ref}) + return EvalAuthorResult( + train_dataset=train_dataset, + validation_dataset=validation_dataset, + insight_suite=materialized_dataset, + summary=summary, + ) raise AssertionError("unreachable") diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py index af7211324e..96219b5cd7 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py @@ -17,12 +17,7 @@ import tomlkit from harbor.models.task.task import Task as HarborTask from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset -from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import ( - DatasetRef, - Task, - local_path_from_uri, -) -from nemo_platform import AsyncNeMoPlatform +from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import Task, local_path_from_uri _MANIFEST_SCHEMA_VERSION = 1 _SLUG_RE = re.compile(r"[^a-z0-9]+") @@ -49,7 +44,7 @@ class StagedInsightTask: class InsightSuite: - """Build and publish one persisted Harbor dataset for an Insight.""" + """Build one experiment-local persisted Harbor dataset for an Insight.""" def __init__(self, *, experiment_dir: Path, insight_id: str, task_template: Task) -> None: """Initialize deterministic paths and template provenance for a suite.""" @@ -95,7 +90,7 @@ def stage(self, trace_refs: list[str]) -> list[StagedInsightTask]: return staged def discard(self) -> None: - """Remove an unpublished candidate suite and reset its staging state.""" + """Remove an unpromoted candidate suite and reset its staging state.""" if self._candidate_root is not None and self._candidate_root.exists(): shutil.rmtree(self._candidate_root) self._candidate_root = None @@ -205,51 +200,3 @@ def record_analysis(self, statuses: dict[str, tuple[str, str | None]]) -> None: pending_path = manifest_path.with_suffix(".json.pending") pending_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") os.replace(pending_path, manifest_path) - - async def publish_fileset(self, client: AsyncNeMoPlatform, workspace: str) -> DatasetRef: - """Upload the complete local suite to a fresh NeMo Platform Fileset.""" - fileset_name = ( - f"nemo-experimentalist-insight-{_slug(self.insight_id, fallback='insight', max_length=80)}-{uuid4().hex}" - ) - fileset = await client.files.filesets.create( - workspace=workspace, - name=fileset_name, - description="Eval Author-built Harbor tasks materialized from Insight production traces.", - purpose="dataset", - ) - try: - await client.files.upload( - local_path=f"{self.suite_dir}{os.sep}", - fileset=fileset.name, - workspace=workspace, - ) - local_files = { - path.relative_to(self.suite_dir).as_posix(): path.stat().st_size - for path in self.suite_dir.rglob("*") - if path.is_file() - } - uploaded = await client.files.list(fileset=fileset.name, workspace=workspace) - remote_files = {file.path: file.size for file in uploaded.data} - if remote_files != local_files: - raise RuntimeError( - f"Uploaded Insight suite Fileset {fileset.name!r} does not match the validated local suite" - ) - except BaseException as exc: - try: - await client.files.filesets.delete(fileset.name, workspace=workspace) - except BaseException as cleanup_exc: - cleanup_exc.add_note(f"Fileset publication also failed before cleanup: {exc!r}") - raise cleanup_exc from exc - raise - - return DatasetRef( - uri=f"fileset://{workspace}/{fileset.name}", - description="Eval Author-built Harbor tasks materialized from Insight production traces.", - metadata={ - "id": f"insight-{_digest(self.insight_id, 12)}", - "insight_id": self.insight_id, - "fileset_id": fileset.id, - "fileset_name": fileset.name, - "workspace": workspace, - }, - ) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py index a158481ffa..6dffe420e0 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py @@ -3,7 +3,7 @@ """Shared models for the top-level Eval Author.""" -from nemo_experimentalist_plugin.experimentalist.components.evaluator import Dataset, DatasetRef +from nemo_experimentalist_plugin.experimentalist.components.evaluator import Dataset from pydantic import BaseModel, ConfigDict, Field @@ -22,7 +22,7 @@ class EvalAuthorConfig(BaseModel): default=5, ge=0, le=10, - description="Max Eval Author repair attempts after mandatory dataset validation fails.", + description="Max Eval Author repair attempts after mandatory Insight verifier validation fails.", ) @@ -33,8 +33,8 @@ class EvalAuthorResult(BaseModel): train_dataset: Dataset validation_dataset: Dataset - insight_suite: DatasetRef | None = Field( + insight_suite: Dataset | None = Field( default=None, - description="NeMo Platform Fileset reference to tasks materialized from the Insight's production traces.", + description="Materialized Insight dataset for immediate use by the optimization loop.", ) summary: str diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py index e6cc0b5f2d..ba34a072c9 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py @@ -392,6 +392,7 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: train_dataset_ref = deps.train_dataset validation_dataset_ref = deps.validation_dataset task_template_ref = deps.task_template + insight_eval_dataset: Dataset | None = None if deps.insight is not None: if task_template_ref is None: raise ValueError("Task template is required for insight trace analysis") @@ -466,6 +467,7 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: ) train_eval_dataset = eval_author_result.train_dataset validation_eval_dataset = eval_author_result.validation_dataset + insight_eval_dataset = eval_author_result.insight_suite else: # Mode 2: local agent directory as baseline, no insight required. insight = None @@ -547,6 +549,21 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: run_id = run_entity.id or "" + if insight_eval_dataset is not None: + try: + await self._evaluate_and_persist_insight_candidates( + dataset=insight_eval_dataset, + evaluator=evaluator, + candidates=candidates, + workspace=workspace, + backend=backend, + run_id=run_entity.id or "", + ) + except Exception: + run_entity.status = "failed" + await backend.update_run(workspace=workspace, run=run_entity) + raise + # ---- Initial goal tree (idempotent) ------------------------------ await self._generate_initial_goal_tree( dataset=train_eval_dataset, @@ -678,6 +695,15 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: backend=backend, run_id=run_id, ) + if insight_eval_dataset is not None: + await self._evaluate_and_persist_insight_candidates( + dataset=insight_eval_dataset, + evaluator=evaluator, + candidates=new_candidates, + workspace=workspace, + backend=backend, + run_id=run_id, + ) for c in new_candidates: evolution_tree.add(c) @@ -947,7 +973,7 @@ def _delete_all_artifacts(self, from_round: int) -> None: continue if (meta.get("round") or 0) > from_round: shutil.rmtree(agent_dir) - for suffix in ("-train", "-validation"): + for suffix in ("-train", "-validation", "-insight"): rd = results_dir / f"{agent_dir.name}{suffix}" if rd.exists(): shutil.rmtree(rd) @@ -1274,6 +1300,59 @@ async def _evaluate_validation_candidates( if candidate_result is not None } + async def _evaluate_insight_candidates( + self, + *, + dataset: Dataset, + evaluator: Evaluator, + candidates: list[Candidate], + ) -> dict[str, EvaluationResult]: + """Evaluate candidates that do not yet have metrics for this Insight suite.""" + if not list(dataset.list_tasks()): + return {} + pending = [candidate for candidate in candidates if candidate.insight_reward is None] + evaluated = await asyncio.gather( + *[self._evaluate_agent(candidate, dataset, evaluator) for candidate in pending] + ) + return {candidate.label: result for candidate, result in evaluated} + + async def _evaluate_and_persist_insight_candidates( + self, + *, + dataset: Dataset, + evaluator: Evaluator, + candidates: list[Candidate], + workspace: str, + backend: ExperimentalistBackend, + run_id: str, + ) -> None: + """Evaluate and persist Insight-suite metrics for the supplied candidates.""" + results = await self._evaluate_insight_candidates( + dataset=dataset, + evaluator=evaluator, + candidates=candidates, + ) + for candidate in candidates: + result = results.get(candidate.label) + if result is None: + continue + await backend.persist_evaluation( + workspace=workspace, + result=result, + candidate=candidate, + split="insight", + ) + await self._update_candidate( + candidate, + updates={ + "insight_reward": result.aggregate_metrics, + "insight_reward_details": result.trials, + }, + workspace=workspace, + backend=backend, + run_id=run_id, + ) + async def _generate_initial_goal_tree( self, *, diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py index f0c2409116..6e39a6d0ce 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) -SPLITS: tuple[str, str] = ("train", "validation") +SPLITS: tuple[str, ...] = ("train", "validation", "insight") _NAME_RE = re.compile(r"[^a-z0-9-]+") @@ -94,10 +94,15 @@ def experiment_metadata(candidate: Candidate, split: str) -> dict[str, str]: def _split_reward(candidate: Candidate, split: str) -> Any: - """The candidate's reward object for *split* — an explicit lookup over the two known - split fields (``train_reward``/``validation_reward``) rather than a dynamic attribute - read. Used only as a presence check: the reward value itself is never projected.""" - return {"train": candidate.train_reward, "validation": candidate.validation_reward}[split] + """The candidate's reward object for *split* — an explicit lookup over the known + split fields (``train_reward``/``validation_reward``/``insight_reward``) rather + than a dynamic attribute read. Used only as a presence check: the reward value + itself is never projected.""" + return { + "train": candidate.train_reward, + "validation": candidate.validation_reward, + "insight": candidate.insight_reward, + }[split] class ExperimentMirror: diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_eval_author_repair_e2e.py b/plugins/nemo-experimentalist/tests/experimentalist/test_eval_author_repair_e2e.py new file mode 100644 index 0000000000..6dd3a23b71 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_eval_author_repair_e2e.py @@ -0,0 +1,375 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Real-model canaries for Eval Author-authored Harbor verifiers.""" + +import asyncio +import json +import os +from pathlib import Path + +import pytest +from nemo_experimentalist_plugin.eval_author.agent import EvalAuthor +from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig +from nemo_experimentalist_plugin.experimentalist.components.evaluator import ( + DatasetValidationError, + local_path_from_uri, +) +from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import ( + HarborDataset, + HarborEvaluator, + HarborEvaluatorConfig, +) +from nemo_experimentalist_plugin.experimentalist.components.model_config import get_fast_model +from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import Diagnostic +from nemo_insights_plugin.entities import Insight + +_HAS_EXPERIMENTALIST_LLM = bool( + os.environ.get("EXPERIMENTALIST_API_BASE") and os.environ.get("EXPERIMENTALIST_API_KEY") +) +_RUN_EVAL_AUTHOR_REPAIR_E2E = os.environ.get("RUN_EVAL_AUTHOR_REPAIR_E2E") == "1" +_RUN_EVAL_AUTHOR_HARBOR_E2E = os.environ.get("RUN_EVAL_AUTHOR_HARBOR_E2E") == "1" +_MALFORMED_VERIFIER = """\ +def check_tool_hallucination() -> float: + try: + return 1.0 +""" +_KNOWN_FAILING_TRACE = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "name": "generate_response", + "attributes": [ + { + "key": "openinference.span.kind", + "value": {"stringValue": "CHAIN"}, + }, + { + "key": "output.value", + "value": { + "stringValue": ( + "The Denver warehouse has eight units available. " + "I answered from memory without consulting inventory." + ) + }, + }, + ], + } + ] + } + ] + } + ] +} + + +def _write_malformed_task(dataset_dir: Path, task_id: str) -> None: + task_dir = dataset_dir / task_id + verifier_dir = task_dir / "tests" + verifier_dir.mkdir(parents=True) + (task_dir / "task.toml").write_text(f'[task]\nname = "local/{task_id}"\n', encoding="utf-8") + (verifier_dir / "test.sh").write_text( + "#!/usr/bin/env bash\npython /tests/check_tool_hallucination.py\n", + encoding="utf-8", + ) + (verifier_dir / "check_tool_hallucination.py").write_text(_MALFORMED_VERIFIER, encoding="utf-8") + + +def _write_known_failing_task(dataset_dir: Path) -> None: + task_dir = dataset_dir / "known-failing-inventory-lookup" + tests_dir = task_dir / "tests" + environment_dir = task_dir / "environment" + tests_dir.mkdir(parents=True) + environment_dir.mkdir() + (task_dir / "task.toml").write_text( + """\ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +version = "1.0" + +[task] +name = "local/known-failing-inventory-lookup" + +[verifier] +timeout_sec = 60.0 + +[agent] +timeout_sec = 60.0 + +[environment] +build_timeout_sec = 300.0 +cpus = 1 +memory_mb = 512 +storage_mb = 1024 +gpus = 0 +network_mode = "no-network" +mcp_servers = [] + +[verifier.env] + +[solution.env] +""", + encoding="utf-8", + ) + (task_dir / "instruction.md").write_text( + "Use the inventory_lookup tool before reporting how many units are available in the Denver warehouse.\n", + encoding="utf-8", + ) + (environment_dir / "Dockerfile").write_text( + """\ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +FROM python:3.12-slim +WORKDIR /app +""", + encoding="utf-8", + ) + (tests_dir / "test.sh").write_text( + """\ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +mkdir -p /logs/verifier +printf '{"reward": 1.0}\n' > /logs/verifier/reward.json +""", + encoding="utf-8", + ) + + +def _write_known_failing_agent(agent_dir: Path) -> None: + agent_dir.mkdir() + trace_payload = json.dumps(_KNOWN_FAILING_TRACE, separators=(",", ":")) + (agent_dir / "harbor_wrapper.py").write_text( + f"""\ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import shlex + +from harbor import AgentContext, BaseAgent, BaseEnvironment + + +class WrappedAgent(BaseAgent): + @staticmethod + def name() -> str: + return "known-failing-baseline" + + def version(self) -> str: + return "1.0.0" + + async def setup(self, environment: BaseEnvironment) -> None: + pass + + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + trace_payload = {trace_payload!r} + command = ( + "mkdir -p /logs/artifacts/traces && " + f"printf '%s\\\\n' {{shlex.quote(trace_payload)}} " + "> /logs/artifacts/traces/trace.jsonl" + ) + process = await environment.exec(command) + context.metadata = {{ + "instruction": instruction, + "stdout": process.stdout, + "stderr": process.stderr, + "returncode": process.return_code, + }} + if process.return_code != 0: + raise RuntimeError(process.stderr or process.stdout) +""", + encoding="utf-8", + ) + + +@pytest.mark.skipif( + not (_RUN_EVAL_AUTHOR_REPAIR_E2E and _HAS_EXPERIMENTALIST_LLM), + reason=( + "Set RUN_EVAL_AUTHOR_REPAIR_E2E=1 with EXPERIMENTALIST_API_BASE and EXPERIMENTALIST_API_KEY to run the Eval Author repair canary." + ), +) +async def test_gpt5_mini_repairs_malformed_harbor_verifiers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """GPT-5 mini repairs try-without-except failures across an Insight suite.""" + insight_suite_dir = tmp_path / "insight-suite" + _write_malformed_task(insight_suite_dir, "task-a") + _write_malformed_task(insight_suite_dir, "task-b") + insight_suite = HarborDataset.from_path(insight_suite_dir) + + with pytest.raises(DatasetValidationError) as exc_info: + await insight_suite.validate() + + validation_feedback = str(exc_info.value) + assert "task 'task-a'" in validation_feedback + assert "task 'task-b'" in validation_feedback + assert "SyntaxError: expected 'except' or 'finally' block" in validation_feedback + + llm = get_fast_model() + monkeypatch.delenv("EXPERIMENTALIST_API_KEY") + eval_author = EvalAuthor( + experiment_dir=tmp_path, + config=EvalAuthorConfig(), + llm=llm, + ) + insight = Insight( + workspace="local", + title="Harbor verifier contains invalid Python syntax", + description=( + "The shared check_tool_hallucination.py verifier contains a try statement without an except or finally " + "clause. Repair every reported syntax error without changing the intended successful score of 1.0." + ), + agent="eval-author-repair-canary", + ) + runner_conventions = ( + "This is a Harbor dataset. Verifier files live under each task's tests directory. " + "Python verifier files are statically checked by await dataset.validate(); test.sh is checked as Bash. " + "Preserve the existing verifier's intended behavior and repair every validation error." + ) + + summary = await asyncio.wait_for( + eval_author.author_insight_metrics( + insight, + [], + insight_suite, + runner_conventions, + validation_feedback=validation_feedback, + ), + timeout=300, + ) + + assert summary + await insight_suite.validate() + for verifier_path in ( + insight_suite_dir / "task-a" / "tests" / "check_tool_hallucination.py", + insight_suite_dir / "task-b" / "tests" / "check_tool_hallucination.py", + ): + repaired_source = verifier_path.read_text(encoding="utf-8") + assert "def check_tool_hallucination" in repaired_source + assert "return 1.0" in repaired_source + + +@pytest.mark.skipif( + not (_RUN_EVAL_AUTHOR_HARBOR_E2E and _HAS_EXPERIMENTALIST_LLM), + reason=( + "Set RUN_EVAL_AUTHOR_HARBOR_E2E=1 with EXPERIMENTALIST_API_BASE and EXPERIMENTALIST_API_KEY " + "to run the live Harbor metric canary." + ), +) +async def test_eval_author_metric_scores_known_failing_harbor_baseline_low( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An authored root-cause metric scores a known-failing Harbor baseline low.""" + insight_suite_dir = tmp_path / "insight-suite" + agent_dir = tmp_path / "known-failing-agent" + _write_known_failing_task(insight_suite_dir) + _write_known_failing_agent(agent_dir) + insight_suite = HarborDataset.from_path(insight_suite_dir) + await insight_suite.validate() + + llm = get_fast_model() + llm.config["temperature"] = 0.0 + monkeypatch.delenv("EXPERIMENTALIST_API_KEY") + eval_author = EvalAuthor( + experiment_dir=tmp_path, + config=EvalAuthorConfig(), + llm=llm, + ) + eval_author.context.pop("trace_documentation", None) + insight = Insight( + workspace="local", + title="Agent skips the required inventory lookup", + description=( + "The agent reports warehouse availability from memory instead of calling the required " + "inventory_lookup tool. The metric must measure whether the current Harbor execution " + "contains that required tool call, not whether the final answer happens to contain a quantity." + ), + agent="known-failing-baseline", + ) + diagnostic = Diagnostic( + outcome="FAILURE", + summary=( + "The execution trace contains only response generation. It has no tool span and no evidence " + "that live inventory was retrieved before the answer." + ), + failure_point=1, + root_cause="The agent answered from memory without calling the required inventory_lookup tool.", + ) + runner_conventions = ( + "This is a Harbor dataset. Preserve the existing reward metric and add a new numeric root-cause metric. " + "Verifier files live under each task's tests directory. After the agent runs, OTLP JSONL trace files are " + "available under /logs/artifacts/traces. Every metric must be written as a numeric entry in " + "/logs/verifier/reward.json, where higher is better and values are bounded to [0.0, 1.0]. Make the minimal " + "verifier-only edit. Missing tool evidence is the expected failing case: it must score 0.0 while still " + "writing reward.json and exiting successfully. Do not use an unguarded grep pipeline whose no-match status " + "can abort a set -e script; prefer a small Python standard-library checker. Call await " + "insight_suite.validate() once after editing, then return the metric summary as soon as validation passes; " + "do not inspect unrelated files." + ) + + summary = await asyncio.wait_for( + eval_author.author_insight_metrics( + insight, + [("known-failing-trace", diagnostic)], + insight_suite, + runner_conventions, + ), + timeout=600, + ) + + assert summary + await insight_suite.validate() + evaluator = HarborEvaluator(experiment_dir=tmp_path) + result = await asyncio.wait_for( + evaluator.run( + agent=agent_dir, + dataset=insight_suite, + options=HarborEvaluatorConfig( + force_rerun=True, + job_name="known-failing-insight-metric", + jobs_dir=Path("harbor-jobs"), + n_concurrent_trials=1, + quiet=True, + ), + ), + timeout=600, + ) + + assert len(result.trials) == 1 + trial = result.trials[0] + assert trial.status == "completed", trial.error + reward_ref = trial.resources["log:verifier/reward.json"] + reward_path = local_path_from_uri(reward_ref.uri, context="Harbor verifier reward") + reward_payload = json.loads(reward_path.read_text(encoding="utf-8")) + assert reward_payload["reward"] == pytest.approx(1.0) + + insight_metric_names = set(reward_payload) - {"reward"} + assert insight_metric_names + insight_metric_values = { + name: float(reward_payload[name]) + for name in insight_metric_names + if isinstance(reward_payload[name], int | float) and not isinstance(reward_payload[name], bool) + } + assert set(insight_metric_values) == insight_metric_names + assert all(0.0 <= value <= 1.0 for value in insight_metric_values.values()) + assert min(insight_metric_values.values()) <= 0.25 + assert insight_metric_names <= set(trial.metrics) + print( + json.dumps( + { + "authored_summary": summary, + "reward_json": reward_payload, + }, + sort_keys=True, + ) + ) diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py new file mode 100644 index 0000000000..83b016fac6 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py @@ -0,0 +1,263 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock + +import pytest +from nemo_experimentalist_plugin.entities import Candidate +from nemo_experimentalist_plugin.experimentalist.components import loop as loop_module +from nemo_experimentalist_plugin.experimentalist.components.evaluator import Dataset, EvaluationResult, Task +from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef +from nemo_experimentalist_plugin.experimentalist.components.loop import EvolutionaryOptimizer +from nemo_experimentalist_plugin.resolve import EvolutionaryOptimizerConfig + + +class _StopAfterOneRound(Exception): + pass + + +@pytest.mark.asyncio +async def test_insight_run_evaluates_and_persists_baseline_and_new_candidate_metrics( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + train_dataset = Dataset(id="train") + validation_dataset = Dataset(id="validation") + insight_dataset = Dataset(id="insight-suite", tasks=[Task(id="insight-task")]) + datasets = { + "train": train_dataset, + "validation": validation_dataset, + } + + class RecordingDatasetFactory: + def build_dataset(self, evaluator_type: str, ref: DatasetRef) -> Dataset: + return datasets[ref.uri] + + def build_task_template(self, evaluator_type: str, ref: DatasetRef) -> Task: + return Task(id="template", uri=ref.uri) + + class ReturningEvalAuthor: + def __init__(self, **kwargs: object) -> None: + pass + + async def run(self, **kwargs: Any) -> SimpleNamespace: + return SimpleNamespace( + train_dataset=kwargs["train_dataset"], + validation_dataset=kwargs["validation_dataset"], + insight_suite=insight_dataset, + ) + + baseline = Candidate(run_id="run-1", label="agent-0", round=0, optimization="baseline") + new_candidate = Candidate( + run_id="run-1", + label="agent-1", + ancestor="agent-0", + round=1, + optimization="use the required tool", + ) + insight_results = { + "agent-0": EvaluationResult( + id="agent-0-insight", + aggregate_metrics={"uses_required_tool": 0.0}, + ), + "agent-1": EvaluationResult( + id="agent-1-insight", + aggregate_metrics={"uses_required_tool": 1.0}, + ), + } + insight_evaluations: list[tuple[Dataset, list[Candidate]]] = [] + + async def evaluate_insight_candidates( + self: EvolutionaryOptimizer, + *, + dataset: Dataset, + evaluator: object, + candidates: list[Candidate], + ) -> dict[str, EvaluationResult]: + insight_evaluations.append((dataset, candidates)) + return {candidate.label: insight_results[candidate.label] for candidate in candidates} + + async def evaluate_validation_candidates( + self: EvolutionaryOptimizer, + *, + candidates: list[Candidate], + **kwargs: object, + ) -> dict[str, EvaluationResult]: + return { + candidate.label: EvaluationResult( + id=f"{candidate.label}-validation", + aggregate_metrics={"reward": 0.5}, + ) + for candidate in candidates + if candidate.validation_reward is None + } + + async def update_candidate( + self: EvolutionaryOptimizer, + candidate: Candidate, + *, + updates: dict[str, object] | None = None, + **kwargs: object, + ) -> None: + for key, value in (updates or {}).items(): + setattr(candidate, key, value) + + run_entity = SimpleNamespace(id="run-1", status="running", rounds_completed=0) + backend = SimpleNamespace( + client=object(), + get_insight=AsyncMock(return_value=SimpleNamespace(agent="agent-source")), + get_agent_code=AsyncMock(), + persist_evaluation=AsyncMock(), + update_run=AsyncMock(), + ) + evolution_tree = SimpleNamespace(survivors=lambda round_num: [baseline], add=lambda candidate: None) + + class StopAfterOneRoundTerminator: + calls = 0 + + async def run(self, **kwargs: object) -> SimpleNamespace: + self.calls += 1 + if self.calls > 1: + raise _StopAfterOneRound + return SimpleNamespace(stop=False, reason="continue") + + monkeypatch.setattr(loop_module, "DatasetFactory", RecordingDatasetFactory) + monkeypatch.setattr( + loop_module, + "EvaluatorFactory", + lambda: SimpleNamespace(build_evaluator=lambda *args, **kwargs: object()), + ) + monkeypatch.setattr(loop_module, "EvalAuthor", ReturningEvalAuthor) + monkeypatch.setattr( + loop_module, + "stage_eval_author_inputs", + AsyncMock(side_effect=lambda _, **refs: SimpleNamespace(**refs)), + ) + monkeypatch.setattr(loop_module.EvolutionTree, "from_dir", lambda path: evolution_tree) + monkeypatch.setattr(EvolutionaryOptimizer, "_detect_last_round", lambda self: None) + monkeypatch.setattr(EvolutionaryOptimizer, "_create_experiment_run", AsyncMock(return_value=run_entity)) + monkeypatch.setattr(EvolutionaryOptimizer, "_create_baseline_agent", AsyncMock(return_value=baseline)) + monkeypatch.setattr(EvolutionaryOptimizer, "_update_candidate", update_candidate) + monkeypatch.setattr( + EvolutionaryOptimizer, + "_evaluate_validation_candidates", + evaluate_validation_candidates, + ) + monkeypatch.setattr( + EvolutionaryOptimizer, + "_evaluate_insight_candidates", + evaluate_insight_candidates, + raising=False, + ) + monkeypatch.setattr(EvolutionaryOptimizer, "_generate_initial_goal_tree", AsyncMock()) + monkeypatch.setattr( + EvolutionaryOptimizer, + "_evaluate_train_candidates", + AsyncMock( + return_value={ + "agent-0": EvaluationResult(id="agent-0-train", aggregate_metrics={"reward": 0.5}), + } + ), + ) + monkeypatch.setattr(EvolutionaryOptimizer, "_analyze_round", AsyncMock(return_value="round analysis")) + monkeypatch.setattr(EvolutionaryOptimizer, "_update_goal_tree", AsyncMock()) + monkeypatch.setattr(EvolutionaryOptimizer, "_propose_improvements", AsyncMock(return_value=[object()])) + monkeypatch.setattr(EvolutionaryOptimizer, "_create_agent", lambda self, **kwargs: new_candidate) + monkeypatch.setattr( + EvolutionaryOptimizer, + "_implement_candidates", + AsyncMock(side_effect=lambda **kwargs: kwargs["candidates"]), + ) + + config = EvolutionaryOptimizerConfig(disable_trajectory_scoring=True) + optimizer = object.__new__(EvolutionaryOptimizer) + optimizer.working_dir = tmp_path + optimizer.config = config + optimizer.shell = SimpleNamespace(close=AsyncMock()) + optimizer.terminator = StopAfterOneRoundTerminator() + deps = SimpleNamespace( + backend=backend, + workspace="default", + config=config, + evaluator_type="harbor", + train_dataset=DatasetRef(uri="train"), + validation_dataset=DatasetRef(uri="validation"), + task_template=DatasetRef(uri="template"), + insight="insight-1", + agent=None, + agent_spec=None, + ) + + with pytest.raises(_StopAfterOneRound): + await optimizer.run(deps) + + assert insight_evaluations == [ + (insight_dataset, [baseline]), + (insight_dataset, [new_candidate]), + ] + assert baseline.insight_reward == {"uses_required_tool": 0.0} + assert new_candidate.insight_reward == {"uses_required_tool": 1.0} + insight_persistence = [ + call.kwargs for call in backend.persist_evaluation.await_args_list if call.kwargs["split"] == "insight" + ] + assert insight_persistence == [ + { + "workspace": "default", + "result": insight_results["agent-0"], + "candidate": baseline, + "split": "insight", + }, + { + "workspace": "default", + "result": insight_results["agent-1"], + "candidate": new_candidate, + "split": "insight", + }, + ] + + +@pytest.mark.asyncio +async def test_insight_evaluation_skips_cached_candidates_and_empty_suites( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cached = Candidate( + run_id="run-1", + label="agent-0", + round=0, + optimization="baseline", + insight_reward={"uses_required_tool": 0.0}, + ) + pending = Candidate( + run_id="run-1", + label="agent-1", + round=1, + optimization="use the required tool", + ) + result = EvaluationResult( + id="agent-1-insight", + aggregate_metrics={"uses_required_tool": 1.0}, + ) + evaluate_agent = AsyncMock(return_value=(pending, result)) + monkeypatch.setattr(EvolutionaryOptimizer, "_evaluate_agent", evaluate_agent) + optimizer = object.__new__(EvolutionaryOptimizer) + + evaluated = await optimizer._evaluate_insight_candidates( + dataset=Dataset(id="insight-suite", tasks=[Task(id="insight-task")]), + evaluator=object(), # type: ignore[arg-type] + candidates=[cached, pending], + ) + + assert evaluated == {"agent-1": result} + assert evaluate_agent.await_args is not None + assert evaluate_agent.await_args.args[0] is pending + + empty = await optimizer._evaluate_insight_candidates( + dataset=Dataset(id="empty-insight-suite"), + evaluator=object(), # type: ignore[arg-type] + candidates=[pending], + ) + assert empty == {} + assert evaluate_agent.await_count == 1 diff --git a/plugins/nemo-experimentalist/tests/test_eval_author_agent.py b/plugins/nemo-experimentalist/tests/test_eval_author_agent.py index 52a758e52b..2079ad688b 100644 --- a/plugins/nemo-experimentalist/tests/test_eval_author_agent.py +++ b/plugins/nemo-experimentalist/tests/test_eval_author_agent.py @@ -12,11 +12,10 @@ import pytest from nemo_experimentalist_plugin.eval_author import agent as eval_author_module -from nemo_experimentalist_plugin.eval_author.agent import EvalAuthor, EvalAuthorDatasetValidationError -from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult +from nemo_experimentalist_plugin.eval_author.agent import EvalAuthor +from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig from nemo_experimentalist_plugin.experimentalist.components.evaluator import ( Dataset, - DatasetRef, DatasetValidationError, Task, TrialResult, @@ -27,7 +26,6 @@ TraceAnalyzerConfig, ) from nemo_insights_plugin.entities import Insight -from nemo_platform import AsyncNeMoPlatform @dataclass @@ -60,9 +58,8 @@ class _PipelineCalls: fill_task_template: list[_FillTaskTemplateCall] analyzer_init: list[_AnalyzerInitCall] analyzer_run: list[_AnalyzerRunCall] - fileset_publications: list[tuple[AsyncNeMoPlatform, str]] discovered_datasets: list[Dataset] - augment_args: list[tuple[Insight, list[tuple[str, Diagnostic]], Dataset, Dataset, str, str | None]] + author_args: list[tuple[Insight, list[tuple[str, Diagnostic]], Dataset, str, str | None]] suite_discards: int @@ -115,28 +112,30 @@ def _prompt(method: Any) -> str: return " ".join(prompt.split()) -def test_eval_author_prompts_retain_dataset_inspection_and_suite_wide_guidance() -> None: +def test_eval_author_prompts_scope_metrics_to_materialized_insight_suite() -> None: discover_prompt = _prompt(EvalAuthor.discover_runner) - augment_prompt = _prompt(EvalAuthor.augment_dataset) + author_prompt = _prompt(EvalAuthor.author_insight_metrics) assert "read it first" in discover_prompt assert "inspect the actual files" in discover_prompt - assert "authoritative reference for what artifacts exist at evaluation runtime" in augment_prompt - assert "how tasks are structured, and how to add metrics" in augment_prompt - assert "Add the metric to every task in ``train_dataset`` and ``validation_dataset``" in augment_prompt - assert "call ``await train_dataset.validate()``" in augment_prompt - assert "``await validation_dataset.validate()``" in augment_prompt - assert "fix all reported failures and revalidate both datasets" in augment_prompt + assert "authoritative reference for what artifacts exist at evaluation runtime" in author_prompt + assert "how tasks are structured, and how to add metrics" in author_prompt + assert "Add at least one new Insight-specific metric key to every task in ``insight_suite``" in author_prompt + assert "Preserve every existing verifier metric" in author_prompt + assert "Do not modify the user's train or validation datasets" in author_prompt + assert "call ``await insight_suite.validate()``" in author_prompt + assert "fix every reported failure and revalidate the suite" in author_prompt def test_eval_author_prompts_retain_root_cause_and_normalized_scoring_guidance() -> None: - prompt = _prompt(EvalAuthor.augment_dataset) + prompt = _prompt(EvalAuthor.author_insight_metrics) assert "Focus the metric on the root cause, not the surface symptom" in prompt assert "Every metric value must be a float in ``[0.0, 1.0]``" in prompt assert "Error rate → ``max(0.0, 1.0 - errors / total_calls)``" in prompt assert "Presence of a behavior → ``1.0`` if present, ``0.0`` if absent" in prompt assert "Partial credit → fraction of required steps completed correctly" in prompt + assert "Do not hard-code scores for the production traces" in prompt def test_eval_author_prompts_retain_template_path_and_harbor_name_guidance() -> None: @@ -153,14 +152,15 @@ def _install_pipeline( monkeypatch: pytest.MonkeyPatch, outcomes: Sequence[Diagnostic | BaseException], eval_author: EvalAuthor, + *, + materialized_dataset: Dataset | None = None, ) -> _PipelineCalls: calls = _PipelineCalls( fill_task_template=[], analyzer_init=[], analyzer_run=[], - fileset_publications=[], discovered_datasets=[], - augment_args=[], + author_args=[], suite_discards=0, ) next_analyzer = 0 @@ -186,7 +186,11 @@ def validate(self, staged: Any) -> None: def promote_local(self, trace_refs: list[str], staged_tasks: list[Any]) -> Dataset: assert trace_refs == [staged.trace_ref for staged in staged_tasks] - return Dataset(id="insight-suite", tasks=[staged.result for staged in staged_tasks]) + tasks = [staged.result for staged in staged_tasks] + if materialized_dataset is not None: + materialized_dataset.tasks = tasks + return materialized_dataset + return Dataset(id="insight-suite", tasks=tasks) def discard(self) -> None: calls.suite_discards += 1 @@ -194,13 +198,6 @@ def discard(self) -> None: def record_analysis(self, statuses: dict[str, tuple[str, str | None]]) -> None: pass - async def publish_fileset(self, client: AsyncNeMoPlatform, workspace: str) -> DatasetRef: - calls.fileset_publications.append((client, workspace)) - return DatasetRef( - uri=f"fileset://{workspace}/insight-suite", - metadata={"id": "insight-suite", "fileset_id": "fileset-1"}, - ) - class FillTaskTemplate: async def __call__( self, @@ -258,35 +255,29 @@ async def __call__(self, dataset: Dataset) -> str: calls.discovered_datasets.append(dataset) return "runner conventions" - class AugmentDataset: + class AuthorInsightMetrics: async def __call__( self, insight: Insight, diagnostics: list[tuple[str, Diagnostic]], - train_dataset: Dataset, - validation_dataset: Dataset, + insight_suite: Dataset, runner_conventions: str, validation_feedback: str | None = None, - ) -> EvalAuthorResult: - calls.augment_args.append( + ) -> str: + calls.author_args.append( ( insight, diagnostics, - train_dataset, - validation_dataset, + insight_suite, runner_conventions, validation_feedback, ) ) - return EvalAuthorResult( - train_dataset=train_dataset, - validation_dataset=validation_dataset, - summary="augmented", - ) + return "authored insight metrics" - eval_author.fill_task_template = FillTaskTemplate() # type: ignore[method-assign,assignment] - eval_author.discover_runner = DiscoverRunner() # type: ignore[method-assign,assignment] - eval_author.augment_dataset = AugmentDataset() # type: ignore[method-assign,assignment] + eval_author.fill_task_template = cast(Any, FillTaskTemplate()) + eval_author.discover_runner = cast(Any, DiscoverRunner()) + eval_author.author_insight_metrics = cast(Any, AuthorInsightMetrics()) monkeypatch.setattr(eval_author_module, "TraceAnalyzer", FakeTraceAnalyzer) monkeypatch.setattr(eval_author_module, "InsightSuite", FakeInsightSuite) return calls @@ -369,7 +360,7 @@ class FailedFillTaskTemplate: async def __call__(self, *_: Any) -> Task: raise RuntimeError("fill failed") - eval_author.fill_task_template = FailedFillTaskTemplate() # type: ignore[method-assign,assignment] + eval_author.fill_task_template = cast(Any, FailedFillTaskTemplate()) with pytest.raises(RuntimeError, match="fill failed"): await eval_author.run( @@ -452,10 +443,9 @@ async def test_run_skips_failed_trace_analysis_and_keeps_successes( client=cast(Any, object()), ) - assert result.summary == "augmented" + assert result.summary == "authored insight metrics" assert stored == [successful] - assert calls.augment_args[0][1] == [("trace-good", successful)] - assert len(calls.fileset_publications) == 1 + assert calls.author_args[0][1] == [("trace-good", successful)] assert "Trace analysis failed for trace-bad: analysis failed" in caplog.text @@ -478,11 +468,11 @@ async def test_run_propagates_trace_analysis_cancellation( client=cast(Any, object()), ) - assert calls.augment_args == [] + assert calls.author_args == [] @pytest.mark.asyncio -async def test_run_documents_dataset_discovers_runner_and_forwards_augmentation_arguments( +async def test_run_authors_metrics_on_materialized_insight_suite( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -513,12 +503,7 @@ def fake_doc(dataset_type: type[Dataset], *, inline_depth: int) -> object: client=client, ) - assert result.summary == "augmented" - assert result.insight_suite == DatasetRef( - uri="fileset://workspace-a/insight-suite", - metadata={"id": "insight-suite", "fileset_id": "fileset-1"}, - ) - assert calls.fileset_publications == [(client, "workspace-a")] + assert result.summary == "authored insight metrics" assert len(calls.fill_task_template) == 1 fill_call = calls.fill_task_template[0] assert fill_call.trace_ref == "trace-1" @@ -549,22 +534,27 @@ def fake_doc(dataset_type: type[Dataset], *, inline_depth: int) -> object: assert doc_calls == [(Dataset, 1)] assert eval_author.context["dataset_documentation"] is documentation - assert calls.discovered_datasets == [train_dataset] - assert len(calls.augment_args) == 1 + assert len(calls.discovered_datasets) == 1 + materialized_dataset = calls.discovered_datasets[0] + assert result.insight_suite is materialized_dataset + assert materialized_dataset.id == "insight-suite" + assert materialized_dataset is not train_dataset + assert materialized_dataset is not validation_dataset + assert len(calls.author_args) == 1 ( - augmented_insight, + authored_insight, diagnostics, - augmented_train, - augmented_validation, + authored_suite, runner_conventions, validation_feedback, - ) = calls.augment_args[0] - assert augmented_insight is insight + ) = calls.author_args[0] + assert authored_insight is insight assert diagnostics == [("trace-1", diagnostic)] - assert augmented_train is train_dataset - assert augmented_validation is validation_dataset + assert authored_suite is materialized_dataset assert runner_conventions == "runner conventions" assert validation_feedback is None + assert result.train_dataset is train_dataset + assert result.validation_dataset is validation_dataset class _RepairableDataset(Dataset): @@ -585,29 +575,34 @@ async def test_run_feeds_validation_failures_back_for_one_repair_attempt( monkeypatch: pytest.MonkeyPatch, ) -> None: eval_author = _eval_author(tmp_path) - _install_pipeline(monkeypatch, [_diagnostic("diagnostic")], eval_author) + insight_dataset = _RepairableDataset("insight-suite", "task 'insight-a': check.py:2:1: invalid syntax") + _install_pipeline( + monkeypatch, + [_diagnostic("diagnostic")], + eval_author, + materialized_dataset=insight_dataset, + ) monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None) - train_dataset = _RepairableDataset("train", "task 'train-a': check.py:2:1: invalid syntax") - validation_dataset = _RepairableDataset("validation", "task 'validation-a': test.sh:4: syntax error") + train_dataset = Dataset(id="train") + validation_dataset = Dataset(id="validation") + client = cast(Any, object()) feedback: list[str | None] = [] - class RepairDataset: + class RepairInsightMetrics: async def __call__( self, insight: Insight, diagnostics: list[tuple[str, Diagnostic]], - train: Dataset, - validation: Dataset, + insight_suite: Dataset, runner_conventions: str, validation_feedback: str | None = None, - ) -> EvalAuthorResult: + ) -> str: feedback.append(validation_feedback) if validation_feedback is not None: - train_dataset.error = None - validation_dataset.error = None - return EvalAuthorResult(train_dataset=train, validation_dataset=validation, summary="augmented") + insight_dataset.error = None + return "repaired insight metric" - eval_author.augment_dataset = RepairDataset() # type: ignore[method-assign,assignment] + eval_author.author_insight_metrics = cast(Any, RepairInsightMetrics()) result = await eval_author.run( _insight(["trace-1"]), @@ -615,19 +610,15 @@ async def __call__( Task(id="template"), train_dataset, validation_dataset, - client=cast(Any, object()), + client=client, ) assert result.train_dataset is train_dataset assert result.validation_dataset is validation_dataset assert feedback[0] is None assert feedback[1] is not None - assert "train dataset" in feedback[1] - assert "task 'train-a': check.py:2:1: invalid syntax" in feedback[1] - assert "validation dataset" in feedback[1] - assert "task 'validation-a': test.sh:4: syntax error" in feedback[1] - assert train_dataset.validate_calls == 2 - assert validation_dataset.validate_calls == 2 + assert "task 'insight-a': check.py:2:1: invalid syntax" in feedback[1] + assert insight_dataset.validate_calls == 2 @pytest.mark.asyncio @@ -636,25 +627,30 @@ async def test_run_raises_after_validation_repair_budget_is_exhausted( monkeypatch: pytest.MonkeyPatch, ) -> None: eval_author = _eval_author(tmp_path, max_validation_repair_attempts=1) - calls = _install_pipeline(monkeypatch, [_diagnostic("diagnostic")], eval_author) + insight_dataset = _RepairableDataset("insight-suite", "task 'insight-a': check.py:2:1: invalid syntax") + calls = _install_pipeline( + monkeypatch, + [_diagnostic("diagnostic")], + eval_author, + materialized_dataset=insight_dataset, + ) monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None) - train_dataset = _RepairableDataset("train", "task 'train-a': check.py:2:1: invalid syntax") - with pytest.raises(EvalAuthorDatasetValidationError) as exc_info: + with pytest.raises(DatasetValidationError) as exc_info: await eval_author.run( _insight(["trace-1"]), Path("agent"), Task(id="template"), - train_dataset, + Dataset(id="train"), Dataset(id="validation"), client=cast(Any, object()), ) - assert "task 'train-a': check.py:2:1: invalid syntax" in str(exc_info.value) - assert len(calls.augment_args) == 2 - assert calls.augment_args[0][-1] is None - assert calls.augment_args[1][-1] is not None - assert train_dataset.validate_calls == 2 + assert "task 'insight-a': check.py:2:1: invalid syntax" in str(exc_info.value) + assert len(calls.author_args) == 2 + assert calls.author_args[0][-1] is None + assert calls.author_args[1][-1] is not None + assert insight_dataset.validate_calls == 2 def test_evolutionary_optimizer_uses_top_level_eval_author_config() -> None: diff --git a/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py b/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py index 9cae930292..04184bb920 100644 --- a/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py +++ b/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py @@ -7,83 +7,13 @@ import json import tomllib -from dataclasses import dataclass from pathlib import Path -from typing import Any, cast import pytest from nemo_experimentalist_plugin.eval_author import materialization as materialization_module from nemo_experimentalist_plugin.eval_author.materialization import InsightSuite from nemo_experimentalist_plugin.experimentalist.components.evaluator import Task from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset -from nemo_platform import AsyncNeMoPlatform - - -@dataclass(frozen=True, slots=True) -class _FakeFileset: - id: str - name: str - workspace: str - - -@dataclass(frozen=True, slots=True) -class _FakeRemoteFile: - path: str - size: int - - -@dataclass(frozen=True, slots=True) -class _FakeFileList: - data: list[_FakeRemoteFile] - - -class _FakeFilesets: - def __init__(self, files: _FakeFiles) -> None: - self._files = files - self.created: list[str] = [] - self.deleted: list[str] = [] - - async def create(self, *, workspace: str, name: str, **_: Any) -> _FakeFileset: - fileset = _FakeFileset(id=f"fileset-id-{len(self.created) + 1}", name=name, workspace=workspace) - self.created.append(name) - self._files.contents[name] = {} - return fileset - - async def delete(self, name: str, *, workspace: str) -> _FakeFileset: - self.deleted.append(name) - self._files.contents.pop(name) - return _FakeFileset(id="deleted", name=name, workspace=workspace) - - -class _FakeFiles: - def __init__(self) -> None: - self.contents: dict[str, dict[str, int]] = {} - self.fail_upload = False - self.omit_from_listing: str | None = None - self.filesets = _FakeFilesets(self) - - async def upload(self, *, local_path: str, fileset: str, **_: Any) -> _FakeFileset: - if self.fail_upload: - raise RuntimeError("upload failed") - root = Path(local_path) - self.contents[fileset] = { - path.relative_to(root).as_posix(): path.stat().st_size for path in root.rglob("*") if path.is_file() - } - return _FakeFileset(id="uploaded", name=fileset, workspace="workspace-a") - - async def list(self, *, fileset: str, **_: Any) -> _FakeFileList: - return _FakeFileList( - data=[ - _FakeRemoteFile(path=path, size=size) - for path, size in self.contents[fileset].items() - if path != self.omit_from_listing - ] - ) - - -class _FakeClient: - def __init__(self) -> None: - self.files = _FakeFiles() def _write_template(root: Path) -> Task: @@ -116,7 +46,7 @@ def _write_template(root: Path) -> Task: return Task(id="task-template", uri=root.as_uri()) -def test_insight_suite_publishes_discoverable_tasks_with_provenance(tmp_path: Path) -> None: +def test_insight_suite_materializes_discoverable_tasks_with_provenance(tmp_path: Path) -> None: template = _write_template(tmp_path / "template") refs = ["intake/traces/unsafe ref", "intake/traces/unsafe ref"] suite = InsightSuite(experiment_dir=tmp_path, insight_id="insight/unsafe id", task_template=template) @@ -154,74 +84,7 @@ def test_insight_suite_publishes_discoverable_tasks_with_provenance(tmp_path: Pa assert [task["source_trace_ref"] for task in manifest["tasks"]] == refs -@pytest.mark.asyncio -async def test_insight_suite_uploads_complete_suite_to_fresh_filesets(tmp_path: Path) -> None: - template = _write_template(tmp_path / "template") - suite = InsightSuite(experiment_dir=tmp_path, insight_id="insight/unsafe id", task_template=template) - staged = suite.stage(["trace-1"]) - (staged[0].path / "instruction.md").write_text("Generated instruction.\n", encoding="utf-8") - suite.validate(staged[0]) - suite.promote_local(["trace-1"], staged) - client = _FakeClient() - - first_ref = await suite.publish_fileset(cast(AsyncNeMoPlatform, client), "workspace-a") - second_ref = await suite.publish_fileset(cast(AsyncNeMoPlatform, client), "workspace-a") - - assert first_ref.uri.startswith("fileset://workspace-a/nemo-experimentalist-insight-insight-unsafe-id-") - assert second_ref.uri.startswith("fileset://workspace-a/nemo-experimentalist-insight-insight-unsafe-id-") - assert first_ref.uri != second_ref.uri - assert first_ref.metadata["insight_id"] == "insight/unsafe id" - assert first_ref.metadata["fileset_id"] == "fileset-id-1" - assert first_ref.metadata["workspace"] == "workspace-a" - first_name = cast(str, first_ref.metadata["fileset_name"]) - assert "manifest.json" in client.files.contents[first_name] - assert any(path.endswith("/task.toml") for path in client.files.contents[first_name]) - assert client.files.filesets.deleted == [] - - -@pytest.mark.asyncio -async def test_failed_fileset_upload_removes_only_incomplete_artifact(tmp_path: Path) -> None: - template = _write_template(tmp_path / "template") - suite = InsightSuite(experiment_dir=tmp_path, insight_id="insight-1", task_template=template) - staged = suite.stage(["trace-1"]) - (staged[0].path / "instruction.md").write_text("Generated instruction.\n", encoding="utf-8") - suite.validate(staged[0]) - suite.promote_local(["trace-1"], staged) - client = _FakeClient() - published_ref = await suite.publish_fileset(cast(AsyncNeMoPlatform, client), "workspace-a") - published_name = cast(str, published_ref.metadata["fileset_name"]) - published_contents = dict(client.files.contents[published_name]) - client.files.fail_upload = True - - with pytest.raises(RuntimeError, match="upload failed"): - await suite.publish_fileset(cast(AsyncNeMoPlatform, client), "workspace-a") - - failed_name = client.files.filesets.created[-1] - assert client.files.filesets.deleted == [failed_name] - assert failed_name not in client.files.contents - assert client.files.contents[published_name] == published_contents - - -@pytest.mark.asyncio -async def test_fileset_inventory_mismatch_fails_and_removes_incomplete_artifact(tmp_path: Path) -> None: - template = _write_template(tmp_path / "template") - suite = InsightSuite(experiment_dir=tmp_path, insight_id="insight-1", task_template=template) - staged = suite.stage(["trace-1"]) - (staged[0].path / "instruction.md").write_text("Generated instruction.\n", encoding="utf-8") - suite.validate(staged[0]) - suite.promote_local(["trace-1"], staged) - client = _FakeClient() - client.files.omit_from_listing = "manifest.json" - - with pytest.raises(RuntimeError, match="does not match the validated local suite"): - await suite.publish_fileset(cast(AsyncNeMoPlatform, client), "workspace-a") - - failed_name = client.files.filesets.created[-1] - assert client.files.filesets.deleted == [failed_name] - assert failed_name not in client.files.contents - - -def test_insight_suite_second_publication_replaces_first_at_stable_path(tmp_path: Path) -> None: +def test_insight_suite_second_materialization_replaces_first_at_stable_path(tmp_path: Path) -> None: template = _write_template(tmp_path / "template") refs = ["trace-1"] first_suite = InsightSuite(experiment_dir=tmp_path, insight_id="insight-1", task_template=template) @@ -229,7 +92,7 @@ def test_insight_suite_second_publication_replaces_first_at_stable_path(tmp_path (first_staged[0].path / "instruction.md").write_text("First generated instruction.\n", encoding="utf-8") first_suite.validate(first_staged[0]) first_suite.promote_local(refs, first_staged) - published_path = first_suite.suite_dir + materialized_path = first_suite.suite_dir second_suite = InsightSuite(experiment_dir=tmp_path, insight_id="insight-1", task_template=template) second_staged = second_suite.stage(refs) @@ -237,18 +100,18 @@ def test_insight_suite_second_publication_replaces_first_at_stable_path(tmp_path second_suite.validate(second_staged[0]) second_suite.promote_local(refs, second_staged) - assert second_suite.suite_dir == published_path - task = list(HarborDataset.from_path(published_path).list_tasks())[0] + assert second_suite.suite_dir == materialized_path + task = list(HarborDataset.from_path(materialized_path).list_tasks())[0] task_dir = Path(task.uri.removeprefix("file://")) assert (task_dir / "instruction.md").read_text(encoding="utf-8") == "Second generated instruction.\n" -def test_failed_rebuild_preserves_previous_published_suite(tmp_path: Path) -> None: +def test_failed_rebuild_preserves_previous_materialized_suite(tmp_path: Path) -> None: template = _write_template(tmp_path / "template") refs = ["trace-1"] first_suite = InsightSuite(experiment_dir=tmp_path, insight_id="insight-1", task_template=template) first_staged = first_suite.stage(refs) - (first_staged[0].path / "instruction.md").write_text("Valid published instruction.\n", encoding="utf-8") + (first_staged[0].path / "instruction.md").write_text("Valid materialized instruction.\n", encoding="utf-8") first_suite.validate(first_staged[0]) first_suite.promote_local(refs, first_staged) @@ -261,7 +124,7 @@ def test_failed_rebuild_preserves_previous_published_suite(tmp_path: Path) -> No task = list(HarborDataset.from_path(first_suite.suite_dir).list_tasks())[0] task_dir = Path(task.uri.removeprefix("file://")) - assert (task_dir / "instruction.md").read_text(encoding="utf-8") == "Valid published instruction.\n" + assert (task_dir / "instruction.md").read_text(encoding="utf-8") == "Valid materialized instruction.\n" def test_post_promotion_validation_failure_restores_previous_suite( @@ -272,7 +135,7 @@ def test_post_promotion_validation_failure_restores_previous_suite( refs = ["trace-1"] first_suite = InsightSuite(experiment_dir=tmp_path, insight_id="insight-1", task_template=template) first_staged = first_suite.stage(refs) - (first_staged[0].path / "instruction.md").write_text("Previously published.\n", encoding="utf-8") + (first_staged[0].path / "instruction.md").write_text("Previously materialized.\n", encoding="utf-8") first_suite.validate(first_staged[0]) first_suite.promote_local(refs, first_staged) @@ -291,11 +154,11 @@ def fail_post_promotion_validation(_: Path) -> None: task = list(HarborDataset.from_path(first_suite.suite_dir).list_tasks())[0] task_dir = Path(task.uri.removeprefix("file://")) - assert (task_dir / "instruction.md").read_text(encoding="utf-8") == "Previously published.\n" + assert (task_dir / "instruction.md").read_text(encoding="utf-8") == "Previously materialized.\n" assert list(first_suite.root.glob(".insight-suite-backup-*")) == [] -def test_insight_suite_rejects_empty_instruction_before_publication(tmp_path: Path) -> None: +def test_insight_suite_rejects_empty_instruction_before_materialization(tmp_path: Path) -> None: template = _write_template(tmp_path / "template") suite = InsightSuite(experiment_dir=tmp_path, insight_id="insight-1", task_template=template) staged = suite.stage(["trace-1"]) diff --git a/plugins/nemo-experimentalist/tests/test_experiment_mirror.py b/plugins/nemo-experimentalist/tests/test_experiment_mirror.py index 7d1bc5fec0..0db23f07db 100644 --- a/plugins/nemo-experimentalist/tests/test_experiment_mirror.py +++ b/plugins/nemo-experimentalist/tests/test_experiment_mirror.py @@ -1,13 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from types import SimpleNamespace +from typing import Any, cast from unittest.mock import AsyncMock import httpx import pytest from nemo_experimentalist_plugin.entities import Candidate, ExperimentRun from nemo_experimentalist_plugin.experimentalist.experiment_mirror import ExperimentMirror, group_metadata -from nemo_platform import ConflictError, NotFoundError, omit # VERIFY-2 +from nemo_platform import AsyncNeMoPlatform, ConflictError, NotFoundError, omit # VERIFY-2 pytestmark = pytest.mark.asyncio @@ -19,8 +20,8 @@ ) -def _client(groups: object, experiments: object) -> object: - return SimpleNamespace(experiment_groups=groups, evaluations=experiments) +def _client(groups: object, experiments: object) -> AsyncNeMoPlatform: + return cast(AsyncNeMoPlatform, SimpleNamespace(experiment_groups=groups, evaluations=experiments)) class _StatefulGroups: @@ -52,8 +53,8 @@ async def update(self, path_name: str, **kwargs: object) -> object: return SimpleNamespace(id="grp-1", name=self.name, **self.body) -def _run(**kw) -> ExperimentRun: - base = dict( +def _run(**kw: Any) -> ExperimentRun: + base: dict[str, Any] = dict( workspace="default", agent="a", insight="ins-1", @@ -68,8 +69,8 @@ def _run(**kw) -> ExperimentRun: return run -def _cand(**kw) -> Candidate: - base = dict(run_id="run-1", label="agent-0", round=0, optimization="baseline") +def _cand(**kw: Any) -> Candidate: + base: dict[str, Any] = dict(run_id="run-1", label="agent-0", round=0, optimization="baseline") base.update(kw) return Candidate(**base) @@ -116,6 +117,20 @@ async def test_project_candidate_skips_when_no_reward(): experiments.create.assert_not_awaited() +async def test_project_candidate_creates_insight_experiment_when_evaluated(): + experiments = AsyncMock() + experiments.create.return_value = SimpleNamespace(id="exp-insight") + mirror = ExperimentMirror(_client(AsyncMock(), experiments), workspace="default") + candidate = _cand(insight_reward={"uses_required_tool": 0.5}, insight_reward_details=[]) + + await mirror.project_candidate(candidate) + + kwargs = experiments.create.await_args.kwargs + assert kwargs["name"] == "opt-run-1-agent-0-insight" + assert kwargs["dataset_name"] == "insight" + assert kwargs["metadata"] == {"round": "0", "candidate_id": "agent-0", "split": "insight"} + + async def test_project_candidate_conflict_updates_experiment(): groups = AsyncMock() groups.retrieve.return_value = SimpleNamespace(id="grp-1", name="opt-run-1") From f7c539bb368ef8044d9646311e8b8dc09a5a694c Mon Sep 17 00:00:00 2001 From: Ryan Angilly Date: Mon, 27 Jul 2026 10:16:02 -0600 Subject: [PATCH 2/5] feat: report insight suite optimization metrics Reissues NVIDIA-dev/NeMo-Optimizer#95 in the nemo-platform monorepo. Signed-off-by: Ryan Angilly --- .../experimentalist/components/loop.py | 66 ++++++++++++++--- .../experimentalist_backend.py | 14 +++- .../experimentalist/test_loop_reporting.py | 74 +++++++++++++++++++ .../tests/test_experimentalist_backend.py | 35 +++++++-- 4 files changed, 168 insertions(+), 21 deletions(-) create mode 100644 plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py index ba34a072c9..27c8d85f52 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py @@ -183,8 +183,19 @@ class AnalysisSkill(Skill): | agent-1 | 0.48 | 0.58 | 0.41 | ... | agent-0 | -0.10 | | agent-0 | 0.45 | 0.55 | 0.40 | ... | --- | baseline | + Insight Suite Reward: + | Agent | | | ... | vs. Baseline | + | ----- | -------------- | -------------- | --- | ------------ | + | agent-3 | 0.80 | 0.67 | ... | +0.40 | + | agent-1 | 0.60 | 0.50 | ... | +0.20 | + | agent-0 | 0.40 | 0.33 | ... | baseline | + [Columns are the actual reward dimension keys from metadata. Order by any dimension that - helps comparison — no dimension is privileged.] + helps comparison — no dimension is privileged. Read Insight Suite Reward from + `candidate.insight_reward`. Omit that table when `insight_reward` is absent or empty + for every agent. Keep Insight Suite Reward separate from train and validation rewards: + it reports performance on scenarios authored for the motivating Insight and is not a + ranking or Pareto-selection input.] ## Trajectory Rewards @@ -783,8 +794,16 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: agent_name=agent_name, ) + baseline_entity = next( + (node.candidate for node in evolution_tree.nodes.values() if node.round == 0), + None, + ) result = ExperimentalistResult( - summary=self._render_summary(rounds_completed=round_num, winner=winner_entity), + summary=self._render_summary( + rounds_completed=round_num, + baseline=baseline_entity, + winner=winner_entity, + ), run_id=run_id, rounds_completed=round_num, winner=winner_entity, @@ -849,10 +868,19 @@ async def merge_analysis( ```python rewards = {c.id: self.workspace.get_metadata(c.name).train_reward or {} for c in agent_ids} + insight_rewards = { + c.id: self.workspace.get_metadata(c.name).insight_reward or {} for c in agent_ids + } + all_candidates = [ + self.workspace.get_metadata(agent_id).slim() for agent_id in self.workspace.list_agents() + ] + baseline = next((candidate for candidate in all_candidates if candidate.round == 0), None) ``` - Compare siblings: which optimization strategy worked better this round? - Compare to ancestors: did the change actually fix the targeted root cause? + - When any Insight Suite rewards are present, compare those dimensions to the + round-zero baseline separately from train and validation rewards. ## Step 2: Analyze divergent and complementary patterns @@ -867,13 +895,18 @@ async def merge_analysis( candidate = self.workspace.get_metadata(agent_ids[0].name).slim() train_reward = candidate.train_reward or {} dim_keys = sorted(train_reward.keys()) + insight_dim_keys = sorted({key for reward in insight_rewards.values() for key in reward}) ``` - Follow the `ext.analysis_skill` format exactly for every section (Rewards table, - Trajectory Rewards, Divergent Trial Analysis, Complementary Failures, Failure - Patterns, Root Causes, Mechanical/Infrastructure Errors). + Follow the `ext.analysis_skill` format exactly for every section (Rewards tables, + including the conditional Insight Suite Reward table; Trajectory Rewards; Divergent + Trial Analysis; Complementary Failures; Failure Patterns; Root Causes; + Mechanical/Infrastructure Errors). - Fill in every section with real data. No placeholders. + If at least one agent has a non-empty `insight_reward`, the round analysis must name + every available Insight Suite dimension and show its values in the separate Insight + Suite Reward table. Never blend those metrics into train/validation rewards or imply + that they affected ranking. Fill in every included section with real data. No placeholders. Return the complete markdown content as a string. """ ... @@ -887,6 +920,7 @@ async def write_final_report(self, best_agent_id: str) -> None: # pyright: igno ```python agent_ids = self.workspace.list_agents() candidate = self.workspace.get_metadata(agent_id).slim() + insight_reward = candidate.insight_reward or {} analysis = self.workspace.read_analysis_file(n) ``` @@ -897,10 +931,17 @@ async def write_final_report(self, best_agent_id: str) -> None: # pyright: igno 4. Write eval-and-optimize/OPTIMIZATION.md with format: - Summary (baseline vs best rewards, rounds completed, total agents) - Reward Breakdown table (one row per agent, per-dimension columns) + - Insight Suite Metrics table when available - Lineage Tree (ASCII tree with rewards and optimization type) - Round-by-Round Analysis - Optimization Insights + When both the round-zero baseline and best agent have non-empty `insight_reward`, + the Summary must state whether the Insight-specific scenarios improved and the + Insight Suite Metrics table must show every available dimension with baseline, + winner, and signed delta columns. Keep this table separate from generic train and + validation rewards. Omit it only when Insight Suite rewards are unavailable. + Fill in every section with real data. Every agent must appear in the lineage tree. Mark the best agent with * BEST. """ @@ -1741,13 +1782,16 @@ async def _finalize( def _render_summary( self, rounds_completed: int, + baseline: Candidate | None, winner: Candidate | None, ) -> str: """Render a human-readable summary of the run outcome.""" winner_str = winner.name if winner else "none" - val_reward = "" + details: list[str] = [] if winner: - vr = getattr(winner, "validation_reward", None) - if vr: - val_reward = f", validation_reward={vr}" - return f"Optimization complete: {rounds_completed} round(s) completed, winner={winner_str}{val_reward}" + if winner.validation_reward: + details.append(f"validation_reward={winner.validation_reward}") + if baseline is not None and baseline.insight_reward and winner.insight_reward: + details.append(f"insight_suite=(baseline={baseline.insight_reward}, winner={winner.insight_reward})") + suffix = f", {', '.join(details)}" if details else "" + return f"Optimization complete: {rounds_completed} round(s) completed, winner={winner_str}{suffix}" diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py index 999c17275b..a66d2e227a 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py @@ -20,7 +20,7 @@ from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable from pathlib import Path -from typing import Any, Literal, cast +from typing import Any, Literal, TypeVar, cast from urllib.parse import urlparse import httpx @@ -50,8 +50,10 @@ ) from nemo_insights_plugin.entities import Insight from nemo_platform import AsyncNeMoPlatform +from pydantic import BaseModel logger = logging.getLogger(__name__) +_ModelT = TypeVar("_ModelT", bound=BaseModel) async def _upload_trace_otlp( @@ -368,13 +370,13 @@ async def get_agent_spec(self, *, workspace: str, spec: str, dest: Path) -> Path return await self._files.get_agent_spec(workspace=workspace, spec=spec, dest=dest) -def _load_entity(cls: type, path: Path) -> Any: +def _load_entity(cls: type[_ModelT], path: Path) -> _ModelT: """Deserialize *path* as JSON into *cls*, restoring the private ``_id`` field.""" data = json.loads(path.read_text()) entity_id = data.get("id", "") obj = cls.model_validate(data) if entity_id: - obj._id = entity_id # PrivateAttr set the same way as the real entity client + cast(Any, obj)._id = entity_id # PrivateAttr set the same way as the real entity client return obj @@ -854,7 +856,11 @@ async def get_experiment_id(self, *, workspace: str, candidate: Candidate, split return "" async def persist_result(self, *, workspace: str, result: ExperimentalistResult) -> None: - (self._eo / "OPTIMIZATION.md").write_text(result.summary) + report_path = self._eo / "OPTIMIZATION.md" + # The optimizer's report writer creates the full document before this call. + # Use the compact result summary only as a fallback when no report was produced. + if not report_path.exists() or not report_path.read_text().strip(): + report_path.write_text(result.summary) run_path = self._eo / "run.json" if run_path.exists(): run: ExperimentRun = _load_entity(ExperimentRun, run_path) diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py new file mode 100644 index 0000000000..3a12a63d04 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from nemo_experimentalist_plugin.entities import Candidate +from nemo_experimentalist_plugin.experimentalist.components.loop import AnalysisSkill, EvolutionaryOptimizer + + +def _candidate( + label: str, + *, + round_num: int, + insight_reward: dict[str, float] | None = None, + validation_reward: dict[str, float] | None = None, +) -> Candidate: + return Candidate( + run_id="run-1", + label=label, + round=round_num, + optimization="baseline" if round_num == 0 else "improve required tool use", + insight_reward=insight_reward, + validation_reward=validation_reward, + ) + + +def test_round_analysis_contract_requires_separate_insight_suite_dimensions() -> None: + skill_prompt = " ".join((AnalysisSkill.__doc__ or "").split()) + merge_prompt = " ".join((EvolutionaryOptimizer.merge_analysis.__doc__ or "").split()) + + assert "Insight Suite Reward" in skill_prompt + assert "candidate.insight_reward" in skill_prompt + assert "separate from train and validation rewards" in skill_prompt + assert "insight_dim_keys" in merge_prompt + assert "candidate.round == 0" in merge_prompt + assert "must name every available Insight Suite dimension" in merge_prompt + assert "Never blend those metrics into train/validation rewards" in merge_prompt + + +def test_final_report_contract_requires_baseline_winner_insight_comparison() -> None: + report_prompt = " ".join((EvolutionaryOptimizer.write_final_report.__doc__ or "").split()) + + assert "Insight Suite Metrics table" in report_prompt + assert "baseline, winner, and signed delta columns" in report_prompt + assert "Keep this table separate from generic train and validation rewards" in report_prompt + + +def test_terminal_summary_includes_baseline_and_winner_insight_metrics() -> None: + baseline = _candidate( + "agent-0", + round_num=0, + insight_reward={"uses_required_tool": 0.0}, + ) + winner = _candidate( + "agent-1", + round_num=1, + insight_reward={"uses_required_tool": 1.0}, + validation_reward={"reward": 0.75}, + ) + optimizer = object.__new__(EvolutionaryOptimizer) + + summary = optimizer._render_summary(rounds_completed=1, baseline=baseline, winner=winner) + + assert "validation_reward={'reward': 0.75}" in summary + assert "insight_suite=(baseline={'uses_required_tool': 0.0}" in summary + assert "winner={'uses_required_tool': 1.0})" in summary + + +def test_terminal_summary_omits_insight_comparison_when_unavailable() -> None: + baseline = _candidate("agent-0", round_num=0) + winner = _candidate("agent-1", round_num=1, validation_reward={"reward": 0.75}) + optimizer = object.__new__(EvolutionaryOptimizer) + + summary = optimizer._render_summary(rounds_completed=1, baseline=baseline, winner=winner) + + assert "insight_suite" not in summary diff --git a/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py b/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py index a27a1d4aa2..331572e8a5 100644 --- a/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py +++ b/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py @@ -13,6 +13,7 @@ import json import traceback from pathlib import Path +from typing import cast from unittest.mock import AsyncMock import httpx @@ -27,12 +28,17 @@ RemoteExperimentalistBackend, ) from nemo_insights_plugin.entities import Insight +from nemo_platform import AsyncNeMoPlatform def _local_backend(tmp_path: Path) -> LocalExperimentalistBackend: return LocalExperimentalistBackend(path=tmp_path / "backend") +def _as_platform_client(value: object) -> AsyncNeMoPlatform: + return cast(AsyncNeMoPlatform, value) + + # --------------------------------------------------------------------------- # get_insight — local file (offline) vs platform id fetch # --------------------------------------------------------------------------- @@ -82,7 +88,7 @@ async def test_get_insight_reads_local_file(tmp_path: Path) -> None: async def test_get_insight_fetches_platform_id_via_client(tmp_path: Path) -> None: platform_insight = Insight(workspace="ws", title="platform", description="d", agent="a") client = _StubClient(platform_insight) - backend = LocalExperimentalistBackend(client=client, path=tmp_path / "backend") # type: ignore[arg-type] + backend = LocalExperimentalistBackend(client=_as_platform_client(client), path=tmp_path / "backend") insight = await backend.get_insight(workspace="ws", insight_id="insight-remote-123") @@ -100,7 +106,10 @@ async def test_get_insight_platform_404_raises_value_error(tmp_path: Path) -> No request = httpx.Request("GET", "http://platform.test/insights/missing") response = httpx.Response(404, request=request) error = httpx.HTTPStatusError("not found", request=request, response=response) - backend = LocalExperimentalistBackend(client=_ErrorStubClient(error), path=tmp_path / "backend") # type: ignore[arg-type] + backend = LocalExperimentalistBackend( + client=_as_platform_client(_ErrorStubClient(error)), + path=tmp_path / "backend", + ) with pytest.raises(ValueError, match="Insight not found on the platform"): await backend.get_insight(workspace="w", insight_id="missing") @@ -140,7 +149,7 @@ def fake_clone(spec: str, dest: Path, *, clone_depth: int | None = None) -> Agen backend = ( _local_backend(tmp_path) if backend_factory == "local" - else RemoteExperimentalistBackend(client=None, path=tmp_path / "backend") # type: ignore[arg-type] + else RemoteExperimentalistBackend(client=_as_platform_client(None), path=tmp_path / "backend") ) spec = "ssh://git@h/g/r.git@main" dest = tmp_path / "agent-src" @@ -202,7 +211,7 @@ def fail_clone(spec: str, *_args, **_kwargs) -> AgentSource: # noqa: ANN002, AN async def test_get_agent_code_remote_nongit_raises(tmp_path: Path) -> None: - backend = RemoteExperimentalistBackend(client=None, path=tmp_path / "backend") # type: ignore[arg-type] + backend = RemoteExperimentalistBackend(client=_as_platform_client(None), path=tmp_path / "backend") with pytest.raises(NotImplementedError): await backend.get_agent_code(workspace="w", agent="some-live-agent-name", dest=tmp_path / "d") @@ -239,7 +248,7 @@ async def test_get_agent_spec_remote_delegates_to_files(tmp_path: Path) -> None: spec_file = tmp_path / "AGENT-SPEC.md" spec_file.write_text("# Remote Agent\n") dest = tmp_path / "workspace" / "AGENT-SPEC.md" - backend = RemoteExperimentalistBackend(client=None, path=tmp_path / "backend") # type: ignore[arg-type] + backend = RemoteExperimentalistBackend(client=_as_platform_client(None), path=tmp_path / "backend") result = await backend.get_agent_spec(workspace="w", spec=str(spec_file), dest=dest) @@ -344,6 +353,20 @@ async def test_persist_result_writes_run_summary(tmp_path: Path) -> None: saved = json.loads((backend._eo / "run.json").read_text()) assert saved["summary"] == "the real run summary" + assert (backend._eo / "OPTIMIZATION.md").read_text() == "the real run summary" + + +async def test_persist_result_preserves_generated_optimization_report(tmp_path: Path) -> None: + from nemo_experimentalist_plugin.experimentalist.result import ExperimentalistResult + + backend = _local_backend(tmp_path) + report_path = backend._eo / "OPTIMIZATION.md" + report_path.write_text("# Full optimization report\n\nInsight Suite Metrics") + + result = ExperimentalistResult(summary="compact run summary", run_id="run-1", rounds_completed=2, winner=None) + await backend.persist_result(workspace="w", result=result) + + assert report_path.read_text() == "# Full optimization report\n\nInsight Suite Metrics" # --------------------------------------------------------------------------- @@ -353,7 +376,7 @@ async def test_persist_result_writes_run_summary(tmp_path: Path) -> None: async def test_remote_forwards_to_local(tmp_path: Path) -> None: """Remote delegates persist_evaluation verbatim to its local file backend.""" - backend = RemoteExperimentalistBackend(client=None, path=tmp_path / "backend") # type: ignore[arg-type] + backend = RemoteExperimentalistBackend(client=_as_platform_client(None), path=tmp_path / "backend") delegate = AsyncMock() backend._files.persist_evaluation = delegate result = EvaluationResult(id="r1") From 431c336236ad2bc3038734262ef884210965c920 Mon Sep 17 00:00:00 2001 From: Ryan Angilly Date: Mon, 27 Jul 2026 10:20:12 -0600 Subject: [PATCH 3/5] feat: suggest insight tasks for validation Reissues NVIDIA-dev/NeMo-Optimizer#96 for the Experimentalist plugin.\n\nLinear: ASE-625, ASE-699 Signed-off-by: Ryan Angilly --- .../components/insight_promotion.py | 260 ++++++++++++++++++ .../experimentalist/components/loop.py | 16 ++ .../experimentalist/test_loop_reporting.py | 104 +++++++ 3 files changed, 380 insertions(+) create mode 100644 plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py new file mode 100644 index 0000000000..737a995458 --- /dev/null +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Rank Insight-suite tasks for possible manual promotion into validation.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, replace +from pathlib import Path + +from nemo_experimentalist_plugin.entities import Candidate +from nemo_experimentalist_plugin.experimentalist.components.evaluator import ( + Dataset, + Task, + TrialResult, + local_path_from_uri, +) + +_GENERIC_METRIC_NAMES = frozenset({"reward", "score"}) +_MAX_REPEAT_SPREAD = 0.1 +_MIN_DISCRIMINATION = 1e-9 +_REPORT_SECTION_START = "" +_REPORT_SECTION_END = "" + + +@dataclass(frozen=True, slots=True) +class InsightPromotionSuggestion: + """Evidence-backed recommendation to review one Insight-suite task.""" + + task_id: str + task_path: str + metric_name: str + discrimination: float + completed_attempts: int + total_attempts: int + repeat_spread: float + candidate_count: int + diversity_score: float | None = None + + +@dataclass(frozen=True, slots=True) +class _TaskEvidence: + suggestion: InsightPromotionSuggestion + profile: dict[tuple[str, str], float] + + +def _completed_metric_values( + trials: Sequence[TrialResult], +) -> dict[str, list[float]]: + values: dict[str, list[float]] = {} + for trial in trials: + if trial.status != "completed": + continue + for metric_name, metric in trial.metrics.items(): + values.setdefault(metric_name, []).append(float(metric.value)) + return values + + +def _task_evidence(task: Task, candidates: Sequence[Candidate]) -> _TaskEvidence | None: + trials_by_candidate = { + candidate.label: [trial for trial in candidate.insight_reward_details or () if trial.task_id == task.id] + for candidate in candidates + } + values_by_candidate = {label: _completed_metric_values(trials) for label, trials in trials_by_candidate.items()} + common_metrics = set.intersection(*(set(metric_values) for metric_values in values_by_candidate.values())) + if not common_metrics: + return None + + insight_metrics = common_metrics - _GENERIC_METRIC_NAMES + selected_metrics = sorted(insight_metrics or common_metrics) + total_attempts = sum(max(len(trials), 1) for trials in trials_by_candidate.values()) + completed_attempts = sum( + 1 + for trials in trials_by_candidate.values() + for trial in trials + if trial.status == "completed" and all(metric in trial.metrics for metric in selected_metrics) + ) + if completed_attempts != total_attempts: + return None + + profile: dict[tuple[str, str], float] = {} + repeat_spread = 0.0 + metric_ranges: dict[str, float] = {} + for metric_name in selected_metrics: + candidate_means: list[float] = [] + for candidate_label in sorted(values_by_candidate): + metric_values = values_by_candidate[candidate_label][metric_name] + candidate_mean = sum(metric_values) / len(metric_values) + profile[(candidate_label, metric_name)] = candidate_mean + candidate_means.append(candidate_mean) + repeat_spread = max(repeat_spread, max(metric_values) - min(metric_values)) + metric_ranges[metric_name] = max(candidate_means) - min(candidate_means) + + if repeat_spread > _MAX_REPEAT_SPREAD: + return None + metric_name, discrimination = max( + metric_ranges.items(), + key=lambda item: (item[1], item[0]), + ) + if discrimination <= _MIN_DISCRIMINATION: + return None + + return _TaskEvidence( + suggestion=InsightPromotionSuggestion( + task_id=task.id, + task_path=_task_path(task), + metric_name=metric_name, + discrimination=discrimination, + completed_attempts=completed_attempts, + total_attempts=total_attempts, + repeat_spread=repeat_spread, + candidate_count=len(candidates), + ), + profile=profile, + ) + + +def _task_path(task: Task) -> str: + if not task.uri: + return "-" + try: + return str(local_path_from_uri(task.uri, context=f"Insight task {task.id!r}")) + except ValueError: + return task.uri + + +def _profile_distance(left: _TaskEvidence, right: _TaskEvidence) -> float: + common_keys = set(left.profile) & set(right.profile) + if not common_keys: + return 1.0 + return sum(min(abs(left.profile[key] - right.profile[key]), 1.0) for key in common_keys) / len(common_keys) + + +def select_insight_promotion_suggestions( + dataset: Dataset, + candidates: Sequence[Candidate], + *, + limit: int = 3, +) -> list[InsightPromotionSuggestion]: + """Select stable, discriminative tasks with distinct observed score profiles.""" + if limit <= 0: + return [] + evaluated_candidates = [candidate for candidate in candidates if candidate.insight_reward_details is not None] + if len(evaluated_candidates) < 2: + return [] + + remaining = [ + evidence + for task in dataset.list_tasks() + if (evidence := _task_evidence(task, evaluated_candidates)) is not None + ] + remaining.sort( + key=lambda evidence: ( + -evidence.suggestion.discrimination, + evidence.suggestion.repeat_spread, + evidence.suggestion.task_id, + ) + ) + if not remaining: + return [] + + selected = [remaining.pop(0)] + while remaining and len(selected) < limit: + ranked: list[tuple[float, _TaskEvidence]] = [ + ( + min(_profile_distance(evidence, chosen) for chosen in selected), + evidence, + ) + for evidence in remaining + ] + diversity_score, next_evidence = max( + ranked, + key=lambda item: ( + item[0], + item[1].suggestion.discrimination, + -item[1].suggestion.repeat_spread, + item[1].suggestion.task_id, + ), + ) + if diversity_score <= _MIN_DISCRIMINATION: + break + selected.append( + replace( + next_evidence, + suggestion=replace( + next_evidence.suggestion, + diversity_score=diversity_score, + ), + ) + ) + remaining.remove(next_evidence) + + return [evidence.suggestion for evidence in selected] + + +def _markdown_cell(value: str) -> str: + return value.replace("|", r"\|").replace("\n", " ") + + +def render_insight_promotion_section( + suggestions: Sequence[InsightPromotionSuggestion], +) -> str: + """Render an advisory-only final-report section.""" + lines = [ + "## Insight Suite Promotion Suggestions", + "", + ( + "Advisory only: these tasks were not copied into the validation dataset. " + "Review them manually before changing the canonical validation set." + ), + "", + ] + if not suggestions: + lines.append( + "No task had complete, repeat-consistent results that discriminated between at least two candidates." + ) + return "\n".join(lines) + + lines.extend( + [ + "| Task | Path | Evidence |", + "| --- | --- | --- |", + ] + ) + for suggestion in suggestions: + diversity = ( + "highest discriminative signal" + if suggestion.diversity_score is None + else f"score-profile distance {suggestion.diversity_score:.2f}" + ) + evidence = ( + f"{suggestion.metric_name} range {suggestion.discrimination:.2f} across " + f"{suggestion.candidate_count} candidates; " + f"{suggestion.completed_attempts}/{suggestion.total_attempts} attempts completed; " + f"repeat spread {suggestion.repeat_spread:.2f}; {diversity}" + ) + lines.append( + f"| `{_markdown_cell(suggestion.task_id)}` | " + f"`{_markdown_cell(suggestion.task_path)}` | {_markdown_cell(evidence)} |" + ) + return "\n".join(lines) + + +def write_insight_promotion_section( + report_path: Path, + suggestions: Sequence[InsightPromotionSuggestion], +) -> None: + """Append or replace the advisory promotion section in the final report.""" + report = report_path.read_text() if report_path.exists() else "# Optimization Report\n" + rendered = render_insight_promotion_section(suggestions) + section = f"{_REPORT_SECTION_START}\n{rendered}\n{_REPORT_SECTION_END}" + if _REPORT_SECTION_START in report and _REPORT_SECTION_END in report: + before, _, marked = report.partition(_REPORT_SECTION_START) + _, _, after = marked.partition(_REPORT_SECTION_END) + report = f"{before.rstrip()}\n\n{section}{after}" + else: + report = f"{report.rstrip()}\n\n{section}\n" + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text(f"{report.rstrip()}\n") diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py index 27c8d85f52..8547ce7d6e 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py @@ -41,6 +41,10 @@ ensure_heldout_hidden, restore_heldout_splits, ) +from nemo_experimentalist_plugin.experimentalist.components.insight_promotion import ( + select_insight_promotion_suggestions, + write_insight_promotion_section, +) from nemo_experimentalist_plugin.experimentalist.components.model_config import ( get_fast_model, get_smart_model, @@ -792,6 +796,7 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: run_entity=run_entity, evolution_tree=evolution_tree, agent_name=agent_name, + insight_dataset=insight_eval_dataset, ) baseline_entity = next( @@ -1750,6 +1755,7 @@ async def _finalize( run_entity: ExperimentRun, evolution_tree: EvolutionTree, agent_name: str, + insight_dataset: Dataset | None, ) -> Candidate | None: """Select the winner, copy to workspace root, write final report.""" # Only survivors that actually have a validation reward are eligible winners. @@ -1773,6 +1779,16 @@ async def _finalize( except Exception as exc: # noqa: BLE001 logger.warning(f"[FINAL] Failed to write final report: {exc}") + if insight_dataset is not None: + suggestions = select_insight_promotion_suggestions( + insight_dataset, + [node.candidate for node in evolution_tree.nodes.values()], + ) + write_insight_promotion_section( + self.working_dir / "eval-and-optimize" / "OPTIMIZATION.md", + suggestions, + ) + run_entity.status = "completed" run_entity.winner_agent = best_id await backend.update_run(workspace=workspace, run=run_entity) diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py index 3a12a63d04..211ceea130 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py @@ -1,7 +1,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from pathlib import Path + from nemo_experimentalist_plugin.entities import Candidate +from nemo_experimentalist_plugin.experimentalist.components.evaluator import ( + Dataset, + MetricResult, + Task, + TrialResult, + TrialStatus, +) +from nemo_experimentalist_plugin.experimentalist.components.insight_promotion import ( + render_insight_promotion_section, + select_insight_promotion_suggestions, + write_insight_promotion_section, +) from nemo_experimentalist_plugin.experimentalist.components.loop import AnalysisSkill, EvolutionaryOptimizer @@ -72,3 +86,93 @@ def test_terminal_summary_omits_insight_comparison_when_unavailable() -> None: summary = optimizer._render_summary(rounds_completed=1, baseline=baseline, winner=winner) assert "insight_suite" not in summary + + +def _insight_trial( + task_id: str, + score: float, + *, + attempt: int = 1, + status: TrialStatus = "completed", +) -> TrialResult: + return TrialResult( + id=f"{task_id}-{attempt}", + task_id=task_id, + attempt=attempt, + status=status, + metrics={ + "reward": MetricResult(name="reward", value=1.0), + "uses_required_tool": MetricResult(name="uses_required_tool", value=score), + }, + ) + + +def test_insight_promotion_suggestions_are_stable_discriminative_and_diverse( + tmp_path: Path, +) -> None: + tasks = [ + Task(id="task-a", uri=(tmp_path / "task-a").as_uri()), + Task(id="task-b", uri=(tmp_path / "task-b").as_uri()), + Task(id="task-c", uri=(tmp_path / "task-c").as_uri()), + Task(id="task-flaky", uri=(tmp_path / "task-flaky").as_uri()), + Task(id="task-flat", uri=(tmp_path / "task-flat").as_uri()), + ] + baseline = _candidate("agent-0", round_num=0) + baseline.insight_reward_details = [ + _insight_trial("task-a", 0.0), + _insight_trial("task-b", 0.0), + _insight_trial("task-c", 0.8), + _insight_trial("task-flaky", 0.0, attempt=1), + _insight_trial("task-flaky", 1.0, attempt=2), + _insight_trial("task-flat", 0.5), + ] + winner = _candidate("agent-1", round_num=1) + winner.insight_reward_details = [ + _insight_trial("task-a", 1.0), + _insight_trial("task-b", 1.0), + _insight_trial("task-c", 0.2), + _insight_trial("task-flaky", 1.0), + _insight_trial("task-flat", 0.5), + ] + + suggestions = select_insight_promotion_suggestions( + Dataset(id="insight", tasks=tasks), + [baseline, winner], + ) + + assert [suggestion.task_id for suggestion in suggestions] == ["task-a", "task-c"] + assert suggestions[0].metric_name == "uses_required_tool" + assert suggestions[0].discrimination == 1.0 + assert suggestions[0].diversity_score is None + assert suggestions[1].diversity_score == 0.8 + + section = render_insight_promotion_section(suggestions) + assert "Advisory only: these tasks were not copied into the validation dataset." in section + assert "`task-a`" in section + assert f"`{tmp_path / 'task-a'}`" in section + assert "task-b" not in section + assert "task-flaky" not in section + assert "task-flat" not in section + + +def test_insight_promotion_section_explains_when_no_task_qualifies() -> None: + section = render_insight_promotion_section([]) + + assert "## Insight Suite Promotion Suggestions" in section + assert "No task had complete, repeat-consistent results" in section + + +def test_insight_promotion_section_is_appended_without_rewriting_report( + tmp_path: Path, +) -> None: + report_path = tmp_path / "eval-and-optimize" / "OPTIMIZATION.md" + report_path.parent.mkdir(parents=True) + report_path.write_text("# Optimization\n\nExisting analysis.\n") + + write_insight_promotion_section(report_path, []) + first_report = report_path.read_text() + write_insight_promotion_section(report_path, []) + + assert report_path.read_text() == first_report + assert first_report.startswith("# Optimization\n\nExisting analysis.") + assert first_report.count("## Insight Suite Promotion Suggestions") == 1 From e45f78af484cf859ee1c457e82dbd0e502198e7b Mon Sep 17 00:00:00 2001 From: Ryan Angilly Date: Mon, 27 Jul 2026 10:25:35 -0600 Subject: [PATCH 4/5] fix: make insight evaluation promotion-safe Reissues NVIDIA-dev/NeMo-Optimizer#97 for the Experimentalist plugin.\n\nLinear: ASE-684, ASE-699 Signed-off-by: Ryan Angilly --- .../nemo_experimentalist_plugin/entities.py | 12 + .../eval_author/README.md | 27 +- .../eval_author/agent.py | 5 +- .../eval_author/materialization.py | 236 +++++++++++- .../eval_author/models.py | 10 +- .../components/insight_promotion.py | 361 +++++++++++++++--- .../experimentalist/components/loop.py | 86 ++++- .../test_loop_insight_suite.py | 152 ++++++-- .../experimentalist/test_loop_reporting.py | 322 +++++++++++++++- .../tests/test_eval_author_agent.py | 33 +- .../tests/test_eval_author_materialization.py | 70 +++- 11 files changed, 1199 insertions(+), 115 deletions(-) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py index e6e7070fe1..a2346a70f7 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py @@ -157,6 +157,18 @@ class Candidate(NemoEntity, entity_type="candidate"): default=None, description="Insight-suite trial results from the last evaluation run.", ) + insight_suite_identity: str | None = Field( + default=None, + description="Content identity of the Insight suite associated with insight_reward.", + ) + insight_suite_artifact_ref: str | None = Field( + default=None, + description="Portable reference to the immutable Insight suite associated with insight_reward.", + ) + insight_metric_keys: list[str] | None = Field( + default=None, + description="Validated runtime metric keys associated with insight_reward.", + ) validation_trajectory_reward: dict[str, float] | None = Field( default=None, description="Validation trajectory reward: aggregate + per-node scores.", diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md index 810d0ae0ea..0f7319316a 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md @@ -58,13 +58,36 @@ scoped to the Insight suite; the user's train and validation datasets remain unchanged. The authored verifiers must pass static Harbor validation before the local suite is returned to the optimization loop. +After authoring and validation, Eval Author hashes every task file and verifier +file, derives deterministic suite and scorer identities, and freezes the exact +content beneath: + +```text +eval-and-optimize/eval_author//artifacts//insight-suite/ +``` + +The returned dataset points at this immutable artifact and carries a portable +`nemo-experimentalist-insight-suite://.../sha256/...` reference. Candidate Insight +results persist the same suite identity and artifact reference. Resume reuses +those results only when the identity still matches; changed task or verifier +content is re-evaluated. + Task-template inputs may be local paths, `file://` URIs, or NeMo Platform `fileset:///` references. Fileset-backed templates are downloaded into the experiment-local staging directory before Harbor parses them. The staged template is refreshed on every invocation rather than reused. -`EvalAuthorResult.insight_suite` contains the experiment-local materialized -`Dataset` for immediate evaluation by the optimization loop. +`EvalAuthorResult.insight_suite` contains the finalized content-addressed +`Dataset` for immediate evaluation by the optimization loop. Its identity and +portable artifact reference are also available as +`EvalAuthorResult.insight_suite_identity` and +`EvalAuthorResult.insight_suite_artifact_ref`. + +Insight metrics remain adaptive development feedback. They may steer round +analysis, goal-tree updates, and proposals, but validation remains the direct +Pareto and winner-selection criterion. Promotion suggestions require complete +repeated baseline-to-winner improvement evidence, remain advisory, and never +mutate the canonical validation dataset. ## Intended Invocation diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py index 6c13623ec2..af86c6c260 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py @@ -366,10 +366,13 @@ async def _run( validation_feedback=str(exc), ) else: + artifact = insight_suite.finalize_artifact() return EvalAuthorResult( train_dataset=train_dataset, validation_dataset=validation_dataset, - insight_suite=materialized_dataset, + insight_suite=artifact.dataset, + insight_suite_identity=artifact.identity, + insight_suite_artifact_ref=artifact.ref, summary=summary, ) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py index 96219b5cd7..dae2757e21 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py @@ -10,8 +10,11 @@ import os import re import shutil +import tomllib from dataclasses import dataclass from pathlib import Path +from typing import cast +from urllib.parse import urlparse from uuid import uuid4 import tomlkit @@ -19,8 +22,12 @@ from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import Task, local_path_from_uri -_MANIFEST_SCHEMA_VERSION = 1 +_MANIFEST_SCHEMA_VERSION = 2 +_CONTENT_HASH_SCHEMA_VERSION = 1 +_METRIC_CONTRACT_VERSION = 1 +_ARTIFACT_SCHEME = "nemo-experimentalist-insight-suite" _SLUG_RE = re.compile(r"[^a-z0-9]+") +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") def _slug(value: str, *, fallback: str, max_length: int = 48) -> str: @@ -32,6 +39,104 @@ def _digest(value: str, length: int = 10) -> str: return hashlib.sha256(value.encode("utf-8")).hexdigest()[:length] +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _canonical_digest(value: object) -> str: + return _sha256_bytes(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")) + + +def _file_hashes(root: Path) -> dict[str, str]: + return { + path.relative_to(root).as_posix(): _sha256_bytes(path.read_bytes()) + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +def _verifier_dir(task_dir: Path) -> Path: + config = tomllib.loads((task_dir / "task.toml").read_text(encoding="utf-8")) + verifier = config.get("verifier") + if isinstance(verifier, dict): + configured = verifier.get("directory") + if isinstance(configured, str) and configured.strip(): + path = Path(configured) + return path if path.is_absolute() else task_dir / path + for name in ("tests", "test"): + path = task_dir / name + if path.is_dir(): + return path + raise ValueError(f"Materialized task has no verifier directory: {task_dir}") + + +def _content_provenance(suite_dir: Path, manifest: dict[str, object]) -> tuple[list[dict[str, object]], str, str]: + raw_tasks = manifest.get("tasks") + if not isinstance(raw_tasks, list): + raise ValueError(f"Insight suite manifest has invalid tasks: {suite_dir / 'manifest.json'}") + + tasks: list[dict[str, object]] = [] + scorer_inputs: list[dict[str, str]] = [] + suite_inputs: list[dict[str, str]] = [] + for raw_task in raw_tasks: + if not isinstance(raw_task, dict): + raise ValueError(f"Insight suite manifest has invalid task entry: {raw_task!r}") + if not all(isinstance(key, str) for key in raw_task): + raise ValueError(f"Insight suite manifest task has invalid keys: {raw_task!r}") + task_entry = cast(dict[str, object], raw_task) + relative_path = task_entry.get("path") + if not isinstance(relative_path, str) or not relative_path: + raise ValueError(f"Insight suite manifest task has invalid path: {relative_path!r}") + task_dir = (suite_dir / relative_path).resolve() + try: + task_dir.relative_to(suite_dir.resolve()) + except ValueError as exc: + raise ValueError(f"Insight suite manifest task escapes the suite: {relative_path!r}") from exc + if not task_dir.is_dir(): + raise ValueError(f"Insight suite manifest task path is missing: {task_dir}") + + files = _file_hashes(task_dir) + verifier_dir = _verifier_dir(task_dir).resolve() + try: + verifier_path = verifier_dir.relative_to(task_dir).as_posix() + except ValueError as exc: + raise ValueError(f"Insight suite verifier must be contained in its task: {verifier_dir}") from exc + verifier_files = _file_hashes(verifier_dir) + content_hash = f"sha256:{_canonical_digest(files)}" + verifier_hash = f"sha256:{_canonical_digest(verifier_files)}" + tasks.append( + { + **task_entry, + "content_hash": content_hash, + "verifier": { + "path": verifier_path, + "content_hash": verifier_hash, + "files": verifier_files, + }, + "files": files, + } + ) + scorer_inputs.append({"path": relative_path, "verifier_hash": verifier_hash}) + suite_inputs.append( + { + "path": relative_path, + "task_hash": content_hash, + "verifier_hash": verifier_hash, + } + ) + + scorer_identity = f"sha256:{_canonical_digest(scorer_inputs)}" + suite_payload = { + "schema_version": _CONTENT_HASH_SCHEMA_VERSION, + "insight_id": manifest.get("insight_id"), + "metric_contract_version": _METRIC_CONTRACT_VERSION, + "scorer_identity": scorer_identity, + "tasks": suite_inputs, + } + suite_identity = f"sha256:{_canonical_digest(suite_payload)}" + return tasks, scorer_identity, suite_identity + + @dataclass(frozen=True, slots=True) class StagedInsightTask: """One copied task template waiting to be filled and validated.""" @@ -43,6 +148,53 @@ class StagedInsightTask: task: Task +@dataclass(frozen=True, slots=True) +class InsightSuiteArtifact: + """Immutable, content-addressed result of an authored Insight suite.""" + + identity: str + scorer_identity: str + ref: str + path: Path + dataset: HarborDataset + + +def resolve_insight_suite_artifact(experiment_dir: Path, artifact_ref: str) -> Path: + """Resolve and verify a portable Insight-suite artifact reference.""" + parsed = urlparse(artifact_ref) + parts = parsed.path.strip("/").split("/") + if ( + parsed.scheme != _ARTIFACT_SCHEME + or not parsed.netloc + or len(parts) != 2 + or parts[0] != "sha256" + or not _SHA256_RE.fullmatch(parts[1]) + ): + raise ValueError(f"Invalid Insight suite artifact reference: {artifact_ref!r}") + + suite_dir = ( + experiment_dir.resolve() + / "eval-and-optimize" + / "eval_author" + / parsed.netloc + / "artifacts" + / parts[1] + / "insight-suite" + ) + manifest_path = suite_dir / "manifest.json" + if not manifest_path.is_file(): + raise FileNotFoundError(f"Insight suite artifact not found: {artifact_ref}") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + _, scorer_identity, suite_identity = _content_provenance(suite_dir, manifest) + expected_identity = f"sha256:{parts[1]}" + if manifest.get("suite_identity") != expected_identity or suite_identity != expected_identity: + raise ValueError(f"Insight suite artifact content does not match reference: {artifact_ref}") + scorer = manifest.get("scorer") + if not isinstance(scorer, dict) or scorer.get("identity") != scorer_identity: + raise ValueError(f"Insight suite scorer content does not match reference: {artifact_ref}") + return suite_dir + + class InsightSuite: """Build one experiment-local persisted Harbor dataset for an Insight.""" @@ -53,6 +205,7 @@ def __init__(self, *, experiment_dir: Path, insight_id: str, task_template: Task if not task_template.uri: raise ValueError("Task template URI is required to materialize an insight suite") + self.experiment_dir = experiment_dir.resolve() self.insight_id = insight_id self.template_dir = local_path_from_uri( task_template.uri, @@ -62,7 +215,7 @@ def __init__(self, *, experiment_dir: Path, insight_id: str, task_template: Task raise ValueError(f"Eval Author task template is not a directory: {self.template_dir}") self.template_uri = self.template_dir.as_uri() insight_slug = f"{_slug(insight_id, fallback='insight')}-{_digest(insight_id)}" - self.root = experiment_dir.resolve() / "eval-and-optimize" / "eval_author" / insight_slug + self.root = self.experiment_dir / "eval-and-optimize" / "eval_author" / insight_slug self.suite_dir = self.root / "insight-suite" self._candidate_root: Path | None = None self._candidate_suite: Path | None = None @@ -200,3 +353,82 @@ def record_analysis(self, statuses: dict[str, tuple[str, str | None]]) -> None: pending_path = manifest_path.with_suffix(".json.pending") pending_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") os.replace(pending_path, manifest_path) + + def finalize_artifact(self) -> InsightSuiteArtifact: + """Freeze the authored suite under a verified content-addressed reference.""" + manifest_path = self.suite_dir / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + tasks, scorer_identity, suite_identity = _content_provenance(self.suite_dir, manifest) + digest = suite_identity.removeprefix("sha256:") + artifact_ref = f"{_ARTIFACT_SCHEME}://{self.root.name}/sha256/{digest}" + artifact_path = self.root / "artifacts" / digest / "insight-suite" + manifest.update( + { + "schema_version": _MANIFEST_SCHEMA_VERSION, + "content_hash_schema_version": _CONTENT_HASH_SCHEMA_VERSION, + "metric_contract_version": _METRIC_CONTRACT_VERSION, + "suite_identity": suite_identity, + "scorer": { + "identity": scorer_identity, + "metric_contract_version": _METRIC_CONTRACT_VERSION, + }, + "artifact": { + "ref": artifact_ref, + "relative_path": artifact_path.relative_to(self.experiment_dir).as_posix(), + }, + "tasks": tasks, + } + ) + pending_path = manifest_path.with_suffix(".json.pending") + pending_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(pending_path, manifest_path) + + if artifact_path.exists(): + resolved = resolve_insight_suite_artifact(self.experiment_dir, artifact_ref) + if resolved != artifact_path.resolve(): + raise ValueError(f"Insight suite artifact resolved to unexpected path: {resolved}") + else: + artifact_path.parent.mkdir(parents=True, exist_ok=True) + candidate_artifact = artifact_path.parent / f".candidate-{uuid4().hex}" + shutil.copytree(self.suite_dir, candidate_artifact) + try: + os.replace(candidate_artifact, artifact_path) + finally: + if candidate_artifact.exists(): + shutil.rmtree(candidate_artifact) + + dataset = HarborDataset.from_path( + artifact_path, + dataset_id=f"insight-{digest[:12]}", + ) + task_hashes: dict[str, dict[str, str]] = {} + for task in tasks: + task_path = task.get("path") + content_hash = task.get("content_hash") + verifier = task.get("verifier") + verifier_hash = verifier.get("content_hash") if isinstance(verifier, dict) else None + if ( + not isinstance(task_path, str) + or not isinstance(content_hash, str) + or not isinstance(verifier_hash, str) + ): + raise ValueError(f"Finalized Insight suite has invalid task provenance: {task!r}") + task_hashes[task_path] = { + "content_hash": content_hash, + "verifier_hash": verifier_hash, + } + dataset.metadata.update( + { + "insight_suite_identity": suite_identity, + "insight_suite_scorer_identity": scorer_identity, + "insight_suite_artifact_ref": artifact_ref, + "insight_suite_task_hashes": task_hashes, + } + ) + return InsightSuiteArtifact( + identity=suite_identity, + scorer_identity=scorer_identity, + ref=artifact_ref, + path=artifact_path, + dataset=dataset, + ) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py index 6dffe420e0..f3652736cc 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py @@ -35,6 +35,14 @@ class EvalAuthorResult(BaseModel): validation_dataset: Dataset insight_suite: Dataset | None = Field( default=None, - description="Materialized Insight dataset for immediate use by the optimization loop.", + description="Finalized content-addressed Insight dataset for use by the optimization loop.", + ) + insight_suite_identity: str | None = Field( + default=None, + description="SHA-256 identity of the finalized Insight task and verifier content.", + ) + insight_suite_artifact_ref: str | None = Field( + default=None, + description="Portable reference resolving to the immutable finalized Insight suite.", ) summary: str diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py index 737a995458..a4993133fd 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py @@ -5,6 +5,7 @@ from __future__ import annotations +import math from collections.abc import Sequence from dataclasses import dataclass, replace from pathlib import Path @@ -12,9 +13,9 @@ from nemo_experimentalist_plugin.entities import Candidate from nemo_experimentalist_plugin.experimentalist.components.evaluator import ( Dataset, + EvaluationResult, Task, TrialResult, - local_path_from_uri, ) _GENERIC_METRIC_NAMES = frozenset({"reward", "score"}) @@ -22,6 +23,18 @@ _MIN_DISCRIMINATION = 1e-9 _REPORT_SECTION_START = "" _REPORT_SECTION_END = "" +_COMPARISON_SECTION_START = "" +_COMPARISON_SECTION_END = "" + + +@dataclass(frozen=True, slots=True) +class InsightSuiteProvenance: + """Runtime identity and portable location for one finalized Insight suite.""" + + identity: str + scorer_identity: str + artifact_ref: str + task_hashes: dict[str, dict[str, str]] @dataclass(frozen=True, slots=True) @@ -29,9 +42,13 @@ class InsightPromotionSuggestion: """Evidence-backed recommendation to review one Insight-suite task.""" task_id: str - task_path: str + suite_artifact_ref: str + task_content_hash: str + verifier_hash: str metric_name: str discrimination: float + baseline_score: float + winner_score: float completed_attempts: int total_attempts: int repeat_spread: float @@ -45,68 +62,208 @@ class _TaskEvidence: profile: dict[tuple[str, str], float] -def _completed_metric_values( +def insight_suite_provenance(dataset: Dataset) -> InsightSuiteProvenance: + """Return validated content provenance carried by a finalized suite dataset.""" + identity = dataset.metadata.get("insight_suite_identity") + scorer_identity = dataset.metadata.get("insight_suite_scorer_identity") + artifact_ref = dataset.metadata.get("insight_suite_artifact_ref") + raw_task_hashes = dataset.metadata.get("insight_suite_task_hashes") + if not isinstance(identity, str) or not identity.startswith("sha256:"): + raise ValueError("Finalized Insight suite is missing its content identity") + if not isinstance(scorer_identity, str) or not scorer_identity.startswith("sha256:"): + raise ValueError("Finalized Insight suite is missing its scorer identity") + if not isinstance(artifact_ref, str) or not artifact_ref: + raise ValueError("Finalized Insight suite is missing its portable artifact reference") + if not isinstance(raw_task_hashes, dict): + raise ValueError("Finalized Insight suite is missing task and verifier hashes") + task_hashes: dict[str, dict[str, str]] = {} + for task_id, raw_hashes in raw_task_hashes.items(): + if not isinstance(task_id, str) or not isinstance(raw_hashes, dict): + raise ValueError("Finalized Insight suite has invalid task hash provenance") + content_hash = raw_hashes.get("content_hash") + verifier_hash = raw_hashes.get("verifier_hash") + if not isinstance(content_hash, str) or not isinstance(verifier_hash, str): + raise ValueError(f"Finalized Insight task {task_id!r} has invalid content hashes") + task_hashes[task_id] = { + "content_hash": content_hash, + "verifier_hash": verifier_hash, + } + return InsightSuiteProvenance( + identity=identity, + scorer_identity=scorer_identity, + artifact_ref=artifact_ref, + task_hashes=task_hashes, + ) + + +def _validated_metric_value(value: float | int, *, context: str) -> float: + metric_value = float(value) + if not math.isfinite(metric_value) or not 0.0 <= metric_value <= 1.0: + raise ValueError(f"{context} must be finite and within [0, 1], got {value!r}") + return metric_value + + +def validate_insight_evaluation_result( + result: EvaluationResult, + *, + expected_metric_keys: Sequence[str] | None = None, +) -> tuple[str, ...]: + """Validate metrics before they become adaptive analysis or promotion evidence.""" + aggregate_keys = set(result.aggregate_metrics) + if not aggregate_keys: + raise ValueError("Insight evaluation produced no aggregate metrics") + if not aggregate_keys - _GENERIC_METRIC_NAMES: + raise ValueError("Insight evaluation produced no Insight-specific metric") + if expected_metric_keys is not None and aggregate_keys != set(expected_metric_keys): + raise ValueError( + "Insight evaluation aggregate metric keys are inconsistent: " + f"expected {sorted(expected_metric_keys)}, got {sorted(aggregate_keys)}" + ) + for metric_name, value in result.aggregate_metrics.items(): + _validated_metric_value(value, context=f"Insight aggregate metric {metric_name!r}") + + completed = [trial for trial in result.trials if trial.status == "completed"] + if not completed: + raise ValueError("Insight evaluation produced no completed trial evidence") + for trial in completed: + trial_keys = set(trial.metrics) + if trial_keys != aggregate_keys: + raise ValueError( + f"Insight trial {trial.id!r} metric keys are inconsistent: " + f"expected {sorted(aggregate_keys)}, got {sorted(trial_keys)}" + ) + for metric_name, metric in trial.metrics.items(): + _validated_metric_value( + metric.value, + context=f"Insight trial {trial.id!r} metric {metric_name!r}", + ) + return tuple(sorted(aggregate_keys)) + + +def stamp_insight_evaluation_result( + result: EvaluationResult, + provenance: InsightSuiteProvenance, +) -> EvaluationResult: + """Attach suite identity to aggregate and per-trial evidence.""" + suite_metadata = { + "insight_suite_identity": provenance.identity, + "insight_suite_scorer_identity": provenance.scorer_identity, + "insight_suite_artifact_ref": provenance.artifact_ref, + } + return result.model_copy( + update={ + "metadata": {**result.metadata, **suite_metadata}, + "trials": [ + trial.model_copy(update={"metadata": {**trial.metadata, **suite_metadata}}) for trial in result.trials + ], + } + ) + + +def _task_metric_values( trials: Sequence[TrialResult], -) -> dict[str, list[float]]: - values: dict[str, list[float]] = {} + *, + required_metrics: set[str], +) -> dict[str, list[float]] | None: + if len(trials) < 2 or any(trial.status != "completed" for trial in trials): + return None + values = {metric_name: [] for metric_name in required_metrics} for trial in trials: - if trial.status != "completed": - continue + if set(trial.metrics) != required_metrics: + return None for metric_name, metric in trial.metrics.items(): - values.setdefault(metric_name, []).append(float(metric.value)) + try: + value = _validated_metric_value( + metric.value, + context=f"Insight trial {trial.id!r} metric {metric_name!r}", + ) + except ValueError: + return None + values[metric_name].append(value) return values -def _task_evidence(task: Task, candidates: Sequence[Candidate]) -> _TaskEvidence | None: +def _task_evidence( + task: Task, + candidates: Sequence[Candidate], + *, + baseline: Candidate, + winner: Candidate, + provenance: InsightSuiteProvenance, +) -> _TaskEvidence | None: + metric_key_sets = { + tuple(candidate.insight_metric_keys or ()) + for candidate in candidates + if candidate.insight_suite_identity == provenance.identity + } + if len(metric_key_sets) != 1: + return None + required_metrics = set(next(iter(metric_key_sets), ())) + insight_metrics = required_metrics - _GENERIC_METRIC_NAMES + if not insight_metrics: + return None + trials_by_candidate = { candidate.label: [trial for trial in candidate.insight_reward_details or () if trial.task_id == task.id] for candidate in candidates } - values_by_candidate = {label: _completed_metric_values(trials) for label, trials in trials_by_candidate.items()} - common_metrics = set.intersection(*(set(metric_values) for metric_values in values_by_candidate.values())) - if not common_metrics: + values_by_candidate: dict[str, dict[str, list[float]]] = {} + for label, trials in trials_by_candidate.items(): + values = _task_metric_values(trials, required_metrics=required_metrics) + if values is None: + return None + values_by_candidate[label] = values + + if baseline.label not in values_by_candidate or winner.label not in values_by_candidate: return None - - insight_metrics = common_metrics - _GENERIC_METRIC_NAMES - selected_metrics = sorted(insight_metrics or common_metrics) - total_attempts = sum(max(len(trials), 1) for trials in trials_by_candidate.values()) + total_attempts = sum(len(trials) for trials in trials_by_candidate.values()) completed_attempts = sum( - 1 - for trials in trials_by_candidate.values() - for trial in trials - if trial.status == "completed" and all(metric in trial.metrics for metric in selected_metrics) + 1 for trials in trials_by_candidate.values() for trial in trials if trial.status == "completed" ) - if completed_attempts != total_attempts: + if not total_attempts or completed_attempts != total_attempts: return None profile: dict[tuple[str, str], float] = {} repeat_spread = 0.0 - metric_ranges: dict[str, float] = {} - for metric_name in selected_metrics: - candidate_means: list[float] = [] + metric_improvements: dict[str, tuple[float, float, float]] = {} + for metric_name in sorted(insight_metrics): + candidate_means: dict[str, float] = {} for candidate_label in sorted(values_by_candidate): metric_values = values_by_candidate[candidate_label][metric_name] candidate_mean = sum(metric_values) / len(metric_values) profile[(candidate_label, metric_name)] = candidate_mean - candidate_means.append(candidate_mean) + candidate_means[candidate_label] = candidate_mean repeat_spread = max(repeat_spread, max(metric_values) - min(metric_values)) - metric_ranges[metric_name] = max(candidate_means) - min(candidate_means) + baseline_score = candidate_means[baseline.label] + winner_score = candidate_means[winner.label] + metric_improvements[metric_name] = ( + winner_score - baseline_score, + baseline_score, + winner_score, + ) if repeat_spread > _MAX_REPEAT_SPREAD: return None - metric_name, discrimination = max( - metric_ranges.items(), - key=lambda item: (item[1], item[0]), + metric_name, (discrimination, baseline_score, winner_score) = max( + metric_improvements.items(), + key=lambda item: (item[1][0], item[0]), ) - if discrimination <= _MIN_DISCRIMINATION: + if baseline_score >= 1.0 or discrimination <= _MIN_DISCRIMINATION: + return None + hashes = provenance.task_hashes.get(task.id) + if hashes is None: return None return _TaskEvidence( suggestion=InsightPromotionSuggestion( task_id=task.id, - task_path=_task_path(task), + suite_artifact_ref=provenance.artifact_ref, + task_content_hash=hashes["content_hash"], + verifier_hash=hashes["verifier_hash"], metric_name=metric_name, discrimination=discrimination, + baseline_score=baseline_score, + winner_score=winner_score, completed_attempts=completed_attempts, total_attempts=total_attempts, repeat_spread=repeat_spread, @@ -116,15 +273,6 @@ def _task_evidence(task: Task, candidates: Sequence[Candidate]) -> _TaskEvidence ) -def _task_path(task: Task) -> str: - if not task.uri: - return "-" - try: - return str(local_path_from_uri(task.uri, context=f"Insight task {task.id!r}")) - except ValueError: - return task.uri - - def _profile_distance(left: _TaskEvidence, right: _TaskEvidence) -> float: common_keys = set(left.profile) & set(right.profile) if not common_keys: @@ -136,19 +284,37 @@ def select_insight_promotion_suggestions( dataset: Dataset, candidates: Sequence[Candidate], *, + winner: Candidate | None = None, limit: int = 3, ) -> list[InsightPromotionSuggestion]: - """Select stable, discriminative tasks with distinct observed score profiles.""" - if limit <= 0: + """Select repeated, complete baseline-to-winner improvements for manual review.""" + if limit <= 0 or winner is None: return [] - evaluated_candidates = [candidate for candidate in candidates if candidate.insight_reward_details is not None] + provenance = insight_suite_provenance(dataset) + evaluated_candidates = [ + candidate + for candidate in candidates + if candidate.insight_reward_details is not None and candidate.insight_suite_identity == provenance.identity + ] if len(evaluated_candidates) < 2: return [] + baseline = next((candidate for candidate in evaluated_candidates if candidate.round == 0), None) + if baseline is None or winner not in evaluated_candidates: + return [] remaining = [ evidence for task in dataset.list_tasks() - if (evidence := _task_evidence(task, evaluated_candidates)) is not None + if ( + evidence := _task_evidence( + task, + evaluated_candidates, + baseline=baseline, + winner=winner, + provenance=provenance, + ) + ) + is not None ] remaining.sort( key=lambda evidence: ( @@ -206,20 +372,21 @@ def render_insight_promotion_section( "## Insight Suite Promotion Suggestions", "", ( - "Advisory only: these tasks were not copied into the validation dataset. " - "Review them manually before changing the canonical validation set." + "Advisory adaptive/development evidence only, not independent validation evidence. " + "These tasks were not copied into the validation dataset; review them manually before " + "changing the canonical validation set." ), "", ] if not suggestions: lines.append( - "No task had complete, repeat-consistent results that discriminated between at least two candidates." + "No task had complete repeated evidence reproducing a baseline failure and showing a winner improvement." ) return "\n".join(lines) lines.extend( [ - "| Task | Path | Evidence |", + "| Task | Content-addressed suite | Evidence |", "| --- | --- | --- |", ] ) @@ -230,31 +397,107 @@ def render_insight_promotion_section( else f"score-profile distance {suggestion.diversity_score:.2f}" ) evidence = ( - f"{suggestion.metric_name} range {suggestion.discrimination:.2f} across " + f"{suggestion.metric_name} baseline {suggestion.baseline_score:.2f} → " + f"winner {suggestion.winner_score:.2f} ({suggestion.discrimination:+.2f}) across " f"{suggestion.candidate_count} candidates; " f"{suggestion.completed_attempts}/{suggestion.total_attempts} attempts completed; " - f"repeat spread {suggestion.repeat_spread:.2f}; {diversity}" + f"repeat spread {suggestion.repeat_spread:.2f}; " + f"task {suggestion.task_content_hash}; verifier {suggestion.verifier_hash}; {diversity}" ) lines.append( f"| `{_markdown_cell(suggestion.task_id)}` | " - f"`{_markdown_cell(suggestion.task_path)}` | {_markdown_cell(evidence)} |" + f"`{_markdown_cell(suggestion.suite_artifact_ref)}` | {_markdown_cell(evidence)} |" ) return "\n".join(lines) -def write_insight_promotion_section( +def _write_marked_section( report_path: Path, - suggestions: Sequence[InsightPromotionSuggestion], + *, + rendered: str, + start_marker: str, + end_marker: str, ) -> None: - """Append or replace the advisory promotion section in the final report.""" report = report_path.read_text() if report_path.exists() else "# Optimization Report\n" - rendered = render_insight_promotion_section(suggestions) - section = f"{_REPORT_SECTION_START}\n{rendered}\n{_REPORT_SECTION_END}" - if _REPORT_SECTION_START in report and _REPORT_SECTION_END in report: - before, _, marked = report.partition(_REPORT_SECTION_START) - _, _, after = marked.partition(_REPORT_SECTION_END) + section = f"{start_marker}\n{rendered}\n{end_marker}" + if start_marker in report and end_marker in report: + before, _, marked = report.partition(start_marker) + _, _, after = marked.partition(end_marker) report = f"{before.rstrip()}\n\n{section}{after}" else: report = f"{report.rstrip()}\n\n{section}\n" report_path.parent.mkdir(parents=True, exist_ok=True) report_path.write_text(f"{report.rstrip()}\n") + + +def write_insight_promotion_section( + report_path: Path, + suggestions: Sequence[InsightPromotionSuggestion], +) -> None: + """Append or replace the advisory promotion section in the final report.""" + _write_marked_section( + report_path, + rendered=render_insight_promotion_section(suggestions), + start_marker=_REPORT_SECTION_START, + end_marker=_REPORT_SECTION_END, + ) + + +def render_insight_comparison_section( + baseline: Candidate, + winner: Candidate, + provenance: InsightSuiteProvenance, +) -> str: + """Render the deterministic baseline-versus-winner Insight comparison.""" + for candidate in (baseline, winner): + if ( + candidate.insight_suite_identity != provenance.identity + or candidate.insight_suite_artifact_ref != provenance.artifact_ref + ): + raise ValueError( + f"Candidate {candidate.label!r} Insight evidence does not match finalized suite {provenance.identity}" + ) + baseline_reward = baseline.insight_reward or {} + winner_reward = winner.insight_reward or {} + metric_names = sorted(set(baseline_reward) | set(winner_reward)) + lines = [ + "## Deterministic Insight Suite Comparison", + "", + ( + "Adaptive/development evidence only; canonical validation remains the direct " + "Pareto and winner-selection criterion." + ), + "", + f"Suite: `{provenance.artifact_ref}` (`{provenance.identity}`)", + "", + "| Metric | Baseline | Winner | Delta |", + "| --- | ---: | ---: | ---: |", + ] + for metric_name in metric_names: + baseline_value = baseline_reward.get(metric_name) + winner_value = winner_reward.get(metric_name) + if baseline_value is None or winner_value is None: + baseline_text = "—" if baseline_value is None else f"{baseline_value:.3f}" + winner_text = "—" if winner_value is None else f"{winner_value:.3f}" + delta_text = "—" + else: + baseline_text = f"{baseline_value:.3f}" + winner_text = f"{winner_value:.3f}" + delta_text = f"{winner_value - baseline_value:+.3f}" + lines.append(f"| `{_markdown_cell(metric_name)}` | {baseline_text} | {winner_text} | {delta_text} |") + return "\n".join(lines) + + +def write_insight_comparison_section( + report_path: Path, + baseline: Candidate, + winner: Candidate, + provenance: InsightSuiteProvenance, +) -> None: + """Append or replace the deterministic baseline-versus-winner section.""" + _write_marked_section( + report_path, + rendered=render_insight_comparison_section(baseline, winner, provenance), + start_marker=_COMPARISON_SECTION_START, + end_marker=_COMPARISON_SECTION_END, + ) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py index 8547ce7d6e..f4d2160b53 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py @@ -42,7 +42,11 @@ restore_heldout_splits, ) from nemo_experimentalist_plugin.experimentalist.components.insight_promotion import ( + insight_suite_provenance, select_insight_promotion_suggestions, + stamp_insight_evaluation_result, + validate_insight_evaluation_result, + write_insight_comparison_section, write_insight_promotion_section, ) from nemo_experimentalist_plugin.experimentalist.components.model_config import ( @@ -199,7 +203,10 @@ class AnalysisSkill(Skill): `candidate.insight_reward`. Omit that table when `insight_reward` is absent or empty for every agent. Keep Insight Suite Reward separate from train and validation rewards: it reports performance on scenarios authored for the motivating Insight and is not a - ranking or Pareto-selection input.] + ranking or Pareto-selection input. Insight Suite metrics may steer round analysis, + goal-tree updates, and the proposer only as adaptive/development feedback. Label any + resulting claim accordingly; never present this adaptive evidence as independent + validation evidence.] ## Trajectory Rewards @@ -911,7 +918,10 @@ async def merge_analysis( If at least one agent has a non-empty `insight_reward`, the round analysis must name every available Insight Suite dimension and show its values in the separate Insight Suite Reward table. Never blend those metrics into train/validation rewards or imply - that they affected ranking. Fill in every included section with real data. No placeholders. + that they affected ranking. These metrics may steer this analysis, the goal tree, and + the proposer only as adaptive/development feedback; label claims accordingly and never + present them as independent validation evidence. Fill in every included section with + real data. No placeholders. Return the complete markdown content as a string. """ ... @@ -1356,7 +1366,16 @@ async def _evaluate_insight_candidates( """Evaluate candidates that do not yet have metrics for this Insight suite.""" if not list(dataset.list_tasks()): return {} - pending = [candidate for candidate in candidates if candidate.insight_reward is None] + provenance = insight_suite_provenance(dataset) + pending = [ + candidate + for candidate in candidates + if candidate.insight_reward is None + or candidate.insight_reward_details is None + or candidate.insight_suite_identity != provenance.identity + or candidate.insight_suite_artifact_ref != provenance.artifact_ref + or not candidate.insight_metric_keys + ] evaluated = await asyncio.gather( *[self._evaluate_agent(candidate, dataset, evaluator) for candidate in pending] ) @@ -1373,15 +1392,41 @@ async def _evaluate_and_persist_insight_candidates( run_id: str, ) -> None: """Evaluate and persist Insight-suite metrics for the supplied candidates.""" + provenance = insight_suite_provenance(dataset) results = await self._evaluate_insight_candidates( dataset=dataset, evaluator=evaluator, candidates=candidates, ) + dataset_metric_keys = dataset.metadata.get("insight_metric_keys") + if dataset_metric_keys is not None and ( + not isinstance(dataset_metric_keys, list) or not all(isinstance(key, str) for key in dataset_metric_keys) + ): + raise ValueError("Insight suite runtime metric keys have invalid metadata") + cached_metric_key_sets = { + tuple(candidate.insight_metric_keys or ()) + for candidate in candidates + if candidate.insight_suite_identity == provenance.identity and candidate.insight_metric_keys + } + if isinstance(dataset_metric_keys, list): + cached_metric_key_sets.add(tuple(dataset_metric_keys)) + if len(cached_metric_key_sets) > 1: + raise ValueError( + f"Cached Insight evaluations disagree on required metric keys: {sorted(cached_metric_key_sets)}" + ) + expected_metric_keys = next(iter(cached_metric_key_sets), None) for candidate in candidates: result = results.get(candidate.label) if result is None: continue + metric_keys = validate_insight_evaluation_result( + result, + expected_metric_keys=expected_metric_keys, + ) + if expected_metric_keys is None: + expected_metric_keys = metric_keys + dataset.metadata["insight_metric_keys"] = list(metric_keys) + result = stamp_insight_evaluation_result(result, provenance) await backend.persist_evaluation( workspace=workspace, result=result, @@ -1393,11 +1438,16 @@ async def _evaluate_and_persist_insight_candidates( updates={ "insight_reward": result.aggregate_metrics, "insight_reward_details": result.trials, + "insight_suite_identity": provenance.identity, + "insight_suite_artifact_ref": provenance.artifact_ref, + "insight_metric_keys": list(metric_keys), }, workspace=workspace, backend=backend, run_id=run_id, ) + if expected_metric_keys is not None: + dataset.metadata["insight_metric_keys"] = list(expected_metric_keys) async def _generate_initial_goal_tree( self, @@ -1774,18 +1824,44 @@ async def _finalize( evolution_tree.mark_best(best_id) self._copy_best_to_workspace(best_id) + winner = evolution_tree.nodes[best_id].candidate + baseline = next( + (node.candidate for node in evolution_tree.nodes.values() if node.round == 0), + None, + ) + report_path = self.working_dir / "eval-and-optimize" / "OPTIMIZATION.md" + final_report_failed = False try: await self.write_final_report(best_id) except Exception as exc: # noqa: BLE001 logger.warning(f"[FINAL] Failed to write final report: {exc}") + final_report_failed = True + if not report_path.exists() or not report_path.read_text().strip(): + final_report_failed = True + if final_report_failed: + summary = self._render_summary( + rounds_completed=run_entity.rounds_completed, + baseline=baseline, + winner=winner, + ) + report_path.write_text(f"# Optimization Report\n\n## Compact Run Summary\n\n{summary}\n") if insight_dataset is not None: + provenance = insight_suite_provenance(insight_dataset) + if baseline is not None: + write_insight_comparison_section( + report_path, + baseline, + winner, + provenance, + ) suggestions = select_insight_promotion_suggestions( insight_dataset, [node.candidate for node in evolution_tree.nodes.values()], + winner=winner, ) write_insight_promotion_section( - self.working_dir / "eval-and-optimize" / "OPTIMIZATION.md", + report_path, suggestions, ) @@ -1793,7 +1869,7 @@ async def _finalize( run_entity.winner_agent = best_id await backend.update_run(workspace=workspace, run=run_entity) - return evolution_tree.nodes[best_id].candidate + return winner def _render_summary( self, diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py index 83b016fac6..e2ac7a27e7 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py @@ -9,8 +9,14 @@ import pytest from nemo_experimentalist_plugin.entities import Candidate from nemo_experimentalist_plugin.experimentalist.components import loop as loop_module -from nemo_experimentalist_plugin.experimentalist.components.evaluator import Dataset, EvaluationResult, Task -from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef +from nemo_experimentalist_plugin.experimentalist.components.evaluator import ( + Dataset, + EvaluationResult, + MetricResult, + Task, + TrialResult, +) +from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef, DataValue from nemo_experimentalist_plugin.experimentalist.components.loop import EvolutionaryOptimizer from nemo_experimentalist_plugin.resolve import EvolutionaryOptimizerConfig @@ -19,6 +25,42 @@ class _StopAfterOneRound(Exception): pass +def _suite_metadata(identity_char: str = "a") -> dict[str, DataValue]: + identity = f"sha256:{identity_char * 64}" + return { + "insight_suite_identity": identity, + "insight_suite_scorer_identity": f"sha256:{'b' * 64}", + "insight_suite_artifact_ref": (f"nemo-experimentalist-insight-suite://insight-1/sha256/{identity_char * 64}"), + "insight_suite_task_hashes": { + "insight-task": { + "content_hash": f"sha256:{'c' * 64}", + "verifier_hash": f"sha256:{'d' * 64}", + } + }, + } + + +def _insight_result(label: str, score: float) -> EvaluationResult: + return EvaluationResult( + id=f"{label}-insight", + aggregate_metrics={"uses_required_tool": score}, + trials=[ + TrialResult( + id=f"{label}-insight-task-1", + task_id="insight-task", + attempt=1, + status="completed", + metrics={ + "uses_required_tool": MetricResult( + name="uses_required_tool", + value=score, + ) + }, + ) + ], + ) + + @pytest.mark.asyncio async def test_insight_run_evaluates_and_persists_baseline_and_new_candidate_metrics( monkeypatch: pytest.MonkeyPatch, @@ -26,7 +68,11 @@ async def test_insight_run_evaluates_and_persists_baseline_and_new_candidate_met ) -> None: train_dataset = Dataset(id="train") validation_dataset = Dataset(id="validation") - insight_dataset = Dataset(id="insight-suite", tasks=[Task(id="insight-task")]) + insight_dataset = Dataset( + id="insight-suite", + tasks=[Task(id="insight-task")], + metadata=_suite_metadata(), + ) datasets = { "train": train_dataset, "validation": validation_dataset, @@ -59,14 +105,8 @@ async def run(self, **kwargs: Any) -> SimpleNamespace: optimization="use the required tool", ) insight_results = { - "agent-0": EvaluationResult( - id="agent-0-insight", - aggregate_metrics={"uses_required_tool": 0.0}, - ), - "agent-1": EvaluationResult( - id="agent-1-insight", - aggregate_metrics={"uses_required_tool": 1.0}, - ), + "agent-0": _insight_result("agent-0", 0.0), + "agent-1": _insight_result("agent-1", 1.0), } insight_evaluations: list[tuple[Dataset, list[Candidate]]] = [] @@ -200,23 +240,25 @@ async def run(self, **kwargs: object) -> SimpleNamespace: ] assert baseline.insight_reward == {"uses_required_tool": 0.0} assert new_candidate.insight_reward == {"uses_required_tool": 1.0} + assert baseline.insight_suite_identity == f"sha256:{'a' * 64}" + assert new_candidate.insight_suite_identity == f"sha256:{'a' * 64}" + assert baseline.insight_metric_keys == ["uses_required_tool"] insight_persistence = [ call.kwargs for call in backend.persist_evaluation.await_args_list if call.kwargs["split"] == "insight" ] - assert insight_persistence == [ - { - "workspace": "default", - "result": insight_results["agent-0"], - "candidate": baseline, - "split": "insight", - }, - { - "workspace": "default", - "result": insight_results["agent-1"], - "candidate": new_candidate, - "split": "insight", - }, + assert [call["candidate"] for call in insight_persistence] == [baseline, new_candidate] + assert [call["result"].id for call in insight_persistence] == [ + insight_results["agent-0"].id, + insight_results["agent-1"].id, ] + assert all( + call["result"].metadata["insight_suite_identity"] == f"sha256:{'a' * 64}" for call in insight_persistence + ) + assert all( + trial.metadata["insight_suite_artifact_ref"].startswith("nemo-experimentalist-insight-suite://") + for call in insight_persistence + for trial in call["result"].trials + ) @pytest.mark.asyncio @@ -229,6 +271,10 @@ async def test_insight_evaluation_skips_cached_candidates_and_empty_suites( round=0, optimization="baseline", insight_reward={"uses_required_tool": 0.0}, + insight_reward_details=[], + insight_suite_identity=f"sha256:{'a' * 64}", + insight_suite_artifact_ref=f"nemo-experimentalist-insight-suite://insight-1/sha256/{'a' * 64}", + insight_metric_keys=["uses_required_tool"], ) pending = Candidate( run_id="run-1", @@ -236,16 +282,17 @@ async def test_insight_evaluation_skips_cached_candidates_and_empty_suites( round=1, optimization="use the required tool", ) - result = EvaluationResult( - id="agent-1-insight", - aggregate_metrics={"uses_required_tool": 1.0}, - ) + result = _insight_result("agent-1", 1.0) evaluate_agent = AsyncMock(return_value=(pending, result)) monkeypatch.setattr(EvolutionaryOptimizer, "_evaluate_agent", evaluate_agent) optimizer = object.__new__(EvolutionaryOptimizer) evaluated = await optimizer._evaluate_insight_candidates( - dataset=Dataset(id="insight-suite", tasks=[Task(id="insight-task")]), + dataset=Dataset( + id="insight-suite", + tasks=[Task(id="insight-task")], + metadata=_suite_metadata(), + ), evaluator=object(), # type: ignore[arg-type] candidates=[cached, pending], ) @@ -261,3 +308,50 @@ async def test_insight_evaluation_skips_cached_candidates_and_empty_suites( ) assert empty == {} assert evaluate_agent.await_count == 1 + + +@pytest.mark.asyncio +async def test_insight_evaluation_reuses_only_matching_suite_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cached = Candidate( + run_id="run-1", + label="agent-0", + round=0, + optimization="baseline", + insight_reward={"uses_required_tool": 0.0}, + insight_reward_details=[], + insight_suite_identity=f"sha256:{'a' * 64}", + insight_suite_artifact_ref=f"nemo-experimentalist-insight-suite://insight-1/sha256/{'a' * 64}", + insight_metric_keys=["uses_required_tool"], + ) + result = _insight_result("agent-0", 0.5) + evaluate_agent = AsyncMock(return_value=(cached, result)) + monkeypatch.setattr(EvolutionaryOptimizer, "_evaluate_agent", evaluate_agent) + optimizer = object.__new__(EvolutionaryOptimizer) + + matching = Dataset( + id="insight-suite", + tasks=[Task(id="insight-task")], + metadata=_suite_metadata("a"), + ) + changed = Dataset( + id="insight-suite", + tasks=[Task(id="insight-task")], + metadata=_suite_metadata("e"), + ) + + assert ( + await optimizer._evaluate_insight_candidates( + dataset=matching, + evaluator=object(), # type: ignore[arg-type] + candidates=[cached], + ) + == {} + ) + assert await optimizer._evaluate_insight_candidates( + dataset=changed, + evaluator=object(), # type: ignore[arg-type] + candidates=[cached], + ) == {"agent-0": result} + evaluate_agent.assert_awaited_once() diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py index 211ceea130..833beb3dae 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py @@ -1,22 +1,34 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import math from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock +import pytest from nemo_experimentalist_plugin.entities import Candidate from nemo_experimentalist_plugin.experimentalist.components.evaluator import ( Dataset, + EvaluationResult, MetricResult, Task, TrialResult, TrialStatus, ) from nemo_experimentalist_plugin.experimentalist.components.insight_promotion import ( + insight_suite_provenance, render_insight_promotion_section, select_insight_promotion_suggestions, + validate_insight_evaluation_result, + write_insight_comparison_section, write_insight_promotion_section, ) from nemo_experimentalist_plugin.experimentalist.components.loop import AnalysisSkill, EvolutionaryOptimizer +from nemo_experimentalist_plugin.experimentalist.components.models import EvolutionTree + +_SUITE_IDENTITY = f"sha256:{'a' * 64}" +_SUITE_ARTIFACT_REF = f"nemo-experimentalist-insight-suite://insight-1/sha256/{'a' * 64}" def _candidate( @@ -33,6 +45,28 @@ def _candidate( optimization="baseline" if round_num == 0 else "improve required tool use", insight_reward=insight_reward, validation_reward=validation_reward, + insight_suite_identity=_SUITE_IDENTITY, + insight_suite_artifact_ref=_SUITE_ARTIFACT_REF, + insight_metric_keys=["reward", "uses_required_tool"], + ) + + +def _insight_dataset(tasks: list[Task]) -> Dataset: + return Dataset( + id="insight", + tasks=tasks, + metadata={ + "insight_suite_identity": _SUITE_IDENTITY, + "insight_suite_scorer_identity": f"sha256:{'b' * 64}", + "insight_suite_artifact_ref": _SUITE_ARTIFACT_REF, + "insight_suite_task_hashes": { + task.id: { + "content_hash": f"sha256:{'c' * 64}", + "verifier_hash": f"sha256:{'d' * 64}", + } + for task in tasks + }, + }, ) @@ -47,6 +81,8 @@ def test_round_analysis_contract_requires_separate_insight_suite_dimensions() -> assert "candidate.round == 0" in merge_prompt assert "must name every available Insight Suite dimension" in merge_prompt assert "Never blend those metrics into train/validation rewards" in merge_prompt + assert "adaptive/development feedback" in merge_prompt + assert "never present them as independent validation evidence" in merge_prompt def test_final_report_contract_requires_baseline_winner_insight_comparison() -> None: @@ -119,37 +155,49 @@ def test_insight_promotion_suggestions_are_stable_discriminative_and_diverse( ] baseline = _candidate("agent-0", round_num=0) baseline.insight_reward_details = [ - _insight_trial("task-a", 0.0), - _insight_trial("task-b", 0.0), - _insight_trial("task-c", 0.8), + _insight_trial("task-a", 0.0, attempt=1), + _insight_trial("task-a", 0.0, attempt=2), + _insight_trial("task-b", 0.0, attempt=1), + _insight_trial("task-b", 0.0, attempt=2), + _insight_trial("task-c", 0.8, attempt=1), + _insight_trial("task-c", 0.8, attempt=2), _insight_trial("task-flaky", 0.0, attempt=1), _insight_trial("task-flaky", 1.0, attempt=2), - _insight_trial("task-flat", 0.5), + _insight_trial("task-flat", 0.5, attempt=1), + _insight_trial("task-flat", 0.5, attempt=2), ] winner = _candidate("agent-1", round_num=1) winner.insight_reward_details = [ - _insight_trial("task-a", 1.0), - _insight_trial("task-b", 1.0), - _insight_trial("task-c", 0.2), - _insight_trial("task-flaky", 1.0), - _insight_trial("task-flat", 0.5), + _insight_trial("task-a", 1.0, attempt=1), + _insight_trial("task-a", 1.0, attempt=2), + _insight_trial("task-b", 1.0, attempt=1), + _insight_trial("task-b", 1.0, attempt=2), + _insight_trial("task-c", 0.9, attempt=1), + _insight_trial("task-c", 0.9, attempt=2), + _insight_trial("task-flaky", 1.0, attempt=1), + _insight_trial("task-flaky", 1.0, attempt=2), + _insight_trial("task-flat", 0.5, attempt=1), + _insight_trial("task-flat", 0.5, attempt=2), ] suggestions = select_insight_promotion_suggestions( - Dataset(id="insight", tasks=tasks), + _insight_dataset(tasks), [baseline, winner], + winner=winner, ) assert [suggestion.task_id for suggestion in suggestions] == ["task-a", "task-c"] assert suggestions[0].metric_name == "uses_required_tool" assert suggestions[0].discrimination == 1.0 assert suggestions[0].diversity_score is None - assert suggestions[1].diversity_score == 0.8 + assert suggestions[1].diversity_score == pytest.approx(0.45) section = render_insight_promotion_section(suggestions) - assert "Advisory only: these tasks were not copied into the validation dataset." in section + assert "Advisory adaptive/development evidence only" in section assert "`task-a`" in section - assert f"`{tmp_path / 'task-a'}`" in section + assert f"`{_SUITE_ARTIFACT_REF}`" in section + assert "baseline 0.00 → winner 1.00" in section + assert f"task sha256:{'c' * 64}" in section assert "task-b" not in section assert "task-flaky" not in section assert "task-flat" not in section @@ -159,7 +207,7 @@ def test_insight_promotion_section_explains_when_no_task_qualifies() -> None: section = render_insight_promotion_section([]) assert "## Insight Suite Promotion Suggestions" in section - assert "No task had complete, repeat-consistent results" in section + assert "No task had complete repeated evidence" in section def test_insight_promotion_section_is_appended_without_rewriting_report( @@ -176,3 +224,249 @@ def test_insight_promotion_section_is_appended_without_rewriting_report( assert report_path.read_text() == first_report assert first_report.startswith("# Optimization\n\nExisting analysis.") assert first_report.count("## Insight Suite Promotion Suggestions") == 1 + + +@pytest.mark.parametrize("score", [1.1, -0.1, math.inf, -math.inf, math.nan]) +def test_runtime_insight_metrics_reject_out_of_range_and_non_finite_values(score: float) -> None: + result = EvaluationResult( + id="invalid", + aggregate_metrics={"reward": 1.0, "uses_required_tool": score}, + trials=[_insight_trial("task-a", score)], + ) + + with pytest.raises(ValueError, match=r"finite and within \[0, 1\]"): + validate_insight_evaluation_result(result) + + +def test_runtime_insight_metrics_reject_missing_or_inconsistent_keys() -> None: + missing_trial_key = EvaluationResult( + id="missing", + aggregate_metrics={"reward": 1.0, "uses_required_tool": 0.5}, + trials=[ + TrialResult( + id="task-a-1", + task_id="task-a", + status="completed", + metrics={"reward": MetricResult(name="reward", value=1.0)}, + ) + ], + ) + + with pytest.raises(ValueError, match="metric keys are inconsistent"): + validate_insight_evaluation_result(missing_trial_key) + + with pytest.raises(ValueError, match="aggregate metric keys are inconsistent"): + validate_insight_evaluation_result( + EvaluationResult( + id="changed", + aggregate_metrics={"reward": 1.0, "different_metric": 0.5}, + trials=[ + TrialResult( + id="task-a-1", + task_id="task-a", + status="completed", + metrics={ + "reward": MetricResult(name="reward", value=1.0), + "different_metric": MetricResult(name="different_metric", value=0.5), + }, + ) + ], + ), + expected_metric_keys=["reward", "uses_required_tool"], + ) + + +@pytest.mark.parametrize( + ("invalid_score", "missing_key"), + [(1.1, False), (math.nan, False), (0.5, True)], +) +def test_invalid_runtime_metrics_cannot_be_promotion_evidence( + invalid_score: float, + missing_key: bool, +) -> None: + task = Task(id="task-a") + baseline = _candidate("agent-0", round_num=0) + winner = _candidate("agent-1", round_num=1) + baseline.insight_reward_details = [ + _insight_trial("task-a", 0.0, attempt=1), + _insight_trial("task-a", 0.0, attempt=2), + ] + invalid_trial = _insight_trial("task-a", invalid_score, attempt=1) + if missing_key: + invalid_trial.metrics.pop("uses_required_tool") + winner.insight_reward_details = [ + invalid_trial, + _insight_trial("task-a", 1.0, attempt=2), + ] + + assert ( + select_insight_promotion_suggestions( + _insight_dataset([task]), + [baseline, winner], + winner=winner, + ) + == [] + ) + + +def test_one_attempt_failed_and_incomplete_evidence_do_not_qualify_as_stable() -> None: + task = Task(id="task-a") + baseline = _candidate("agent-0", round_num=0) + winner = _candidate("agent-1", round_num=1) + baseline.insight_reward_details = [_insight_trial("task-a", 0.0)] + winner.insight_reward_details = [_insight_trial("task-a", 1.0)] + + assert ( + select_insight_promotion_suggestions( + _insight_dataset([task]), + [baseline, winner], + winner=winner, + ) + == [] + ) + + baseline.insight_reward_details.append(_insight_trial("task-a", 0.0, attempt=2)) + winner.insight_reward_details.append(_insight_trial("task-a", 1.0, attempt=2, status="failed")) + assert ( + select_insight_promotion_suggestions( + _insight_dataset([task]), + [baseline, winner], + winner=winner, + ) + == [] + ) + + winner.insight_reward_details = [] + assert ( + select_insight_promotion_suggestions( + _insight_dataset([task]), + [baseline, winner], + winner=winner, + ) + == [] + ) + + +@pytest.mark.parametrize( + ("baseline_score", "winner_score", "bad_score"), + [ + (0.5, 0.5, None), + (0.8, 0.2, None), + (0.5, 0.5, 0.0), + ], +) +def test_promotion_requires_baseline_to_winner_improvement( + baseline_score: float, + winner_score: float, + bad_score: float | None, +) -> None: + task = Task(id="task-a") + baseline = _candidate("agent-0", round_num=0) + winner = _candidate("agent-1", round_num=1) + candidates = [baseline, winner] + baseline.insight_reward_details = [ + _insight_trial("task-a", baseline_score, attempt=1), + _insight_trial("task-a", baseline_score, attempt=2), + ] + winner.insight_reward_details = [ + _insight_trial("task-a", winner_score, attempt=1), + _insight_trial("task-a", winner_score, attempt=2), + ] + if bad_score is not None: + bad = _candidate("agent-bad", round_num=1) + bad.insight_reward_details = [ + _insight_trial("task-a", bad_score, attempt=1), + _insight_trial("task-a", bad_score, attempt=2), + ] + candidates.append(bad) + + assert ( + select_insight_promotion_suggestions( + _insight_dataset([task]), + candidates, + winner=winner, + ) + == [] + ) + + +def test_deterministic_insight_comparison_section_uses_content_addressed_suite( + tmp_path: Path, +) -> None: + baseline = _candidate( + "agent-0", + round_num=0, + insight_reward={"reward": 0.5, "uses_required_tool": 0.0}, + ) + winner = _candidate( + "agent-1", + round_num=1, + insight_reward={"reward": 0.75, "uses_required_tool": 1.0}, + ) + report_path = tmp_path / "OPTIMIZATION.md" + provenance = insight_suite_provenance(_insight_dataset([Task(id="task-a")])) + + write_insight_comparison_section(report_path, baseline, winner, provenance) + report = report_path.read_text() + + assert "## Deterministic Insight Suite Comparison" in report + assert _SUITE_ARTIFACT_REF in report + assert "| `uses_required_tool` | 0.000 | 1.000 | +1.000 |" in report + + +@pytest.mark.asyncio +async def test_final_report_failure_preserves_compact_summary_and_deterministic_sections( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + baseline = _candidate( + "agent-0", + round_num=0, + insight_reward={"reward": 0.5, "uses_required_tool": 0.0}, + validation_reward={"reward": 0.5}, + ) + winner = _candidate( + "agent-1", + round_num=1, + insight_reward={"reward": 0.75, "uses_required_tool": 1.0}, + validation_reward={"reward": 0.75}, + ) + tree = EvolutionTree() + tree.add(baseline) + tree.add(winner) + optimizer = object.__new__(EvolutionaryOptimizer) + optimizer.working_dir = tmp_path + (tmp_path / "eval-and-optimize").mkdir() + monkeypatch.setattr(optimizer, "_copy_best_to_workspace", lambda best_id: None) + original_report_writer = EvolutionaryOptimizer.write_final_report + type.__setattr__( + EvolutionaryOptimizer, + "write_final_report", + AsyncMock(side_effect=RuntimeError("LLM report failed")), + ) + run = SimpleNamespace(status="running", winner_agent=None, rounds_completed=1) + backend = SimpleNamespace(update_run=AsyncMock()) + + try: + finalized = await optimizer._finalize( + workspace="default", + backend=backend, + agents_dir=tmp_path / "eval-and-optimize" / "agents", + run_entity=run, + evolution_tree=tree, + agent_name="agent", + insight_dataset=_insight_dataset([Task(id="task-a")]), + ) + finally: + type.__setattr__( + EvolutionaryOptimizer, + "write_final_report", + original_report_writer, + ) + + report = (tmp_path / "eval-and-optimize" / "OPTIMIZATION.md").read_text() + assert finalized is winner + assert "## Compact Run Summary" in report + assert "Optimization complete: 1 round(s) completed" in report + assert "## Deterministic Insight Suite Comparison" in report + assert "## Insight Suite Promotion Suggestions" in report diff --git a/plugins/nemo-experimentalist/tests/test_eval_author_agent.py b/plugins/nemo-experimentalist/tests/test_eval_author_agent.py index 2079ad688b..26c678f70f 100644 --- a/plugins/nemo-experimentalist/tests/test_eval_author_agent.py +++ b/plugins/nemo-experimentalist/tests/test_eval_author_agent.py @@ -8,6 +8,7 @@ from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path +from types import SimpleNamespace from typing import Any, cast import pytest @@ -189,8 +190,10 @@ def promote_local(self, trace_refs: list[str], staged_tasks: list[Any]) -> Datas tasks = [staged.result for staged in staged_tasks] if materialized_dataset is not None: materialized_dataset.tasks = tasks + self.materialized_dataset = materialized_dataset return materialized_dataset - return Dataset(id="insight-suite", tasks=tasks) + self.materialized_dataset = Dataset(id="insight-suite", tasks=tasks) + return self.materialized_dataset def discard(self) -> None: calls.suite_discards += 1 @@ -198,6 +201,31 @@ def discard(self) -> None: def record_analysis(self, statuses: dict[str, tuple[str, str | None]]) -> None: pass + def finalize_artifact(self) -> SimpleNamespace: + identity = "sha256:" + "a" * 64 + scorer_identity = "sha256:" + "b" * 64 + artifact_ref = f"nemo-experimentalist-insight-suite://insight-1/sha256/{'a' * 64}" + self.materialized_dataset.metadata.update( + { + "insight_suite_identity": identity, + "insight_suite_scorer_identity": scorer_identity, + "insight_suite_artifact_ref": artifact_ref, + "insight_suite_task_hashes": { + task.id: { + "content_hash": "sha256:" + "c" * 64, + "verifier_hash": "sha256:" + "d" * 64, + } + for task in self.materialized_dataset.list_tasks() + }, + } + ) + return SimpleNamespace( + dataset=self.materialized_dataset, + identity=identity, + scorer_identity=scorer_identity, + ref=artifact_ref, + ) + class FillTaskTemplate: async def __call__( self, @@ -537,6 +565,9 @@ def fake_doc(dataset_type: type[Dataset], *, inline_depth: int) -> object: assert len(calls.discovered_datasets) == 1 materialized_dataset = calls.discovered_datasets[0] assert result.insight_suite is materialized_dataset + assert result.insight_suite_identity == f"sha256:{'a' * 64}" + assert result.insight_suite_artifact_ref is not None + assert result.insight_suite_artifact_ref.startswith("nemo-experimentalist-insight-suite://") assert materialized_dataset.id == "insight-suite" assert materialized_dataset is not train_dataset assert materialized_dataset is not validation_dataset diff --git a/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py b/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py index 04184bb920..39695b11e7 100644 --- a/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py +++ b/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py @@ -11,7 +11,10 @@ import pytest from nemo_experimentalist_plugin.eval_author import materialization as materialization_module -from nemo_experimentalist_plugin.eval_author.materialization import InsightSuite +from nemo_experimentalist_plugin.eval_author.materialization import ( + InsightSuite, + resolve_insight_suite_artifact, +) from nemo_experimentalist_plugin.experimentalist.components.evaluator import Task from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset @@ -207,3 +210,68 @@ def test_insight_suite_records_analysis_without_removing_failed_tasks(tmp_path: {"error": "analysis failed", "status": "failed"}, ] assert len(HarborDataset.from_path(suite.suite_dir).list_tasks()) == 2 + + +def test_finalized_suite_is_content_addressed_durable_and_resolvable(tmp_path: Path) -> None: + template = _write_template(tmp_path / "template") + refs = ["trace-1"] + suite = InsightSuite(experiment_dir=tmp_path, insight_id="insight-1", task_template=template) + staged = suite.stage(refs) + (staged[0].path / "instruction.md").write_text("Reproduce the motivating failure.\n") + suite.validate(staged[0]) + suite.promote_local(refs, staged) + + artifact = suite.finalize_artifact() + manifest = json.loads((artifact.path / "manifest.json").read_text(encoding="utf-8")) + + assert artifact.identity.startswith("sha256:") + assert artifact.scorer_identity.startswith("sha256:") + assert artifact.ref.startswith("nemo-experimentalist-insight-suite://") + assert manifest["suite_identity"] == artifact.identity + assert manifest["scorer"] == { + "identity": artifact.scorer_identity, + "metric_contract_version": 1, + } + assert manifest["tasks"][0]["content_hash"].startswith("sha256:") + assert manifest["tasks"][0]["verifier"]["content_hash"].startswith("sha256:") + assert resolve_insight_suite_artifact(tmp_path, artifact.ref) == artifact.path + assert artifact.dataset.metadata["insight_suite_identity"] == artifact.identity + assert artifact.dataset.metadata["insight_suite_artifact_ref"] == artifact.ref + assert list(artifact.dataset.list_tasks())[0].uri.startswith(artifact.path.as_uri()) + + artifact_instruction = next(path for path in artifact.path.iterdir() if path.is_dir()) / "instruction.md" + artifact_instruction.write_text("tampered\n") + with pytest.raises(ValueError, match="content does not match reference"): + resolve_insight_suite_artifact(tmp_path, artifact.ref) + + +def test_finalized_suite_identity_is_stable_and_changes_with_task_or_verifier_content( + tmp_path: Path, +) -> None: + template = _write_template(tmp_path / "template") + + def build(instruction: str, verifier_suffix: str = "") -> tuple[str, str]: + suite = InsightSuite( + experiment_dir=tmp_path, + insight_id="insight-1", + task_template=template, + ) + staged = suite.stage(["trace-1"]) + (staged[0].path / "instruction.md").write_text(instruction) + if verifier_suffix: + verifier_path = staged[0].path / "tests" / "test.sh" + verifier_path.write_text(verifier_path.read_text() + verifier_suffix) + suite.validate(staged[0]) + suite.promote_local(["trace-1"], staged) + artifact = suite.finalize_artifact() + return artifact.identity, artifact.ref + + first_identity, first_ref = build("Same authored task.\n") + identical_identity, identical_ref = build("Same authored task.\n") + changed_task_identity, _ = build("Changed authored task.\n") + changed_verifier_identity, _ = build("Same authored task.\n", "\n# changed scorer\n") + + assert (identical_identity, identical_ref) == (first_identity, first_ref) + assert changed_task_identity != first_identity + assert changed_verifier_identity != first_identity + assert resolve_insight_suite_artifact(tmp_path, first_ref).is_dir() From 06fbf9ead22df00225b162ce80af72e5f63c473a Mon Sep 17 00:00:00 2001 From: Ryan Angilly Date: Mon, 27 Jul 2026 12:57:40 -0600 Subject: [PATCH 5/5] fix(experimentalist): address insight suite review feedback Signed-off-by: Ryan Angilly --- .../nemo_experimentalist_plugin/entities.py | 4 - .../eval_author/README.md | 21 +-- .../eval_author/REFERENCE.md | 23 +++ .../eval_author/agent.py | 7 +- .../eval_author/materialization.py | 86 ++-------- .../eval_author/models.py | 6 +- .../components/insight_promotion.py | 62 ++++---- .../experimentalist/components/loop.py | 63 +++++--- .../test_loop_insight_suite.py | 147 +++++++++++++++++- .../experimentalist/test_loop_reporting.py | 109 ++++++++++++- .../tests/test_eval_author_agent.py | 7 +- .../tests/test_eval_author_materialization.py | 55 +++---- .../tests/test_experiment_mirror.py | 2 +- 13 files changed, 388 insertions(+), 204 deletions(-) create mode 100644 plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/REFERENCE.md diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py index a2346a70f7..139a3a1e75 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py @@ -161,10 +161,6 @@ class Candidate(NemoEntity, entity_type="candidate"): default=None, description="Content identity of the Insight suite associated with insight_reward.", ) - insight_suite_artifact_ref: str | None = Field( - default=None, - description="Portable reference to the immutable Insight suite associated with insight_reward.", - ) insight_metric_keys: list[str] | None = Field( default=None, description="Validated runtime metric keys associated with insight_reward.", diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md index 0f7319316a..09acb7f706 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md @@ -18,6 +18,7 @@ runs the top-level Eval Author before beginning optimization. - `agent.py` defines the canonical `EvalAuthor` agent. - `materialization.py` stages, validates, and persists Insight suites locally. - `models.py` defines the lightweight `EvalAuthorConfig` and `EvalAuthorResult` models. +- `REFERENCE.md` documents the Python return contract. - `run.py` defines `run_eval_author(...)`, a reusable orchestration function for Python callers. - `config.yaml` is a default run preset for future CLI or job wiring. @@ -59,16 +60,9 @@ unchanged. The authored verifiers must pass static Harbor validation before the local suite is returned to the optimization loop. After authoring and validation, Eval Author hashes every task file and verifier -file, derives deterministic suite and scorer identities, and freezes the exact -content beneath: - -```text -eval-and-optimize/eval_author//artifacts//insight-suite/ -``` - -The returned dataset points at this immutable artifact and carries a portable -`nemo-experimentalist-insight-suite://.../sha256/...` reference. Candidate Insight -results persist the same suite identity and artifact reference. Resume reuses +file and persists deterministic suite and scorer identities in the local suite's +manifest. The returned dataset continues to point at the single experiment-local +suite. Candidate Insight results persist the suite identity, and resume reuses those results only when the identity still matches; changed task or verifier content is re-evaluated. @@ -77,11 +71,8 @@ Task-template inputs may be local paths, `file://` URIs, or NeMo Platform downloaded into the experiment-local staging directory before Harbor parses them. The staged template is refreshed on every invocation rather than reused. -`EvalAuthorResult.insight_suite` contains the finalized content-addressed -`Dataset` for immediate evaluation by the optimization loop. Its identity and -portable artifact reference are also available as -`EvalAuthorResult.insight_suite_identity` and -`EvalAuthorResult.insight_suite_artifact_ref`. +The returned Python contract is documented in the +[Eval Author Python Reference](REFERENCE.md#evalauthorresult). Insight metrics remain adaptive development feedback. They may steer round analysis, goal-tree updates, and proposals, but validation remains the direct diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/REFERENCE.md b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/REFERENCE.md new file mode 100644 index 0000000000..aa3c584557 --- /dev/null +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/REFERENCE.md @@ -0,0 +1,23 @@ + + +# Eval Author Python Reference + +## `EvalAuthorResult` + +`run_eval_author(...)` returns an `EvalAuthorResult` with these fields: + +| Field | Type | Description | +| --- | --- | --- | +| `train_dataset` | `Dataset` | Training dataset supplied to the run. Eval Author does not mutate it. | +| `validation_dataset` | `Dataset` | Validation dataset supplied to the run. Eval Author does not mutate it. | +| `insight_suite` | `Dataset \| None` | Finalized experiment-local Insight dataset for immediate evaluation by the optimization loop. | +| `insight_suite_identity` | `str \| None` | SHA-256 identity of the finalized Insight task and verifier content. | +| `summary` | `str` | Eval Author's analysis summary. | + +When an Insight suite is materialized successfully, `insight_suite` and +`insight_suite_identity` are both populated. Callers can persist the identity +with candidate results and reuse those results only while the suite identity +continues to match. diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py index af86c6c260..33fe71da8c 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py @@ -366,13 +366,12 @@ async def _run( validation_feedback=str(exc), ) else: - artifact = insight_suite.finalize_artifact() + finalized_suite = insight_suite.finalize() return EvalAuthorResult( train_dataset=train_dataset, validation_dataset=validation_dataset, - insight_suite=artifact.dataset, - insight_suite_identity=artifact.identity, - insight_suite_artifact_ref=artifact.ref, + insight_suite=finalized_suite.dataset, + insight_suite_identity=finalized_suite.identity, summary=summary, ) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py index dae2757e21..4e084dccb8 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py @@ -14,7 +14,6 @@ from dataclasses import dataclass from pathlib import Path from typing import cast -from urllib.parse import urlparse from uuid import uuid4 import tomlkit @@ -22,12 +21,10 @@ from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import Task, local_path_from_uri -_MANIFEST_SCHEMA_VERSION = 2 +_MANIFEST_SCHEMA_VERSION = 3 _CONTENT_HASH_SCHEMA_VERSION = 1 _METRIC_CONTRACT_VERSION = 1 -_ARTIFACT_SCHEME = "nemo-experimentalist-insight-suite" _SLUG_RE = re.compile(r"[^a-z0-9]+") -_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") def _slug(value: str, *, fallback: str, max_length: int = 48) -> str: @@ -104,16 +101,17 @@ def _content_provenance(suite_dir: Path, manifest: dict[str, object]) -> tuple[l verifier_files = _file_hashes(verifier_dir) content_hash = f"sha256:{_canonical_digest(files)}" verifier_hash = f"sha256:{_canonical_digest(verifier_files)}" + task_metadata = { + key: value for key, value in task_entry.items() if key not in {"content_hash", "files", "verifier"} + } tasks.append( { - **task_entry, + **task_metadata, "content_hash": content_hash, "verifier": { "path": verifier_path, "content_hash": verifier_hash, - "files": verifier_files, }, - "files": files, } ) scorer_inputs.append({"path": relative_path, "verifier_hash": verifier_hash}) @@ -149,52 +147,15 @@ class StagedInsightTask: @dataclass(frozen=True, slots=True) -class InsightSuiteArtifact: - """Immutable, content-addressed result of an authored Insight suite.""" +class FinalizedInsightSuite: + """Experiment-local Insight suite with deterministic content identities.""" identity: str scorer_identity: str - ref: str path: Path dataset: HarborDataset -def resolve_insight_suite_artifact(experiment_dir: Path, artifact_ref: str) -> Path: - """Resolve and verify a portable Insight-suite artifact reference.""" - parsed = urlparse(artifact_ref) - parts = parsed.path.strip("/").split("/") - if ( - parsed.scheme != _ARTIFACT_SCHEME - or not parsed.netloc - or len(parts) != 2 - or parts[0] != "sha256" - or not _SHA256_RE.fullmatch(parts[1]) - ): - raise ValueError(f"Invalid Insight suite artifact reference: {artifact_ref!r}") - - suite_dir = ( - experiment_dir.resolve() - / "eval-and-optimize" - / "eval_author" - / parsed.netloc - / "artifacts" - / parts[1] - / "insight-suite" - ) - manifest_path = suite_dir / "manifest.json" - if not manifest_path.is_file(): - raise FileNotFoundError(f"Insight suite artifact not found: {artifact_ref}") - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - _, scorer_identity, suite_identity = _content_provenance(suite_dir, manifest) - expected_identity = f"sha256:{parts[1]}" - if manifest.get("suite_identity") != expected_identity or suite_identity != expected_identity: - raise ValueError(f"Insight suite artifact content does not match reference: {artifact_ref}") - scorer = manifest.get("scorer") - if not isinstance(scorer, dict) or scorer.get("identity") != scorer_identity: - raise ValueError(f"Insight suite scorer content does not match reference: {artifact_ref}") - return suite_dir - - class InsightSuite: """Build one experiment-local persisted Harbor dataset for an Insight.""" @@ -354,14 +315,13 @@ def record_analysis(self, statuses: dict[str, tuple[str, str | None]]) -> None: pending_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") os.replace(pending_path, manifest_path) - def finalize_artifact(self) -> InsightSuiteArtifact: - """Freeze the authored suite under a verified content-addressed reference.""" + def finalize(self) -> FinalizedInsightSuite: + """Persist content identities on the experiment-local authored suite.""" manifest_path = self.suite_dir / "manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) tasks, scorer_identity, suite_identity = _content_provenance(self.suite_dir, manifest) digest = suite_identity.removeprefix("sha256:") - artifact_ref = f"{_ARTIFACT_SCHEME}://{self.root.name}/sha256/{digest}" - artifact_path = self.root / "artifacts" / digest / "insight-suite" + manifest.pop("artifact", None) manifest.update( { "schema_version": _MANIFEST_SCHEMA_VERSION, @@ -372,10 +332,6 @@ def finalize_artifact(self) -> InsightSuiteArtifact: "identity": scorer_identity, "metric_contract_version": _METRIC_CONTRACT_VERSION, }, - "artifact": { - "ref": artifact_ref, - "relative_path": artifact_path.relative_to(self.experiment_dir).as_posix(), - }, "tasks": tasks, } ) @@ -383,22 +339,8 @@ def finalize_artifact(self) -> InsightSuiteArtifact: pending_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") os.replace(pending_path, manifest_path) - if artifact_path.exists(): - resolved = resolve_insight_suite_artifact(self.experiment_dir, artifact_ref) - if resolved != artifact_path.resolve(): - raise ValueError(f"Insight suite artifact resolved to unexpected path: {resolved}") - else: - artifact_path.parent.mkdir(parents=True, exist_ok=True) - candidate_artifact = artifact_path.parent / f".candidate-{uuid4().hex}" - shutil.copytree(self.suite_dir, candidate_artifact) - try: - os.replace(candidate_artifact, artifact_path) - finally: - if candidate_artifact.exists(): - shutil.rmtree(candidate_artifact) - dataset = HarborDataset.from_path( - artifact_path, + self.suite_dir, dataset_id=f"insight-{digest[:12]}", ) task_hashes: dict[str, dict[str, str]] = {} @@ -421,14 +363,12 @@ def finalize_artifact(self) -> InsightSuiteArtifact: { "insight_suite_identity": suite_identity, "insight_suite_scorer_identity": scorer_identity, - "insight_suite_artifact_ref": artifact_ref, "insight_suite_task_hashes": task_hashes, } ) - return InsightSuiteArtifact( + return FinalizedInsightSuite( identity=suite_identity, scorer_identity=scorer_identity, - ref=artifact_ref, - path=artifact_path, + path=self.suite_dir, dataset=dataset, ) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py index f3652736cc..83f5e6f86b 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py @@ -35,14 +35,10 @@ class EvalAuthorResult(BaseModel): validation_dataset: Dataset insight_suite: Dataset | None = Field( default=None, - description="Finalized content-addressed Insight dataset for use by the optimization loop.", + description="Finalized experiment-local Insight dataset for use by the optimization loop.", ) insight_suite_identity: str | None = Field( default=None, description="SHA-256 identity of the finalized Insight task and verifier content.", ) - insight_suite_artifact_ref: str | None = Field( - default=None, - description="Portable reference resolving to the immutable finalized Insight suite.", - ) summary: str diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py index a4993133fd..660aa94b5a 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py @@ -17,6 +17,7 @@ Task, TrialResult, ) +from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import local_path_from_uri _GENERIC_METRIC_NAMES = frozenset({"reward", "score"}) _MAX_REPEAT_SPREAD = 0.1 @@ -29,11 +30,11 @@ @dataclass(frozen=True, slots=True) class InsightSuiteProvenance: - """Runtime identity and portable location for one finalized Insight suite.""" + """Runtime identities and local location for one finalized Insight suite.""" identity: str scorer_identity: str - artifact_ref: str + suite_path: Path task_hashes: dict[str, dict[str, str]] @@ -42,7 +43,8 @@ class InsightPromotionSuggestion: """Evidence-backed recommendation to review one Insight-suite task.""" task_id: str - suite_artifact_ref: str + task_path: str + suite_identity: str task_content_hash: str verifier_hash: str metric_name: str @@ -66,14 +68,14 @@ def insight_suite_provenance(dataset: Dataset) -> InsightSuiteProvenance: """Return validated content provenance carried by a finalized suite dataset.""" identity = dataset.metadata.get("insight_suite_identity") scorer_identity = dataset.metadata.get("insight_suite_scorer_identity") - artifact_ref = dataset.metadata.get("insight_suite_artifact_ref") raw_task_hashes = dataset.metadata.get("insight_suite_task_hashes") if not isinstance(identity, str) or not identity.startswith("sha256:"): raise ValueError("Finalized Insight suite is missing its content identity") if not isinstance(scorer_identity, str) or not scorer_identity.startswith("sha256:"): raise ValueError("Finalized Insight suite is missing its scorer identity") - if not isinstance(artifact_ref, str) or not artifact_ref: - raise ValueError("Finalized Insight suite is missing its portable artifact reference") + if dataset.source is None: + raise ValueError("Finalized Insight suite is missing its local source path") + suite_path = local_path_from_uri(dataset.source.uri, context="Finalized Insight suite").resolve() if not isinstance(raw_task_hashes, dict): raise ValueError("Finalized Insight suite is missing task and verifier hashes") task_hashes: dict[str, dict[str, str]] = {} @@ -91,7 +93,7 @@ def insight_suite_provenance(dataset: Dataset) -> InsightSuiteProvenance: return InsightSuiteProvenance( identity=identity, scorer_identity=scorer_identity, - artifact_ref=artifact_ref, + suite_path=suite_path, task_hashes=task_hashes, ) @@ -148,7 +150,6 @@ def stamp_insight_evaluation_result( suite_metadata = { "insight_suite_identity": provenance.identity, "insight_suite_scorer_identity": provenance.scorer_identity, - "insight_suite_artifact_ref": provenance.artifact_ref, } return result.model_copy( update={ @@ -167,7 +168,7 @@ def _task_metric_values( ) -> dict[str, list[float]] | None: if len(trials) < 2 or any(trial.status != "completed" for trial in trials): return None - values = {metric_name: [] for metric_name in required_metrics} + values: dict[str, list[float]] = {metric_name: [] for metric_name in required_metrics} for trial in trials: if set(trial.metrics) != required_metrics: return None @@ -191,11 +192,10 @@ def _task_evidence( winner: Candidate, provenance: InsightSuiteProvenance, ) -> _TaskEvidence | None: - metric_key_sets = { - tuple(candidate.insight_metric_keys or ()) - for candidate in candidates - if candidate.insight_suite_identity == provenance.identity - } + suite_candidates = [ + candidate for candidate in candidates if candidate.insight_suite_identity == provenance.identity + ] + metric_key_sets = {tuple(sorted(candidate.insight_metric_keys or ())) for candidate in suite_candidates} if len(metric_key_sets) != 1: return None required_metrics = set(next(iter(metric_key_sets), ())) @@ -205,7 +205,7 @@ def _task_evidence( trials_by_candidate = { candidate.label: [trial for trial in candidate.insight_reward_details or () if trial.task_id == task.id] - for candidate in candidates + for candidate in suite_candidates } values_by_candidate: dict[str, dict[str, list[float]]] = {} for label, trials in trials_by_candidate.items(): @@ -217,10 +217,7 @@ def _task_evidence( if baseline.label not in values_by_candidate or winner.label not in values_by_candidate: return None total_attempts = sum(len(trials) for trials in trials_by_candidate.values()) - completed_attempts = sum( - 1 for trials in trials_by_candidate.values() for trial in trials if trial.status == "completed" - ) - if not total_attempts or completed_attempts != total_attempts: + if not total_attempts: return None profile: dict[tuple[str, str], float] = {} @@ -251,23 +248,28 @@ def _task_evidence( if baseline_score >= 1.0 or discrimination <= _MIN_DISCRIMINATION: return None hashes = provenance.task_hashes.get(task.id) - if hashes is None: + if hashes is None or not task.uri: + return None + try: + task_path = str(local_path_from_uri(task.uri, context=f"Insight task {task.id!r}").resolve()) + except ValueError: return None return _TaskEvidence( suggestion=InsightPromotionSuggestion( task_id=task.id, - suite_artifact_ref=provenance.artifact_ref, + task_path=task_path, + suite_identity=provenance.identity, task_content_hash=hashes["content_hash"], verifier_hash=hashes["verifier_hash"], metric_name=metric_name, discrimination=discrimination, baseline_score=baseline_score, winner_score=winner_score, - completed_attempts=completed_attempts, + completed_attempts=total_attempts, total_attempts=total_attempts, repeat_spread=repeat_spread, - candidate_count=len(candidates), + candidate_count=len(suite_candidates), ), profile=profile, ) @@ -386,7 +388,7 @@ def render_insight_promotion_section( lines.extend( [ - "| Task | Content-addressed suite | Evidence |", + "| Task | Local task path | Evidence |", "| --- | --- | --- |", ] ) @@ -402,11 +404,12 @@ def render_insight_promotion_section( f"{suggestion.candidate_count} candidates; " f"{suggestion.completed_attempts}/{suggestion.total_attempts} attempts completed; " f"repeat spread {suggestion.repeat_spread:.2f}; " - f"task {suggestion.task_content_hash}; verifier {suggestion.verifier_hash}; {diversity}" + f"suite {suggestion.suite_identity}; task {suggestion.task_content_hash}; " + f"verifier {suggestion.verifier_hash}; {diversity}" ) lines.append( f"| `{_markdown_cell(suggestion.task_id)}` | " - f"`{_markdown_cell(suggestion.suite_artifact_ref)}` | {_markdown_cell(evidence)} |" + f"`{_markdown_cell(suggestion.task_path)}` | {_markdown_cell(evidence)} |" ) return "\n".join(lines) @@ -450,10 +453,7 @@ def render_insight_comparison_section( ) -> str: """Render the deterministic baseline-versus-winner Insight comparison.""" for candidate in (baseline, winner): - if ( - candidate.insight_suite_identity != provenance.identity - or candidate.insight_suite_artifact_ref != provenance.artifact_ref - ): + if candidate.insight_suite_identity != provenance.identity: raise ValueError( f"Candidate {candidate.label!r} Insight evidence does not match finalized suite {provenance.identity}" ) @@ -468,7 +468,7 @@ def render_insight_comparison_section( "Pareto and winner-selection criterion." ), "", - f"Suite: `{provenance.artifact_ref}` (`{provenance.identity}`)", + (f"Suite: `{provenance.suite_path}` (suite `{provenance.identity}`; scorer `{provenance.scorer_identity}`)"), "", "| Metric | Baseline | Winner | Delta |", "| --- | ---: | ---: | ---: |", diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py index f4d2160b53..80649146f9 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py @@ -1029,10 +1029,9 @@ def _delete_all_artifacts(self, from_round: int) -> None: continue if (meta.get("round") or 0) > from_round: shutil.rmtree(agent_dir) - for suffix in ("-train", "-validation", "-insight"): - rd = results_dir / f"{agent_dir.name}{suffix}" - if rd.exists(): - shutil.rmtree(rd) + for result_dir in results_dir.glob(f"{agent_dir.name}-*"): + if result_dir.is_dir(): + shutil.rmtree(result_dir) for rd in ( smoke_results_dir / agent_dir.name, smoke_dataset_dir / agent_dir.name, @@ -1263,6 +1262,7 @@ async def _evaluate_agent( dataset: Dataset, evaluator: Evaluator, task_ids: list[str] | None = None, + minimum_attempts: int | None = None, ) -> tuple[Candidate, EvaluationResult]: """Run evaluator for one candidate and return the candidate/result pair.""" eval_dataset = dataset.subset(task_ids) if task_ids is not None else dataset @@ -1270,6 +1270,11 @@ async def _evaluate_agent( # collide on the same results directory when the user sets a fixed job_name. options_dict = evaluator.options.model_dump() options_dict["job_name"] = f"{candidate.label}-{eval_dataset.id}" + if minimum_attempts is not None: + configured_attempts = options_dict.get("n_attempts") + if not isinstance(configured_attempts, int): + raise ValueError("Insight evaluator options must define integer n_attempts") + options_dict["n_attempts"] = max(configured_attempts, minimum_attempts) per_candidate_options = type(evaluator.options).model_validate(options_dict) result = await evaluator.run( agent=self.working_dir / "eval-and-optimize" / "agents" / candidate.label, @@ -1373,11 +1378,18 @@ async def _evaluate_insight_candidates( if candidate.insight_reward is None or candidate.insight_reward_details is None or candidate.insight_suite_identity != provenance.identity - or candidate.insight_suite_artifact_ref != provenance.artifact_ref or not candidate.insight_metric_keys ] evaluated = await asyncio.gather( - *[self._evaluate_agent(candidate, dataset, evaluator) for candidate in pending] + *[ + self._evaluate_agent( + candidate, + dataset, + evaluator, + minimum_attempts=2, + ) + for candidate in pending + ] ) return {candidate.label: result for candidate, result in evaluated} @@ -1404,12 +1416,12 @@ async def _evaluate_and_persist_insight_candidates( ): raise ValueError("Insight suite runtime metric keys have invalid metadata") cached_metric_key_sets = { - tuple(candidate.insight_metric_keys or ()) + tuple(sorted(candidate.insight_metric_keys or ())) for candidate in candidates if candidate.insight_suite_identity == provenance.identity and candidate.insight_metric_keys } if isinstance(dataset_metric_keys, list): - cached_metric_key_sets.add(tuple(dataset_metric_keys)) + cached_metric_key_sets.add(tuple(sorted(dataset_metric_keys))) if len(cached_metric_key_sets) > 1: raise ValueError( f"Cached Insight evaluations disagree on required metric keys: {sorted(cached_metric_key_sets)}" @@ -1425,7 +1437,6 @@ async def _evaluate_and_persist_insight_candidates( ) if expected_metric_keys is None: expected_metric_keys = metric_keys - dataset.metadata["insight_metric_keys"] = list(metric_keys) result = stamp_insight_evaluation_result(result, provenance) await backend.persist_evaluation( workspace=workspace, @@ -1439,7 +1450,6 @@ async def _evaluate_and_persist_insight_candidates( "insight_reward": result.aggregate_metrics, "insight_reward_details": result.trials, "insight_suite_identity": provenance.identity, - "insight_suite_artifact_ref": provenance.artifact_ref, "insight_metric_keys": list(metric_keys), }, workspace=workspace, @@ -1847,23 +1857,26 @@ async def _finalize( report_path.write_text(f"# Optimization Report\n\n## Compact Run Summary\n\n{summary}\n") if insight_dataset is not None: - provenance = insight_suite_provenance(insight_dataset) - if baseline is not None: - write_insight_comparison_section( + try: + provenance = insight_suite_provenance(insight_dataset) + if baseline is not None: + write_insight_comparison_section( + report_path, + baseline, + winner, + provenance, + ) + suggestions = select_insight_promotion_suggestions( + insight_dataset, + [node.candidate for node in evolution_tree.nodes.values()], + winner=winner, + ) + write_insight_promotion_section( report_path, - baseline, - winner, - provenance, + suggestions, ) - suggestions = select_insight_promotion_suggestions( - insight_dataset, - [node.candidate for node in evolution_tree.nodes.values()], - winner=winner, - ) - write_insight_promotion_section( - report_path, - suggestions, - ) + except ValueError as exc: + logger.warning(f"[FINAL] Skipping Insight Suite report sections: {exc}") run_entity.status = "completed" run_entity.winner_agent = best_id diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py index e2ac7a27e7..e959de65be 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py @@ -16,7 +16,12 @@ Task, TrialResult, ) -from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef, DataValue +from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborEvaluatorConfig +from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import ( + DatasetRef, + DataValue, + ResourceRef, +) from nemo_experimentalist_plugin.experimentalist.components.loop import EvolutionaryOptimizer from nemo_experimentalist_plugin.resolve import EvolutionaryOptimizerConfig @@ -30,7 +35,6 @@ def _suite_metadata(identity_char: str = "a") -> dict[str, DataValue]: return { "insight_suite_identity": identity, "insight_suite_scorer_identity": f"sha256:{'b' * 64}", - "insight_suite_artifact_ref": (f"nemo-experimentalist-insight-suite://insight-1/sha256/{identity_char * 64}"), "insight_suite_task_hashes": { "insight-task": { "content_hash": f"sha256:{'c' * 64}", @@ -70,6 +74,7 @@ async def test_insight_run_evaluates_and_persists_baseline_and_new_candidate_met validation_dataset = Dataset(id="validation") insight_dataset = Dataset( id="insight-suite", + source=ResourceRef(uri="file:///experiment/eval-and-optimize/eval_author/insight-1/insight-suite"), tasks=[Task(id="insight-task")], metadata=_suite_metadata(), ) @@ -255,7 +260,7 @@ async def run(self, **kwargs: object) -> SimpleNamespace: call["result"].metadata["insight_suite_identity"] == f"sha256:{'a' * 64}" for call in insight_persistence ) assert all( - trial.metadata["insight_suite_artifact_ref"].startswith("nemo-experimentalist-insight-suite://") + trial.metadata["insight_suite_scorer_identity"] == f"sha256:{'b' * 64}" for call in insight_persistence for trial in call["result"].trials ) @@ -273,7 +278,6 @@ async def test_insight_evaluation_skips_cached_candidates_and_empty_suites( insight_reward={"uses_required_tool": 0.0}, insight_reward_details=[], insight_suite_identity=f"sha256:{'a' * 64}", - insight_suite_artifact_ref=f"nemo-experimentalist-insight-suite://insight-1/sha256/{'a' * 64}", insight_metric_keys=["uses_required_tool"], ) pending = Candidate( @@ -290,6 +294,7 @@ async def test_insight_evaluation_skips_cached_candidates_and_empty_suites( evaluated = await optimizer._evaluate_insight_candidates( dataset=Dataset( id="insight-suite", + source=ResourceRef(uri="file:///experiment/eval-and-optimize/eval_author/insight-1/insight-suite"), tasks=[Task(id="insight-task")], metadata=_suite_metadata(), ), @@ -300,6 +305,7 @@ async def test_insight_evaluation_skips_cached_candidates_and_empty_suites( assert evaluated == {"agent-1": result} assert evaluate_agent.await_args is not None assert evaluate_agent.await_args.args[0] is pending + assert evaluate_agent.await_args.kwargs["minimum_attempts"] == 2 empty = await optimizer._evaluate_insight_candidates( dataset=Dataset(id="empty-insight-suite"), @@ -322,7 +328,6 @@ async def test_insight_evaluation_reuses_only_matching_suite_identity( insight_reward={"uses_required_tool": 0.0}, insight_reward_details=[], insight_suite_identity=f"sha256:{'a' * 64}", - insight_suite_artifact_ref=f"nemo-experimentalist-insight-suite://insight-1/sha256/{'a' * 64}", insight_metric_keys=["uses_required_tool"], ) result = _insight_result("agent-0", 0.5) @@ -332,11 +337,13 @@ async def test_insight_evaluation_reuses_only_matching_suite_identity( matching = Dataset( id="insight-suite", + source=ResourceRef(uri="file:///experiment/eval-and-optimize/eval_author/insight-1/insight-suite"), tasks=[Task(id="insight-task")], metadata=_suite_metadata("a"), ) changed = Dataset( id="insight-suite", + source=ResourceRef(uri="file:///experiment/eval-and-optimize/eval_author/insight-1/insight-suite"), tasks=[Task(id="insight-task")], metadata=_suite_metadata("e"), ) @@ -355,3 +362,133 @@ async def test_insight_evaluation_reuses_only_matching_suite_identity( candidates=[cached], ) == {"agent-0": result} evaluate_agent.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_cached_insight_metric_keys_are_order_independent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + identity = f"sha256:{'a' * 64}" + candidates = [ + Candidate( + run_id="run-1", + label="agent-0", + round=0, + optimization="baseline", + insight_reward={"reward": 0.5, "uses_required_tool": 0.0}, + insight_reward_details=[], + insight_suite_identity=identity, + insight_metric_keys=["uses_required_tool", "reward"], + ), + Candidate( + run_id="run-1", + label="agent-1", + round=1, + optimization="improve tool use", + insight_reward={"reward": 0.75, "uses_required_tool": 1.0}, + insight_reward_details=[], + insight_suite_identity=identity, + insight_metric_keys=["reward", "uses_required_tool"], + ), + ] + dataset = Dataset( + id="insight-suite", + source=ResourceRef(uri="file:///experiment/eval-and-optimize/eval_author/insight-1/insight-suite"), + tasks=[Task(id="insight-task")], + metadata={ + **_suite_metadata(), + "insight_metric_keys": ["uses_required_tool", "reward"], + }, + ) + optimizer = object.__new__(EvolutionaryOptimizer) + monkeypatch.setattr( + optimizer, + "_evaluate_insight_candidates", + AsyncMock(return_value={}), + ) + + await optimizer._evaluate_and_persist_insight_candidates( + dataset=dataset, + evaluator=object(), # type: ignore[arg-type] + candidates=candidates, + workspace="default", + backend=SimpleNamespace(), + run_id="run-1", + ) + + assert dataset.metadata["insight_metric_keys"] == ["reward", "uses_required_tool"] + + +@pytest.mark.asyncio +async def test_insight_evaluation_uses_at_least_two_attempts_without_changing_other_splits( + tmp_path: Path, +) -> None: + candidate = Candidate( + run_id="run-1", + label="agent-0", + round=0, + optimization="baseline", + ) + received_attempts: list[int] = [] + + class RecordingEvaluator: + options = HarborEvaluatorConfig(n_attempts=1) + + async def run(self, **kwargs: object) -> EvaluationResult: + options = kwargs["options"] + assert isinstance(options, HarborEvaluatorConfig) + received_attempts.append(options.n_attempts) + return _insight_result(candidate.label, 0.5) + + optimizer = object.__new__(EvolutionaryOptimizer) + optimizer.working_dir = tmp_path + evaluator = RecordingEvaluator() + dataset = Dataset(id="insight-suite") + + await optimizer._evaluate_agent(candidate, dataset, evaluator) # type: ignore[arg-type] + await optimizer._evaluate_agent( + candidate, + dataset, + evaluator, # type: ignore[arg-type] + minimum_attempts=2, + ) + + assert received_attempts == [1, 2] + assert evaluator.options.n_attempts == 1 + + +def test_rollback_removes_digest_named_insight_results(tmp_path: Path) -> None: + root = tmp_path / "eval-and-optimize" + agents_dir = root / "agents" + results_dir = root / "results" + analysis_dir = root / "analysis" + smoke_dataset_dir = root / "smoke-dataset" + smoke_results_dir = root / "smoke-results" + for directory in (agents_dir, results_dir, analysis_dir, smoke_dataset_dir, smoke_results_dir): + directory.mkdir(parents=True) + + removed_agent = agents_dir / "agent-2" + removed_agent.mkdir() + (removed_agent / "metadata.json").write_text('{"round": 2}\n') + surviving_agent = agents_dir / "agent-20" + surviving_agent.mkdir() + (surviving_agent / "metadata.json").write_text('{"round": 1}\n') + + removed_results = [ + results_dir / "agent-2-train", + results_dir / "agent-2-validation", + results_dir / "agent-2-insight-abcdef123456", + ] + for result_dir in removed_results: + result_dir.mkdir() + surviving_result = results_dir / "agent-20-insight-abcdef123456" + surviving_result.mkdir() + + optimizer = object.__new__(EvolutionaryOptimizer) + optimizer.working_dir = tmp_path + optimizer._delete_all_artifacts(from_round=1) + + assert not removed_agent.exists() + assert all(not result_dir.exists() for result_dir in removed_results) + assert surviving_agent.is_dir() + assert surviving_result.is_dir() diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py index 833beb3dae..b5cbef7d0d 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py @@ -16,7 +16,9 @@ TrialResult, TrialStatus, ) +from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import ResourceRef from nemo_experimentalist_plugin.experimentalist.components.insight_promotion import ( + _task_evidence, insight_suite_provenance, render_insight_promotion_section, select_insight_promotion_suggestions, @@ -28,7 +30,7 @@ from nemo_experimentalist_plugin.experimentalist.components.models import EvolutionTree _SUITE_IDENTITY = f"sha256:{'a' * 64}" -_SUITE_ARTIFACT_REF = f"nemo-experimentalist-insight-suite://insight-1/sha256/{'a' * 64}" +_SUITE_PATH = Path("/experiment/eval-and-optimize/eval_author/insight-1/insight-suite") def _candidate( @@ -46,25 +48,27 @@ def _candidate( insight_reward=insight_reward, validation_reward=validation_reward, insight_suite_identity=_SUITE_IDENTITY, - insight_suite_artifact_ref=_SUITE_ARTIFACT_REF, insight_metric_keys=["reward", "uses_required_tool"], ) def _insight_dataset(tasks: list[Task]) -> Dataset: + local_tasks = [ + task if task.uri else task.model_copy(update={"uri": (_SUITE_PATH / task.id).as_uri()}) for task in tasks + ] return Dataset( id="insight", - tasks=tasks, + source=ResourceRef(uri=_SUITE_PATH.as_uri()), + tasks=local_tasks, metadata={ "insight_suite_identity": _SUITE_IDENTITY, "insight_suite_scorer_identity": f"sha256:{'b' * 64}", - "insight_suite_artifact_ref": _SUITE_ARTIFACT_REF, "insight_suite_task_hashes": { task.id: { "content_hash": f"sha256:{'c' * 64}", "verifier_hash": f"sha256:{'d' * 64}", } - for task in tasks + for task in local_tasks }, }, ) @@ -167,6 +171,7 @@ def test_insight_promotion_suggestions_are_stable_discriminative_and_diverse( _insight_trial("task-flat", 0.5, attempt=2), ] winner = _candidate("agent-1", round_num=1) + winner.insight_metric_keys = ["uses_required_tool", "reward"] winner.insight_reward_details = [ _insight_trial("task-a", 1.0, attempt=1), _insight_trial("task-a", 1.0, attempt=2), @@ -195,7 +200,8 @@ def test_insight_promotion_suggestions_are_stable_discriminative_and_diverse( section = render_insight_promotion_section(suggestions) assert "Advisory adaptive/development evidence only" in section assert "`task-a`" in section - assert f"`{_SUITE_ARTIFACT_REF}`" in section + assert str(tmp_path / "task-a") in section + assert _SUITE_IDENTITY in section assert "baseline 0.00 → winner 1.00" in section assert f"task sha256:{'c' * 64}" in section assert "task-b" not in section @@ -203,6 +209,36 @@ def test_insight_promotion_suggestions_are_stable_discriminative_and_diverse( assert "task-flat" not in section +def test_task_evidence_excludes_candidates_from_other_suites(tmp_path: Path) -> None: + dataset = _insight_dataset([Task(id="task-a", uri=(tmp_path / "task-a").as_uri())]) + task = dataset.list_tasks()[0] + baseline = _candidate("agent-0", round_num=0) + winner = _candidate("agent-1", round_num=1) + stale = _candidate("agent-stale", round_num=1) + stale.insight_suite_identity = f"sha256:{'e' * 64}" + for candidate, score in ((baseline, 0.0), (winner, 1.0), (stale, 0.5)): + candidate.insight_reward_details = [ + _insight_trial(task.id, score, attempt=1), + _insight_trial(task.id, score, attempt=2), + ] + + evidence = _task_evidence( + task, + [baseline, winner, stale], + baseline=baseline, + winner=winner, + provenance=insight_suite_provenance(dataset), + ) + + assert evidence is not None + assert evidence.suggestion.candidate_count == 2 + assert evidence.suggestion.total_attempts == 4 + assert set(candidate_label for candidate_label, _ in evidence.profile) == { + baseline.label, + winner.label, + } + + def test_insight_promotion_section_explains_when_no_task_qualifies() -> None: section = render_insight_promotion_section([]) @@ -390,7 +426,7 @@ def test_promotion_requires_baseline_to_winner_improvement( ) -def test_deterministic_insight_comparison_section_uses_content_addressed_suite( +def test_deterministic_insight_comparison_section_uses_local_suite_identity( tmp_path: Path, ) -> None: baseline = _candidate( @@ -410,7 +446,8 @@ def test_deterministic_insight_comparison_section_uses_content_addressed_suite( report = report_path.read_text() assert "## Deterministic Insight Suite Comparison" in report - assert _SUITE_ARTIFACT_REF in report + assert str(_SUITE_PATH) in report + assert _SUITE_IDENTITY in report assert "| `uses_required_tool` | 0.000 | 1.000 | +1.000 |" in report @@ -470,3 +507,59 @@ async def test_final_report_failure_preserves_compact_summary_and_deterministic_ assert "Optimization complete: 1 round(s) completed" in report assert "## Deterministic Insight Suite Comparison" in report assert "## Insight Suite Promotion Suggestions" in report + + +@pytest.mark.asyncio +async def test_insight_report_mismatch_does_not_fail_completed_run( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + baseline = _candidate( + "agent-0", + round_num=0, + insight_reward={"reward": 0.5, "uses_required_tool": 0.0}, + validation_reward={"reward": 0.5}, + ) + winner = _candidate( + "agent-1", + round_num=1, + insight_reward={"reward": 0.75, "uses_required_tool": 1.0}, + validation_reward={"reward": 0.75}, + ) + winner.insight_suite_identity = f"sha256:{'e' * 64}" + tree = EvolutionTree() + tree.add(baseline) + tree.add(winner) + optimizer = object.__new__(EvolutionaryOptimizer) + optimizer.working_dir = tmp_path + (tmp_path / "eval-and-optimize").mkdir() + monkeypatch.setattr(optimizer, "_copy_best_to_workspace", lambda best_id: None) + original_report_writer = EvolutionaryOptimizer.write_final_report + type.__setattr__(EvolutionaryOptimizer, "write_final_report", AsyncMock()) + run = SimpleNamespace(status="running", winner_agent=None, rounds_completed=1) + backend = SimpleNamespace(update_run=AsyncMock()) + + try: + with caplog.at_level("WARNING"): + finalized = await optimizer._finalize( + workspace="default", + backend=backend, + agents_dir=tmp_path / "eval-and-optimize" / "agents", + run_entity=run, + evolution_tree=tree, + agent_name="agent", + insight_dataset=_insight_dataset([Task(id="task-a")]), + ) + finally: + type.__setattr__( + EvolutionaryOptimizer, + "write_final_report", + original_report_writer, + ) + + assert finalized is winner + assert run.status == "completed" + assert run.winner_agent == winner.label + backend.update_run.assert_awaited_once() + assert "Skipping Insight Suite report sections" in caplog.text diff --git a/plugins/nemo-experimentalist/tests/test_eval_author_agent.py b/plugins/nemo-experimentalist/tests/test_eval_author_agent.py index 26c678f70f..943b9775bb 100644 --- a/plugins/nemo-experimentalist/tests/test_eval_author_agent.py +++ b/plugins/nemo-experimentalist/tests/test_eval_author_agent.py @@ -201,15 +201,13 @@ def discard(self) -> None: def record_analysis(self, statuses: dict[str, tuple[str, str | None]]) -> None: pass - def finalize_artifact(self) -> SimpleNamespace: + def finalize(self) -> SimpleNamespace: identity = "sha256:" + "a" * 64 scorer_identity = "sha256:" + "b" * 64 - artifact_ref = f"nemo-experimentalist-insight-suite://insight-1/sha256/{'a' * 64}" self.materialized_dataset.metadata.update( { "insight_suite_identity": identity, "insight_suite_scorer_identity": scorer_identity, - "insight_suite_artifact_ref": artifact_ref, "insight_suite_task_hashes": { task.id: { "content_hash": "sha256:" + "c" * 64, @@ -223,7 +221,6 @@ def finalize_artifact(self) -> SimpleNamespace: dataset=self.materialized_dataset, identity=identity, scorer_identity=scorer_identity, - ref=artifact_ref, ) class FillTaskTemplate: @@ -566,8 +563,6 @@ def fake_doc(dataset_type: type[Dataset], *, inline_depth: int) -> object: materialized_dataset = calls.discovered_datasets[0] assert result.insight_suite is materialized_dataset assert result.insight_suite_identity == f"sha256:{'a' * 64}" - assert result.insight_suite_artifact_ref is not None - assert result.insight_suite_artifact_ref.startswith("nemo-experimentalist-insight-suite://") assert materialized_dataset.id == "insight-suite" assert materialized_dataset is not train_dataset assert materialized_dataset is not validation_dataset diff --git a/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py b/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py index 39695b11e7..da415afd0b 100644 --- a/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py +++ b/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py @@ -11,10 +11,7 @@ import pytest from nemo_experimentalist_plugin.eval_author import materialization as materialization_module -from nemo_experimentalist_plugin.eval_author.materialization import ( - InsightSuite, - resolve_insight_suite_artifact, -) +from nemo_experimentalist_plugin.eval_author.materialization import InsightSuite from nemo_experimentalist_plugin.experimentalist.components.evaluator import Task from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset @@ -212,7 +209,7 @@ def test_insight_suite_records_analysis_without_removing_failed_tasks(tmp_path: assert len(HarborDataset.from_path(suite.suite_dir).list_tasks()) == 2 -def test_finalized_suite_is_content_addressed_durable_and_resolvable(tmp_path: Path) -> None: +def test_finalized_suite_persists_identities_in_the_local_suite(tmp_path: Path) -> None: template = _write_template(tmp_path / "template") refs = ["trace-1"] suite = InsightSuite(experiment_dir=tmp_path, insight_id="insight-1", task_template=template) @@ -221,28 +218,31 @@ def test_finalized_suite_is_content_addressed_durable_and_resolvable(tmp_path: P suite.validate(staged[0]) suite.promote_local(refs, staged) - artifact = suite.finalize_artifact() - manifest = json.loads((artifact.path / "manifest.json").read_text(encoding="utf-8")) + finalized = suite.finalize() + manifest = json.loads((finalized.path / "manifest.json").read_text(encoding="utf-8")) - assert artifact.identity.startswith("sha256:") - assert artifact.scorer_identity.startswith("sha256:") - assert artifact.ref.startswith("nemo-experimentalist-insight-suite://") - assert manifest["suite_identity"] == artifact.identity + assert finalized.identity.startswith("sha256:") + assert finalized.scorer_identity.startswith("sha256:") + assert finalized.path == suite.suite_dir + assert manifest["suite_identity"] == finalized.identity assert manifest["scorer"] == { - "identity": artifact.scorer_identity, + "identity": finalized.scorer_identity, "metric_contract_version": 1, } assert manifest["tasks"][0]["content_hash"].startswith("sha256:") assert manifest["tasks"][0]["verifier"]["content_hash"].startswith("sha256:") - assert resolve_insight_suite_artifact(tmp_path, artifact.ref) == artifact.path - assert artifact.dataset.metadata["insight_suite_identity"] == artifact.identity - assert artifact.dataset.metadata["insight_suite_artifact_ref"] == artifact.ref - assert list(artifact.dataset.list_tasks())[0].uri.startswith(artifact.path.as_uri()) + assert "files" not in manifest["tasks"][0] + assert "files" not in manifest["tasks"][0]["verifier"] + assert "artifact" not in manifest + assert not (suite.root / "artifacts").exists() + assert finalized.dataset.metadata["insight_suite_identity"] == finalized.identity + assert "insight_suite_artifact_ref" not in finalized.dataset.metadata + assert list(finalized.dataset.list_tasks())[0].uri.startswith(finalized.path.as_uri()) - artifact_instruction = next(path for path in artifact.path.iterdir() if path.is_dir()) / "instruction.md" - artifact_instruction.write_text("tampered\n") - with pytest.raises(ValueError, match="content does not match reference"): - resolve_insight_suite_artifact(tmp_path, artifact.ref) + instruction = next(path for path in finalized.path.iterdir() if path.is_dir()) / "instruction.md" + instruction.write_text("changed after finalization\n") + changed = suite.finalize() + assert changed.identity != finalized.identity def test_finalized_suite_identity_is_stable_and_changes_with_task_or_verifier_content( @@ -263,15 +263,16 @@ def build(instruction: str, verifier_suffix: str = "") -> tuple[str, str]: verifier_path.write_text(verifier_path.read_text() + verifier_suffix) suite.validate(staged[0]) suite.promote_local(["trace-1"], staged) - artifact = suite.finalize_artifact() - return artifact.identity, artifact.ref + finalized = suite.finalize() + return finalized.identity, finalized.scorer_identity - first_identity, first_ref = build("Same authored task.\n") - identical_identity, identical_ref = build("Same authored task.\n") + first_identity, first_scorer_identity = build("Same authored task.\n") + identical_identity, identical_scorer_identity = build("Same authored task.\n") changed_task_identity, _ = build("Changed authored task.\n") - changed_verifier_identity, _ = build("Same authored task.\n", "\n# changed scorer\n") + changed_verifier_identity, changed_scorer_identity = build("Same authored task.\n", "\n# changed scorer\n") - assert (identical_identity, identical_ref) == (first_identity, first_ref) + assert (identical_identity, identical_scorer_identity) == (first_identity, first_scorer_identity) assert changed_task_identity != first_identity assert changed_verifier_identity != first_identity - assert resolve_insight_suite_artifact(tmp_path, first_ref).is_dir() + assert changed_scorer_identity != first_scorer_identity + assert list((tmp_path / "eval-and-optimize" / "eval_author").glob("*/artifacts")) == [] diff --git a/plugins/nemo-experimentalist/tests/test_experiment_mirror.py b/plugins/nemo-experimentalist/tests/test_experiment_mirror.py index 0db23f07db..cd4df4bd5c 100644 --- a/plugins/nemo-experimentalist/tests/test_experiment_mirror.py +++ b/plugins/nemo-experimentalist/tests/test_experiment_mirror.py @@ -8,7 +8,7 @@ import pytest from nemo_experimentalist_plugin.entities import Candidate, ExperimentRun from nemo_experimentalist_plugin.experimentalist.experiment_mirror import ExperimentMirror, group_metadata -from nemo_platform import AsyncNeMoPlatform, ConflictError, NotFoundError, omit # VERIFY-2 +from nemo_platform import AsyncNeMoPlatform, ConflictError, NotFoundError, omit pytestmark = pytest.mark.asyncio