feat: author and evaluate insight suite metrics

plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py

@@ -149,6 +149,22 @@ class Candidate(NemoEntity, entity_type="candidate"):
         default=None,
         description="Validation split trial results from the last evaluation run.",
     )
+    insight_reward: dict[str, float] | None = Field(
+        default=None,
+        description="Multi-dimensional reward on the materialized Insight suite.",
+    )
+    insight_reward_details: Sequence[TrialResult] | None = Field(
+        default=None,
+        description="Insight-suite trial results from the last evaluation run.",
+    )
+    insight_suite_identity: str | None = Field(
+        default=None,
+        description="Content identity of the Insight suite associated with insight_reward.",
+    )
+    insight_metric_keys: list[str] | None = Field(
+        default=None,
+        description="Validated runtime metric keys associated with insight_reward.",
+    )
     validation_trajectory_reward: dict[str, float] | None = Field(
         default=None,
         description="Validation trajectory reward: aggregate + per-node scores.",
@@ -180,6 +196,9 @@ def __repr__(self) -> str:
         if self.validation_reward:
             scores = ", ".join(f"{k}={v:.3f}" for k, v in self.validation_reward.items())
             parts.append(f", validation_reward={{{scores}}}")
+        if self.insight_reward:
+            scores = ", ".join(f"{k}={v:.3f}" for k, v in self.insight_reward.items())
+            parts.append(f", insight_reward={{{scores}}}")
         if self.killed_round is not None:
             parts.append(f", killed_round={self.killed_round}")
         parts.append(")")
@@ -191,6 +210,7 @@ def slim(self) -> "Candidate":
             update={
                 "train_reward_details": None,
                 "validation_reward_details": None,
+                "insight_reward_details": None,
                 "validation_trajectory_reward_details": None,
             }
         )

plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md

@@ -16,8 +16,9 @@ runs the top-level Eval Author before beginning optimization.
 ## Current Files
 
 - `agent.py` defines the canonical `EvalAuthor` agent.
-- `materialization.py` stages, validates, and publishes Insight suites.
+- `materialization.py` stages, validates, and persists Insight suites locally.
 - `models.py` defines the lightweight `EvalAuthorConfig` and `EvalAuthorResult` models.
+- `REFERENCE.md` documents the Python return contract.
 - `run.py` defines `run_eval_author(...)`, a reusable orchestration function for
   Python callers.
 - `config.yaml` is a default run preset for future CLI or job wiring.
@@ -51,21 +52,33 @@ eval-and-optimize/eval_author/<insight-slug>/insight-suite/
 Each Eval Author invocation fills a fresh candidate suite from the current template
 and traces. The complete suite is Harbor-validated locally, promoted to the
 experiment-local working copy with backup-and-restore failure handling, and
-uploaded to a newly created NeMo Platform Fileset. Eval Author verifies the remote
-file inventory before returning.
-An incomplete Fileset is deleted if upload or verification fails; Filesets from
-earlier successful invocations are never modified or reused.
+analyzed for the Insight's root cause. Eval Author then adds normalized
+Insight-specific verifier metric keys to every materialized task while
+preserving the template's existing task metrics. The metric authoring step is
+scoped to the Insight suite; the user's train and validation datasets remain
+unchanged. The authored verifiers must pass static Harbor validation before the
+local suite is returned to the optimization loop.
+
+After authoring and validation, Eval Author hashes every task file and verifier
+file and persists deterministic suite and scorer identities in the local suite's
+manifest. The returned dataset continues to point at the single experiment-local
+suite. Candidate Insight results persist the suite identity, and resume reuses
+those results only when the identity still matches; changed task or verifier
+content is re-evaluated.
 
 Task-template inputs may be local paths, `file://` URIs, or NeMo Platform
 `fileset://<workspace>/<fileset>` references. Fileset-backed templates are
 downloaded into the experiment-local staging directory before Harbor parses
 them. The staged template is refreshed on every invocation rather than reused.
 
-`EvalAuthorResult.insight_suite` contains a durable `DatasetRef` whose URI uses the
-`fileset://<workspace>/<fileset>` form. Downstream agents may store and pass that
-reference without understanding its storage. A component that needs Harbor's
-local filesystem layout must hydrate the Fileset into its own working directory
-before calling `HarborDataset.from_path`.
+The returned Python contract is documented in the
+[Eval Author Python Reference](REFERENCE.md#evalauthorresult).
+
+Insight metrics remain adaptive development feedback. They may steer round
+analysis, goal-tree updates, and proposals, but validation remains the direct
+Pareto and winner-selection criterion. Promotion suggestions require complete
+repeated baseline-to-winner improvement evidence, remain advisory, and never
+mutate the canonical validation dataset.
 
 ## Intended Invocation
 

plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/REFERENCE.md

@@ -0,0 +1,23 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# Eval Author Python Reference
+
+## `EvalAuthorResult`
+
+`run_eval_author(...)` returns an `EvalAuthorResult` with these fields:
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `train_dataset` | `Dataset` | Training dataset supplied to the run. Eval Author does not mutate it. |
+| `validation_dataset` | `Dataset` | Validation dataset supplied to the run. Eval Author does not mutate it. |
+| `insight_suite` | `Dataset \| None` | Finalized experiment-local Insight dataset for immediate evaluation by the optimization loop. |
+| `insight_suite_identity` | `str \| None` | SHA-256 identity of the finalized Insight task and verifier content. |
+| `summary` | `str` | Eval Author's analysis summary. |
+
+When an Insight suite is materialized successfully, `insight_suite` and
+`insight_suite_identity` are both populated. Callers can persist the identity
+with candidate results and reuse those results only while the suite identity
+continues to match.

plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py

@@ -9,7 +9,6 @@
 
 import asyncio
 import logging
-from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
@@ -49,37 +48,6 @@
 logger = logging.getLogger(__name__)
 
 
-@dataclass(frozen=True)
-class EvalAuthorDatasetValidationFailure:
-    """Validation failure for one Eval Author dataset split."""
-
-    split: str
-    error: DatasetValidationError
-
-
-class EvalAuthorDatasetValidationError(DatasetValidationError):
-    """Validation failures from datasets returned by the Eval Author."""
-
-    def __init__(self, failures: list[EvalAuthorDatasetValidationFailure]) -> None:
-        self.failures = tuple(failures)
-        details = "\n".join(f"{failure.split} dataset:\n{failure.error}" for failure in failures)
-        super().__init__(f"Eval Author dataset validation failed:\n{details}")
-
-
-async def _validate_eval_author_result(result: EvalAuthorResult) -> None:
-    """Validate both Eval Author output splits and aggregate authoring failures."""
-    failures: list[EvalAuthorDatasetValidationFailure] = []
-    for split, dataset in (("train", result.train_dataset), ("validation", result.validation_dataset)):
-        try:
-            await dataset.validate()
-        except DatasetValidationError as exc:
-            failures.append(EvalAuthorDatasetValidationFailure(split=split, error=exc))
-
-    if failures:
-        first_failure = failures[0]
-        raise EvalAuthorDatasetValidationError(failures) from first_failure.error
-
-
 class EvalAuthor(Agent, llm=get_smart_model()):
     """Insights are failure modes of an agent in production.
 
@@ -131,47 +99,57 @@ async def discover_runner(self, dataset: Dataset) -> str:
         ...
 
     @strategy(CodeActStrategy(config=CodeActConfig(max_iterations=60, cell_timeout=3600.0)))
-    async def augment_dataset(
+    async def author_insight_metrics(
         self,
         insight: Insight,
         diagnostics: list[tuple[str, Diagnostic]],
-        train_dataset: Dataset,
-        validation_dataset: Dataset,
+        insight_suite: Dataset,
         runner_conventions: str,
         validation_feedback: str | None = None,
-    ) -> EvalAuthorResult:
-        """Augment existing dataset tasks with evaluation metrics that capture the insight.
+    ) -> str:
+        """Author verifier metrics for the materialized tasks that capture the insight.
 
         Args:
             insight: The insight whose failure mode the tasks should detect.
             diagnostics: Per-trace ``(trace_ref, Diagnostic)`` pairs for concrete evidence.
-            train_dataset: The train dataset to augment for optimization feedback.
-            validation_dataset: The validation dataset to augment for scoring.
+            insight_suite: The materialized tasks recreated from the Insight's production traces.
             runner_conventions: Summary of how this dataset's runner works (from ``discover_runner``).
                 Use this as the authoritative reference for what artifacts exist at
                 evaluation runtime, how tasks are structured, and how to add metrics.
             validation_feedback: Actionable failures from mandatory validation of
-                the previous augmentation attempt. If provided, repair every reported
+                the previous metric-authoring attempt. If provided, repair every reported
                 file before returning.
 
         Refer to ``self.context["dataset_documentation"]`` for the dataset-specific API
         and metric authoring conventions (file layout, how to add/remove/modify a metric).
 
-        **Scope: every task in both datasets**
+        **Scope: new grades on the materialized Insight tasks**
+
+        Add at least one new Insight-specific metric key to every task in ``insight_suite``.
+        Use the same new metric key set and shared scoring semantics across the entire
+        suite. Preserve every existing verifier metric, including the task's ordinary
+        ``reward`` or ``score``; append the Insight signal instead of replacing the
+        task's original notion of success.
 
-        Add the metric to every task in ``train_dataset`` and ``validation_dataset``.
-        A metric is only useful as a suite-wide signal, not a per-sample patch.
+        Only edit verifier files in the materialized Insight suite. Do not modify the
+        user's train or validation datasets, and do not change task instructions,
+        environments, solutions, or other agent-visible inputs. This work adds new
+        grades to the new rows; it does not add new agent output to old benchmark rows.
+
+        Name each new metric after the root-cause behavior, not a trace id or surface
+        symptom. Measure the current Harbor run from runtime artifacts such as OTLP
+        traces or agent outputs. Do not hard-code scores for the production traces that
+        motivated the Insight.
 
         **Validate while authoring**
 
-        After every verifier edit, call ``await train_dataset.validate()`` and
-        ``await validation_dataset.validate()``. These validation tools perform
+        After every verifier edit, call ``await insight_suite.validate()``. This performs
         evaluator-specific static checks without launching trials or executing verifier
-        code. If either raises ``DatasetValidationError``, use its task, path, and source
-        location diagnostics to repair the files, then call the tools again. Do not return
-        until both datasets pass validation. If ``validation_feedback`` is provided, it
-        means the previous result failed the mandatory validation performed by the caller;
-        fix all reported failures and revalidate both datasets.
+        code. If it raises ``DatasetValidationError``, use its task, path, and source
+        location diagnostics to repair the files, then call it again. Do not return until
+        the suite passes validation. If ``validation_feedback`` is provided, the caller's
+        mandatory validation found errors in the previous attempt; fix every reported
+        failure and revalidate the suite.
 
         **Metric quality**
 
@@ -197,8 +175,9 @@ async def augment_dataset(
         objects for Y, so X is missing from its context.  Measure whether the agent retrieves
         all required objects, not merely whether X appears in the final answer.
 
-        Return an ``EvalAuthorResult(train_dataset=..., validation_dataset=..., summary=...)``
-        with the same dataset objects and a summary of what was added.
+        Return a concise summary naming the new metric key(s), what they measure, and
+        which runtime evidence they score. The caller retains the materialized suite and
+        the user's unchanged train and validation datasets.
         """  # noqa: D413
         ...
 
@@ -291,8 +270,8 @@ async def _run(
             insight: The Insight to investigate with relevant traces.
             agent_path: Agent root, relative to ``experiment_dir`` or absolute.
             task_template: Parsed evaluator task containing explicit placeholders.
-            train_dataset: The train dataset to augment.
-            validation_dataset: The validation dataset to augment.
+            train_dataset: The train dataset, returned unchanged.
+            validation_dataset: The validation dataset, returned unchanged.
             client: Existing NeMo Platform client used for Intake requests.
         """
         resolved_agent = self.experiment_dir / agent_path
@@ -358,39 +337,43 @@ async def _run(
             diagnostics.append((ref, result))
             analysis_statuses[task.id] = ("completed", None)
         insight_suite.record_analysis(analysis_statuses)
-        insight_suite_ref = await insight_suite.publish_fileset(client, insight.workspace)
 
-        self.context["dataset_documentation"] = doc(type(train_dataset), inline_depth=1)
-        runner_conventions = await self.discover_runner(train_dataset)
-        result = await self.augment_dataset(
+        self.context["dataset_documentation"] = doc(type(materialized_dataset), inline_depth=1)
+        runner_conventions = await self.discover_runner(materialized_dataset)
+        summary = await self.author_insight_metrics(
             insight,
             diagnostics,
-            train_dataset,
-            validation_dataset,
+            materialized_dataset,
             runner_conventions,
         )
         for repair_attempt in range(self._config.max_validation_repair_attempts + 1):
             try:
-                await _validate_eval_author_result(result)
+                await materialized_dataset.validate()
             except DatasetValidationError as exc:
                 if repair_attempt >= self._config.max_validation_repair_attempts:
                     raise
                 logger.warning(
-                    "Eval Author dataset validation failed; requesting repair attempt %d/%d: %s",
+                    "Eval Author Insight metric validation failed; requesting repair attempt %d/%d: %s",
                     repair_attempt + 1,
                     self._config.max_validation_repair_attempts,
                     exc,
                 )
-                result = await self.augment_dataset(
+                summary = await self.author_insight_metrics(
                     insight,
                     diagnostics,
-                    result.train_dataset,
-                    result.validation_dataset,
+                    materialized_dataset,
                     runner_conventions,
                     validation_feedback=str(exc),
                 )
             else:
-                return result.model_copy(update={"insight_suite": insight_suite_ref})
+                finalized_suite = insight_suite.finalize()
+                return EvalAuthorResult(
+                    train_dataset=train_dataset,
+                    validation_dataset=validation_dataset,
+                    insight_suite=finalized_suite.dataset,
+                    insight_suite_identity=finalized_suite.identity,
+                    summary=summary,
+                )
 
         raise AssertionError("unreachable")