From 6b7b74dc6b045a2153a2a40732754348bed748f9 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 17 Aug 2026 21:50:18 +0000
Subject: [PATCH 01/10] chore(experimentalist): add a reward-pipeline inspector
 for missing verifier rewards
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A Harbor trial that writes no /logs/verifier/reward.txt surfaces only as
'reward n/a', and the run continues. This inspector prints the state at every
layer the reward passes through — the trial directory, trials_from_job_dir,
aggregate_results, reward_scalar/RunReporter, and has_metric_dimensions — so the
layer that drops the value is visible.

It runs against synthetic trial shapes or a real Harbor job directory. Read-only
and not imported by the plugin.

Signed-off-by: Cursor Agent <cursoragent@cursor.com>

Co-authored-by: Aditya Pandey <aditya@autospace.co>
---
 .../tools/reward_pipeline_inspector.py        | 260 ++++++++++++++++++
 1 file changed, 260 insertions(+)
 create mode 100644 plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py

diff --git a/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py b/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
new file mode 100644
index 00000000000..5c645bbd8ed
--- /dev/null
+++ b/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
@@ -0,0 +1,260 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Step through the four layers between a Harbor verifier and the reward the loop ranks on.
+
+A missing ``/logs/verifier/reward.txt`` is invisible by the time it reaches the
+operator: the run prints ``reward n/a`` and keeps going. This tool prints the
+state at every layer the value passes through, so you can see which one drops it
+and which one is entitled to complain.
+
+The layers, in order:
+
+1. **on disk** — what the verifier left in the trial directory.
+2. ``trials_from_job_dir`` — the ``TrialResult`` status and metrics parsed from it.
+3. ``Evaluator.aggregate_results`` — the per-candidate aggregate the loop stores.
+4. ``reward_scalar`` / ``RunReporter`` — the number a strategy ranks on and the
+   line an operator reads.
+5. ``has_metric_dimensions`` — the predicate the selector and terminator share,
+   which decides whether the loop has anything to compare and therefore whether
+   it can stop early.
+
+Every layer is called through production code. Nothing here is imported by the
+plugin; this is a read-only inspector for humans.
+
+Usage::
+
+    # Compare synthetic trial shapes side by side.
+    uv run python plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
+
+    # One shape only.
+    uv run python plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py \
+        --scenario verifier-silent
+
+    # Inspect a real Harbor job directory from a run that reported `reward n/a`.
+    uv run python plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py \
+        --job-dir eval-and-optimize/results/agent-0-validation
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import io
+import json
+import tempfile
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+from nemo_experimentalist_plugin.entities import Dataset, MetricTarget, Task, TrialResult
+from nemo_experimentalist_plugin.experimentalist.components.evaluator.base import (
+    Evaluator,
+    EvaluatorConfig,
+)
+from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import trials_from_job_dir
+from nemo_experimentalist_plugin.experimentalist.components.models import has_metric_dimensions
+from nemo_experimentalist_plugin.experimentalist.reporting import RunReporter, reward_scalar
+
+_OBJECTIVES = [MetricTarget(name="reward", direction="maximize")]
+
+
+class _AggregateOnlyEvaluator(Evaluator):
+    """The real ``aggregate_results``, with the abstract run step stubbed out."""
+
+    evaluator_type = "reward-pipeline-inspector"
+
+    async def _run(self, agent: Path, dataset: Dataset, options: EvaluatorConfig) -> Sequence[TrialResult]:
+        raise NotImplementedError("The inspector reads an existing job directory; it never runs an evaluation.")
+
+
+@dataclass(frozen=True)
+class Scenario:
+    """One synthetic trial shape, plus what the verifier did to produce it."""
+
+    name: str
+    verifier_behaviour: str
+    trials: tuple[dict, ...]
+
+
+def _trial(
+    task: str,
+    *,
+    reward_text: str | None = None,
+    exception_type: str | None = None,
+) -> dict:
+    """Describe one trial: its task, whether a reward file exists, and any Harbor exception."""
+    return {"task": task, "reward_text": reward_text, "exception_type": exception_type}
+
+
+SCENARIOS = (
+    Scenario(
+        name="reward-written",
+        verifier_behaviour="test.sh writes /logs/verifier/reward.txt. The control case.",
+        trials=(_trial("task-a", reward_text="1.0"), _trial("task-b", reward_text="0.0")),
+    ),
+    Scenario(
+        name="verifier-raised",
+        verifier_behaviour=(
+            "test.sh exits 0 without writing a reward file, so Harbor's verifier raises "
+            "RewardFileNotFoundError and the trial records exception_info."
+        ),
+        trials=(_trial("task-a", exception_type="RewardFileNotFoundError"),),
+    ),
+    Scenario(
+        name="verifier-silent",
+        verifier_behaviour=(
+            "The trial finishes with no exception and no reward file — a disabled verifier, "
+            "a bind mount that never reflected the write, or a reward stripped during log collection."
+        ),
+        trials=(_trial("task-a"), _trial("task-b")),
+    ),
+    Scenario(
+        name="partial",
+        verifier_behaviour="One task writes a reward and another does not.",
+        trials=(_trial("task-a", reward_text="1.0"), _trial("task-b")),
+    ),
+)
+
+
+def materialize(scenario: Scenario, root: Path) -> Path:
+    """Write *scenario* as a Harbor job directory and return it."""
+    job_dir = root / scenario.name
+    for index, spec in enumerate(scenario.trials):
+        trial_dir = job_dir / f"{spec['task']}__{index}"
+        trial_dir.mkdir(parents=True)
+        exception_type = spec["exception_type"]
+        (trial_dir / "result.json").write_text(
+            json.dumps(
+                {
+                    "task_name": f"harbor/{spec['task']}",
+                    "trial_name": trial_dir.name,
+                    "verifier_result": None,
+                    "exception_info": (
+                        None
+                        if exception_type is None
+                        else {
+                            "exception_type": exception_type,
+                            "exception_message": "No reward file found at /logs/verifier/reward.txt",
+                            "exception_traceback": "<elided>",
+                        }
+                    ),
+                }
+            ),
+            encoding="utf-8",
+        )
+        verifier_dir = trial_dir / "verifier"
+        verifier_dir.mkdir()
+        (verifier_dir / "test-stdout.txt").write_text("tests passed\n", encoding="utf-8")
+        if spec["reward_text"] is not None:
+            (verifier_dir / "reward.txt").write_text(spec["reward_text"], encoding="utf-8")
+    return job_dir
+
+
+def _describe_on_disk(job_dir: Path) -> list[str]:
+    lines = []
+    for trial_dir in sorted(path for path in job_dir.iterdir() if path.is_dir()):
+        result_path = trial_dir / "result.json"
+        if not result_path.is_file():
+            lines.append(f"{trial_dir.name}: no result.json (this directory is skipped entirely)")
+            continue
+        result = json.loads(result_path.read_text(encoding="utf-8"))
+        rewards = (result.get("verifier_result") or {}).get("rewards")
+        exception_info = result.get("exception_info") or {}
+        files = [
+            name
+            for name in ("verifier/reward.json", "verifier/reward.txt", "reward.json", "reward.txt")
+            if (trial_dir / name).is_file()
+        ]
+        lines.append(
+            f"{trial_dir.name}: reward files={files or 'none'} · "
+            f"result.json rewards={rewards if rewards else 'none'} · "
+            f"exception={exception_info.get('exception_type', 'none')}"
+        )
+    return lines
+
+
+def _reporter_line(metrics: dict[str, float | int]) -> str:
+    sink = io.StringIO()
+    RunReporter(sink=sink).candidate_evaluated(
+        label="agent-0",
+        split="validation",
+        metrics=metrics,
+        objective_metrics=_OBJECTIVES,
+        artifacts=Path("eval-and-optimize/results/agent-0-validation"),
+    )
+    return sink.getvalue().strip()
+
+
+async def inspect(job_dir: Path, *, behaviour: str | None = None) -> None:
+    """Print the state of every layer between *job_dir* and the loop's scalar reward."""
+    print(f"\n{'=' * 78}\n{job_dir.name}\n{'=' * 78}")
+    if behaviour is not None:
+        print(f"verifier: {behaviour}\n")
+
+    print("[1] on disk")
+    for line in _describe_on_disk(job_dir):
+        print(f"    {line}")
+
+    task_ids = sorted({path.name.rsplit("__", 1)[0] for path in job_dir.iterdir() if path.is_dir()})
+    trials = trials_from_job_dir(job_dir, [Task(id=task_id) for task_id in task_ids])
+    print("\n[2] trials_from_job_dir")
+    for trial in trials:
+        metrics = {name: metric.value for name, metric in trial.metrics.items()}
+        error = (trial.error or {}).get("type", "none")
+        print(f"    {trial.id}: status={trial.status} metrics={metrics or '{}'} error={error}")
+
+    print("\n[3] Evaluator.aggregate_results")
+    try:
+        aggregate = await _AggregateOnlyEvaluator(EvaluatorConfig()).aggregate_results(trials)
+    except ValueError as exc:
+        print(f"    raises ValueError: {exc}")
+        print("\n[4] reward_scalar / RunReporter")
+        print("    not reached — the run fails here, which is the loud case")
+        return
+    completed = [trial for trial in trials if trial.status == "completed"]
+    print(f"    completed trials={len(completed)}/{len(trials)} · aggregate={aggregate or '{}'}")
+
+    print("\n[4] reward_scalar / RunReporter")
+    print(f"    reward_scalar={reward_scalar(aggregate)!r}   (the number a strategy ranks on)")
+    print(f"    operator sees: {_reporter_line(aggregate)}")
+    if "reward" not in aggregate:
+        print("    ^ no reward reached the loop, and nothing on this path said so")
+
+    print("\n[5] has_metric_dimensions (shared by the selector and the terminator)")
+    comparable = has_metric_dimensions({name: float(value) for name, value in aggregate.items()}, _OBJECTIVES)
+    print(f"    comparable={comparable}")
+    if comparable:
+        print("    the selector can rank, the terminator can detect a stagnant front, and a winner exists")
+    else:
+        print("    the terminator sees nothing scored, so it never stops early;")
+        print("    the selector still calls a model each round, and finalization ends with no winner")
+
+
+async def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument(
+        "--job-dir",
+        type=Path,
+        help="Inspect a real Harbor job directory instead of the synthetic scenarios.",
+    )
+    parser.add_argument(
+        "--scenario",
+        choices=[scenario.name for scenario in SCENARIOS],
+        help="Run one synthetic scenario instead of all of them.",
+    )
+    args = parser.parse_args()
+
+    if args.job_dir is not None:
+        await inspect(args.job_dir.expanduser().resolve())
+        return
+
+    selected = [s for s in SCENARIOS if args.scenario is None or s.name == args.scenario]
+    with tempfile.TemporaryDirectory(prefix="reward-pipeline-") as tmp:
+        for scenario in selected:
+            job_dir = materialize(scenario, Path(tmp))
+            await inspect(job_dir, behaviour=scenario.verifier_behaviour)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

From e1490fe66a658862400b9943a567d48b61b2db77 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 17 Aug 2026 22:03:46 +0000
Subject: [PATCH 02/10] chore(experimentalist): show the real ranking
 projection in the reward inspector

reward_scalar looks like the loop's scalar reward but has no caller in the loop.
Label it as such, and print pareto_objectives alongside has_metric_dimensions so
the layer that actually decides comparability is the one on screen.

Signed-off-by: Cursor Agent <cursoragent@cursor.com>

Co-authored-by: Aditya Pandey <aditya@autospace.co>
---
 .../tools/reward_pipeline_inspector.py        | 31 +++++++++++--------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py b/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
index 5c645bbd8ed..1d13c21d7d0 100644
--- a/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
+++ b/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
@@ -13,11 +13,12 @@
 1. **on disk** — what the verifier left in the trial directory.
 2. ``trials_from_job_dir`` — the ``TrialResult`` status and metrics parsed from it.
 3. ``Evaluator.aggregate_results`` — the per-candidate aggregate the loop stores.
-4. ``reward_scalar`` / ``RunReporter`` — the number a strategy ranks on and the
-   line an operator reads.
-5. ``has_metric_dimensions`` — the predicate the selector and terminator share,
-   which decides whether the loop has anything to compare and therefore whether
-   it can stop early.
+4. ``reward_scalar`` / ``RunReporter`` — the line an operator reads. ``reward_scalar``
+   is printed alongside it because it looks like the loop's scalar reward, but no
+   caller in the loop uses it.
+5. ``pareto_objectives`` / ``has_metric_dimensions`` — the projection the selector
+   ranks on and the predicate the selector and terminator share, which together
+   decide whether the loop can compare candidates or stop early.
 
 Every layer is called through production code. Nothing here is imported by the
 plugin; this is a read-only inspector for humans.
@@ -53,7 +54,10 @@
     EvaluatorConfig,
 )
 from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import trials_from_job_dir
-from nemo_experimentalist_plugin.experimentalist.components.models import has_metric_dimensions
+from nemo_experimentalist_plugin.experimentalist.components.models import (
+    has_metric_dimensions,
+    pareto_objectives,
+)
 from nemo_experimentalist_plugin.experimentalist.reporting import RunReporter, reward_scalar
 
 _OBJECTIVES = [MetricTarget(name="reward", direction="maximize")]
@@ -216,19 +220,20 @@ async def inspect(job_dir: Path, *, behaviour: str | None = None) -> None:
     print(f"    completed trials={len(completed)}/{len(trials)} · aggregate={aggregate or '{}'}")
 
     print("\n[4] reward_scalar / RunReporter")
-    print(f"    reward_scalar={reward_scalar(aggregate)!r}   (the number a strategy ranks on)")
+    print(f"    reward_scalar={reward_scalar(aggregate)!r}   (defined in reporting.py; no caller in the loop)")
     print(f"    operator sees: {_reporter_line(aggregate)}")
     if "reward" not in aggregate:
         print("    ^ no reward reached the loop, and nothing on this path said so")
 
-    print("\n[5] has_metric_dimensions (shared by the selector and the terminator)")
-    comparable = has_metric_dimensions({name: float(value) for name, value in aggregate.items()}, _OBJECTIVES)
-    print(f"    comparable={comparable}")
-    if comparable:
+    metrics = {name: float(value) for name, value in aggregate.items()}
+    print("\n[5] pareto_objectives / has_metric_dimensions (what the loop really ranks on)")
+    print(f"    pareto_objectives={pareto_objectives(metrics, _OBJECTIVES) or '{}'}")
+    print(f"    has_metric_dimensions={has_metric_dimensions(metrics, _OBJECTIVES)}")
+    if pareto_objectives(metrics, _OBJECTIVES):
         print("    the selector can rank, the terminator can detect a stagnant front, and a winner exists")
     else:
-        print("    the terminator sees nothing scored, so it never stops early;")
-        print("    the selector still calls a model each round, and finalization ends with no winner")
+        print("    every candidate projects to {}, so _dominates is always False and one front holds them all;")
+        print("    ranking is a no-op, the terminator never stops early, and finalization finds no winner")
 
 
 async def main() -> None:

From ac41c67470454f61fdc4d39a13620d47c39459fe Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 17 Aug 2026 22:15:48 +0000
Subject: [PATCH 03/10] chore(experimentalist): add a thin-denominator scenario
 to the reward inspector

aggregate_results averages over completed trials only, so a candidate that
completes 1 of 4 tasks reports the surviving trial's reward as its score. That
number is plausible and comparable, so no later gate questions it. The scenario
puts the completed/total count next to the aggregate that was derived from it.

Signed-off-by: Cursor Agent <cursoragent@cursor.com>

Co-authored-by: Aditya Pandey <aditya@autospace.co>
---
 .../tools/reward_pipeline_inspector.py              | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py b/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
index 1d13c21d7d0..4fb37cebe6d 100644
--- a/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
+++ b/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
@@ -118,6 +118,19 @@ def _trial(
         verifier_behaviour="One task writes a reward and another does not.",
         trials=(_trial("task-a", reward_text="1.0"), _trial("task-b")),
     ),
+    Scenario(
+        name="thin-denominator",
+        verifier_behaviour=(
+            "One easy task scores 1.0 and three harder tasks crash before their verifier runs. "
+            "The reward survives; the sample it was averaged over does not."
+        ),
+        trials=(
+            _trial("task-easy", reward_text="1.0"),
+            _trial("task-hard-1", exception_type="AgentTimeoutError"),
+            _trial("task-hard-2", exception_type="AgentTimeoutError"),
+            _trial("task-hard-3", exception_type="NonZeroAgentExitCodeError"),
+        ),
+    ),
 )
 
 

From 9c92b013c984bcba3fd75616a40576f6d15a68af Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 17 Aug 2026 22:21:04 +0000
Subject: [PATCH 04/10] feat(experimentalist): explain an absent objective
 metric instead of printing bare n/a

An evaluation that produces no objective metric rendered as 'reward n/a', which
reads as a score of nothing rather than a measurement that never happened. The
evidence for which one occurred lives on the trials -- their status and, for a
Harbor verifier that wrote no reward file, a RewardFileNotFoundError already
recorded in TrialResult.error -- and was discarded once a caller held only the
aggregate.

missing_objective_reason recovers it while the trials are still in hand, and
ExperimentContext.evaluate passes it to the reporter, which now prints the
explanation under the n/a line. The helper names only targets and trial
statuses, so no evaluator's vocabulary leaks into the metric contract.

Signed-off-by: Cursor Agent <cursoragent@cursor.com>

Co-authored-by: Aditya Pandey <aditya@autospace.co>
---
 .../experimentalist/components/models.py      | 54 ++++++++++++++++-
 .../experimentalist/context.py                |  2 +
 .../experimentalist/reporting.py              |  8 +++
 .../tests/experimentalist/test_reporting.py   | 17 ++++++
 .../tests/test_metric_contract.py             | 59 ++++++++++++++++++-
 5 files changed, 137 insertions(+), 3 deletions(-)

diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/models.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/models.py
index c4ffe2eb8d9..7e5f3196424 100644
--- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/models.py
+++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/models.py
@@ -10,10 +10,11 @@
 
 from __future__ import annotations
 
-from collections.abc import Callable, Iterable
+from collections import Counter
+from collections.abc import Callable, Iterable, Sequence
 from typing import Literal, TypeVar
 
-from nemo_experimentalist_plugin.entities import Candidate, MetricTarget
+from nemo_experimentalist_plugin.entities import Candidate, MetricTarget, TrialResult
 from pydantic import BaseModel
 
 # ---------------------------------------------------------------------------
@@ -334,3 +335,52 @@ def pareto_objectives(metrics: dict[str, float], objective_function: list[Metric
 def has_metric_dimensions(metrics: dict[str, float], targets: list[MetricTarget]) -> bool:
     """Return whether an evaluator result contains every required metric target."""
     return all(target.name in metrics for target in targets)
+
+
+def missing_objective_reason(
+    trials: Sequence[TrialResult],
+    metrics: dict[str, float | int],
+    targets: list[MetricTarget],
+) -> str | None:
+    """Explain why *metrics* omits a required target, or None when none is omitted.
+
+    An absent objective is indistinguishable from a measured one in every
+    downstream check: :func:`pareto_objectives` projects both an unmeasured and a
+    zero-scored candidate onto a dict, and only one of them is comparable. The
+    evidence for *which* happened lives in the trials, and by the time a caller
+    holds only the aggregate it is gone. This recovers it from the trials while
+    they are still in hand.
+
+    Deliberately free of any evaluator's vocabulary: it names the absent target
+    and the trial statuses behind it. A caller that knows which harness produced
+    the trials is the one that can point at a directory.
+
+    Args:
+        trials: Trials the aggregate was computed from.
+        metrics: The aggregate, as ``Evaluator.aggregate_results`` returned it.
+        targets: Metrics the run is scored against.
+
+    Returns:
+        str | None: A one-line explanation, or None when every target is present.
+    """
+    absent = [target.name for target in targets if target.name not in metrics]
+    if not absent:
+        return None
+
+    named = ", ".join(repr(name) for name in absent)
+    if not trials:
+        return f"the evaluator produced no trials, so {named} was never measured"
+
+    completed = [trial for trial in trials if trial.status == "completed"]
+    if not completed:
+        return f"no trial completed ({len(trials)} failed: {_trial_error_summary(trials)}), so {named} is absent"
+    return (
+        f"{len(completed)}/{len(trials)} trials completed but none reported {named}; "
+        "the verifier ran without emitting the metric"
+    )
+
+
+def _trial_error_summary(trials: Sequence[TrialResult]) -> str:
+    """Name the error types across *trials*, most frequent first."""
+    counts = Counter(str((trial.error or {}).get("type") or "unknown") for trial in trials)
+    return ", ".join(name if count == 1 else f"{name} ×{count}" for name, count in counts.most_common())
diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/context.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/context.py
index 08f4e2f8e95..f32003fc8f8 100644
--- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/context.py
+++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/context.py
@@ -43,6 +43,7 @@
     local_path_from_uri,
 )
 from nemo_experimentalist_plugin.experimentalist.components.evaluator import Evaluator
+from nemo_experimentalist_plugin.experimentalist.components.models import missing_objective_reason
 from nemo_experimentalist_plugin.experimentalist.components.trace_explorer import TraceExplorer
 from nemo_experimentalist_plugin.experimentalist.experimentalist_backend import (
     ExperimentalistBackend,
@@ -454,6 +455,7 @@ async def evaluate(
                 metrics=result.aggregate_metrics,
                 objective_metrics=self._objective_metrics,
                 artifacts=self.root / "eval-and-optimize" / "results" / result.id,
+                reason=missing_objective_reason(result.trials, result.aggregate_metrics, self._objective_metrics),
             )
         return result
 
diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py
index 0110b8ee02a..e79c7b4860b 100644
--- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py
+++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py
@@ -120,12 +120,18 @@ def candidate_evaluated(
         metrics: dict[str, float | int],
         objective_metrics: list[MetricTarget],
         artifacts: Path,
+        reason: str | None = None,
     ) -> None:
         """Narrate an evaluation using all configured objective metrics.
 
         Objective directions determine whether validation deltas are displayed
         as improvements or declines. Regression metrics are guardrails and are
         intentionally not presented as progress dimensions.
+
+        *reason* explains an ``n/a`` when the caller could work out why one
+        happened. Without it the line states that a metric is missing and stops
+        there, which reads as a score of nothing rather than a measurement that
+        never occurred.
         """
         if self._verbosity is Verbosity.QUIET:
             return
@@ -148,6 +154,8 @@ def candidate_evaluated(
                     rendered += f" {'▲' if improved else '▼'}{delta:+.3f}"
                 rendered_metrics.append(rendered)
             self._emit(f"   {label} · {split:<10} · {', '.join(rendered_metrics)}   → {artifacts}")
+            if reason is not None:
+                self._emit(f"     ↳ {reason}")
         except Exception:  # noqa: BLE001
             pass
 
diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py b/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py
index faa418a0bce..dc59902e6e4 100644
--- a/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py
+++ b/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py
@@ -77,6 +77,23 @@ def test_candidate_evaluated_shows_all_objectives_and_direction_aware_deltas() -
     assert "tokens 80.000 ▲-20.000" in lines[1]
 
 
+def test_candidate_evaluated_explains_an_na_when_the_caller_knows_why() -> None:
+    # Without the reason, `n/a` reads as a score of nothing rather than a
+    # measurement that never happened, and the two lead to opposite actions.
+    r, sink = _reporter()
+    r.candidate_evaluated(
+        label="agent-0",
+        split="validation",
+        metrics={"tokens": 100.0},
+        objective_metrics=OBJECTIVES,
+        artifacts=Path("/exp/results/agent-0-validation"),
+        reason="no trial completed (2 failed: RewardFileNotFoundError), so 'success' is absent",
+    )
+    lines = sink.getvalue().splitlines()
+    assert "success n/a" in lines[0]
+    assert "RewardFileNotFoundError" in lines[1]
+
+
 def test_seed_baseline_sets_delta_reference_silently_for_resume() -> None:
     # On resume agent-0 is not re-evaluated; seed_baseline sets the delta
     # reference from its cached reward without emitting a line, so the first
diff --git a/plugins/nemo-experimentalist/tests/test_metric_contract.py b/plugins/nemo-experimentalist/tests/test_metric_contract.py
index 1b5f0f424a3..445ab50763b 100644
--- a/plugins/nemo-experimentalist/tests/test_metric_contract.py
+++ b/plugins/nemo-experimentalist/tests/test_metric_contract.py
@@ -8,7 +8,8 @@
     pareto_objectives,
     with_insight_objective,
 )
-from nemo_experimentalist_plugin.entities import MetricTarget
+from nemo_experimentalist_plugin.entities import MetricTarget, TrialResult, TrialStatus
+from nemo_experimentalist_plugin.experimentalist.components.models import missing_objective_reason
 
 
 def test_metric_contract_supports_multiple_objective_metrics() -> None:
@@ -64,6 +65,62 @@ def test_metric_dimensions_require_only_the_objectives_used_for_pareto_ranking()
     assert not has_metric_dimensions({"quality": 0.9}, [*objectives, *regressions])
 
 
+def _trial(status: TrialStatus, *, error_type: str | None = None) -> TrialResult:
+    return TrialResult(
+        id=f"task__{status}",
+        task_id="task",
+        status=status,
+        error=None if error_type is None else {"type": error_type},
+    )
+
+
+@pytest.mark.parametrize(
+    ("trials", "metrics", "expected"),
+    [
+        # Measured: nothing to explain.
+        ([_trial("completed")], {"reward": 0.0}, None),
+        # No trials at all: the evaluator itself produced nothing.
+        ([], {}, "the evaluator produced no trials, so 'reward' was never measured"),
+        # Every trial failed. Harbor's own diagnosis is on the trials; name it.
+        (
+            [_trial("failed", error_type="RewardFileNotFoundError")],
+            {},
+            "no trial completed (1 failed: RewardFileNotFoundError), so 'reward' is absent",
+        ),
+        # Mixed failures are counted, most frequent first.
+        (
+            [
+                _trial("failed", error_type="AgentTimeoutError"),
+                _trial("failed", error_type="AgentTimeoutError"),
+                _trial("failed"),
+            ],
+            {},
+            "no trial completed (3 failed: AgentTimeoutError ×2, unknown), so 'reward' is absent",
+        ),
+        # The silent shape: the verifier ran, exited cleanly, and emitted no metric.
+        (
+            [_trial("completed"), _trial("completed")],
+            {},
+            "2/2 trials completed but none reported 'reward'; the verifier ran without emitting the metric",
+        ),
+        # A trial can carry other metrics and still miss the objective.
+        (
+            [_trial("completed")],
+            {"latency": 3.0},
+            "1/1 trials completed but none reported 'reward'; the verifier ran without emitting the metric",
+        ),
+    ],
+)
+def test_missing_objective_reason_names_the_evidence_behind_an_absent_metric(
+    trials: list[TrialResult],
+    metrics: dict[str, float],
+    expected: str | None,
+) -> None:
+    objectives = [MetricTarget(name="reward", direction="maximize")]
+
+    assert missing_objective_reason(trials, metrics, objectives) == expected
+
+
 @pytest.mark.parametrize(
     "config",
     [

From f5c209d95a6eceb2294e13027c9efcefde157dc4 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 17 Aug 2026 22:23:34 +0000
Subject: [PATCH 05/10] fix(experimentalist): fail the run when the baseline
 carries no objective metric

The baseline's validation reward is the one measurement every later comparison is
against. When it is absent the loop did not notice: pareto_objectives projects an
unmeasured candidate to {}, _dominates then returns False against every peer, so
one Pareto front held the whole population and ranking became a no-op. The
terminator could not stop either, because its warm-up guard reads 'nothing scored'
as 'not enough scored generations yet'. The run therefore spent every round on
analysis, proposals, and builds it could not rank, and finished with no winner.

_record_baseline_validation now refuses to record such a baseline and raises. The
runner already marks the run failed and re-raises, so the operator gets the reason
at the cost of one evaluation rather than max_rounds rounds. Resume is unaffected:
the check only sees a baseline this round actually measured.

Signed-off-by: Cursor Agent <cursoragent@cursor.com>

Co-authored-by: Aditya Pandey <aditya@autospace.co>
---
 .../strategies/evolutionary.py                | 23 ++++++++++-
 .../experimentalist/test_loop_helpers.py      | 41 ++++++++++++++++++-
 2 files changed, 60 insertions(+), 4 deletions(-)

diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/strategies/evolutionary.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/strategies/evolutionary.py
index 7d0091161d4..87ba72468d1 100644
--- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/strategies/evolutionary.py
+++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/strategies/evolutionary.py
@@ -31,6 +31,7 @@
 from nemo_experimentalist_plugin.experimentalist.components.importer import IMPORT, import_proposal
 from nemo_experimentalist_plugin.experimentalist.components.models import (
     EvolutionTree,
+    missing_objective_reason,
 )
 from nemo_experimentalist_plugin.experimentalist.components.tools import (
     GuardedShellTools,
@@ -644,10 +645,28 @@ async def _record_baseline_validation(
         and `_evaluate_validation_candidates` only returns candidates it actually ran. A
         round-0 resume therefore has nothing to record, and indexing the map raised
         KeyError on a round-0 resume.
+
+        Raises:
+            ValueError: if the baseline measured here carries no objective metric.
+                Every later comparison is against this one measurement, so a run that
+                cannot score its own starting point has nothing to optimize toward,
+                and no later gate says so: `pareto_objectives` projects an unmeasured
+                candidate to `{}`, `_dominates` then returns False against every peer,
+                so one Pareto front holds the whole population and ranking silently
+                becomes a no-op. Failing here costs one evaluation instead of
+                `max_rounds` rounds of analysis, proposals, and builds.
         """
         measured = results.get(baseline.label)
-        if measured is not None:
-            await ctx.record_reward(baseline, channel="validation", result=measured)
+        if measured is None:
+            return
+        reason = missing_objective_reason(measured.trials, measured.aggregate_metrics, ctx.objective_metrics)
+        if reason is not None:
+            raise ValueError(
+                f"Baseline {baseline.label} produced no objective metric: {reason}. The run is scored "
+                "against this measurement, so there is nothing to optimize toward. Check that the "
+                "evaluation emits every objective metric for the unmodified agent before running again."
+            )
+        await ctx.record_reward(baseline, channel="validation", result=measured)
 
     async def _ensure_baseline(self, *, ctx: StrategyContext, config: EvolutionaryOptimizerConfig) -> None:
         """Create the baseline candidate unless this run already has one.
diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_helpers.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_helpers.py
index bc947193917..b31307be20c 100644
--- a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_helpers.py
+++ b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_helpers.py
@@ -1,13 +1,13 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-"""Rolling a round back so the loop can re-enter it cleanly."""
+"""Loop helpers: rolling a round back, and refusing a baseline nothing could measure."""
 
 from pathlib import Path
 
 import pytest
 from doubles import FakeBackend, make_context
-from nemo_experimentalist_plugin.entities import Proposal
+from nemo_experimentalist_plugin.entities import EvaluationResult, Proposal, TrialResult
 from nemo_experimentalist_plugin.experimentalist.strategies.evolutionary import EvolutionaryStrategy
 
 
@@ -50,3 +50,40 @@ async def test_rollback_discards_later_candidates_and_clears_their_evaluator_scr
     assert ctx.candidate_dir(discarded).exists(), "the artifact must survive a rollback"
     assert not any(d.exists() for d in scratch), "stale evaluator scratch would be read on re-run"
     assert survivor_scratch.exists(), "a survivor's results must not be swept up"
+
+
+@pytest.mark.asyncio
+async def test_baseline_validation_refuses_an_unmeasurable_baseline_and_records_a_measured_one(
+    tmp_path: Path,
+) -> None:
+    """The run's one reference point must exist before anything is compared against it.
+
+    An empty aggregate is indistinguishable from a scored one downstream, so a run
+    that cannot measure its own starting point would spend every round ranking
+    candidates that are all mutually incomparable and then report no winner.
+    """
+    ctx = make_context(root=tmp_path, backend=FakeBackend())
+    baseline = await _commit(ctx, "baseline: the agent under test, unchanged", generation=0)
+
+    unmeasurable = EvaluationResult(
+        id=f"{baseline.label}-validation",
+        aggregate_metrics={},
+        trials=[
+            TrialResult(id="task-a__0", task_id="task-a", status="failed", error={"type": "RewardFileNotFoundError"})
+        ],
+    )
+    with pytest.raises(ValueError, match="RewardFileNotFoundError"):
+        await EvolutionaryStrategy._record_baseline_validation(
+            ctx=ctx, baseline=baseline, results={baseline.label: unmeasurable}
+        )
+    assert "validation" not in (await ctx.candidates())[0].rewards, "a refused baseline must not be recorded"
+
+    measured = EvaluationResult(
+        id=f"{baseline.label}-validation",
+        aggregate_metrics={"reward": 0.4},
+        trials=[TrialResult(id="task-a__0", task_id="task-a", status="completed")],
+    )
+    await EvolutionaryStrategy._record_baseline_validation(
+        ctx=ctx, baseline=baseline, results={baseline.label: measured}
+    )
+    assert (await ctx.candidates())[0].rewards["validation"].metrics == {"reward": 0.4}

From f4f88a5d90c36cc8df77c4e55065dac8add2031f Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 17 Aug 2026 22:24:41 +0000
Subject: [PATCH 06/10] docs(experimentalist): state the reward-file contract
 in the task authoring guide

The HarborDataset docstring is the task authoring guide: EvalAuthor hands it to
the model verbatim as context['dataset_documentation']. It documented reward.json
thoroughly and never said what happens when a verifier writes nothing, and it
mentioned reward.txt only as a resource label.

Say that writing one of the two files is mandatory, that exiting 0 without either
scores nothing rather than zero, and that Harbor raises RewardFileNotFoundError.
The taught test.sh pattern also produced exactly this failure -- under set -e a
failing check aborts before merge_metrics runs -- so it now seeds a reward before
anything can fail.

Signed-off-by: Cursor Agent <cursoragent@cursor.com>

Co-authored-by: Aditya Pandey <aditya@autospace.co>
---
 .../components/evaluator/harbor.py            | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py
index a7840899f5a..e810c39fa46 100644
--- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py
+++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py
@@ -861,6 +861,24 @@ class HarborDataset(Dataset):
 
         {"reward": 0.8, "my_metric": 1.0}
 
+    **Writing a reward file is mandatory, not optional.** Harbor accepts either
+    ``/logs/verifier/reward.json`` or ``/logs/verifier/reward.txt`` (one bare
+    number, and nothing else).  ``reward.json`` wins when both exist.  A
+    ``test.sh`` that exits ``0`` without writing either scores nothing at all —
+    it does **not** score zero.  Harbor raises ``RewardFileNotFoundError``, the
+    trial is recorded as failed, and the task contributes no metric to the run.
+    A run whose baseline emits no metric this way is refused outright, because
+    there is then no measurement to optimize toward.
+
+    So create the directory and write the file unconditionally, on every exit
+    path — including the ones where the agent failed::
+
+        mkdir -p /logs/verifier
+        echo 0 > /logs/verifier/reward.txt   # replaced below once the score is known
+
+    Under ``set -e`` a failing check aborts the script, so a reward written only
+    at the end is never written on the path that matters most.
+
     **To add a new metric** — write it as an additional key in ``reward.json``::
 
         {"reward": 0.8, "my_metric": 1.0, "another_metric": 0.5}
@@ -901,6 +919,9 @@ class HarborDataset(Dataset):
         #!/usr/bin/env bash
         set -euo pipefail
         mkdir -p /logs/verifier
+        # Seed a reward before anything can fail. Under `set -e` a failing check
+        # aborts the script, and reward.json below wins whenever it is written.
+        echo 0 > /logs/verifier/reward.txt
         SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 
         # Run each metric script and collect its output.

From 8d57d64aa959c6aa067d3c756ca292bd8eb425c6 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 18 Aug 2026 17:42:14 +0000
Subject: [PATCH 07/10] docs(experimentalist): warn against seeding a
 placeholder reward instead of advising one

The authoring guide told task authors to write `echo 0 > /logs/verifier/reward.txt`
before running any check, so a crashing verifier would still leave a reward behind.
That defeats the rest of this change. Harbor ignores test.sh's exit code and looks
only for a reward file, so a seeded zero makes a verifier that died half-way report
a confident 0.0: the trial is 'completed', has_metric_dimensions passes, and the
baseline guard never fires. A broken harness becomes indistinguishable from an
agent that genuinely scored zero, which is the distinction the contract exists to
keep.

Say the opposite: score every agent outcome including failure, and let a verifier
that cannot finish fail the trial. Drop the seed from the taught test.sh.

Signed-off-by: Cursor Agent <cursoragent@cursor.com>

Co-authored-by: Aditya Pandey <aditya@autospace.co>
---
 .../components/evaluator/harbor.py            | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py
index e810c39fa46..9e04e710f68 100644
--- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py
+++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py
@@ -870,14 +870,19 @@ class HarborDataset(Dataset):
     A run whose baseline emits no metric this way is refused outright, because
     there is then no measurement to optimize toward.
 
-    So create the directory and write the file unconditionally, on every exit
-    path — including the ones where the agent failed::
-
-        mkdir -p /logs/verifier
-        echo 0 > /logs/verifier/reward.txt   # replaced below once the score is known
-
-    Under ``set -e`` a failing check aborts the script, so a reward written only
-    at the end is never written on the path that matters most.
+    So score **every agent outcome**, failures included: an agent that did
+    nothing scores ``0``, and that is a real measurement.  What must never
+    happen is a *task* that emits no number at all.
+
+    **Do not seed a placeholder reward to keep a trial alive.** Harbor ignores
+    ``test.sh``'s exit code and looks only for a reward file, so a pre-written
+    ``echo 0 > /logs/verifier/reward.txt`` turns a verifier that crashed
+    half-way into a confident score of ``0.0`` — indistinguishable from an agent
+    that genuinely earned zero, and comparable enough that nothing downstream
+    questions it.  That is the one distinction this contract exists to keep.
+    A verifier that could not finish should fail the trial; under ``set -e`` a
+    crashing check aborts before the merge step below, which is the intended
+    outcome and is reported as a failed trial naming the error.
 
     **To add a new metric** — write it as an additional key in ``reward.json``::
 
@@ -919,9 +924,6 @@ class HarborDataset(Dataset):
         #!/usr/bin/env bash
         set -euo pipefail
         mkdir -p /logs/verifier
-        # Seed a reward before anything can fail. Under `set -e` a failing check
-        # aborts the script, and reward.json below wins whenever it is written.
-        echo 0 > /logs/verifier/reward.txt
         SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 
         # Run each metric script and collect its output.

From a21575ad5a3bcc1911d2274eaabc9278f8f2f878 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 18 Aug 2026 19:33:05 +0000
Subject: [PATCH 08/10] refactor(experimentalist): tighten the missing-metric
 diagnosis and its docs

Fold the reward-file contract into the existing 'must write to' bullet list in the
HarborDataset guide instead of appending three paragraphs after it. The warning
against seeding a placeholder went with them: it argued against a pattern the guide
no longer shows.

Trim missing_objective_reason to the constraint a future editor needs -- why it
reads the trials rather than the aggregate, and why it stays free of evaluator
vocabulary. Both failure branches now share the completed/total prefix, which
reads correctly at any trial count where 'all 1 trials failed' did not.

Signed-off-by: Cursor Agent <cursoragent@cursor.com>

Co-authored-by: Aditya Pandey <aditya@autospace.co>
---
 .../components/evaluator/harbor.py            | 31 +++----------
 .../experimentalist/components/models.py      | 46 ++++++-------------
 .../experimentalist/reporting.py              |  6 +--
 .../strategies/evolutionary.py                | 16 +++----
 .../tests/experimentalist/test_reporting.py   |  2 +-
 .../tests/test_metric_contract.py             | 16 ++-----
 6 files changed, 35 insertions(+), 82 deletions(-)

diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py
index 9e04e710f68..1b2b22cf64b 100644
--- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py
+++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py
@@ -851,39 +851,22 @@ class HarborDataset(Dataset):
       Use only when the OTLP traces don't capture what you need.
     - ``/tests/``         — any helper files you place in ``tests/``
 
-    and must write to:
+    and must write **one of** (``reward.json`` wins when both exist):
 
     - ``/logs/verifier/reward.json`` — flat JSON object; **every value must be a
       plain number** (int or float).  Harbor calls ``float(value)`` on each entry
       and silently drops non-numeric values (nested objects, booleans, strings).
+    - ``/logs/verifier/reward.txt`` — one bare number, and nothing else.
+
+    Writing one is mandatory.  Exiting ``0`` without either scores *nothing* — not
+    zero: Harbor raises ``RewardFileNotFoundError`` and the trial is recorded as
+    failed.  So score every agent outcome, failures included; an agent that did
+    nothing scores ``0``, and that is a real measurement.
 
     Correct format::
 
         {"reward": 0.8, "my_metric": 1.0}
 
-    **Writing a reward file is mandatory, not optional.** Harbor accepts either
-    ``/logs/verifier/reward.json`` or ``/logs/verifier/reward.txt`` (one bare
-    number, and nothing else).  ``reward.json`` wins when both exist.  A
-    ``test.sh`` that exits ``0`` without writing either scores nothing at all —
-    it does **not** score zero.  Harbor raises ``RewardFileNotFoundError``, the
-    trial is recorded as failed, and the task contributes no metric to the run.
-    A run whose baseline emits no metric this way is refused outright, because
-    there is then no measurement to optimize toward.
-
-    So score **every agent outcome**, failures included: an agent that did
-    nothing scores ``0``, and that is a real measurement.  What must never
-    happen is a *task* that emits no number at all.
-
-    **Do not seed a placeholder reward to keep a trial alive.** Harbor ignores
-    ``test.sh``'s exit code and looks only for a reward file, so a pre-written
-    ``echo 0 > /logs/verifier/reward.txt`` turns a verifier that crashed
-    half-way into a confident score of ``0.0`` — indistinguishable from an agent
-    that genuinely earned zero, and comparable enough that nothing downstream
-    questions it.  That is the one distinction this contract exists to keep.
-    A verifier that could not finish should fail the trial; under ``set -e`` a
-    crashing check aborts before the merge step below, which is the intended
-    outcome and is reported as a failed trial naming the error.
-
     **To add a new metric** — write it as an additional key in ``reward.json``::
 
         {"reward": 0.8, "my_metric": 1.0, "another_metric": 0.5}
diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/models.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/models.py
index 7e5f3196424..2738ff31aa7 100644
--- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/models.py
+++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/models.py
@@ -344,43 +344,27 @@ def missing_objective_reason(
 ) -> str | None:
     """Explain why *metrics* omits a required target, or None when none is omitted.
 
-    An absent objective is indistinguishable from a measured one in every
-    downstream check: :func:`pareto_objectives` projects both an unmeasured and a
-    zero-scored candidate onto a dict, and only one of them is comparable. The
-    evidence for *which* happened lives in the trials, and by the time a caller
-    holds only the aggregate it is gone. This recovers it from the trials while
-    they are still in hand.
-
-    Deliberately free of any evaluator's vocabulary: it names the absent target
-    and the trial statuses behind it. A caller that knows which harness produced
-    the trials is the one that can point at a directory.
-
-    Args:
-        trials: Trials the aggregate was computed from.
-        metrics: The aggregate, as ``Evaluator.aggregate_results`` returned it.
-        targets: Metrics the run is scored against.
-
-    Returns:
-        str | None: A one-line explanation, or None when every target is present.
+    An aggregate cannot say whether a target is absent because nothing ran or
+    because nothing scored, and every later check reads the two the same way. The
+    trials still carry that difference, so it is read off them here rather than
+    inferred later from the aggregate alone.
+
+    Names only targets and trial statuses: evaluator-specific detail belongs to
+    whichever caller knows which harness ran.
     """
-    absent = [target.name for target in targets if target.name not in metrics]
+    absent = ", ".join(repr(target.name) for target in targets if target.name not in metrics)
     if not absent:
         return None
-
-    named = ", ".join(repr(name) for name in absent)
     if not trials:
-        return f"the evaluator produced no trials, so {named} was never measured"
+        return f"the evaluator produced no trials, so {absent} was never measured"
 
-    completed = [trial for trial in trials if trial.status == "completed"]
+    completed = sum(1 for trial in trials if trial.status == "completed")
     if not completed:
-        return f"no trial completed ({len(trials)} failed: {_trial_error_summary(trials)}), so {named} is absent"
-    return (
-        f"{len(completed)}/{len(trials)} trials completed but none reported {named}; "
-        "the verifier ran without emitting the metric"
-    )
+        return f"0/{len(trials)} trials completed ({_error_types(trials)}), so {absent} was never measured"
+    return f"{completed}/{len(trials)} trials completed, but none reported {absent}"
 
 
-def _trial_error_summary(trials: Sequence[TrialResult]) -> str:
-    """Name the error types across *trials*, most frequent first."""
+def _error_types(trials: Sequence[TrialResult]) -> str:
+    """Error types across *trials*, most frequent first."""
     counts = Counter(str((trial.error or {}).get("type") or "unknown") for trial in trials)
-    return ", ".join(name if count == 1 else f"{name} ×{count}" for name, count in counts.most_common())
+    return ", ".join(name if n == 1 else f"{name} ×{n}" for name, n in counts.most_common())
diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py
index e79c7b4860b..42ab39af755 100644
--- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py
+++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py
@@ -128,10 +128,8 @@ def candidate_evaluated(
         as improvements or declines. Regression metrics are guardrails and are
         intentionally not presented as progress dimensions.
 
-        *reason* explains an ``n/a`` when the caller could work out why one
-        happened. Without it the line states that a metric is missing and stops
-        there, which reads as a score of nothing rather than a measurement that
-        never occurred.
+        *reason* explains an ``n/a``, which otherwise reads as a score of nothing
+        rather than a measurement that never happened.
         """
         if self._verbosity is Verbosity.QUIET:
             return
diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/strategies/evolutionary.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/strategies/evolutionary.py
index 87ba72468d1..7381361c58e 100644
--- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/strategies/evolutionary.py
+++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/strategies/evolutionary.py
@@ -648,13 +648,10 @@ async def _record_baseline_validation(
 
         Raises:
             ValueError: if the baseline measured here carries no objective metric.
-                Every later comparison is against this one measurement, so a run that
-                cannot score its own starting point has nothing to optimize toward,
-                and no later gate says so: `pareto_objectives` projects an unmeasured
-                candidate to `{}`, `_dominates` then returns False against every peer,
-                so one Pareto front holds the whole population and ranking silently
-                becomes a no-op. Failing here costs one evaluation instead of
-                `max_rounds` rounds of analysis, proposals, and builds.
+                Every later comparison is against this one measurement, and no later
+                gate notices its absence -- unmeasured candidates are mutually
+                incomparable, so ranking becomes a no-op and the run pays for every
+                round before reporting no winner.
         """
         measured = results.get(baseline.label)
         if measured is None:
@@ -662,9 +659,8 @@ async def _record_baseline_validation(
         reason = missing_objective_reason(measured.trials, measured.aggregate_metrics, ctx.objective_metrics)
         if reason is not None:
             raise ValueError(
-                f"Baseline {baseline.label} produced no objective metric: {reason}. The run is scored "
-                "against this measurement, so there is nothing to optimize toward. Check that the "
-                "evaluation emits every objective metric for the unmodified agent before running again."
+                f"Baseline {baseline.label} produced no objective metric: {reason}. The run is scored against "
+                "this measurement, so fix the evaluation before running again."
             )
         await ctx.record_reward(baseline, channel="validation", result=measured)
 
diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py b/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py
index dc59902e6e4..dc12886ea2b 100644
--- a/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py
+++ b/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py
@@ -87,7 +87,7 @@ def test_candidate_evaluated_explains_an_na_when_the_caller_knows_why() -> None:
         metrics={"tokens": 100.0},
         objective_metrics=OBJECTIVES,
         artifacts=Path("/exp/results/agent-0-validation"),
-        reason="no trial completed (2 failed: RewardFileNotFoundError), so 'success' is absent",
+        reason="0/2 trials completed (RewardFileNotFoundError), so 'success' was never measured",
     )
     lines = sink.getvalue().splitlines()
     assert "success n/a" in lines[0]
diff --git a/plugins/nemo-experimentalist/tests/test_metric_contract.py b/plugins/nemo-experimentalist/tests/test_metric_contract.py
index 445ab50763b..2211779a1d0 100644
--- a/plugins/nemo-experimentalist/tests/test_metric_contract.py
+++ b/plugins/nemo-experimentalist/tests/test_metric_contract.py
@@ -85,7 +85,7 @@ def _trial(status: TrialStatus, *, error_type: str | None = None) -> TrialResult
         (
             [_trial("failed", error_type="RewardFileNotFoundError")],
             {},
-            "no trial completed (1 failed: RewardFileNotFoundError), so 'reward' is absent",
+            "0/1 trials completed (RewardFileNotFoundError), so 'reward' was never measured",
         ),
         # Mixed failures are counted, most frequent first.
         (
@@ -95,20 +95,12 @@ def _trial(status: TrialStatus, *, error_type: str | None = None) -> TrialResult
                 _trial("failed"),
             ],
             {},
-            "no trial completed (3 failed: AgentTimeoutError ×2, unknown), so 'reward' is absent",
+            "0/3 trials completed (AgentTimeoutError ×2, unknown), so 'reward' was never measured",
         ),
         # The silent shape: the verifier ran, exited cleanly, and emitted no metric.
-        (
-            [_trial("completed"), _trial("completed")],
-            {},
-            "2/2 trials completed but none reported 'reward'; the verifier ran without emitting the metric",
-        ),
+        ([_trial("completed"), _trial("completed")], {}, "2/2 trials completed, but none reported 'reward'"),
         # A trial can carry other metrics and still miss the objective.
-        (
-            [_trial("completed")],
-            {"latency": 3.0},
-            "1/1 trials completed but none reported 'reward'; the verifier ran without emitting the metric",
-        ),
+        ([_trial("completed")], {"latency": 3.0}, "1/1 trials completed, but none reported 'reward'"),
     ],
 )
 def test_missing_objective_reason_names_the_evidence_behind_an_absent_metric(

From c4c7fcee082a9e0f7acf57588a0efacc6baf0bd0 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 18 Aug 2026 19:47:42 +0000
Subject: [PATCH 09/10] chore(experimentalist): address review on the reward
 inspector

- Invoke the script directly with `uv run <path>.py`, per the repo's Python guidance.
- Replace the untyped trial-spec dict with a frozen TrialSpec dataclass, which also
  removes the _trial factory: the constructor already was one.
- Stop describing pre-fix behaviour. Layer 4 now passes the reason through the
  reporter, as ExperimentContext.evaluate does, and layer 5 no longer claims the run
  reaches ranking and finds no winner -- as a baseline it now stops outright, and the
  job directory alone cannot say which candidate it belongs to, so both are stated.
- Docstring the three helpers that lacked one.

Signed-off-by: Cursor Agent <cursoragent@cursor.com>

Co-authored-by: Aditya Pandey <aditya@autospace.co>
---
 .../tools/reward_pipeline_inspector.py        | 90 ++++++++++---------
 1 file changed, 49 insertions(+), 41 deletions(-)

diff --git a/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py b/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
index 4fb37cebe6d..ca3b5b4e131 100644
--- a/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
+++ b/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
@@ -13,9 +13,9 @@
 1. **on disk** — what the verifier left in the trial directory.
 2. ``trials_from_job_dir`` — the ``TrialResult`` status and metrics parsed from it.
 3. ``Evaluator.aggregate_results`` — the per-candidate aggregate the loop stores.
-4. ``reward_scalar`` / ``RunReporter`` — the line an operator reads. ``reward_scalar``
-   is printed alongside it because it looks like the loop's scalar reward, but no
-   caller in the loop uses it.
+4. ``missing_objective_reason`` / ``RunReporter`` — the lines an operator reads.
+   ``reward_scalar`` is printed alongside them because it looks like the loop's
+   scalar reward, but no caller in the loop uses it.
 5. ``pareto_objectives`` / ``has_metric_dimensions`` — the projection the selector
    ranks on and the predicate the selector and terminator share, which together
    decide whether the loop can compare candidates or stop early.
@@ -26,14 +26,14 @@
 Usage::
 
     # Compare synthetic trial shapes side by side.
-    uv run python plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
+    uv run plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
 
     # One shape only.
-    uv run python plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py \
+    uv run plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py \
         --scenario verifier-silent
 
     # Inspect a real Harbor job directory from a run that reported `reward n/a`.
-    uv run python plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py \
+    uv run plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py \
         --job-dir eval-and-optimize/results/agent-0-validation
 """
 
@@ -56,6 +56,7 @@
 from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import trials_from_job_dir
 from nemo_experimentalist_plugin.experimentalist.components.models import (
     has_metric_dimensions,
+    missing_objective_reason,
     pareto_objectives,
 )
 from nemo_experimentalist_plugin.experimentalist.reporting import RunReporter, reward_scalar
@@ -69,33 +70,33 @@ class _AggregateOnlyEvaluator(Evaluator):
     evaluator_type = "reward-pipeline-inspector"
 
     async def _run(self, agent: Path, dataset: Dataset, options: EvaluatorConfig) -> Sequence[TrialResult]:
+        """Never called: the inspector adapts a job directory that already exists."""
         raise NotImplementedError("The inspector reads an existing job directory; it never runs an evaluation.")
 
 
+@dataclass(frozen=True)
+class TrialSpec:
+    """One synthetic trial: its task, the reward it wrote, and any Harbor exception."""
+
+    task: str
+    reward_text: str | None = None
+    exception_type: str | None = None
+
+
 @dataclass(frozen=True)
 class Scenario:
     """One synthetic trial shape, plus what the verifier did to produce it."""
 
     name: str
     verifier_behaviour: str
-    trials: tuple[dict, ...]
-
-
-def _trial(
-    task: str,
-    *,
-    reward_text: str | None = None,
-    exception_type: str | None = None,
-) -> dict:
-    """Describe one trial: its task, whether a reward file exists, and any Harbor exception."""
-    return {"task": task, "reward_text": reward_text, "exception_type": exception_type}
+    trials: tuple[TrialSpec, ...]
 
 
 SCENARIOS = (
     Scenario(
         name="reward-written",
         verifier_behaviour="test.sh writes /logs/verifier/reward.txt. The control case.",
-        trials=(_trial("task-a", reward_text="1.0"), _trial("task-b", reward_text="0.0")),
+        trials=(TrialSpec("task-a", reward_text="1.0"), TrialSpec("task-b", reward_text="0.0")),
     ),
     Scenario(
         name="verifier-raised",
@@ -103,7 +104,7 @@ def _trial(
             "test.sh exits 0 without writing a reward file, so Harbor's verifier raises "
             "RewardFileNotFoundError and the trial records exception_info."
         ),
-        trials=(_trial("task-a", exception_type="RewardFileNotFoundError"),),
+        trials=(TrialSpec("task-a", exception_type="RewardFileNotFoundError"),),
     ),
     Scenario(
         name="verifier-silent",
@@ -111,12 +112,12 @@ def _trial(
             "The trial finishes with no exception and no reward file — a disabled verifier, "
             "a bind mount that never reflected the write, or a reward stripped during log collection."
         ),
-        trials=(_trial("task-a"), _trial("task-b")),
+        trials=(TrialSpec("task-a"), TrialSpec("task-b")),
     ),
     Scenario(
         name="partial",
         verifier_behaviour="One task writes a reward and another does not.",
-        trials=(_trial("task-a", reward_text="1.0"), _trial("task-b")),
+        trials=(TrialSpec("task-a", reward_text="1.0"), TrialSpec("task-b")),
     ),
     Scenario(
         name="thin-denominator",
@@ -125,10 +126,10 @@ def _trial(
             "The reward survives; the sample it was averaged over does not."
         ),
         trials=(
-            _trial("task-easy", reward_text="1.0"),
-            _trial("task-hard-1", exception_type="AgentTimeoutError"),
-            _trial("task-hard-2", exception_type="AgentTimeoutError"),
-            _trial("task-hard-3", exception_type="NonZeroAgentExitCodeError"),
+            TrialSpec("task-easy", reward_text="1.0"),
+            TrialSpec("task-hard-1", exception_type="AgentTimeoutError"),
+            TrialSpec("task-hard-2", exception_type="AgentTimeoutError"),
+            TrialSpec("task-hard-3", exception_type="NonZeroAgentExitCodeError"),
         ),
     ),
 )
@@ -138,20 +139,19 @@ def materialize(scenario: Scenario, root: Path) -> Path:
     """Write *scenario* as a Harbor job directory and return it."""
     job_dir = root / scenario.name
     for index, spec in enumerate(scenario.trials):
-        trial_dir = job_dir / f"{spec['task']}__{index}"
+        trial_dir = job_dir / f"{spec.task}__{index}"
         trial_dir.mkdir(parents=True)
-        exception_type = spec["exception_type"]
         (trial_dir / "result.json").write_text(
             json.dumps(
                 {
-                    "task_name": f"harbor/{spec['task']}",
+                    "task_name": f"harbor/{spec.task}",
                     "trial_name": trial_dir.name,
                     "verifier_result": None,
                     "exception_info": (
                         None
-                        if exception_type is None
+                        if spec.exception_type is None
                         else {
-                            "exception_type": exception_type,
+                            "exception_type": spec.exception_type,
                             "exception_message": "No reward file found at /logs/verifier/reward.txt",
                             "exception_traceback": "<elided>",
                         }
@@ -163,12 +163,13 @@ def materialize(scenario: Scenario, root: Path) -> Path:
         verifier_dir = trial_dir / "verifier"
         verifier_dir.mkdir()
         (verifier_dir / "test-stdout.txt").write_text("tests passed\n", encoding="utf-8")
-        if spec["reward_text"] is not None:
-            (verifier_dir / "reward.txt").write_text(spec["reward_text"], encoding="utf-8")
+        if spec.reward_text is not None:
+            (verifier_dir / "reward.txt").write_text(spec.reward_text, encoding="utf-8")
     return job_dir
 
 
 def _describe_on_disk(job_dir: Path) -> list[str]:
+    """One line per trial directory: its reward files, recorded rewards, and exception."""
     lines = []
     for trial_dir in sorted(path for path in job_dir.iterdir() if path.is_dir()):
         result_path = trial_dir / "result.json"
@@ -191,7 +192,8 @@ def _describe_on_disk(job_dir: Path) -> list[str]:
     return lines
 
 
-def _reporter_line(metrics: dict[str, float | int]) -> str:
+def _reporter_lines(metrics: dict[str, float | int], reason: str | None) -> list[str]:
+    """The lines `ExperimentContext.evaluate` would narrate for this aggregate."""
     sink = io.StringIO()
     RunReporter(sink=sink).candidate_evaluated(
         label="agent-0",
@@ -199,8 +201,9 @@ def _reporter_line(metrics: dict[str, float | int]) -> str:
         metrics=metrics,
         objective_metrics=_OBJECTIVES,
         artifacts=Path("eval-and-optimize/results/agent-0-validation"),
+        reason=reason,
     )
-    return sink.getvalue().strip()
+    return [line.strip() for line in sink.getvalue().splitlines()]
 
 
 async def inspect(job_dir: Path, *, behaviour: str | None = None) -> None:
@@ -232,24 +235,29 @@ async def inspect(job_dir: Path, *, behaviour: str | None = None) -> None:
     completed = [trial for trial in trials if trial.status == "completed"]
     print(f"    completed trials={len(completed)}/{len(trials)} · aggregate={aggregate or '{}'}")
 
+    reason = missing_objective_reason(trials, aggregate, _OBJECTIVES)
     print("\n[4] reward_scalar / RunReporter")
     print(f"    reward_scalar={reward_scalar(aggregate)!r}   (defined in reporting.py; no caller in the loop)")
-    print(f"    operator sees: {_reporter_line(aggregate)}")
-    if "reward" not in aggregate:
-        print("    ^ no reward reached the loop, and nothing on this path said so")
+    print("    operator sees:")
+    for line in _reporter_lines(aggregate, reason):
+        print(f"      {line}")
 
     metrics = {name: float(value) for name, value in aggregate.items()}
     print("\n[5] pareto_objectives / has_metric_dimensions (what the loop really ranks on)")
     print(f"    pareto_objectives={pareto_objectives(metrics, _OBJECTIVES) or '{}'}")
     print(f"    has_metric_dimensions={has_metric_dimensions(metrics, _OBJECTIVES)}")
-    if pareto_objectives(metrics, _OBJECTIVES):
+    if reason is None:
         print("    the selector can rank, the terminator can detect a stagnant front, and a winner exists")
-    else:
-        print("    every candidate projects to {}, so _dominates is always False and one front holds them all;")
-        print("    ranking is a no-op, the terminator never stops early, and finalization finds no winner")
+        return
+    # Which of these applies depends on whose evaluation this is, and the job
+    # directory alone cannot say.
+    print("    as the baseline's validation: the run stops here, because nothing measured the starting point")
+    print("    as a later candidate:         it projects to {}, so _dominates is False against every peer")
+    print("                                  and it can never be ranked or win, but the run continues")
 
 
 async def main() -> None:
+    """Inspect a real job directory, or the synthetic scenarios when none is given."""
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.add_argument(
         "--job-dir",

From 8d502de91bece8a84ea3cfdc7c701064b57cbe66 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 18 Aug 2026 19:58:39 +0000
Subject: [PATCH 10/10] chore(experimentalist): drop the reward-pipeline
 inspector

The tool existed to locate which layer dropped a missing verifier reward. That
question is answered and the behaviour it demonstrated is now pinned by tests, so
keeping it means maintaining a second description of the same pipeline that can
drift from the code -- it already had, twice, describing pre-fix narration and a
pre-fix ranking outcome.

Signed-off-by: Cursor Agent <cursoragent@cursor.com>

Co-authored-by: Aditya Pandey <aditya@autospace.co>
---
 .../tools/reward_pipeline_inspector.py        | 286 ------------------
 1 file changed, 286 deletions(-)
 delete mode 100644 plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py

diff --git a/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py b/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
deleted file mode 100644
index ca3b5b4e131..00000000000
--- a/plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
+++ /dev/null
@@ -1,286 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Step through the four layers between a Harbor verifier and the reward the loop ranks on.
-
-A missing ``/logs/verifier/reward.txt`` is invisible by the time it reaches the
-operator: the run prints ``reward n/a`` and keeps going. This tool prints the
-state at every layer the value passes through, so you can see which one drops it
-and which one is entitled to complain.
-
-The layers, in order:
-
-1. **on disk** — what the verifier left in the trial directory.
-2. ``trials_from_job_dir`` — the ``TrialResult`` status and metrics parsed from it.
-3. ``Evaluator.aggregate_results`` — the per-candidate aggregate the loop stores.
-4. ``missing_objective_reason`` / ``RunReporter`` — the lines an operator reads.
-   ``reward_scalar`` is printed alongside them because it looks like the loop's
-   scalar reward, but no caller in the loop uses it.
-5. ``pareto_objectives`` / ``has_metric_dimensions`` — the projection the selector
-   ranks on and the predicate the selector and terminator share, which together
-   decide whether the loop can compare candidates or stop early.
-
-Every layer is called through production code. Nothing here is imported by the
-plugin; this is a read-only inspector for humans.
-
-Usage::
-
-    # Compare synthetic trial shapes side by side.
-    uv run plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py
-
-    # One shape only.
-    uv run plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py \
-        --scenario verifier-silent
-
-    # Inspect a real Harbor job directory from a run that reported `reward n/a`.
-    uv run plugins/nemo-experimentalist/tools/reward_pipeline_inspector.py \
-        --job-dir eval-and-optimize/results/agent-0-validation
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import io
-import json
-import tempfile
-from collections.abc import Sequence
-from dataclasses import dataclass
-from pathlib import Path
-
-from nemo_experimentalist_plugin.entities import Dataset, MetricTarget, Task, TrialResult
-from nemo_experimentalist_plugin.experimentalist.components.evaluator.base import (
-    Evaluator,
-    EvaluatorConfig,
-)
-from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import trials_from_job_dir
-from nemo_experimentalist_plugin.experimentalist.components.models import (
-    has_metric_dimensions,
-    missing_objective_reason,
-    pareto_objectives,
-)
-from nemo_experimentalist_plugin.experimentalist.reporting import RunReporter, reward_scalar
-
-_OBJECTIVES = [MetricTarget(name="reward", direction="maximize")]
-
-
-class _AggregateOnlyEvaluator(Evaluator):
-    """The real ``aggregate_results``, with the abstract run step stubbed out."""
-
-    evaluator_type = "reward-pipeline-inspector"
-
-    async def _run(self, agent: Path, dataset: Dataset, options: EvaluatorConfig) -> Sequence[TrialResult]:
-        """Never called: the inspector adapts a job directory that already exists."""
-        raise NotImplementedError("The inspector reads an existing job directory; it never runs an evaluation.")
-
-
-@dataclass(frozen=True)
-class TrialSpec:
-    """One synthetic trial: its task, the reward it wrote, and any Harbor exception."""
-
-    task: str
-    reward_text: str | None = None
-    exception_type: str | None = None
-
-
-@dataclass(frozen=True)
-class Scenario:
-    """One synthetic trial shape, plus what the verifier did to produce it."""
-
-    name: str
-    verifier_behaviour: str
-    trials: tuple[TrialSpec, ...]
-
-
-SCENARIOS = (
-    Scenario(
-        name="reward-written",
-        verifier_behaviour="test.sh writes /logs/verifier/reward.txt. The control case.",
-        trials=(TrialSpec("task-a", reward_text="1.0"), TrialSpec("task-b", reward_text="0.0")),
-    ),
-    Scenario(
-        name="verifier-raised",
-        verifier_behaviour=(
-            "test.sh exits 0 without writing a reward file, so Harbor's verifier raises "
-            "RewardFileNotFoundError and the trial records exception_info."
-        ),
-        trials=(TrialSpec("task-a", exception_type="RewardFileNotFoundError"),),
-    ),
-    Scenario(
-        name="verifier-silent",
-        verifier_behaviour=(
-            "The trial finishes with no exception and no reward file — a disabled verifier, "
-            "a bind mount that never reflected the write, or a reward stripped during log collection."
-        ),
-        trials=(TrialSpec("task-a"), TrialSpec("task-b")),
-    ),
-    Scenario(
-        name="partial",
-        verifier_behaviour="One task writes a reward and another does not.",
-        trials=(TrialSpec("task-a", reward_text="1.0"), TrialSpec("task-b")),
-    ),
-    Scenario(
-        name="thin-denominator",
-        verifier_behaviour=(
-            "One easy task scores 1.0 and three harder tasks crash before their verifier runs. "
-            "The reward survives; the sample it was averaged over does not."
-        ),
-        trials=(
-            TrialSpec("task-easy", reward_text="1.0"),
-            TrialSpec("task-hard-1", exception_type="AgentTimeoutError"),
-            TrialSpec("task-hard-2", exception_type="AgentTimeoutError"),
-            TrialSpec("task-hard-3", exception_type="NonZeroAgentExitCodeError"),
-        ),
-    ),
-)
-
-
-def materialize(scenario: Scenario, root: Path) -> Path:
-    """Write *scenario* as a Harbor job directory and return it."""
-    job_dir = root / scenario.name
-    for index, spec in enumerate(scenario.trials):
-        trial_dir = job_dir / f"{spec.task}__{index}"
-        trial_dir.mkdir(parents=True)
-        (trial_dir / "result.json").write_text(
-            json.dumps(
-                {
-                    "task_name": f"harbor/{spec.task}",
-                    "trial_name": trial_dir.name,
-                    "verifier_result": None,
-                    "exception_info": (
-                        None
-                        if spec.exception_type is None
-                        else {
-                            "exception_type": spec.exception_type,
-                            "exception_message": "No reward file found at /logs/verifier/reward.txt",
-                            "exception_traceback": "<elided>",
-                        }
-                    ),
-                }
-            ),
-            encoding="utf-8",
-        )
-        verifier_dir = trial_dir / "verifier"
-        verifier_dir.mkdir()
-        (verifier_dir / "test-stdout.txt").write_text("tests passed\n", encoding="utf-8")
-        if spec.reward_text is not None:
-            (verifier_dir / "reward.txt").write_text(spec.reward_text, encoding="utf-8")
-    return job_dir
-
-
-def _describe_on_disk(job_dir: Path) -> list[str]:
-    """One line per trial directory: its reward files, recorded rewards, and exception."""
-    lines = []
-    for trial_dir in sorted(path for path in job_dir.iterdir() if path.is_dir()):
-        result_path = trial_dir / "result.json"
-        if not result_path.is_file():
-            lines.append(f"{trial_dir.name}: no result.json (this directory is skipped entirely)")
-            continue
-        result = json.loads(result_path.read_text(encoding="utf-8"))
-        rewards = (result.get("verifier_result") or {}).get("rewards")
-        exception_info = result.get("exception_info") or {}
-        files = [
-            name
-            for name in ("verifier/reward.json", "verifier/reward.txt", "reward.json", "reward.txt")
-            if (trial_dir / name).is_file()
-        ]
-        lines.append(
-            f"{trial_dir.name}: reward files={files or 'none'} · "
-            f"result.json rewards={rewards if rewards else 'none'} · "
-            f"exception={exception_info.get('exception_type', 'none')}"
-        )
-    return lines
-
-
-def _reporter_lines(metrics: dict[str, float | int], reason: str | None) -> list[str]:
-    """The lines `ExperimentContext.evaluate` would narrate for this aggregate."""
-    sink = io.StringIO()
-    RunReporter(sink=sink).candidate_evaluated(
-        label="agent-0",
-        split="validation",
-        metrics=metrics,
-        objective_metrics=_OBJECTIVES,
-        artifacts=Path("eval-and-optimize/results/agent-0-validation"),
-        reason=reason,
-    )
-    return [line.strip() for line in sink.getvalue().splitlines()]
-
-
-async def inspect(job_dir: Path, *, behaviour: str | None = None) -> None:
-    """Print the state of every layer between *job_dir* and the loop's scalar reward."""
-    print(f"\n{'=' * 78}\n{job_dir.name}\n{'=' * 78}")
-    if behaviour is not None:
-        print(f"verifier: {behaviour}\n")
-
-    print("[1] on disk")
-    for line in _describe_on_disk(job_dir):
-        print(f"    {line}")
-
-    task_ids = sorted({path.name.rsplit("__", 1)[0] for path in job_dir.iterdir() if path.is_dir()})
-    trials = trials_from_job_dir(job_dir, [Task(id=task_id) for task_id in task_ids])
-    print("\n[2] trials_from_job_dir")
-    for trial in trials:
-        metrics = {name: metric.value for name, metric in trial.metrics.items()}
-        error = (trial.error or {}).get("type", "none")
-        print(f"    {trial.id}: status={trial.status} metrics={metrics or '{}'} error={error}")
-
-    print("\n[3] Evaluator.aggregate_results")
-    try:
-        aggregate = await _AggregateOnlyEvaluator(EvaluatorConfig()).aggregate_results(trials)
-    except ValueError as exc:
-        print(f"    raises ValueError: {exc}")
-        print("\n[4] reward_scalar / RunReporter")
-        print("    not reached — the run fails here, which is the loud case")
-        return
-    completed = [trial for trial in trials if trial.status == "completed"]
-    print(f"    completed trials={len(completed)}/{len(trials)} · aggregate={aggregate or '{}'}")
-
-    reason = missing_objective_reason(trials, aggregate, _OBJECTIVES)
-    print("\n[4] reward_scalar / RunReporter")
-    print(f"    reward_scalar={reward_scalar(aggregate)!r}   (defined in reporting.py; no caller in the loop)")
-    print("    operator sees:")
-    for line in _reporter_lines(aggregate, reason):
-        print(f"      {line}")
-
-    metrics = {name: float(value) for name, value in aggregate.items()}
-    print("\n[5] pareto_objectives / has_metric_dimensions (what the loop really ranks on)")
-    print(f"    pareto_objectives={pareto_objectives(metrics, _OBJECTIVES) or '{}'}")
-    print(f"    has_metric_dimensions={has_metric_dimensions(metrics, _OBJECTIVES)}")
-    if reason is None:
-        print("    the selector can rank, the terminator can detect a stagnant front, and a winner exists")
-        return
-    # Which of these applies depends on whose evaluation this is, and the job
-    # directory alone cannot say.
-    print("    as the baseline's validation: the run stops here, because nothing measured the starting point")
-    print("    as a later candidate:         it projects to {}, so _dominates is False against every peer")
-    print("                                  and it can never be ranked or win, but the run continues")
-
-
-async def main() -> None:
-    """Inspect a real job directory, or the synthetic scenarios when none is given."""
-    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument(
-        "--job-dir",
-        type=Path,
-        help="Inspect a real Harbor job directory instead of the synthetic scenarios.",
-    )
-    parser.add_argument(
-        "--scenario",
-        choices=[scenario.name for scenario in SCENARIOS],
-        help="Run one synthetic scenario instead of all of them.",
-    )
-    args = parser.parse_args()
-
-    if args.job_dir is not None:
-        await inspect(args.job_dir.expanduser().resolve())
-        return
-
-    selected = [s for s in SCENARIOS if args.scenario is None or s.name == args.scenario]
-    with tempfile.TemporaryDirectory(prefix="reward-pipeline-") as tmp:
-        for scenario in selected:
-            job_dir = materialize(scenario, Path(tmp))
-            await inspect(job_dir, behaviour=scenario.verifier_behaviour)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())