callingmedic911 · Aug 19, 2026 · Aug 17, 2026 · Aug 17, 2026 · Aug 17, 2026 · Aug 17, 2026
@@ -861,6 +861,24 @@ class HarborDataset(Dataset):
 
         {"reward": 0.8, "my_metric": 1.0}
 
+    **Writing a reward file is mandatory, not optional.** Harbor accepts either
+    ``/logs/verifier/reward.json`` or ``/logs/verifier/reward.txt`` (one bare
+    number, and nothing else).  ``reward.json`` wins when both exist.  A
+    ``test.sh`` that exits ``0`` without writing either scores nothing at all —
+    it does **not** score zero.  Harbor raises ``RewardFileNotFoundError``, the
+    trial is recorded as failed, and the task contributes no metric to the run.
+    A run whose baseline emits no metric this way is refused outright, because
+    there is then no measurement to optimize toward.
+
+    So create the directory and write the file unconditionally, on every exit
+    path — including the ones where the agent failed::
+
+        mkdir -p /logs/verifier
+        echo 0 > /logs/verifier/reward.txt   # replaced below once the score is known
+
+    Under ``set -e`` a failing check aborts the script, so a reward written only
+    at the end is never written on the path that matters most.
+
     **To add a new metric** — write it as an additional key in ``reward.json``::
 
         {"reward": 0.8, "my_metric": 1.0, "another_metric": 0.5}
@@ -901,6 +919,9 @@ class HarborDataset(Dataset):
         #!/usr/bin/env bash
         set -euo pipefail
         mkdir -p /logs/verifier
+        # Seed a reward before anything can fail. Under `set -e` a failing check
+        # aborts the script, and reward.json below wins whenever it is written.
+        echo 0 > /logs/verifier/reward.txt
         SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 
         # Run each metric script and collect its output.

@@ -10,10 +10,11 @@
 
 from __future__ import annotations
 
-from collections.abc import Callable, Iterable
+from collections import Counter
+from collections.abc import Callable, Iterable, Sequence
 from typing import Literal, TypeVar
 
-from nemo_experimentalist_plugin.entities import Candidate, MetricTarget
+from nemo_experimentalist_plugin.entities import Candidate, MetricTarget, TrialResult
 from pydantic import BaseModel
 
 # ---------------------------------------------------------------------------
@@ -334,3 +335,52 @@ def pareto_objectives(metrics: dict[str, float], objective_function: list[Metric
 def has_metric_dimensions(metrics: dict[str, float], targets: list[MetricTarget]) -> bool:
     """Return whether an evaluator result contains every required metric target."""
     return all(target.name in metrics for target in targets)
+
+
+def missing_objective_reason(
+    trials: Sequence[TrialResult],
+    metrics: dict[str, float | int],
+    targets: list[MetricTarget],
+) -> str | None:
+    """Explain why *metrics* omits a required target, or None when none is omitted.
+
+    An absent objective is indistinguishable from a measured one in every
+    downstream check: :func:`pareto_objectives` projects both an unmeasured and a
+    zero-scored candidate onto a dict, and only one of them is comparable. The
+    evidence for *which* happened lives in the trials, and by the time a caller
+    holds only the aggregate it is gone. This recovers it from the trials while
+    they are still in hand.
+
+    Deliberately free of any evaluator's vocabulary: it names the absent target
+    and the trial statuses behind it. A caller that knows which harness produced
+    the trials is the one that can point at a directory.
+
+    Args:
+        trials: Trials the aggregate was computed from.
+        metrics: The aggregate, as ``Evaluator.aggregate_results`` returned it.
+        targets: Metrics the run is scored against.
+
+    Returns:
+        str | None: A one-line explanation, or None when every target is present.
+    """
+    absent = [target.name for target in targets if target.name not in metrics]
+    if not absent:
+        return None
+
+    named = ", ".join(repr(name) for name in absent)
+    if not trials:
+        return f"the evaluator produced no trials, so {named} was never measured"
+
+    completed = [trial for trial in trials if trial.status == "completed"]
+    if not completed:
+        return f"no trial completed ({len(trials)} failed: {_trial_error_summary(trials)}), so {named} is absent"
+    return (
+        f"{len(completed)}/{len(trials)} trials completed but none reported {named}; "
+        "the verifier ran without emitting the metric"
+    )
+
+
+def _trial_error_summary(trials: Sequence[TrialResult]) -> str:
+    """Name the error types across *trials*, most frequent first."""
+    counts = Counter(str((trial.error or {}).get("type") or "unknown") for trial in trials)
+    return ", ".join(name if count == 1 else f"{name} ×{count}" for name, count in counts.most_common())
@@ -43,6 +43,7 @@
     local_path_from_uri,
 )
 from nemo_experimentalist_plugin.experimentalist.components.evaluator import Evaluator
+from nemo_experimentalist_plugin.experimentalist.components.models import missing_objective_reason
 from nemo_experimentalist_plugin.experimentalist.components.trace_explorer import TraceExplorer
 from nemo_experimentalist_plugin.experimentalist.experimentalist_backend import (
     ExperimentalistBackend,
@@ -454,6 +455,7 @@ async def evaluate(
                 metrics=result.aggregate_metrics,
                 objective_metrics=self._objective_metrics,
                 artifacts=self.root / "eval-and-optimize" / "results" / result.id,
+                reason=missing_objective_reason(result.trials, result.aggregate_metrics, self._objective_metrics),
             )
         return result
 

@@ -120,12 +120,18 @@ def candidate_evaluated(
         metrics: dict[str, float | int],
         objective_metrics: list[MetricTarget],
         artifacts: Path,
+        reason: str | None = None,
     ) -> None:
         """Narrate an evaluation using all configured objective metrics.
 
         Objective directions determine whether validation deltas are displayed
         as improvements or declines. Regression metrics are guardrails and are
         intentionally not presented as progress dimensions.
+
+        *reason* explains an ``n/a`` when the caller could work out why one
+        happened. Without it the line states that a metric is missing and stops
+        there, which reads as a score of nothing rather than a measurement that
+        never occurred.
         """
         if self._verbosity is Verbosity.QUIET:
             return
@@ -148,6 +154,8 @@ def candidate_evaluated(
                     rendered += f" {'▲' if improved else '▼'}{delta:+.3f}"
                 rendered_metrics.append(rendered)
             self._emit(f"   {label} · {split:<10} · {', '.join(rendered_metrics)}   → {artifacts}")
+            if reason is not None:
+                self._emit(f"     ↳ {reason}")
         except Exception:  # noqa: BLE001
             pass
 

@@ -31,6 +31,7 @@
 from nemo_experimentalist_plugin.experimentalist.components.importer import IMPORT, import_proposal
 from nemo_experimentalist_plugin.experimentalist.components.models import (
     EvolutionTree,
+    missing_objective_reason,
 )
 from nemo_experimentalist_plugin.experimentalist.components.tools import (
     GuardedShellTools,
@@ -644,10 +645,28 @@ async def _record_baseline_validation(
         and `_evaluate_validation_candidates` only returns candidates it actually ran. A
         round-0 resume therefore has nothing to record, and indexing the map raised
         KeyError on a round-0 resume.
+
+        Raises:
+            ValueError: if the baseline measured here carries no objective metric.
+                Every later comparison is against this one measurement, so a run that
+                cannot score its own starting point has nothing to optimize toward,
+                and no later gate says so: `pareto_objectives` projects an unmeasured
+                candidate to `{}`, `_dominates` then returns False against every peer,
+                so one Pareto front holds the whole population and ranking silently
+                becomes a no-op. Failing here costs one evaluation instead of
+                `max_rounds` rounds of analysis, proposals, and builds.
         """
         measured = results.get(baseline.label)
-        if measured is not None:
-            await ctx.record_reward(baseline, channel="validation", result=measured)
+        if measured is None:
+            return
+        reason = missing_objective_reason(measured.trials, measured.aggregate_metrics, ctx.objective_metrics)
+        if reason is not None:
+            raise ValueError(
+                f"Baseline {baseline.label} produced no objective metric: {reason}. The run is scored "
+                "against this measurement, so there is nothing to optimize toward. Check that the "
+                "evaluation emits every objective metric for the unmodified agent before running again."
+            )
+        await ctx.record_reward(baseline, channel="validation", result=measured)
 
     async def _ensure_baseline(self, *, ctx: StrategyContext, config: EvolutionaryOptimizerConfig) -> None:
         """Create the baseline candidate unless this run already has one.

@@ -1,13 +1,13 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-"""Rolling a round back so the loop can re-enter it cleanly."""
+"""Loop helpers: rolling a round back, and refusing a baseline nothing could measure."""
 
 from pathlib import Path
 
 import pytest
 from doubles import FakeBackend, make_context
-from nemo_experimentalist_plugin.entities import Proposal
+from nemo_experimentalist_plugin.entities import EvaluationResult, Proposal, TrialResult
 from nemo_experimentalist_plugin.experimentalist.strategies.evolutionary import EvolutionaryStrategy
 
 
@@ -50,3 +50,40 @@ async def test_rollback_discards_later_candidates_and_clears_their_evaluator_scr
     assert ctx.candidate_dir(discarded).exists(), "the artifact must survive a rollback"
     assert not any(d.exists() for d in scratch), "stale evaluator scratch would be read on re-run"
     assert survivor_scratch.exists(), "a survivor's results must not be swept up"
+
+
+@pytest.mark.asyncio
+async def test_baseline_validation_refuses_an_unmeasurable_baseline_and_records_a_measured_one(
+    tmp_path: Path,
+) -> None:
+    """The run's one reference point must exist before anything is compared against it.
+
+    An empty aggregate is indistinguishable from a scored one downstream, so a run
+    that cannot measure its own starting point would spend every round ranking
+    candidates that are all mutually incomparable and then report no winner.
+    """
+    ctx = make_context(root=tmp_path, backend=FakeBackend())
+    baseline = await _commit(ctx, "baseline: the agent under test, unchanged", generation=0)
+
+    unmeasurable = EvaluationResult(
+        id=f"{baseline.label}-validation",
+        aggregate_metrics={},
+        trials=[
+            TrialResult(id="task-a__0", task_id="task-a", status="failed", error={"type": "RewardFileNotFoundError"})
+        ],
+    )
+    with pytest.raises(ValueError, match="RewardFileNotFoundError"):
+        await EvolutionaryStrategy._record_baseline_validation(
+            ctx=ctx, baseline=baseline, results={baseline.label: unmeasurable}
+        )
+    assert "validation" not in (await ctx.candidates())[0].rewards, "a refused baseline must not be recorded"
+
+    measured = EvaluationResult(
+        id=f"{baseline.label}-validation",
+        aggregate_metrics={"reward": 0.4},
+        trials=[TrialResult(id="task-a__0", task_id="task-a", status="completed")],
+    )
+    await EvolutionaryStrategy._record_baseline_validation(
+        ctx=ctx, baseline=baseline, results={baseline.label: measured}
+    )
+    assert (await ctx.candidates())[0].rewards["validation"].metrics == {"reward": 0.4}
@@ -77,6 +77,23 @@ def test_candidate_evaluated_shows_all_objectives_and_direction_aware_deltas() -
     assert "tokens 80.000 ▲-20.000" in lines[1]
 
 
+def test_candidate_evaluated_explains_an_na_when_the_caller_knows_why() -> None:
+    # Without the reason, `n/a` reads as a score of nothing rather than a
+    # measurement that never happened, and the two lead to opposite actions.
+    r, sink = _reporter()
+    r.candidate_evaluated(
+        label="agent-0",
+        split="validation",
+        metrics={"tokens": 100.0},
+        objective_metrics=OBJECTIVES,
+        artifacts=Path("/exp/results/agent-0-validation"),
+        reason="no trial completed (2 failed: RewardFileNotFoundError), so 'success' is absent",
+    )
+    lines = sink.getvalue().splitlines()
+    assert "success n/a" in lines[0]
+    assert "RewardFileNotFoundError" in lines[1]
+
+
 def test_seed_baseline_sets_delta_reference_silently_for_resume() -> None:
     # On resume agent-0 is not re-evaluated; seed_baseline sets the delta
     # reference from its cached reward without emitting a line, so the first

@@ -8,7 +8,8 @@
     pareto_objectives,
     with_insight_objective,
 )
-from nemo_experimentalist_plugin.entities import MetricTarget
+from nemo_experimentalist_plugin.entities import MetricTarget, TrialResult, TrialStatus
+from nemo_experimentalist_plugin.experimentalist.components.models import missing_objective_reason
 
 
 def test_metric_contract_supports_multiple_objective_metrics() -> None:
@@ -64,6 +65,62 @@ def test_metric_dimensions_require_only_the_objectives_used_for_pareto_ranking()
     assert not has_metric_dimensions({"quality": 0.9}, [*objectives, *regressions])
 
 
+def _trial(status: TrialStatus, *, error_type: str | None = None) -> TrialResult:
+    return TrialResult(
+        id=f"task__{status}",
+        task_id="task",
+        status=status,
+        error=None if error_type is None else {"type": error_type},
+    )
+
+
+@pytest.mark.parametrize(
+    ("trials", "metrics", "expected"),
+    [
+        # Measured: nothing to explain.
+        ([_trial("completed")], {"reward": 0.0}, None),
+        # No trials at all: the evaluator itself produced nothing.
+        ([], {}, "the evaluator produced no trials, so 'reward' was never measured"),
+        # Every trial failed. Harbor's own diagnosis is on the trials; name it.
+        (
+            [_trial("failed", error_type="RewardFileNotFoundError")],
+            {},
+            "no trial completed (1 failed: RewardFileNotFoundError), so 'reward' is absent",
+        ),
+        # Mixed failures are counted, most frequent first.
+        (
+            [
+                _trial("failed", error_type="AgentTimeoutError"),
+                _trial("failed", error_type="AgentTimeoutError"),
+                _trial("failed"),
+            ],
+            {},
+            "no trial completed (3 failed: AgentTimeoutError ×2, unknown), so 'reward' is absent",
+        ),
+        # The silent shape: the verifier ran, exited cleanly, and emitted no metric.
+        (
+            [_trial("completed"), _trial("completed")],
+            {},
+            "2/2 trials completed but none reported 'reward'; the verifier ran without emitting the metric",
+        ),
+        # A trial can carry other metrics and still miss the objective.
+        (
+            [_trial("completed")],
+            {"latency": 3.0},
+            "1/1 trials completed but none reported 'reward'; the verifier ran without emitting the metric",
+        ),
+    ],
+)
+def test_missing_objective_reason_names_the_evidence_behind_an_absent_metric(
+    trials: list[TrialResult],
+    metrics: dict[str, float],
+    expected: str | None,
+) -> None:
+    objectives = [MetricTarget(name="reward", direction="maximize")]
+
+    assert missing_objective_reason(trials, metrics, objectives) == expected
+
+
 @pytest.mark.parametrize(
     "config",
     [