diff --git a/packages/nemo_evaluator_sdk/examples/harbor/README.md b/packages/nemo_evaluator_sdk/examples/harbor/README.md index 618856ea4f..7cd850c095 100644 --- a/packages/nemo_evaluator_sdk/examples/harbor/README.md +++ b/packages/nemo_evaluator_sdk/examples/harbor/README.md @@ -44,7 +44,7 @@ folders: ``` hello_world_dataset/ - hello-world/ # [task] name = "harbor/hello-world" + hello-world/ # [task] name = "harbor/hello-world" task.toml instruction.md environment/Dockerfile @@ -95,10 +95,10 @@ From the repository root: ```bash # Native path: run and print the SDK summary. -python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native +uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native # Optimizer path: run, then rebuild NeMo Optimizer's legacy reward payload. -python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode optimizer +uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode optimizer ``` Both modes call `run_harbor_eval`; the only difference is what they print. diff --git a/packages/nemo_evaluator_sdk/examples/harbor/hello_world_dataset/hello-world/environment/Dockerfile b/packages/nemo_evaluator_sdk/examples/harbor/hello_world_dataset/hello-world/environment/Dockerfile index d14238a06c..a06214dd81 100644 --- a/packages/nemo_evaluator_sdk/examples/harbor/hello_world_dataset/hello-world/environment/Dockerfile +++ b/packages/nemo_evaluator_sdk/examples/harbor/hello_world_dataset/hello-world/environment/Dockerfile @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -FROM alpine:3.22 +# Matches the tag CI pre-pulls in the integration job, so building this task image reuses that +# layer +FROM alpine:3.23 # bash is required for Harbor's docker environment RUN apk add --no-cache bash diff --git a/packages/nemo_evaluator_sdk/examples/harbor/run_harbor_example.py b/packages/nemo_evaluator_sdk/examples/harbor/run_harbor_example.py index f0704cdbe9..1904360b2e 100644 --- a/packages/nemo_evaluator_sdk/examples/harbor/run_harbor_example.py +++ b/packages/nemo_evaluator_sdk/examples/harbor/run_harbor_example.py @@ -9,8 +9,8 @@ runs Harbor's ``JobConfig`` and scores the results; the caller never imports ``harbor`` or assembles a job. -Two modes, both over the bundled ``hello_world_dataset`` (Harbor's ``hello-world`` -task) scored with the deterministic **oracle** agent, so no LLM/API key is needed: +Two modes, both over the bundled ``hello_world_dataset`` scored with the +deterministic **oracle** agent, so no LLM/API key is needed: * ``--mode native`` — print the SDK summary. * ``--mode optimizer`` — collapse the result into NeMo Optimizer's legacy @@ -20,9 +20,9 @@ Run it as a module from the repository root:: - python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native - python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native --n-attempts 2 - python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode optimizer + uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native + uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native --n-attempts 2 + uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode optimizer """ from __future__ import annotations @@ -70,13 +70,16 @@ async def _main(mode: str, jobs_dir: Path, *, n_attempts: int, job_name: str | N for score in result.scores: reward = score.outputs[0].value if score.outputs else None print(f" {score.task_id}: reward={reward} status={score.status.value}") + for trial in result.trials: + if trial.error is not None: + print(f" {trial.id}: error={trial.error.type}: {trial.error.message}") if __name__ == "__main__": if __package__ in {None, ""}: raise SystemExit( "Run this example as a module from the repository root:\n" - " python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native" + " uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native" ) logging.basicConfig(level=logging.INFO, format="%(levelname)s:%(name)s:%(message)s") parser = argparse.ArgumentParser(description=__doc__) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py index 720c94888b..748fe1db9b 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py @@ -192,7 +192,9 @@ async def run( tasks=task_list, trials=trial_list, scores=scores, - summary=AgentEvalSummary.from_scores(scores, tasks=task_list, extra_scores=runner_scores), + summary=AgentEvalSummary.from_scores( + scores, tasks=task_list, trials=trial_list, extra_scores=runner_scores + ), metadata=metadata, work_dir=runtime_config.work_dir, ) @@ -691,6 +693,8 @@ def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]: "id": trial.id, "task_id": trial.task_id, "status": trial.status.value, + # How the trial failed, for a metric that grades on it. None when the producer reported no failure. + "error": trial.error.model_dump(mode="json") if trial.error is not None else None, "metadata": trial.metadata, }, } diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index 9680c0636c..b39418459d 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -398,9 +398,56 @@ class AgentEvalSummary(BaseModel): } ], ) + error_trial_ids: dict[str, list[str]] = Field( + default_factory=dict, + description=( + "Trials that errored, grouped by error type -- Harbor's 'exception_stats' shape. Values " + "are trial ids, not task ids: they join to AgentEvalTrial.id (trials.jsonl), " + "AgentEvalTaskScore.trial_id (scores.jsonl), and TrialMetricValue.trial_id in " + "task_metric_values. Membership is 'the trial carries an error', with no status filter -- " + "an errored Harbor trial is PARTIAL rather than FAILED so that it is still scored, and it " + "belongs here regardless. A trial that both errored and produced a reward therefore " + "appears here AND in task_metric_values, where it may even count as a pass; that is what " + "Harbor does too. Ids are appended in trial order and never deduplicated. Key order is " + "not meaningful -- summary.json is written with sorted keys. Empty is ambiguous and " + "stays that way: it means either no trial errored or no trials were supplied to " + "from_scores(). The field always serializes (it defaults to {}), so the two cases are " + "indistinguishable in summary.json -- read trial_count, or the trials themselves, to " + "tell them apart." + ), + examples=[ + { + "RuntimeError": [ + "contract-review-msa-indemnity__k3f9wq2", + "nda-scope-carveouts__p2hn8sc", + ], + "TimeoutError": ["merger-hsr-filing-threshold__w5db3qy"], + } + ], + ) task_count: int = Field(default=0, description="Number of tasks represented in the run.") trial_count: int = Field(default=0, description="Number of distinct trials scored.") score_count: int = Field(default=0, description="Total number of metric scores.") + error_count: int = Field( + default=0, + description=( + "Number of trials that errored -- Harbor's 'n_errors'. Equals the total ids across " + "error_trial_ids; stated rather than derived so a non-Python reader of summary.json need " + "not sum a nested structure, matching the other counts here." + ), + ) + + @model_validator(mode="after") + def _error_count_matches_rollup(self) -> AgentEvalSummary: + """Keep the two error fields from disagreeing when a summary is built by hand. + + ``from_scores`` derives both from one walk, but the model is public and directly + constructible -- and a count that contradicts the rollup beside it is worse than no count. + """ + total = sum(len(ids) for ids in self.error_trial_ids.values()) + if self.error_count != total: + raise ValueError(f"error_count {self.error_count} does not match {total} ids in error_trial_ids") + return self @property def scores_by_name(self) -> Mapping[str, AggregateScore]: @@ -457,15 +504,23 @@ def from_scores( scores: Sequence[AgentEvalTaskScore], *, tasks: Sequence[AgentEvalTask] | None = None, + trials: Sequence[AgentEvalTrial] | None = None, extra_scores: Sequence[AggregateScore] = (), ) -> AgentEvalSummary: - """Build aggregated scores, task values, and coverage for a set of metric scores. + """Build aggregated scores, task values, coverage, and the error rollup for a set of scores. ``extra_scores`` are already-aggregated scores contributed by the runner (namespaced ``runner..``), merged in so a backend's own figures are addressable the same way as ours. + + ``trials`` supplies the only thing scores cannot carry: what went wrong. Omitting it leaves + :attr:`error_trial_ids` empty rather than raising -- the same silent-skip contract ``tasks`` + already has for pass@k. It may legitimately be *wider* than ``scores`` (a caller + re-aggregating a subset), so the rollup can name trial ids absent from + :attr:`task_metric_values`. """ task_list = list(tasks) if tasks is not None else None task_metric_values = _task_metric_values(scores, task_list) + error_trial_ids = _error_trial_ids(trials) return AgentEvalSummary( scores=_aggregate_scores( scores, @@ -475,9 +530,11 @@ def from_scores( ), metric_coverage=_metric_coverage(scores, task_list), task_metric_values=task_metric_values, + error_trial_ids=error_trial_ids, task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), trial_count=len({score.trial_id for score in scores}), score_count=len(scores), + error_count=sum(len(ids) for ids in error_trial_ids.values()), ) @@ -934,6 +991,32 @@ def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, return scorelike +def _error_trial_ids(trials: Sequence[AgentEvalTrial] | None) -> dict[str, list[str]]: + """Trial ids grouped by error type, in trial order — Harbor's ``exception_stats``. + + Three trials, the middle one fine:: + + in t0 error RuntimeError + t1 (no error) + t2 error RuntimeError + t3 error TimeoutError + + out {"RuntimeError": ["t0", "t2"], "TimeoutError": ["t3"]} + + Selection is on ``trial.error``, never on ``trial.status``: an errored Harbor trial is ``PARTIAL`` + so that it still scores, and filtering by status would drop exactly the trials this exists to name. + + Ids are **appended**, never collected into a set or used as dict keys. Nothing enforces trial-id + uniqueness (Gym derives ids from a rollout index in two separate loops), and losing cardinality + here would understate the error count — the same rule ``task_metric_values`` follows. + """ + grouped: dict[str, list[str]] = {} + for trial in trials or (): + if trial.error is not None: + grouped.setdefault(trial.error.type, []).append(trial.id) + return grouped + + def _task_metric_values( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py index 5b09135d9e..009da57ff4 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py @@ -52,10 +52,12 @@ from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask, AgentEvalTaskset from nemo_evaluator_sdk.agent_eval.trials import ( + UNKNOWN_ERROR_TYPE, AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo, + TrialError, standard_evidence_descriptors, ) from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult @@ -113,6 +115,8 @@ _DIGEST_SKIP_DIRS = frozenset({".git", "__pycache__", ".venv", ".uv", ".mypy_cache", ".pytest_cache"}) _DIGEST_CHUNK_BYTES = 1 << 20 +_MAX_TRACEBACK_CHARS = 8192 + RunJob = Callable[[], Awaitable[None]] @@ -326,10 +330,15 @@ async def run_tasks( stale = _cache_is_stale(job_dir, stamp) if stale or not _all_tasks_cached(job_dir, tasks, n_attempts=self._config.n_attempts): + # Harbor's DatasetConfig matches local folder names, while SDK task ids + # come from `[task] name`. Prefer folder names derived from the tasks + # actually being scored so a filter like `harbor/hello-world` still + # selects the `hello-world/` directory. + harbor_task_names = _harbor_folder_names(tasks) or self._task_names job_dir, run_job = _build_native_job( self._config, dataset_path, - self._task_names, + harbor_task_names, job_name=job_name, # Discard only when the inputs changed. Otherwise leave it off so # Harbor resumes per trial and keeps completed work — including @@ -345,7 +354,7 @@ async def run_tasks( # ran: with no task_names filter that is the whole dataset, and # recording only the requested subset would make the next full-set # run look stale and re-run a complete job dir. - coverage = _stamp_coverage(dataset_path, tasks, self._task_names) + coverage = _stamp_coverage(dataset_path, tasks, harbor_task_names) before = _cache_stamp(self._config, dataset_path, coverage) await run_job() if self._config.job_name is not None: @@ -381,6 +390,25 @@ def _dataset_path_from_tasks(tasks: Sequence[AgentEvalTask]) -> Path: ) +def _harbor_folder_names(tasks: Sequence[AgentEvalTask]) -> list[str] | None: + """Return Harbor local-dataset folder names for ``tasks``, or ``None`` if incomplete. + + Harbor's ``DatasetConfig.task_names`` matches directory names + (``LocalTaskId.get_name()`` → ``path.name``), while SDK task ids come from + ``[task] name``. When every task carries ``metadata['harbor_task_dir']``, + derive the folder list so a filter like ``harbor/hello-world`` still selects + the ``hello-world/`` directory. Return ``None`` when any task is missing that + stamp so callers can fall back to an explicit filter. + """ + names: list[str] = [] + for task in tasks: + stamped = task.metadata.get("harbor_task_dir") + if not isinstance(stamped, str) or not stamped: + return None + names.append(Path(stamped).name) + return names or None + + def _all_tasks_cached(job_dir: Path, tasks: Sequence[AgentEvalTask], *, n_attempts: int) -> bool: """Return True when every requested task already has ``n_attempts`` completed results. @@ -1122,24 +1150,18 @@ def _trial_from_harbor_result(trial_dir: Path, data: Mapping[str, Any], *, rewar trial_id = str(data.get("trial_name") or trial_dir.name) rewards = _rewards_mapping(data) reward = _primary_reward(rewards, reward_key) - exception_type = _exception_type(data.get("exception_info")) + error = _trial_error(data.get("exception_info")) metadata: dict[str, Any] = { "reward": reward, "reward_details": dict(rewards), "harbor_trial_dir": str(trial_dir), } - if exception_type is not None: - metadata["exception_type"] = exception_type metadata.update(_token_measurements(data.get("agent_result"))) # An errored trial (or one with no reward) stays PARTIAL so it is still scored # as 0 and counted in the summary; FAILED would exclude it from scoring. - status = ( - AgentEvalTrialStatus.COMPLETED - if exception_type is None and reward is not None - else AgentEvalTrialStatus.PARTIAL - ) + status = AgentEvalTrialStatus.COMPLETED if error is None and reward is not None else AgentEvalTrialStatus.PARTIAL trace_path = trial_dir / "agent" / "trajectory.json" descriptors = standard_evidence_descriptors( @@ -1155,6 +1177,7 @@ def _trial_from_harbor_result(trial_dir: Path, data: Mapping[str, Any], *, rewar status=status, output=AgentOutput(metadata={"harbor_trial_dir": str(trial_dir)}), evidence=CandidateEvidence(descriptors=descriptors), + error=error, metadata=metadata, ) @@ -1196,16 +1219,64 @@ def _primary_reward(rewards: Mapping[str, float], reward_key: str) -> float | No return None -def _exception_type(exception_info: Any) -> str | None: +def _trial_error(exception_info: Any) -> TrialError | None: + """Harbor's ``exception_info`` as a :class:`TrialError`, for any shape it can arrive in. + + **Total by construction.** :func:`_trial_from_harbor_result` is called outside the only + ``try``/``except`` in :func:`build_trials_from_job_dir` (which guards ``json.loads`` alone), so a + ``ValidationError`` raised here would abort adaptation of the *whole job dir* over one malformed + trial. Every field is therefore normalised rather than trusted: + + - ``type`` -- first non-empty string of ``exception_type``/``type``/``name``/``class``; for a + non-mapping, ``str(value)``; anything left empty becomes :data:`UNKNOWN_ERROR_TYPE` + - ``message``/``traceback`` -- kept only when actually strings; the traceback is truncated + - ``occurred_at`` -- kept only when it parses; never raises + + Returns ``None`` only for a genuinely absent ``exception_info``, which is what marks a trial as + not having errored. + """ if exception_info is None: return None - if isinstance(exception_info, Mapping): - for key in ("exception_type", "type", "name", "class"): - value = exception_info.get(key) - if isinstance(value, str) and value: - return value - return "UnknownException" - return str(exception_info) + if not isinstance(exception_info, Mapping): + return TrialError(type=str(exception_info).strip() or UNKNOWN_ERROR_TYPE) + + error_type = "" + for key in ("exception_type", "type", "name", "class"): + value = exception_info.get(key) + if isinstance(value, str) and value.strip(): + error_type = value + break + + traceback = _first_string(exception_info, ("exception_traceback", "traceback")) + return TrialError( + type=error_type or UNKNOWN_ERROR_TYPE, + message=_first_string(exception_info, ("exception_message", "message")), + traceback=traceback[:_MAX_TRACEBACK_CHARS] if traceback is not None else None, + occurred_at=_error_timestamp(exception_info.get("occurred_at")), + ) + + +def _first_string(payload: Mapping[str, Any], keys: tuple[str, ...]) -> str | None: + """The first value under ``keys`` that is actually a string. Harbor's spelling is tried first.""" + for key in keys: + value = payload.get(key) + if isinstance(value, str): + return value + return None + + +def _error_timestamp(value: Any) -> datetime | None: + """``value`` as a datetime when it plausibly is one, else ``None`` -- never raising. + + Deliberately not normalized to UTC: Harbor writes a naive local wall-clock time here while + stamping trial start/finish in UTC, and inventing an offset would fabricate precision. + """ + if isinstance(value, datetime): + return value + if isinstance(value, str): + with contextlib.suppress(ValueError): + return datetime.fromisoformat(value) + return None def _token_measurements(agent_result: Any) -> dict[str, int | float]: @@ -1368,8 +1439,12 @@ def reward_payload_from_result( * ``reward`` — mean of each metric output, keyed ``"."``. * ``reward_details`` — ``{output: {value_str: [task_id, ...]}}`` grouped from per-trial scores (Harbor's ``reward_stats`` analogue). - * ``exceptions`` — ``{exception_type: [task_id, ...]}`` from trial metadata - (Harbor's ``exception_stats`` analogue). + * ``exceptions`` — ``{error type: [task_id, ...]}`` from ``AgentEvalTrial.error`` + (Harbor's ``exception_stats`` analogue, keyed by task rather than by trial). + + Harbor keys ``exception_stats`` by *trial*, which is what + :attr:`AgentEvalSummary.error_trial_ids` now reproduces exactly. This payload keeps its + task-keyed shape for existing consumers; switching it over is AALGO-441. """ reward = {score.name: score.mean for score in result.summary.scores.scores if score.mean is not None} @@ -1386,9 +1461,8 @@ def reward_payload_from_result( exceptions: dict[str, list[str]] = {} for trial in result.trials: - exc = trial.metadata.get("exception_type") - if isinstance(exc, str) and exc: - exceptions.setdefault(exc, []).append(trial.task_id) + if trial.error is not None: + exceptions.setdefault(trial.error.type, []).append(trial.task_id) return { "reward": reward, diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py index d97669e2d8..cfe55244e2 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py @@ -8,9 +8,10 @@ from __future__ import annotations from collections.abc import Sequence +from datetime import datetime from enum import Enum from pathlib import Path -from typing import Any, Protocol, runtime_checkable +from typing import Annotated, Any, Protocol, runtime_checkable from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask from nemo_evaluator_sdk.values import Agent, Model @@ -26,7 +27,15 @@ EvidenceDescriptor, ) from nemo_evaluator_sdk.values.results import AggregateScore -from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + JsonValue, + WithJsonSchema, + field_validator, + model_validator, +) class AgentEvalTrialStatus(str, Enum): @@ -57,8 +66,73 @@ class AgentOutput(BaseModel): ) +# Type recorded when a producer reported a failure but named no usable type. This +# fires only for hand-built or malformed payloads. +UNKNOWN_ERROR_TYPE = "UnknownException" + + +class TrialError(BaseModel): + """What went wrong producing one trial, as the producer reported it. + + Present means the *producer* reported a failure. It does **not** imply ``status is FAILED``: an + errored trial can be marked as :attr:`AgentEvalTrialStatus.PARTIAL` so it is still scored (ex: HarborRuntime). + It is also unrelated to a score diagnostic's ``exception_type`` detail, which records that the + *metric* raised - a different event. + + Frozen so callers cannot rewrite ``type`` after construction. These objects are returned as-is + (not copied), and ``AgentEvalSummary.error_trial_ids`` groups trial ids by that string. Mutating + ``type`` on a live object would leave the summary keyed on the old value while the trial reports + a new one. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + type: str = Field( + description=( + "Error class name as the producer reported it, e.g. 'RuntimeError'. Rollup key for " + f"AgentEvalSummary.error_trial_ids. Falls back to {UNKNOWN_ERROR_TYPE!r} when the " + "producer reported a failure without a usable type." + ) + ) + message: str | None = Field( + default=None, + description="Short error message, when the producer supplied one.", + ) + traceback: str | None = Field( + default=None, + description=( + "Formatted traceback, when the producer supplied one. May be truncated by the adapter " + "that captured it. Note run bundles are portable: this can carry absolute filesystem " + "paths and other diagnostic text from the machine that ran the trial." + ), + ) + # Schema is a bare string, deliberately not `format: date-time`. RFC 3339 date-time requires a + # UTC offset, and this field may legitimately carry a naive local timestamp (Harbor writes one), + # so claiming the format would be a promise the value cannot keep -- and a client that trusts it + # parses a zoneless string into its *own* zone, silently shifting the instant. A plain string + # says "timestamp as the producer wrote it"; Python callers still get a parsed datetime. + occurred_at: Annotated[datetime | None, WithJsonSchema({"type": "string"})] = Field( + default=None, + description=( + "When the producer recorded the failure, as it reported it. The SDK does not rewrite " + "this: an aware value (UTC, offset) is kept, and a naive value stays naive. Harbor's " + "clock is naive local, e.g. '2026-08-13T17:22:32', while that trial's start in " + "result.json is UTC ('2026-08-14T00:22:25Z') — the same instant on two clocks, so do " + "not subtract them or attach a zone Harbor never wrote. A runner that recorded an " + "offset keeps it. Not RFC 3339 date-time: the offset may be absent." + ), + ) + + @field_validator("type") + @classmethod + def _non_empty_type(cls, value: str) -> str: + if not value.strip(): + raise ValueError("trial error type must not be empty") + return value + + class AgentEvalTrial(BaseModel): - """Durable trial artifact for one task: output, evidence, status, and metadata.""" + """Durable trial artifact for one task: output, evidence, status, error, and metadata.""" model_config = ConfigDict(extra="forbid") @@ -73,6 +147,13 @@ class AgentEvalTrial(BaseModel): default=None, description="Named evidence descriptors (final state, traces, logs, ...) captured for the trial.", ) + error: TrialError | None = Field( + default=None, + description=( + "What went wrong producing this trial, when the producer reported a failure. Populated " + "by a runner runtime. Drives AgentEvalSummary.error_trial_ids." + ), + ) metadata: dict[str, Any] = Field( default_factory=dict, description="Free-form metadata associated with the trial.", diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/environment/Dockerfile b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/environment/Dockerfile new file mode 100644 index 0000000000..73e1b8a2d7 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/environment/Dockerfile @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Matches the tag CI pre-pulls in the integration job, so building this task image reuses that +# layer +FROM alpine:3.23 + +# bash is required for Harbor's docker environment +RUN apk add --no-cache bash + +WORKDIR /app diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/instruction.md b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/instruction.md new file mode 100644 index 0000000000..da3aeaab23 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/instruction.md @@ -0,0 +1,9 @@ + + + +Intentional Harbor failure fixture for exception propagation testing. + +Attempt to create hello.txt with "Hello, world!" as the content. + +The reference solution script deliberately sleeps past the 1s agent timeout so +Harbor records exception_info. diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/solution/solve.sh b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/solution/solve.sh new file mode 100755 index 0000000000..4e19875e8d --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/solution/solve.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Intentional failure fixture: sleep past agent.timeout_sec (1s) so Harbor +# records an agent-timeout exception_info +echo "Intentional Harbor oracle solve.sh hang for exception propagation testing." >&2 +sleep 10 diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/task.toml b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/task.toml new file mode 100644 index 0000000000..cd5ea46e14 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/task.toml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = "1.0" + +[task] +name = "harbor/injected-runtime-error" +authors = [] +keywords = ["debugging", "expected-failure"] + +[metadata] +difficulty = "easy" +category = "debug" +tags = ["exception-propagation"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 1.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 +storage_mb = 10240 +gpus = 0 +mcp_servers = [] + +[verifier.env] + +[solution.env] diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/tests/test.sh b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/tests/test.sh new file mode 100755 index 0000000000..78dc777e39 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_dataset/injected-runtime-error/tests/test.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Never decides this task's outcome: solution/solve.sh sleeps past the 1s agent +# timeout, so Harbor records exception_info before the verifier matters. Kept +# byte-identical to hello-world's so the task shape matches for Harbor discovery. +mkdir -p /logs/verifier + +if [ -f /app/hello.txt ] && [ "$(tr -d '\n' < /app/hello.txt)" = "Hello, world!" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_result.json b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_result.json new file mode 100644 index 0000000000..44a4518bee --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/fixtures/harbor_error_result.json @@ -0,0 +1,115 @@ +{ + "id": "bc1e6caa-1f0b-4a67-85c0-52cc9024dd41", + "task_name": "harbor/injected-runtime-error", + "trial_name": "injected-runtime-error__hDCZYzM", + "trial_uri": "file:///repo/temp/evaluator/data/results/harbor_hello_world-sdk/hello-world-solve-timeout-1s/injected-runtime-error__hDCZYzM", + "task_id": { + "path": "/repo/packages/nemo_evaluator_sdk/examples/harbor/hello_world_dataset/injected-runtime-error" + }, + "source": "hello_world_dataset", + "task_checksum": "890365fc35641165a9be9febc08cdeb71fe7c592bcc137a4f53219a4db1c7e95", + "config": { + "task": { + "path": "/repo/packages/nemo_evaluator_sdk/examples/harbor/hello_world_dataset/injected-runtime-error", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": "hello_world_dataset" + }, + "trial_name": "injected-runtime-error__hDCZYzM", + "trials_dir": "/repo/temp/evaluator/data/results/harbor_hello_world-sdk/hello-world-solve-timeout-1s", + "install_only": false, + "timeout_multiplier": 1.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": "oracle", + "import_path": null, + "model_name": null, + "n_concurrent": null, + "concurrency_group": null, + "skills": [], + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "resume_trajectory": false, + "load_trajectory": null, + "extra_allowed_hosts": [], + "kwargs": {}, + "mcp_servers": [] + }, + "environment": { + "type": "docker", + "import_path": null, + "force_build": false, + "delete": true, + "cpu_enforcement_policy": "auto", + "memory_enforcement_policy": "auto", + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "override_tpu": null, + "mounts": null, + "extra_docker_compose": [], + "kwargs": {}, + "extra_allowed_hosts": [] + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "disable": false + }, + "artifacts": [], + "extra_instruction_paths": [], + "job_id": "67be98a0-6f9b-4eec-9af1-70fa6e911ddd" + }, + "agent_info": { + "name": "oracle", + "version": "1.0.0", + "model_info": null + }, + "agent_result": { + "n_input_tokens": null, + "n_cache_tokens": null, + "n_output_tokens": null, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 1.0 seconds", + "exception_traceback": "Traceback (most recent call last):\n File \"/home/user/.local/share/uv/python/cpython-3.12.12-macos-aarch64-none/lib/python3.12/asyncio/tasks.py\", line 520, in wait_for\n return await fut\n ^^^^^^^^^\n File \"/repo/.venv/lib/python3.12/site-packages/harbor/agents/oracle.py\", line 132, in run\n result = await environment.exec(\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/repo/.venv/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 1096, in exec\n return await self._compose_exec(\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/repo/.venv/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 1173, in _compose_exec\n return await self._run_docker_compose_command(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/repo/.venv/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 649, in _run_docker_compose_command\n result = await self._collect_buffered_output(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/repo/.venv/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 675, in _collect_buffered_output\n stdout_bytes, stderr_bytes = await asyncio.wait_for(\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/user/.local/share/uv/python/cpython-3.12.12-macos-aarch64-none/lib/python3.12/asyncio/tasks.py\", line 520, in wait_for\n return await fut\n ^^^^^^^^^\n File \"/home/user/.local/share/uv/python/cpython-3.12.12-macos-aarch64-none/lib/python3.12/asyncio/subprocess.py\", line 201, in communicate\n stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/user/.local/share/uv/python/cpython-3.12.12-macos-aarch64-none/lib/python3.12/asyncio/subprocess.py\", line 181, in _read_stream\n output = await stream.read()\n ^^^^^^^^^^^^^^^^^^^\n File \"/home/user/.local/share/uv/python/cpython-3.12.12-macos-aarch64-none/lib/python3.12/asyncio/streams.py\", line 706, in read\n block = await self.read(self._limit)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/user/.local/share/uv/python/cpython-3.12.12-macos-aarch64-none/lib/python3.12/asyncio/streams.py\", line 713, in read\n await self._wait_for_data('read')\n File \"/home/user/.local/share/uv/python/cpython-3.12.12-macos-aarch64-none/lib/python3.12/asyncio/streams.py\", line 545, in _wait_for_data\n await self._waiter\nasyncio.exceptions.CancelledError\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/repo/.venv/lib/python3.12/site-packages/harbor/trial/trial.py\", line 450, in _run_agent_phase\n await asyncio.wait_for(\n File \"/home/user/.local/share/uv/python/cpython-3.12.12-macos-aarch64-none/lib/python3.12/asyncio/tasks.py\", line 519, in wait_for\n async with timeouts.timeout(timeout):\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/user/.local/share/uv/python/cpython-3.12.12-macos-aarch64-none/lib/python3.12/asyncio/timeouts.py\", line 115, in __aexit__\n raise TimeoutError from exc_val\nTimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/repo/.venv/lib/python3.12/site-packages/harbor/trial/single_step.py\", line 77, in _run_agent\n await self._run_agent_phase(\n File \"/repo/.venv/lib/python3.12/site-packages/harbor/trial/trial.py\", line 459, in _run_agent_phase\n raise AgentTimeoutError(\nharbor.trial.errors.AgentTimeoutError: Agent execution timed out after 1.0 seconds\n", + "occurred_at": "2026-08-13T23:09:48.095444" + }, + "started_at": "2026-08-14T06:09:44.907370Z", + "finished_at": "2026-08-14T06:09:59.389086Z", + "environment_setup": { + "started_at": "2026-08-14T06:09:44.921358Z", + "finished_at": "2026-08-14T06:09:47.085375Z" + }, + "agent_setup": { + "started_at": "2026-08-14T06:09:47.085425Z", + "finished_at": "2026-08-14T06:09:47.085462Z" + }, + "agent_execution": { + "started_at": "2026-08-14T06:09:47.085534Z", + "finished_at": "2026-08-14T06:09:48.086885Z" + }, + "verifier": { + "started_at": "2026-08-14T06:09:48.461197Z", + "finished_at": "2026-08-14T06:09:48.702603Z" + }, + "step_results": null +} diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py index 8274ddb20d..a6eaaba3c3 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py @@ -28,7 +28,13 @@ SemanticView, ViewSignal, ) -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo +from nemo_evaluator_sdk.agent_eval.trials import ( + AgentEvalTrial, + AgentEvalTrialStatus, + AgentOutput, + RunnerInfo, + TrialError, +) from nemo_evaluator_sdk.agent_inference import AgentInferenceContext, AgentInvocationResult, AgentInvocationStatus from nemo_evaluator_sdk.enums import AgentFormat, ModelFormat from nemo_evaluator_sdk.metrics.protocol import ( @@ -861,3 +867,57 @@ async def test_run_rejects_tasks_without_trials() -> None: with pytest.raises(ValueError, match=r"no trials produced for tasks: \['task-2'\]"): await AgentEvaluator().run(tasks=[_task(), other_task], trials=[_candidate_trial()]) + + +class _ErroringTaskRunner(_TaskRunner): + """A runner whose trials carry a typed error, as the Harbor adapter's do.""" + + async def run_tasks( + self, + tasks: Sequence[AgentEvalTask], + config: AgentEvalRunConfig | None = None, + ) -> list[AgentEvalTrial]: + trials = await super().run_tasks(tasks, config) + return [trial.model_copy(update={"error": TrialError(type="RuntimeError", message="boom")}) for trial in trials] + + +@pytest.mark.asyncio +async def test_run_threads_trials_into_the_summary_error_rollup() -> None: + """Guards the one line wiring ``trials=`` into ``from_scores``. + + Every other rollup test builds the summary directly, so dropping that argument would leave them + all green while silently emptying the rollup for every real run. + """ + result = await AgentEvaluator().run(tasks=[_task()], target=_ErroringTaskRunner()) + + assert result.summary.error_trial_ids == {"RuntimeError": [trial.id for trial in result.trials]} + assert result.summary.error_count == len(result.trials) + + +def test_metric_row_exposes_the_typed_trial_error() -> None: + """The replacement for reading ``metadata["exception_type"]``. + + Harbor no longer writes that key at all (``test_harbor_runtime.py`` asserts its absence), so this + is the only path by which a metric can grade on *how* a trial failed. A pre-``TrialError`` bundle + still carries the old key in its metadata, but nothing reads it. + """ + task = AgentEvalTask(id="task-1", intent="Fix it.", inputs={"instruction": "Q?"}) + trial = _candidate_trial().model_copy( + update={"error": TrialError(type="RuntimeError", message="boom", traceback="Traceback...\n")} + ) + + row = _metric_row(task, trial) + + assert row["trial"]["error"] == { + "type": "RuntimeError", + "message": "boom", + "traceback": "Traceback...\n", + "occurred_at": None, + } + + +def test_metric_row_error_is_none_for_a_trial_that_did_not_fail() -> None: + # Always present, so a metric can read it unconditionally rather than probing for the key. + row = _metric_row(AgentEvalTask(id="task-1", intent="Fix it.", inputs={}), _candidate_trial()) + + assert row["trial"]["error"] is None diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_error_propagation.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_error_propagation.py new file mode 100644 index 0000000000..e9c166824c --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_error_propagation.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Real Harbor, real error: the exception-propagation path end to end. + +The unit half of this lives in ``test_harbor_runtime.py`` and replays a captured ``result.json``, so +it runs on every PR. This one actually runs Harbor in Docker, which is the only way to catch Harbor +changing *what* it stamps rather than the SDK mis-reading what it stamped. + +Marked ``integration`` rather than ``e2e``/``slow`` on purpose: that combination (used by +``test_harbor_runtime_e2e.py``) is selected by no make target and no CI job. ``integration`` at least +runs wherever the plugin's ``test_harbor_plugin_run.py`` does. It still skips in CI today, because +``harbor`` is an optional SDK extra and nothing depends on ``nemo-evaluator-sdk[harbor]``. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest +from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import ( + HarborRuntimeConfig, + reward_payload_from_result, + run_harbor_eval, +) +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrialStatus + +pytestmark = [pytest.mark.integration] + +#: A Harbor dataset owned by the tests, holding exactly one permanently-failing task. Deliberately +#: not in ``examples/``: a dataset users are pointed at should not ship a task that always fails. +_ERROR_DATASET_DIR = Path(__file__).parent / "fixtures" / "harbor_error_dataset" +_ERROR_TASK_NAME = "harbor/injected-runtime-error" + + +def _docker_available() -> bool: + if shutil.which("docker") is None: + return False + try: + # Bound the probe: a wedged daemon can otherwise hang until the test-level timeout. + return subprocess.run(["docker", "info"], capture_output=True, timeout=10).returncode == 0 + except subprocess.TimeoutExpired: + return False + + +@pytest.mark.asyncio +@pytest.mark.timeout(300) +async def test_a_real_harbor_timeout_lands_in_the_summary_error_rollup(tmp_path: Path) -> None: + """The fixture's oracle sleeps past a 1s agent timeout, so Harbor writes ``exception_info``. + + ``AgentTimeoutError`` is deterministic here only because no timeout multiplier is set: a + fractional ``agent_timeout_multiplier`` races Harbor's outer ``wait_for`` against the oracle's + inner Docker timeout, and the inner one raises a plain ``RuntimeError`` instead. + """ + pytest.importorskip("harbor") + if not _docker_available(): + pytest.skip("Docker daemon is required to run a Harbor job") + + config = HarborRuntimeConfig(jobs_dir=tmp_path / "jobs", agent_name="oracle") + result = await run_harbor_eval(config, _ERROR_DATASET_DIR) + + [trial] = result.trials + assert trial.task_id == _ERROR_TASK_NAME + # Errored but still scoreable: FAILED would short-circuit the metric entirely. + assert trial.status is AgentEvalTrialStatus.PARTIAL + assert trial.error is not None + assert trial.error.type == "AgentTimeoutError" + assert trial.error.message is not None and "timed out" in trial.error.message + + # The point of AALGO-428: no re-walking result.trials, no reconstruction helper -- the summary + # already carries Harbor's exception_stats shape, keyed by trial id. + assert result.summary.error_trial_ids == {"AgentTimeoutError": [trial.id]} + assert result.summary.error_count == 1 + + # Harbor runs the verifier even after recording the timeout, so the same trial is also scored. + assert reward_payload_from_result(result)["exceptions"] == {"AgentTimeoutError": [_ERROR_TASK_NAME]} diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py index 39781e57a0..8004a93239 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py @@ -10,6 +10,7 @@ import os import sys from collections.abc import Awaitable, Callable +from datetime import datetime, timezone from pathlib import Path from types import ModuleType @@ -17,22 +18,27 @@ from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import ( + _MAX_TRACEBACK_CHARS, HarborAgentTaskRunner, HarborRewardMetric, HarborRuntimeConfig, HarborTasksetLoader, _build_native_job, + _trial_error, build_trials_from_job_dir, discover_harbor_tasks, reward_payload_from_result, scoped_harbor_agent_import, ) from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrialStatus +from nemo_evaluator_sdk.agent_eval.trials import UNKNOWN_ERROR_TYPE, AgentEvalTrial, AgentEvalTrialStatus, TrialError from nemo_evaluator_sdk.metrics.utils import metric_type_name from pydantic import BaseModel, ValidationError _HELLO_WORLD_DATASET = Path(__file__).resolve().parents[2] / "examples" / "harbor" / "hello_world_dataset" +_FIXTURES = Path(__file__).parent / "fixtures" +# A verbatim Harbor result.json from a real agent-timeout run, host paths scrubbed. +_HARBOR_ERROR_RESULT = _FIXTURES / "harbor_error_result.json" def _write_trial( @@ -81,7 +87,10 @@ async def test_harbor_runner_scores_through_agent_evaluator_and_adapts_legacy_pa assert trials["pass-task"].metadata["prompt_tokens"] == 100 assert trials["pass-task"].evidence is not None assert trials["fail-task"].status == AgentEvalTrialStatus.PARTIAL - assert trials["fail-task"].metadata["exception_type"] == "NonZeroAgentExitCodeError" + assert trials["fail-task"].error is not None + assert trials["fail-task"].error.type == "NonZeroAgentExitCodeError" + # The typed field replaced the metadata stamp outright: one source of truth, no mirror to drift. + assert "exception_type" not in trials["fail-task"].metadata # Missing reward: no explicit reward -> PARTIAL, metadata reward is None, scores as 0.0. assert trials["noreward-task"].status == AgentEvalTrialStatus.PARTIAL assert trials["noreward-task"].metadata["reward"] is None @@ -96,11 +105,17 @@ async def test_harbor_runner_scores_through_agent_evaluator_and_adapts_legacy_pa assert rewards_by_task == {"pass-task": 1.0, "fail-task": 0.0, "noreward-task": 0.0} # Phase-1 legacy adapter reproduces the {reward, reward_details, exceptions} contract. + # Its `exceptions` block now reads AgentEvalTrial.error rather than the retired metadata key, + # while keeping its task-keyed shape -- switching that to trial ids is AALGO-441. payload = reward_payload_from_result(result) assert payload["reward"]["harbor_reward.reward"] == pytest.approx(1.0 / 3) assert payload["reward_details"]["reward"]["1.0"] == ["pass-task"] assert payload["exceptions"] == {"NonZeroAgentExitCodeError": ["fail-task"]} + # ...and the summary carries Harbor's own trial-keyed shape, with no reconstruction needed. + assert result.summary.error_trial_ids == {"NonZeroAgentExitCodeError": ["fail-task__bbb"]} + assert result.summary.error_count == 1 + async def _record(calls: list[str]) -> None: calls.append("ran") @@ -142,9 +157,14 @@ def _reward_stats_from_summary( ``trial_name`` straight onto ``AgentEvalTrial.id``, which is the ``trial_id`` each record now carries, so this reproduces Harbor's shape rather than approximating it. - A ``None`` value is a trial that died before the verifier ran. Harbor has no reward to file it - under either — it lands in ``exception_stats``/``n_errors`` instead — so it is skipped here, as - is any non-numeric value: Harbor keys ``reward_stats`` by the reward value itself. + A ``None`` value is a trial that died before the verifier ran, and is skipped here — as is any + non-numeric value, since Harbor keys ``reward_stats`` by the reward value itself. + + One caveat this does *not* reproduce: Harbor files a trial in ``reward_stats`` only when + ``verifier_result.rewards`` exists, so a trial that crashed before the verifier ran appears in + ``exception_stats`` alone. The SDK synthesises ``0.0`` for it (see ``HarborRewardMetric``), so it + also lands in ``task_metric_values`` — ``gamma__a`` below is exactly that case. Reconciling the + two is AALGO-441, not this helper. """ key = f"{metric_type}.{output_name}" stats: dict[str, dict[float, list[str]]] = {} @@ -165,8 +185,8 @@ async def test_harbor_reward_stats_is_derivable_from_summary_task_metric_values( AALGO-441 can rebuild the real shape without re-walking ``result.scores``. The legacy ``reward_details`` (task-keyed, stringified) stays derivable too, so the rewiring loses nothing. - Still out of scope here: ``exception_stats``, which needs the exception type per dead trial — - AALGO-428. The ``trial_id`` below is the join key that makes it a lookup against ``trials.jsonl``. + ``exception_stats`` is now covered too, by + :func:`test_harbor_exception_stats_is_read_straight_off_the_summary` below (AALGO-428). """ job_dir = tmp_path / "job" job_dir.mkdir() @@ -206,6 +226,55 @@ async def test_harbor_reward_stats_is_derivable_from_summary_task_metric_values( assert _reward_details_from_summary(result.summary) == payload["reward_details"] +@pytest.mark.asyncio +async def test_harbor_exception_stats_is_read_straight_off_the_summary(tmp_path: Path) -> None: + """AALGO-428: ``summary.error_trial_ids`` *is* Harbor's ``exception_stats``, not an approximation. + + The proof is the absence of a helper. ``reward_stats`` needs ``_reward_stats_from_summary`` above + to re-key task-major records into Harbor's shape; this needs nothing — the field is already + ``{exception type: [trial_name, ...]}``, because Harbor's ``trial_name`` is ``AgentEvalTrial.id``. + + Two things this pins that a later refactor could plausibly "tidy" away: + + - ``beta__a`` errored *and* scored 1.0. It appears in the rollup **and** counts as a pass in + ``task_metric_values`` — Harbor double-files it the same way, because its reward and exception + branches are independent. + - ``gamma__a`` is ``PARTIAL``, not ``FAILED`` (an errored Harbor trial stays scoreable). Filtering + this rollup by status would drop exactly the trials it exists to name. + """ + job_dir = tmp_path / "job" + job_dir.mkdir() + _write_trial(job_dir, "alpha__a", "alpha", reward=1.0) # clean + _write_trial(job_dir, "alpha__b", "alpha", reward=0.0, exception={"exception_type": "RuntimeError"}) + _write_trial(job_dir, "beta__a", "beta", reward=1.0, exception={"exception_type": "RuntimeError"}) + _write_trial(job_dir, "gamma__a", "gamma", reward=None, exception={"exception_type": "TimeoutError"}) + + tasks = [ + AgentEvalTask(id=task_id, intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()]) + for task_id in ("alpha", "beta", "gamma") + ] + runner = HarborAgentTaskRunner(job_dir=job_dir, run_job=lambda: _record([])) + result = await AgentEvaluator().run(tasks=tasks, target=runner, config=AgentEvalRunConfig()) + + assert result.summary.error_trial_ids == { + "RuntimeError": ["alpha__b", "beta__a"], + "TimeoutError": ["gamma__a"], + } + assert result.summary.error_count == 3 + + # The errored-but-rewarded trial keeps its reward and its pass, exactly as Harbor reports it. + assert _reward_stats_from_summary(result.summary)["reward"][1.0] == ["alpha__a", "beta__a"] + by_trial = {t.id: t for t in result.trials} + assert by_trial["beta__a"].status is AgentEvalTrialStatus.PARTIAL + assert by_trial["gamma__a"].status is AgentEvalTrialStatus.PARTIAL + + # The legacy task-keyed payload stays derivable from the same typed source. + assert reward_payload_from_result(result)["exceptions"] == { + "RuntimeError": ["alpha", "beta"], + "TimeoutError": ["gamma"], + } + + def test_reward_with_no_matching_reward_key_is_partial_and_warns(tmp_path: Path, caplog) -> None: # Verifier emitted a reward, but under a key we didn't ask for: no guessing — # the trial is treated as having no reward (None -> PARTIAL, scores 0.0) and warns. @@ -226,8 +295,9 @@ def test_task_discovery_and_taskset_loader_over_bundled_dataset() -> None: # Discovery reads the bundled hello-world dataset directory the same way Harbor # does: id comes from [task] name, and each task is scored by a reward metric. tasks = discover_harbor_tasks(_HELLO_WORLD_DATASET) - assert [task.id for task in tasks] == ["harbor/hello-world"] - task = tasks[0] + assert {task.id for task in tasks} == {"harbor/hello-world"} + by_id = {task.id: task for task in tasks} + task = by_id["harbor/hello-world"] # `intent` is the human-facing task name (metadata), NOT the instruction; the instruction the # agent acts on comes from instruction.md and lives in inputs["instruction"]. assert task.intent == "harbor/hello-world" @@ -242,10 +312,18 @@ def test_task_discovery_and_taskset_loader_over_bundled_dataset() -> None: loader = HarborTasksetLoader(_HELLO_WORLD_DATASET) assert loader.name == "harbor" taskset = loader.load() - assert [t.id for t in taskset.tasks] == ["harbor/hello-world"] + assert {t.id for t in taskset.tasks} == {"harbor/hello-world"} assert taskset.metadata["harbor_dataset_path"] == str(_HELLO_WORLD_DATASET) # A limit at/above the task count is a no-op (an empty taskset is invalid). - assert [t.id for t in loader.load(limit=5).tasks] == ["harbor/hello-world"] + assert {t.id for t in loader.load(limit=5).tasks} == {"harbor/hello-world"} + + +def test_harbor_folder_names_prefer_task_directories() -> None: + from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import _harbor_folder_names + + tasks = discover_harbor_tasks(_HELLO_WORLD_DATASET) + assert set(_harbor_folder_names(tasks) or []) == {"hello-world"} + assert _harbor_folder_names([AgentEvalTask(id="x", intent="x", inputs={}, metrics=[])]) is None def test_discovery_fails_loudly_on_malformed_task(tmp_path: Path) -> None: @@ -1399,6 +1477,15 @@ def test_unreadable_result_json_is_skipped_without_raising(tmp_path: Path, caplo # Older/other writers put a bare value there. ("NonZeroAgentExitCodeError", "NonZeroAgentExitCodeError"), (17, "17"), + # Degenerate values must normalise, never raise: _trial_from_harbor_result runs outside the + # only try/except in build_trials_from_job_dir, so a ValidationError here would abandon every + # remaining trial in the job dir. + ("", "UnknownException"), + (" ", "UnknownException"), + ({"exception_type": ""}, "UnknownException"), + ({"exception_type": " "}, "UnknownException"), + ({"exception_type": 123}, "UnknownException"), + ({"exception_type": None}, "UnknownException"), ], ) def test_exception_info_shapes_all_resolve_to_a_type( @@ -1419,10 +1506,91 @@ def test_exception_info_shapes_all_resolve_to_a_type( ) assert len(trials) == 1 - assert trials[0].metadata["exception_type"] == expected_type + assert trials[0].error is not None + assert trials[0].error.type == expected_type assert trials[0].status is AgentEvalTrialStatus.PARTIAL +def test_non_string_message_and_traceback_are_dropped_rather_than_carried(tmp_path: Path) -> None: + # Same totality requirement as the parametrization above: a producer that put a number (or a + # nested object) where a string belongs must not take down the whole job dir. + job_dir = tmp_path / "job" + job_dir.mkdir() + _write_trial( + job_dir, + "t__aaa", + "t", + reward=1.0, + exception={"exception_type": "RuntimeError", "exception_message": 42, "exception_traceback": {"a": 1}}, + ) + + [trial] = build_trials_from_job_dir( + job_dir, [AgentEvalTask(id="t", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()])] + ) + + assert trial.error is not None + assert trial.error.type == "RuntimeError" + assert trial.error.message is None + assert trial.error.traceback is None + + +def test_a_real_harbor_exception_payload_round_trips_every_field(tmp_path: Path) -> None: + """Shaped after a real Harbor result.json: an agent that died with exit 127. + + ``occurred_at`` is naive local wall time while Harbor stamps trial start/finish in UTC, so it is + kept exactly as written rather than normalised — inventing an offset would fabricate precision. + """ + job_dir = tmp_path / "job" + job_dir.mkdir() + _write_trial( + job_dir, + "debug-agent-runtime-error__KFtcHEw", + "t", + reward=None, + exception={ + "exception_type": "RuntimeError", + "exception_message": "Agent process failed with exit code 127: python: command not found\n", + "exception_traceback": 'Traceback (most recent call last):\n File "/Users/x/harbor/trial.py", line 354\n', + "occurred_at": "2026-08-13T17:22:32.230852", + }, + ) + + [trial] = build_trials_from_job_dir( + job_dir, [AgentEvalTask(id="t", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()])] + ) + + assert trial.error is not None + assert trial.error.type == "RuntimeError" + assert trial.error.message is not None and "exit code 127" in trial.error.message + assert trial.error.traceback is not None and trial.error.traceback.startswith("Traceback") + occurred_at = trial.error.occurred_at + assert occurred_at is not None + assert occurred_at == datetime(2026, 8, 13, 17, 22, 32, 230852) + assert occurred_at.tzinfo is None # naive, as Harbor wrote it + + +def test_an_oversized_traceback_is_truncated(tmp_path: Path) -> None: + # Bundles are portable and a traceback is diagnostic text, not something anyone joins on, so it + # is bounded rather than faithful. + job_dir = tmp_path / "job" + job_dir.mkdir() + _write_trial( + job_dir, + "t__aaa", + "t", + reward=1.0, + exception={"exception_type": "RuntimeError", "exception_traceback": "x" * (_MAX_TRACEBACK_CHARS * 3)}, + ) + + [trial] = build_trials_from_job_dir( + job_dir, [AgentEvalTask(id="t", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()])] + ) + + assert trial.error is not None + assert trial.error.traceback is not None + assert len(trial.error.traceback) == _MAX_TRACEBACK_CHARS + + def test_absent_exception_info_leaves_the_trial_completed(tmp_path: Path) -> None: """The negative case that gives the parametrization above its meaning.""" job_dir = tmp_path / "job" @@ -1433,5 +1601,91 @@ def test_absent_exception_info_leaves_the_trial_completed(tmp_path: Path) -> Non job_dir, [AgentEvalTask(id="t", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()])] ) - assert "exception_type" not in trials[0].metadata + assert trials[0].error is None assert trials[0].status is AgentEvalTrialStatus.COMPLETED + + +def test_the_typed_error_is_the_only_carrier(tmp_path: Path) -> None: + """The adapter records the failure once, on ``error`` -- nothing is mirrored into metadata. + + A pre-``TrialError`` bundle put the type in free-form metadata. That is not interpreted on load, + so the two paths do not converge: only a trial adapted by this code carries an error. + """ + job_dir = tmp_path / "job" + job_dir.mkdir() + _write_trial(job_dir, "t__aaa", "t", reward=0.0, exception={"exception_type": "RuntimeError"}) + + [fresh] = build_trials_from_job_dir( + job_dir, [AgentEvalTask(id="t", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()])] + ) + legacy = AgentEvalTrial.model_validate( + {"id": "t__aaa", "task_id": "t", "status": "partial", "metadata": {"exception_type": "RuntimeError"}} + ) + + assert fresh.error is not None and fresh.error.type == "RuntimeError" + assert "exception_type" not in fresh.metadata + assert legacy.error is None # free-form metadata is never promoted to a typed error + + +def test_a_degenerate_exception_type_normalises_to_the_fallback() -> None: + error = _trial_error({"exception_type": ""}) + assert error is not None + assert error.type == UNKNOWN_ERROR_TYPE + + +def test_occurred_at_is_not_advertised_as_rfc_3339() -> None: + # RFC 3339 date-time requires an offset, and Harbor writes a naive local clock. Advertising + # `format: date-time` would make a strict client parse a zoneless string into its own zone, + # silently shifting the instant. + schema = TrialError.model_json_schema()["properties"]["occurred_at"] + assert "format" not in schema + assert schema["type"] == "string" + + +def test_both_naive_and_aware_timestamps_survive_a_round_trip() -> None: + naive = TrialError(type="E", occurred_at=datetime(2026, 8, 13, 17, 22, 32, 230852)) + aware = TrialError(type="E", occurred_at=datetime(2026, 8, 14, 0, 22, 25, tzinfo=timezone.utc)) + + assert TrialError.model_validate(naive.model_dump(mode="json")).occurred_at == naive.occurred_at + assert TrialError.model_validate(aware.model_dump(mode="json")).occurred_at == aware.occurred_at + # The naive one serialises without an offset, which is exactly why the schema cannot claim one. + assert naive.model_dump(mode="json")["occurred_at"] == "2026-08-13T17:22:32.230852" + + +def test_a_real_harbor_error_payload_reaches_the_summary_rollup(tmp_path: Path) -> None: + """The exception path against real Harbor bytes, not a hand-written dict. + + Every other test here builds ``exception_info`` with :func:`_write_trial`, which means the whole + chain is only ever verified against payloads we wrote ourselves. This one replays a captured + ``result.json`` from an actual Harbor run whose agent blew a 1s timeout, so a change to Harbor's + on-disk shape shows up as a failure rather than as silently-empty rollups. + + It also pins the case the synthetic tests can only assert by construction: Harbor ran the verifier + *after* recording the timeout, so this trial carries an error **and** a real reward. It therefore + appears in ``error_trial_ids`` and in ``task_metric_values`` at once. + """ + payload = json.loads(_HARBOR_ERROR_RESULT.read_text(encoding="utf-8")) + trial_name, task_name = payload["trial_name"], payload["task_name"] + + job_dir = tmp_path / "job" + (job_dir / trial_name).mkdir(parents=True) + (job_dir / trial_name / "result.json").write_text(json.dumps(payload), encoding="utf-8") + + task = AgentEvalTask(id=task_name, intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()]) + [trial] = build_trials_from_job_dir(job_dir, [task]) + + assert trial.error is not None + assert trial.error.type == "AgentTimeoutError" + assert trial.error.message == "Agent execution timed out after 1.0 seconds" + assert trial.error.traceback is not None and "AgentTimeoutError" in trial.error.traceback + # Harbor stamps a naive local clock here while writing trial start/finish in UTC; the SDK keeps it + # exactly as written rather than inventing an offset. + occurred_at = trial.error.occurred_at + assert occurred_at is not None and occurred_at.tzinfo is None + # Errored, but still scored -- FAILED would exclude it from scoring entirely. + assert trial.status is AgentEvalTrialStatus.PARTIAL + assert trial.metadata["reward"] == 0.0 + + summary = AgentEvalSummary.from_scores([], trials=[trial]) + assert summary.error_trial_ids == {"AgentTimeoutError": [trial_name]} + assert summary.error_count == 1 diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py index 979fda7cc7..5167acb385 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py @@ -12,7 +12,7 @@ import pytest from nemo_evaluator_sdk.agent_eval.persistence import persist_run, read_trials from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary, TrialMetricValue -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, TrialError from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor @@ -228,3 +228,37 @@ def test_persist_accepts_an_explicit_target_for_an_in_memory_run(tmp_path: Path) assert location.output_dir == tmp_path assert json.loads((tmp_path / "run.json").read_text(encoding="utf-8"))["run_id"] == "r" + + +def test_persist_run_writes_the_error_rollup_to_summary(tmp_path: Path) -> None: + summary = AgentEvalSummary( + error_trial_ids={"RuntimeError": ["task-a__aaa", "task-b__ccc"], "TimeoutError": ["task-a__bbb"]}, + error_count=3, + ) + result = AgentEvalResult(run_id="run", tasks=[], trials=[], scores=[], summary=summary) + + persist_run(result, tmp_path) + + payload = json.loads((tmp_path / "summary.json").read_text(encoding="utf-8")) + assert payload["error_trial_ids"] == { + "RuntimeError": ["task-a__aaa", "task-b__ccc"], + "TimeoutError": ["task-a__bbb"], + } + assert payload["error_count"] == 3 + # Ids keep their order through sort_keys=True, which sorts mapping keys and not list contents. + assert AgentEvalSummary.model_validate(payload).error_trial_ids == summary.error_trial_ids + + +def test_trials_jsonl_round_trips_a_typed_error(tmp_path: Path) -> None: + trial = AgentEvalTrial( + id="task-a__aaa", + task_id="task-a", + status=AgentEvalTrialStatus.PARTIAL, + error=TrialError(type="RuntimeError", message="boom", traceback="Traceback...\n"), + ) + result = AgentEvalResult(run_id="run", tasks=[], trials=[trial], scores=[], summary=AgentEvalSummary()) + + persist_run(result, tmp_path) + [reloaded] = read_trials(tmp_path) + + assert reloaded.error == trial.error diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_trial_error_rollup.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_trial_error_rollup.py new file mode 100644 index 0000000000..0a2811b9b9 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_trial_error_rollup.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""AALGO-428: ``AgentEvalSummary.error_trial_ids`` — Harbor's ``exception_stats`` shape.""" + +from __future__ import annotations + +import pytest +from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary, _error_trial_ids +from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore +from nemo_evaluator_sdk.agent_eval.trials import ( + AgentEvalTrial, + AgentEvalTrialStatus, + AgentOutput, + TrialError, +) +from nemo_evaluator_sdk.metrics.protocol import MetricOutput +from pydantic import ValidationError + + +def _trial( + trial_id: str, + *, + task_id: str = "task-a", + error: str | None = None, + status: AgentEvalTrialStatus = AgentEvalTrialStatus.PARTIAL, +) -> AgentEvalTrial: + return AgentEvalTrial( + id=trial_id, + task_id=task_id, + status=status, + # COMPLETED requires an output; the others tolerate None. + output=AgentOutput(output_text="done") if status is AgentEvalTrialStatus.COMPLETED else None, + error=None if error is None else TrialError(type=error), + ) + + +def _score(task_id: str, trial_id: str, value: float) -> AgentEvalTaskScore: + return AgentEvalTaskScore( + id=f"run:{task_id}:{trial_id}:reward", + run_id="run", + task_id=task_id, + trial_id=trial_id, + metric_type="reward", + status=AgentEvalScoreStatus.COMPLETED, + outputs=[MetricOutput(name="score", value=value)], + ) + + +def test_omitting_trials_leaves_the_rollup_empty_rather_than_raising() -> None: + # Same silent-skip contract `tasks=None` already has for pass@k: a caller that only has scores + # gets a summary, not an error. + summary = AgentEvalSummary.from_scores([_score("task-a", "t0", 1.0)]) + + assert summary.error_trial_ids == {} + assert summary.error_count == 0 + + +def test_errors_group_by_type_with_ids_in_trial_order() -> None: + trials = [ + _trial("t0", error="RuntimeError"), + _trial("t1"), # no error + _trial("t2", error="RuntimeError"), + _trial("t3", error="TimeoutError"), + ] + + summary = AgentEvalSummary.from_scores([], trials=trials) + + assert summary.error_trial_ids == {"RuntimeError": ["t0", "t2"], "TimeoutError": ["t3"]} + assert summary.error_count == 3 + + +def test_membership_ignores_trial_status() -> None: + # An errored Harbor trial is PARTIAL rather than FAILED precisely so it is still scored, so a + # status filter here would drop the trials this rollup exists to name. Harbor keys on the + # presence of exception_info alone. + trials = [ + _trial("done", error="RuntimeError", status=AgentEvalTrialStatus.COMPLETED), + _trial("partial", error="RuntimeError", status=AgentEvalTrialStatus.PARTIAL), + _trial("failed", error="RuntimeError", status=AgentEvalTrialStatus.FAILED), + ] + + summary = AgentEvalSummary.from_scores([], trials=trials) + + assert summary.error_trial_ids == {"RuntimeError": ["done", "partial", "failed"]} + + +def test_duplicate_trial_ids_stay_two_entries() -> None: + # Nothing enforces trial-id uniqueness (Gym derives ids from a rollout index in two separate + # loops), so the rollup must append rather than collect into a set — collapsing them would + # understate the error count. + summary = AgentEvalSummary.from_scores([], trials=[_trial("dup", error="E"), _trial("dup", error="E")]) + + assert summary.error_trial_ids == {"E": ["dup", "dup"]} + assert summary.error_count == 2 + + +def test_trials_may_be_wider_than_the_scores() -> None: + # A caller re-aggregating a subset can hand over more trials than scores. The rollup names them + # regardless: it reads trials, not scores, so the two need not line up. + summary = AgentEvalSummary.from_scores( + [_score("task-a", "t0", 1.0)], + trials=[_trial("t0"), _trial("t1", task_id="task-b", error="RuntimeError")], + ) + + assert summary.error_trial_ids == {"RuntimeError": ["t1"]} + assert "task-b" not in summary.task_metric_values + + +def test_error_count_must_agree_with_the_rollup() -> None: + # The model is public and directly constructible; a count contradicting the rollup beside it is + # worse than no count at all. + with pytest.raises(ValidationError, match="does not match"): + AgentEvalSummary(error_trial_ids={"RuntimeError": ["t0", "t1"]}, error_count=1) + + ok = AgentEvalSummary(error_trial_ids={"RuntimeError": ["t0", "t1"]}, error_count=2) + assert ok.error_count == 2 + + +def test_helper_returns_an_empty_rollup_for_no_trials() -> None: + assert _error_trial_ids(None) == {} + assert _error_trial_ids([]) == {} + + +def test_summary_round_trips_the_rollup_through_json() -> None: + summary = AgentEvalSummary.from_scores([], trials=[_trial("t0", error="RuntimeError")]) + + reloaded = AgentEvalSummary.model_validate(summary.model_dump(mode="json")) + + assert reloaded.error_trial_ids == {"RuntimeError": ["t0"]} + assert reloaded.error_count == 1 + + +def test_vendored_module_exposes_the_error_rollup_surface() -> None: + # The byte-copy pin proves file parity, not that these names are importable through the shipped + # package — which is the path a nemo-platform consumer actually uses. + from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalSummary as VendoredSummary + from nemo_platform.beta.evaluator.agent_eval.trials import ( + AgentEvalTrial as VendoredTrial, + ) + from nemo_platform.beta.evaluator.agent_eval.trials import ( + AgentEvalTrialStatus as VendoredStatus, + ) + from nemo_platform.beta.evaluator.agent_eval.trials import ( + TrialError as VendoredError, + ) + + trial = VendoredTrial( + id="t0", + task_id="task-a", + status=VendoredStatus.PARTIAL, + error=VendoredError(type="RuntimeError", message="boom"), + ) + summary = VendoredSummary.from_scores([], trials=[trial]) + + assert summary.error_trial_ids == {"RuntimeError": ["t0"]} + assert summary.error_count == 1 diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_trials.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_trials.py index 12d81abeac..a5feb42979 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_trials.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_trials.py @@ -1,15 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from datetime import datetime from pathlib import Path import pytest from nemo_evaluator_sdk.agent_eval.trials import ( AgentEvalTrial, AgentEvalTrialStatus, + TrialError, resolve_trial_status, standard_evidence_descriptors, ) +from pydantic import ValidationError def test_trial_accepts_mapping_shaped_evidence_and_serializes_descriptors() -> None: @@ -80,3 +83,63 @@ def test_standard_evidence_descriptors_builds_documented_keys(tmp_path: Path) -> # A missing verifier dir is omitted; trace is optional. minimal = standard_evidence_descriptors(logs_dir=tmp_path / "a", final_state_dir=tmp_path / "w") assert set(minimal) == {"logs", "final_state"} + + +def test_trial_error_rejects_an_empty_type() -> None: + # An empty type would become an empty rollup key, which names nothing. The Harbor adapter + # normalises before it gets here; this is the backstop for every other producer. + for blank in ("", " "): + with pytest.raises(ValidationError, match="must not be empty"): + TrialError(type=blank) + + +def test_trial_error_is_frozen_and_forbids_extras() -> None: + error = TrialError(type="RuntimeError") + + with pytest.raises(ValidationError): + error.type = "TimeoutError" + with pytest.raises(ValidationError): + TrialError(type="RuntimeError", stack="...") # ty: ignore[unknown-argument] + + +def test_trial_error_round_trips_every_field_through_json() -> None: + trial = AgentEvalTrial( + id="t0", + task_id="task-a", + status=AgentEvalTrialStatus.PARTIAL, + error=TrialError( + type="RuntimeError", + message="boom", + traceback="Traceback (most recent call last):\n", + occurred_at=datetime(2026, 8, 13, 17, 22, 32, 230852), + ), + ) + + reloaded = AgentEvalTrial.model_validate(trial.model_dump(mode="json")) + + assert reloaded.error == trial.error + assert AgentEvalTrial.model_validate(reloaded.model_dump(mode="json")).error == trial.error + + +def test_error_is_read_only_from_the_typed_field() -> None: + # `error` is the single carrier. A bundle written before it existed recorded the type in + # free-form metadata, and that is deliberately NOT interpreted: metadata stays opaque, so a + # pre-TrialError bundle re-scores with no error rollup rather than a guessed one. + trial = AgentEvalTrial.model_validate( + { + "id": "t0", + "task_id": "task-a", + "status": "partial", + "metadata": {"exception_type": "TimeoutError", "reward": 0.0}, + } + ) + + assert trial.error is None + assert trial.metadata["exception_type"] == "TimeoutError" # kept verbatim, just not read + + +def test_a_trial_without_any_error_signal_loads_unchanged() -> None: + trial = AgentEvalTrial.model_validate({"id": "t0", "task_id": "task-a", "status": "partial"}) + + assert trial.error is None + assert trial.metadata == {} diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml index 2a7897e6ee..bd559fe543 100644 --- a/plugins/nemo-evaluator/openapi/openapi.yaml +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -2450,6 +2450,11 @@ components: - $ref: '#/components/schemas/CandidateEvidence' description: Named evidence descriptors (final state, traces, logs, ...) captured for the trial. + error: + allOf: + - $ref: '#/components/schemas/TrialError' + description: What went wrong producing this trial, when the producer reported + a failure. Populated by a runner runtime. Drives AgentEvalSummary.error_trial_ids. metadata: additionalProperties: true type: object @@ -2463,7 +2468,7 @@ components: - status title: AgentEvalTrial description: 'Durable trial artifact for one task: output, evidence, status, - and metadata.' + error, and metadata.' AgentEvalTrialStatus: type: string enum: @@ -5686,6 +5691,66 @@ components: required: - data title: TasksetsPage + TrialError: + properties: + type: + type: string + title: Type + description: Error class name as the producer reported it, e.g. 'RuntimeError'. + Rollup key for AgentEvalSummary.error_trial_ids. Falls back to 'UnknownException' + when the producer reported a failure without a usable type. + message: + title: Message + description: Short error message, when the producer supplied one. + type: string + traceback: + title: Traceback + description: 'Formatted traceback, when the producer supplied one. May be + truncated by the adapter that captured it. Note run bundles are portable: + this can carry absolute filesystem paths and other diagnostic text from + the machine that ran the trial.' + type: string + occurred_at: + type: string + title: Occurred At + description: "When the producer recorded the failure, as it reported it.\ + \ The SDK does not rewrite this: an aware value (UTC, offset) is kept,\ + \ and a naive value stays naive. Harbor's clock is naive local, e.g. '2026-08-13T17:22:32',\ + \ while that trial's start in result.json is UTC ('2026-08-14T00:22:25Z')\ + \ \u2014 the same instant on two clocks, so do not subtract them or attach\ + \ a zone Harbor never wrote. A runner that recorded an offset keeps it.\ + \ Not RFC 3339 date-time: the offset may be absent." + additionalProperties: false + type: object + required: + - type + title: TrialError + description: 'What went wrong producing one trial, as the producer reported + it. + + + Present means the *producer* reported a failure. It does **not** imply ``status + is FAILED``: an + + errored trial can be marked as :attr:`AgentEvalTrialStatus.PARTIAL` so it + is still scored (ex: HarborRuntime). + + It is also unrelated to a score diagnostic''s ``exception_type`` detail, which + records that the + + *metric* raised - a different event. + + + Frozen so callers cannot rewrite ``type`` after construction. These objects + are returned as-is + + (not copied), and ``AgentEvalSummary.error_trial_ids`` groups trial ids by + that string. Mutating + + ``type`` on a live object would leave the summary keyed on the old value while + the trial reports + + a new one.' ValidationError: properties: loc: diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py index b4a97fdb05..b9b7c7a1b5 100644 --- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py +++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py @@ -858,3 +858,53 @@ def test_main_returns_setup_exit_code_when_task_sdk_fails(self, mocker: MockerFi assert exit_code == SDK_INITIALIZATION_EXIT_CODE get_task_sdk.assert_called_once_with("evaluator") run_task.assert_not_called() + + +async def test_trial_error_survives_the_job_spec_wire_contract() -> None: + """AALGO-428: ``AgentEvalTrial.error`` is public API, not just an SDK-internal field. + + Precomputed trials are accepted straight off the wire by ``AgentEvalInputSpec.trials``, and + ``AgentEvalTrial`` forbids extras — so a typed error has to survive JSON round-tripping through + the DTO. Regenerating the OpenAPI schema proves the shapes agree; only this proves a request + carrying one actually validates. + """ + payload = { + "id": "debug-agent-runtime-error__KFtcHEw", + "task_id": "fix-bug", + "status": "partial", + "error": { + "type": "RuntimeError", + "message": "Agent process failed with exit code 127", + "traceback": "Traceback (most recent call last):\n", + "occurred_at": "2026-08-13T17:22:32.230852", + }, + } + + input_spec = AgentEvalInputSpec.model_validate( + { + "trials": [payload], + "tasks": [ + { + "id": "fix-bug", + "intent": "Fix the bug.", + "inputs": {"instruction": "Fix calculator.py."}, + "metrics": [_inline_metric().model_dump()], + } + ], + } + ) + + assert input_spec.trials is not None + error = input_spec.trials[0].error + assert error is not None + assert error.type == "RuntimeError" + assert error.message == "Agent process failed with exit code 127" + + spec = await AgentEvalJob.to_spec(input_spec, workspace="dev", entity_client=None, async_sdk=None, is_local=True) + assert isinstance(spec, AgentEvalSpec) + assert spec.trials is not None + assert spec.trials[0].error == error + # And it survives a full serialize -> deserialize hop, which is how the job actually receives it. + round_tripped = AgentEvalSpec.model_validate(json.loads(json.dumps(spec.model_dump(mode="json")))) + assert round_tripped.trials is not None + assert round_tripped.trials[0].error == error diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py index bae3b15c29..9c97f980f1 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py @@ -192,7 +192,9 @@ async def run( tasks=task_list, trials=trial_list, scores=scores, - summary=AgentEvalSummary.from_scores(scores, tasks=task_list, extra_scores=runner_scores), + summary=AgentEvalSummary.from_scores( + scores, tasks=task_list, trials=trial_list, extra_scores=runner_scores + ), metadata=metadata, work_dir=runtime_config.work_dir, ) @@ -691,6 +693,8 @@ def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]: "id": trial.id, "task_id": trial.task_id, "status": trial.status.value, + # How the trial failed, for a metric that grades on it. None when the producer reported no failure. + "error": trial.error.model_dump(mode="json") if trial.error is not None else None, "metadata": trial.metadata, }, } diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index 568ff27bee..19430dd4e5 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -398,9 +398,56 @@ class AgentEvalSummary(BaseModel): } ], ) + error_trial_ids: dict[str, list[str]] = Field( + default_factory=dict, + description=( + "Trials that errored, grouped by error type -- Harbor's 'exception_stats' shape. Values " + "are trial ids, not task ids: they join to AgentEvalTrial.id (trials.jsonl), " + "AgentEvalTaskScore.trial_id (scores.jsonl), and TrialMetricValue.trial_id in " + "task_metric_values. Membership is 'the trial carries an error', with no status filter -- " + "an errored Harbor trial is PARTIAL rather than FAILED so that it is still scored, and it " + "belongs here regardless. A trial that both errored and produced a reward therefore " + "appears here AND in task_metric_values, where it may even count as a pass; that is what " + "Harbor does too. Ids are appended in trial order and never deduplicated. Key order is " + "not meaningful -- summary.json is written with sorted keys. Empty is ambiguous and " + "stays that way: it means either no trial errored or no trials were supplied to " + "from_scores(). The field always serializes (it defaults to {}), so the two cases are " + "indistinguishable in summary.json -- read trial_count, or the trials themselves, to " + "tell them apart." + ), + examples=[ + { + "RuntimeError": [ + "contract-review-msa-indemnity__k3f9wq2", + "nda-scope-carveouts__p2hn8sc", + ], + "TimeoutError": ["merger-hsr-filing-threshold__w5db3qy"], + } + ], + ) task_count: int = Field(default=0, description="Number of tasks represented in the run.") trial_count: int = Field(default=0, description="Number of distinct trials scored.") score_count: int = Field(default=0, description="Total number of metric scores.") + error_count: int = Field( + default=0, + description=( + "Number of trials that errored -- Harbor's 'n_errors'. Equals the total ids across " + "error_trial_ids; stated rather than derived so a non-Python reader of summary.json need " + "not sum a nested structure, matching the other counts here." + ), + ) + + @model_validator(mode="after") + def _error_count_matches_rollup(self) -> AgentEvalSummary: + """Keep the two error fields from disagreeing when a summary is built by hand. + + ``from_scores`` derives both from one walk, but the model is public and directly + constructible -- and a count that contradicts the rollup beside it is worse than no count. + """ + total = sum(len(ids) for ids in self.error_trial_ids.values()) + if self.error_count != total: + raise ValueError(f"error_count {self.error_count} does not match {total} ids in error_trial_ids") + return self @property def scores_by_name(self) -> Mapping[str, AggregateScore]: @@ -457,15 +504,23 @@ def from_scores( scores: Sequence[AgentEvalTaskScore], *, tasks: Sequence[AgentEvalTask] | None = None, + trials: Sequence[AgentEvalTrial] | None = None, extra_scores: Sequence[AggregateScore] = (), ) -> AgentEvalSummary: - """Build aggregated scores, task values, and coverage for a set of metric scores. + """Build aggregated scores, task values, coverage, and the error rollup for a set of scores. ``extra_scores`` are already-aggregated scores contributed by the runner (namespaced ``runner..``), merged in so a backend's own figures are addressable the same way as ours. + + ``trials`` supplies the only thing scores cannot carry: what went wrong. Omitting it leaves + :attr:`error_trial_ids` empty rather than raising -- the same silent-skip contract ``tasks`` + already has for pass@k. It may legitimately be *wider* than ``scores`` (a caller + re-aggregating a subset), so the rollup can name trial ids absent from + :attr:`task_metric_values`. """ task_list = list(tasks) if tasks is not None else None task_metric_values = _task_metric_values(scores, task_list) + error_trial_ids = _error_trial_ids(trials) return AgentEvalSummary( scores=_aggregate_scores( scores, @@ -475,9 +530,11 @@ def from_scores( ), metric_coverage=_metric_coverage(scores, task_list), task_metric_values=task_metric_values, + error_trial_ids=error_trial_ids, task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), trial_count=len({score.trial_id for score in scores}), score_count=len(scores), + error_count=sum(len(ids) for ids in error_trial_ids.values()), ) @@ -934,6 +991,32 @@ def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, return scorelike +def _error_trial_ids(trials: Sequence[AgentEvalTrial] | None) -> dict[str, list[str]]: + """Trial ids grouped by error type, in trial order — Harbor's ``exception_stats``. + + Three trials, the middle one fine:: + + in t0 error RuntimeError + t1 (no error) + t2 error RuntimeError + t3 error TimeoutError + + out {"RuntimeError": ["t0", "t2"], "TimeoutError": ["t3"]} + + Selection is on ``trial.error``, never on ``trial.status``: an errored Harbor trial is ``PARTIAL`` + so that it still scores, and filtering by status would drop exactly the trials this exists to name. + + Ids are **appended**, never collected into a set or used as dict keys. Nothing enforces trial-id + uniqueness (Gym derives ids from a rollout index in two separate loops), and losing cardinality + here would understate the error count — the same rule ``task_metric_values`` follows. + """ + grouped: dict[str, list[str]] = {} + for trial in trials or (): + if trial.error is not None: + grouped.setdefault(trial.error.type, []).append(trial.id) + return grouped + + def _task_metric_values( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py index e96b8132d4..a1837dbce5 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py @@ -52,10 +52,12 @@ from nemo_platform.beta.evaluator.agent_eval.scores import AgentEvalScoreStatus from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask, AgentEvalTaskset from nemo_platform.beta.evaluator.agent_eval.trials import ( + UNKNOWN_ERROR_TYPE, AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo, + TrialError, standard_evidence_descriptors, ) from nemo_platform.beta.evaluator.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult @@ -113,6 +115,8 @@ _DIGEST_SKIP_DIRS = frozenset({".git", "__pycache__", ".venv", ".uv", ".mypy_cache", ".pytest_cache"}) _DIGEST_CHUNK_BYTES = 1 << 20 +_MAX_TRACEBACK_CHARS = 8192 + RunJob = Callable[[], Awaitable[None]] @@ -326,10 +330,15 @@ async def run_tasks( stale = _cache_is_stale(job_dir, stamp) if stale or not _all_tasks_cached(job_dir, tasks, n_attempts=self._config.n_attempts): + # Harbor's DatasetConfig matches local folder names, while SDK task ids + # come from `[task] name`. Prefer folder names derived from the tasks + # actually being scored so a filter like `harbor/hello-world` still + # selects the `hello-world/` directory. + harbor_task_names = _harbor_folder_names(tasks) or self._task_names job_dir, run_job = _build_native_job( self._config, dataset_path, - self._task_names, + harbor_task_names, job_name=job_name, # Discard only when the inputs changed. Otherwise leave it off so # Harbor resumes per trial and keeps completed work — including @@ -345,7 +354,7 @@ async def run_tasks( # ran: with no task_names filter that is the whole dataset, and # recording only the requested subset would make the next full-set # run look stale and re-run a complete job dir. - coverage = _stamp_coverage(dataset_path, tasks, self._task_names) + coverage = _stamp_coverage(dataset_path, tasks, harbor_task_names) before = _cache_stamp(self._config, dataset_path, coverage) await run_job() if self._config.job_name is not None: @@ -381,6 +390,25 @@ def _dataset_path_from_tasks(tasks: Sequence[AgentEvalTask]) -> Path: ) +def _harbor_folder_names(tasks: Sequence[AgentEvalTask]) -> list[str] | None: + """Return Harbor local-dataset folder names for ``tasks``, or ``None`` if incomplete. + + Harbor's ``DatasetConfig.task_names`` matches directory names + (``LocalTaskId.get_name()`` → ``path.name``), while SDK task ids come from + ``[task] name``. When every task carries ``metadata['harbor_task_dir']``, + derive the folder list so a filter like ``harbor/hello-world`` still selects + the ``hello-world/`` directory. Return ``None`` when any task is missing that + stamp so callers can fall back to an explicit filter. + """ + names: list[str] = [] + for task in tasks: + stamped = task.metadata.get("harbor_task_dir") + if not isinstance(stamped, str) or not stamped: + return None + names.append(Path(stamped).name) + return names or None + + def _all_tasks_cached(job_dir: Path, tasks: Sequence[AgentEvalTask], *, n_attempts: int) -> bool: """Return True when every requested task already has ``n_attempts`` completed results. @@ -1122,24 +1150,18 @@ def _trial_from_harbor_result(trial_dir: Path, data: Mapping[str, Any], *, rewar trial_id = str(data.get("trial_name") or trial_dir.name) rewards = _rewards_mapping(data) reward = _primary_reward(rewards, reward_key) - exception_type = _exception_type(data.get("exception_info")) + error = _trial_error(data.get("exception_info")) metadata: dict[str, Any] = { "reward": reward, "reward_details": dict(rewards), "harbor_trial_dir": str(trial_dir), } - if exception_type is not None: - metadata["exception_type"] = exception_type metadata.update(_token_measurements(data.get("agent_result"))) # An errored trial (or one with no reward) stays PARTIAL so it is still scored # as 0 and counted in the summary; FAILED would exclude it from scoring. - status = ( - AgentEvalTrialStatus.COMPLETED - if exception_type is None and reward is not None - else AgentEvalTrialStatus.PARTIAL - ) + status = AgentEvalTrialStatus.COMPLETED if error is None and reward is not None else AgentEvalTrialStatus.PARTIAL trace_path = trial_dir / "agent" / "trajectory.json" descriptors = standard_evidence_descriptors( @@ -1155,6 +1177,7 @@ def _trial_from_harbor_result(trial_dir: Path, data: Mapping[str, Any], *, rewar status=status, output=AgentOutput(metadata={"harbor_trial_dir": str(trial_dir)}), evidence=CandidateEvidence(descriptors=descriptors), + error=error, metadata=metadata, ) @@ -1196,16 +1219,64 @@ def _primary_reward(rewards: Mapping[str, float], reward_key: str) -> float | No return None -def _exception_type(exception_info: Any) -> str | None: +def _trial_error(exception_info: Any) -> TrialError | None: + """Harbor's ``exception_info`` as a :class:`TrialError`, for any shape it can arrive in. + + **Total by construction.** :func:`_trial_from_harbor_result` is called outside the only + ``try``/``except`` in :func:`build_trials_from_job_dir` (which guards ``json.loads`` alone), so a + ``ValidationError`` raised here would abort adaptation of the *whole job dir* over one malformed + trial. Every field is therefore normalised rather than trusted: + + - ``type`` -- first non-empty string of ``exception_type``/``type``/``name``/``class``; for a + non-mapping, ``str(value)``; anything left empty becomes :data:`UNKNOWN_ERROR_TYPE` + - ``message``/``traceback`` -- kept only when actually strings; the traceback is truncated + - ``occurred_at`` -- kept only when it parses; never raises + + Returns ``None`` only for a genuinely absent ``exception_info``, which is what marks a trial as + not having errored. + """ if exception_info is None: return None - if isinstance(exception_info, Mapping): - for key in ("exception_type", "type", "name", "class"): - value = exception_info.get(key) - if isinstance(value, str) and value: - return value - return "UnknownException" - return str(exception_info) + if not isinstance(exception_info, Mapping): + return TrialError(type=str(exception_info).strip() or UNKNOWN_ERROR_TYPE) + + error_type = "" + for key in ("exception_type", "type", "name", "class"): + value = exception_info.get(key) + if isinstance(value, str) and value.strip(): + error_type = value + break + + traceback = _first_string(exception_info, ("exception_traceback", "traceback")) + return TrialError( + type=error_type or UNKNOWN_ERROR_TYPE, + message=_first_string(exception_info, ("exception_message", "message")), + traceback=traceback[:_MAX_TRACEBACK_CHARS] if traceback is not None else None, + occurred_at=_error_timestamp(exception_info.get("occurred_at")), + ) + + +def _first_string(payload: Mapping[str, Any], keys: tuple[str, ...]) -> str | None: + """The first value under ``keys`` that is actually a string. Harbor's spelling is tried first.""" + for key in keys: + value = payload.get(key) + if isinstance(value, str): + return value + return None + + +def _error_timestamp(value: Any) -> datetime | None: + """``value`` as a datetime when it plausibly is one, else ``None`` -- never raising. + + Deliberately not normalized to UTC: Harbor writes a naive local wall-clock time here while + stamping trial start/finish in UTC, and inventing an offset would fabricate precision. + """ + if isinstance(value, datetime): + return value + if isinstance(value, str): + with contextlib.suppress(ValueError): + return datetime.fromisoformat(value) + return None def _token_measurements(agent_result: Any) -> dict[str, int | float]: @@ -1368,8 +1439,12 @@ def reward_payload_from_result( * ``reward`` — mean of each metric output, keyed ``"."``. * ``reward_details`` — ``{output: {value_str: [task_id, ...]}}`` grouped from per-trial scores (Harbor's ``reward_stats`` analogue). - * ``exceptions`` — ``{exception_type: [task_id, ...]}`` from trial metadata - (Harbor's ``exception_stats`` analogue). + * ``exceptions`` — ``{error type: [task_id, ...]}`` from ``AgentEvalTrial.error`` + (Harbor's ``exception_stats`` analogue, keyed by task rather than by trial). + + Harbor keys ``exception_stats`` by *trial*, which is what + :attr:`AgentEvalSummary.error_trial_ids` now reproduces exactly. This payload keeps its + task-keyed shape for existing consumers; switching it over is AALGO-441. """ reward = {score.name: score.mean for score in result.summary.scores.scores if score.mean is not None} @@ -1386,9 +1461,8 @@ def reward_payload_from_result( exceptions: dict[str, list[str]] = {} for trial in result.trials: - exc = trial.metadata.get("exception_type") - if isinstance(exc, str) and exc: - exceptions.setdefault(exc, []).append(trial.task_id) + if trial.error is not None: + exceptions.setdefault(trial.error.type, []).append(trial.task_id) return { "reward": reward, diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py index 68f0f50c40..7e24b41dd5 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py @@ -8,9 +8,10 @@ from __future__ import annotations from collections.abc import Sequence +from datetime import datetime from enum import Enum from pathlib import Path -from typing import Any, Protocol, runtime_checkable +from typing import Annotated, Any, Protocol, runtime_checkable from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask from nemo_platform.beta.evaluator.values import Agent, Model @@ -26,7 +27,15 @@ EvidenceDescriptor, ) from nemo_platform.beta.evaluator.values.results import AggregateScore -from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + JsonValue, + WithJsonSchema, + field_validator, + model_validator, +) class AgentEvalTrialStatus(str, Enum): @@ -57,8 +66,73 @@ class AgentOutput(BaseModel): ) +# Type recorded when a producer reported a failure but named no usable type. This +# fires only for hand-built or malformed payloads. +UNKNOWN_ERROR_TYPE = "UnknownException" + + +class TrialError(BaseModel): + """What went wrong producing one trial, as the producer reported it. + + Present means the *producer* reported a failure. It does **not** imply ``status is FAILED``: an + errored trial can be marked as :attr:`AgentEvalTrialStatus.PARTIAL` so it is still scored (ex: HarborRuntime). + It is also unrelated to a score diagnostic's ``exception_type`` detail, which records that the + *metric* raised - a different event. + + Frozen so callers cannot rewrite ``type`` after construction. These objects are returned as-is + (not copied), and ``AgentEvalSummary.error_trial_ids`` groups trial ids by that string. Mutating + ``type`` on a live object would leave the summary keyed on the old value while the trial reports + a new one. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + type: str = Field( + description=( + "Error class name as the producer reported it, e.g. 'RuntimeError'. Rollup key for " + f"AgentEvalSummary.error_trial_ids. Falls back to {UNKNOWN_ERROR_TYPE!r} when the " + "producer reported a failure without a usable type." + ) + ) + message: str | None = Field( + default=None, + description="Short error message, when the producer supplied one.", + ) + traceback: str | None = Field( + default=None, + description=( + "Formatted traceback, when the producer supplied one. May be truncated by the adapter " + "that captured it. Note run bundles are portable: this can carry absolute filesystem " + "paths and other diagnostic text from the machine that ran the trial." + ), + ) + # Schema is a bare string, deliberately not `format: date-time`. RFC 3339 date-time requires a + # UTC offset, and this field may legitimately carry a naive local timestamp (Harbor writes one), + # so claiming the format would be a promise the value cannot keep -- and a client that trusts it + # parses a zoneless string into its *own* zone, silently shifting the instant. A plain string + # says "timestamp as the producer wrote it"; Python callers still get a parsed datetime. + occurred_at: Annotated[datetime | None, WithJsonSchema({"type": "string"})] = Field( + default=None, + description=( + "When the producer recorded the failure, as it reported it. The SDK does not rewrite " + "this: an aware value (UTC, offset) is kept, and a naive value stays naive. Harbor's " + "clock is naive local, e.g. '2026-08-13T17:22:32', while that trial's start in " + "result.json is UTC ('2026-08-14T00:22:25Z') — the same instant on two clocks, so do " + "not subtract them or attach a zone Harbor never wrote. A runner that recorded an " + "offset keeps it. Not RFC 3339 date-time: the offset may be absent." + ), + ) + + @field_validator("type") + @classmethod + def _non_empty_type(cls, value: str) -> str: + if not value.strip(): + raise ValueError("trial error type must not be empty") + return value + + class AgentEvalTrial(BaseModel): - """Durable trial artifact for one task: output, evidence, status, and metadata.""" + """Durable trial artifact for one task: output, evidence, status, error, and metadata.""" model_config = ConfigDict(extra="forbid") @@ -73,6 +147,13 @@ class AgentEvalTrial(BaseModel): default=None, description="Named evidence descriptors (final state, traces, logs, ...) captured for the trial.", ) + error: TrialError | None = Field( + default=None, + description=( + "What went wrong producing this trial, when the producer reported a failure. Populated " + "by a runner runtime. Drives AgentEvalSummary.error_trial_ids." + ), + ) metadata: dict[str, Any] = Field( default_factory=dict, description="Free-form metadata associated with the trial.",