From bc41a17110f4734d37623dd987b994fa5a0f5889 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Thu, 21 May 2026 09:23:56 -0300 Subject: [PATCH] test(agentic-use): add evaluator agent benchmark matrix --- tests/agentic-use/README.md | 75 +- tests/agentic-use/agent_matrix_benchmark.py | 2499 +++++++++++++++++ .../environment/Dockerfile | 1 + .../instruction.md | 16 + .../task.toml | 48 + .../tests/task_metrics.py | 225 ++ .../tests/test.sh | 2 + .../tests/test_outputs.py | 11 + .../instruction.md | 1 + .../tests/task_metrics.py | 64 +- .../environment/Dockerfile | 1 + .../instruction.md | 17 + .../task.toml | 47 + .../tests/task_metrics.py | 227 ++ .../tests/test.sh | 2 + .../tests/test_outputs.py | 11 + .../tests/task_metrics.py | 46 +- .../evaluator_agent_benchmark_mvp.txt | 4 +- .../matrices/evaluator_agent_benchmark_mvp.py | 56 + tests/agentic-use/nat_runner.py | 393 ++- .../agentic-use/scripts/codex_agent_runner.sh | 81 +- .../shared/evaluator_agent_eval/factory.py | 6 - .../evaluator_agent_eval/metrics/__init__.py | 10 - .../evaluator_agent_eval/metrics/outcome.py | 123 - .../evaluator_agent_eval/pytest_verifier.py | 32 +- .../shared/evaluator_agent_eval/schemas.py | 29 - .../evaluator_agent_eval/task_metric_utils.py | 181 ++ .../evaluator_agent_eval/test_construction.py | 15 - .../evaluator_agent_eval/test_metrics.py | 89 - .../tests/evaluator_agent_eval/test_runner.py | 7 +- .../evaluator_agent_eval/test_schemas.py | 14 - .../evaluator_agent_eval/test_task_config.py | 2 + .../evaluator_agent_eval/test_task_metrics.py | 253 ++ .../tests/test_agent_matrix_benchmark.py | 564 ++++ .../tests/test_nat_runner_config.py | 332 ++- 35 files changed, 4910 insertions(+), 574 deletions(-) create mode 100644 tests/agentic-use/agent_matrix_benchmark.py create mode 100644 tests/agentic-use/evaluator-standalone-sdk-agent-target/environment/Dockerfile create mode 100644 tests/agentic-use/evaluator-standalone-sdk-agent-target/instruction.md create mode 100644 tests/agentic-use/evaluator-standalone-sdk-agent-target/task.toml create mode 100644 tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/task_metrics.py create mode 100755 tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/test.sh create mode 100644 tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/test_outputs.py create mode 100644 tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/environment/Dockerfile create mode 100644 tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/instruction.md create mode 100644 tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/task.toml create mode 100644 tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/task_metrics.py create mode 100755 tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/test.sh create mode 100644 tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/test_outputs.py create mode 100644 tests/agentic-use/matrices/evaluator_agent_benchmark_mvp.py delete mode 100644 tests/agentic-use/shared/evaluator_agent_eval/metrics/outcome.py create mode 100644 tests/agentic-use/shared/evaluator_agent_eval/task_metric_utils.py create mode 100644 tests/agentic-use/tests/test_agent_matrix_benchmark.py diff --git a/tests/agentic-use/README.md b/tests/agentic-use/README.md index 0abe87e9f43..3ebac390374 100644 --- a/tests/agentic-use/README.md +++ b/tests/agentic-use/README.md @@ -126,7 +126,10 @@ Codex auth: - Preferred for CI: set `OPENAI_API_KEY` in the runner environment. - Local development: run `codex login` on the host once, then opt in to using that session by passing `--codex-auth-json ~/.codex/auth.json`. The file is - mounted read-only and copied into the container for the Codex invocation. + mounted read-only and copied into a fresh, task-local `CODEX_HOME` for the + Codex invocation. Host `config.toml`, MCP servers, and project agent + instructions are not copied into the task container; auth is the only + optional host Codex state. Cursor Agent auth: - Set `CURSOR_API_KEY` in the runner environment, for example @@ -134,9 +137,79 @@ Cursor Agent auth: - The container backend does not mount host Cursor session state; use an API key for reproducible local and CI runs. +Claude Code auth: +- Set `ANTHROPIC_API_KEY` in the runner environment. For NVIDIA-hosted Claude + access, set `ANTHROPIC_BASE_URL=https://inference-api.nvidia.com` and pass a + concrete allowed model such as `--agent-model aws/anthropic/bedrock-claude-sonnet-4-6`. +- Smoke the model outside the benchmark before a slow matrix run: + `claude -p 'Reply with OK only.' --model aws/anthropic/bedrock-claude-sonnet-4-6 --output-format json`. + Claude Code's default model may not be allowed by NVIDIA inference keys. + Placeholder values such as `null`, `none`, or an empty string are treated as unset by the runner. +### Agent matrix benchmark runbook + +Use `agent_matrix_benchmark.py` when comparing direct coding agents over the +same task set. It writes `benchmark_summary.json`, `benchmark_summary.md`, and +`benchmark_report.html` under the selected run directory. + +Known-good local flow: + +```bash +# Build or refresh the base and task images first. +docker build -f Dockerfile.agentic-base -t nmp-agentic-base:latest . + +python tests/agentic-use/nat_runner.py \ + --manifest manifests/evaluator_agent_benchmark_mvp.txt \ + --agent-backend codex \ + --build-only + +# Smoke Claude Code against the NVIDIA endpoint before a full matrix run. +export ANTHROPIC_BASE_URL=https://inference-api.nvidia.com +claude -p 'Reply with OK only.' \ + --model aws/anthropic/bedrock-claude-sonnet-4-6 \ + --output-format json + +# Run Codex, Cursor Agent, and Claude Code concurrently, one task at a time per agent. +python tests/agentic-use/agent_matrix_benchmark.py \ + --manifest manifests/evaluator_agent_benchmark_mvp.txt \ + --agent codex:gpt-5.5 \ + --agent cursor-agent \ + --agent claude-code:aws/anthropic/bedrock-claude-sonnet-4-6 \ + --codex-auth-json ~/.codex/auth.json \ + --jobs-dir nat-jobs/agent-matrix \ + --run-id pr470-three-agent \ + --skip-build \ + --allow-dirty \ + --parallel-candidates 3 +``` + +Notes: +- `--parallel-candidates` parallelizes across candidates only. Each candidate's + `nat_runner.py` invocation still runs the selected task list serially. +- Values greater than `1` require `--skip-build`; build the selected task images + before the parallel matrix run to avoid Docker tag races. +- For Codex, the runner copies only auth into a fresh container-local + `CODEX_HOME`; host config and MCP servers are not reproduced. Codex receives + the same task instruction as the other direct agents, runs from `/app`, and + writes task files directly under `/app/workspace`. +- For Claude Code with NVIDIA inference, always pass a concrete allowed model. + The default Claude Code model may not be enabled for a given NVIDIA key. +- Open the generated HTML report at + `nat-jobs/agent-matrix//benchmark_report.html`. + +### Task metric authoring direction + +Task-local verification should prefer Evaluator SDK `Metric` implementations. +Keep reusable mechanics in `shared/evaluator_agent_eval/` helpers and keep +task-specific files focused on task semantics. + +Until the SDK supports enriched row-level metric outputs, task-local diagnostics +can be emitted as additional numeric metric scores. Once the broader +`MetricResult` output contract lands, move richer structured diagnostics into +SDK metric outputs. + ## Planned follow-ups The current runner/gate path is stable for deterministic benchmarking. These diff --git a/tests/agentic-use/agent_matrix_benchmark.py b/tests/agentic-use/agent_matrix_benchmark.py new file mode 100644 index 00000000000..a49d4bbdb46 --- /dev/null +++ b/tests/agentic-use/agent_matrix_benchmark.py @@ -0,0 +1,2499 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run and compare agentic-use tasks across direct agent backends.""" + +from __future__ import annotations + +import argparse +import html +import importlib.util +import json +import os +import re +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass +from dataclasses import field as dataclass_field +from datetime import UTC, datetime +from pathlib import Path +from types import ModuleType +from typing import Literal + +import nat_runner +from pydantic import BaseModel, ConfigDict, Field, model_validator + +TASKS_DIR = Path(__file__).resolve().parent +NAT_RUNNER = TASKS_DIR / "nat_runner.py" +DEFAULT_BACKENDS = ("codex", "claude-code", "cursor-agent") +SUPPORTED_BACKENDS = frozenset({"codex", "claude-code", "cursor-agent"}) +REPORT_JSON = "benchmark_summary.json" +REPORT_MARKDOWN = "benchmark_summary.md" +REPORT_HTML = "benchmark_report.html" +AgentBackend = Literal["codex", "claude-code", "cursor-agent"] +ConfigScalar = str | int | float | bool | None +ConfigValue = ConfigScalar | list[ConfigScalar] +LEADERBOARD_COLUMNS: tuple[tuple[str, str, bool], ...] = ( + ("rank", "Rank", True), + ("candidate", "Candidate", True), + ("backend", "Backend", True), + ("model", "Model", True), + ("params", "Params", True), + ("pass-rate", "Pass Rate", True), + ("passed", "Passed", True), + ("tokens", "Tokens", True), + ("token-coverage", "Token Coverage", True), + ("input", "Input", True), + ("output", "Output", True), + ("cache-read", "Cache Read", True), + ("cache-write", "Cache Write", True), + ("runtime", "Runtime", True), +) +LEADERBOARD_FROZEN_COLUMN_IDS = frozenset({"rank", "candidate"}) + + +class CodexParams(BaseModel): + """Codex-specific benchmark candidate parameters.""" + + model_config = ConfigDict(extra="forbid") + + intelligence: Literal["low", "medium", "high", "xhigh"] | None = None + speed: Literal["standard", "fast"] | None = None + config: dict[str, ConfigValue] = Field(default_factory=dict) + + +class ClaudeCodeParams(BaseModel): + """Claude Code-specific benchmark candidate parameters.""" + + model_config = ConfigDict(extra="forbid") + + permission_mode: Literal["acceptEdits", "bypassPermissions", "default", "delegate", "dontAsk", "plan"] | None = None + max_budget_usd: float | None = None + + +class CursorAgentParams(BaseModel): + """Cursor Agent-specific benchmark candidate parameters.""" + + model_config = ConfigDict(extra="forbid") + + mode: Literal["plan", "ask"] | None = None + sandbox: Literal["enabled", "disabled"] | None = None + + +CandidateParams = CodexParams | ClaudeCodeParams | CursorAgentParams + + +class CandidateConfig(BaseModel): + """Typed Python-config candidate definition.""" + + model_config = ConfigDict(extra="forbid") + + id: str | None = None + backend: AgentBackend + model: str | None = None + params: CandidateParams | None = None + + @model_validator(mode="before") + @classmethod + def _coerce_backend_params(cls, value: object) -> object: + if not isinstance(value, dict): + return value + raw = _object_dict(value) + backend = raw.get("backend") + params = raw.get("params") + if backend == "codex" and not isinstance(params, CodexParams): + raw["params"] = CodexParams.model_validate(params or {}) + elif backend == "claude-code" and not isinstance(params, ClaudeCodeParams): + raw["params"] = ClaudeCodeParams.model_validate(params or {}) + elif backend == "cursor-agent" and not isinstance(params, CursorAgentParams): + raw["params"] = CursorAgentParams.model_validate(params or {}) + return raw + + @model_validator(mode="after") + def _validate_backend_params(self) -> CandidateConfig: + expected_types: dict[str, type[BaseModel]] = { + "codex": CodexParams, + "claude-code": ClaudeCodeParams, + "cursor-agent": CursorAgentParams, + } + expected = expected_types[self.backend] + if self.params is not None and not isinstance(self.params, expected): + raise ValueError(f"{self.backend} candidates must use {expected.__name__}") + return self + + def to_candidate(self) -> Candidate: + params = self.params.model_dump(exclude_none=True) if self.params is not None else {} + return Candidate( + backend=self.backend, + model=self.model, + candidate_id=self.id or _candidate_id(backend=self.backend, model=self.model, params=params), + params=params, + ) + + +class MatrixDefaults(BaseModel): + """Optional operational defaults for a matrix config.""" + + model_config = ConfigDict(extra="forbid") + + timeout: int | None = None + skip_build: bool | None = None + allow_dirty: bool | None = None + parallel_candidates: int | None = None + nmp_base_url: str | None = None + anthropic_base_url: str | None = None + + +class AgentMatrixConfig(BaseModel): + """Python-config schema for an agent/model benchmark matrix.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal["1"] = "1" + manifest: str | None = None + tasks: list[str] = Field(default_factory=list) + all_tasks: bool = False + defaults: MatrixDefaults = Field(default_factory=MatrixDefaults) + candidates: list[CandidateConfig] + + @model_validator(mode="after") + def _validate_task_selection(self) -> AgentMatrixConfig: + selections = sum([self.manifest is not None, bool(self.tasks), self.all_tasks]) + if selections != 1: + raise ValueError("Matrix config must set exactly one of manifest, tasks, or all_tasks") + return self + + +@dataclass(frozen=True) +class Candidate: + """One backend/model candidate in the agent matrix.""" + + backend: str + model: str | None + candidate_id: str + params: dict[str, object] = dataclass_field(default_factory=dict) + + +@dataclass(frozen=True) +class MatrixRow: + """One normalized candidate/task result row.""" + + candidate_id: str + agent_backend: str + agent_model: str + candidate_params: dict[str, object] + task: str + status: str + passed: bool + reward: float + runtime_sec: float | None + total_tokens: int | None + prompt_tokens: int | None + completion_tokens: int | None + cache_read_tokens: int | None + cache_creation_tokens: int | None + token_metrics_status: str + verifier_scores: dict[str, object] | None + result_path: str | None + output_dir: str | None + failure_excerpt: str | None + provenance: dict[str, object] + + +@dataclass(frozen=True) +class CandidateSummary: + """Aggregate result for one candidate.""" + + candidate_id: str + agent_backend: str + agent_model: str + candidate_params: dict[str, object] + total_tasks: int + passed_tasks: int + pass_rate: float + total_tokens_sum: int | None + prompt_tokens_sum: int | None + completion_tokens_sum: int | None + cache_read_tokens_sum: int | None + cache_creation_tokens_sum: int | None + token_metrics_coverage: float + runtime_sec_sum: float | None + rank: int | None = None + + +def parse_candidate(spec: str) -> Candidate: + """Parse ``backend[:model]`` into a matrix candidate.""" + backend, separator, model = spec.partition(":") + backend = backend.strip() + model = model.strip() if separator else None + if not backend or backend not in SUPPORTED_BACKENDS: + raise ValueError(f"Unsupported agent backend {backend!r}; expected one of {sorted(SUPPORTED_BACKENDS)}") + if separator and not model: + raise ValueError(f"Agent candidate {spec!r} has an empty model") + candidate_id = _candidate_id(backend=backend, model=model, params={}) + return Candidate(backend=backend, model=model, candidate_id=candidate_id) + + +def parse_candidates(specs: list[str] | None, model_specs: list[str] | None = None) -> list[Candidate]: + """Parse backend/model candidate specs and reject duplicate ids. + + ``--agent backend[:model]`` remains the terse candidate form. ``--agent + backend`` can also be paired with repeated ``--agent-model backend:model`` + entries to express a grid of sub-models under a selected backend. + """ + if not specs and not model_specs: + candidates = [parse_candidate(spec) for spec in DEFAULT_BACKENDS] + _validate_unique_candidates(candidates) + return candidates + + candidates: list[Candidate] = [] + default_backends: list[str] = [] + model_backends: set[str] = set() + for spec in model_specs or []: + candidate = parse_candidate(spec) + if candidate.model is None: + raise ValueError(f"Agent model spec {spec!r} must use backend:model") + candidates.append(candidate) + model_backends.add(candidate.backend) + + for spec in specs or []: + candidate = parse_candidate(spec) + if candidate.model is None: + default_backends.append(candidate.backend) + else: + candidates.append(candidate) + model_backends.add(candidate.backend) + + for backend in default_backends: + if backend not in model_backends: + candidates.append(parse_candidate(backend)) + + _validate_unique_candidates(candidates) + return candidates + + +def _validate_unique_candidates(candidates: list[Candidate]) -> None: + seen: set[str] = set() + duplicates: list[str] = [] + for candidate in candidates: + if candidate.candidate_id in seen: + duplicates.append(candidate.candidate_id) + seen.add(candidate.candidate_id) + if duplicates: + raise ValueError(f"Duplicate candidate id(s): {', '.join(sorted(set(duplicates)))}") + + +def build_nat_runner_command( + *, + candidate: Candidate, + jobs_dir: Path, + manifest: Path | None, + tasks: list[str], + all_tasks: bool, + skip_build: bool, + allow_dirty: bool, + timeout: int, + codex_auth_json: Path | None, + nmp_base_url: str, + anthropic_base_url: str, + python_executable: str = sys.executable, +) -> list[str]: + """Build the subprocess command used to run one matrix candidate.""" + command = [ + python_executable, + str(NAT_RUNNER), + "--agent-backend", + candidate.backend, + "--candidate-id", + candidate.candidate_id, + "--jobs-dir", + str(jobs_dir), + "--timeout", + str(timeout), + "--nmp-base-url", + nmp_base_url, + ] + if candidate.model is not None: + command.extend(["--agent-model", candidate.model]) + if candidate.params: + command.extend(["--candidate-params", json.dumps(candidate.params, sort_keys=True)]) + if candidate.backend == "claude-code": + command.extend(["--anthropic-base-url", anthropic_base_url]) + if candidate.backend == "codex" and codex_auth_json is not None: + command.extend(["--codex-auth-json", str(codex_auth_json)]) + if skip_build: + command.append("--skip-build") + if allow_dirty: + command.append("--allow-dirty") + if all_tasks: + command.append("--all") + elif manifest is not None: + command.extend(["--manifest", str(manifest)]) + else: + command.extend(tasks) + return command + + +def resolve_selected_tasks(*, manifest: Path | None, tasks: list[str], all_tasks: bool) -> list[str]: + """Resolve the matrix task selection using nat_runner's task resolver.""" + if all_tasks: + return nat_runner.resolve_tasks([]) + if manifest is not None: + patterns = nat_runner._read_manifest(manifest) # noqa: SLF001 - share nat_runner manifest semantics. + return nat_runner.resolve_tasks(patterns) + return nat_runner.resolve_tasks(tasks) + + +def load_matrix_config(path: Path) -> AgentMatrixConfig: + """Load a Python matrix config file containing ``MATRIX``.""" + config_path = path.expanduser().resolve() + module = _load_python_module(config_path) + if not hasattr(module, "MATRIX"): + raise ValueError(f"Matrix config {config_path} must define MATRIX") + raw_config = module.MATRIX + if isinstance(raw_config, BaseModel): + raw_config = raw_config.model_dump() + return AgentMatrixConfig.model_validate(raw_config) + + +def run_candidates( + *, + candidates: list[Candidate], + run_dir: Path, + selected_tasks: list[str], + manifest: Path | None, + tasks: list[str], + all_tasks: bool, + skip_build: bool, + allow_dirty: bool, + timeout: int, + codex_auth_json: Path | None, + nmp_base_url: str, + anthropic_base_url: str, + parallel_candidates: int = 1, +) -> list[dict[str, object]]: + """Run all candidates and return subprocess metadata.""" + if parallel_candidates < 1: + raise ValueError("parallel_candidates must be >= 1") + + run_metadata: list[dict[str, object]] = [] + total_candidates = len(candidates) + if parallel_candidates == 1 or total_candidates <= 1: + for index, candidate in enumerate(candidates, start=1): + run_metadata.append( + _run_candidate( + index=index, + total_candidates=total_candidates, + candidate=candidate, + run_dir=run_dir, + selected_tasks=selected_tasks, + manifest=manifest, + tasks=tasks, + all_tasks=all_tasks, + skip_build=skip_build, + allow_dirty=allow_dirty, + timeout=timeout, + codex_auth_json=codex_auth_json, + nmp_base_url=nmp_base_url, + anthropic_base_url=anthropic_base_url, + ) + ) + return run_metadata + + worker_count = min(parallel_candidates, total_candidates) + _log(f"Running candidates with parallelism={worker_count}; each candidate still runs its tasks serially.") + metadata_by_index: dict[int, dict[str, object]] = {} + with ThreadPoolExecutor(max_workers=worker_count) as executor: + futures = [ + executor.submit( + _run_candidate, + index=index, + total_candidates=total_candidates, + candidate=candidate, + run_dir=run_dir, + selected_tasks=selected_tasks, + manifest=manifest, + tasks=tasks, + all_tasks=all_tasks, + skip_build=skip_build, + allow_dirty=allow_dirty, + timeout=timeout, + codex_auth_json=codex_auth_json, + nmp_base_url=nmp_base_url, + anthropic_base_url=anthropic_base_url, + ) + for index, candidate in enumerate(candidates, start=1) + ] + for future in as_completed(futures): + metadata = future.result() + index_value = metadata.get("candidate_index") + if isinstance(index_value, int): + metadata_by_index[index_value] = metadata + + return [metadata_by_index[index] for index in sorted(metadata_by_index)] + + +def _run_candidate( + *, + index: int, + total_candidates: int, + candidate: Candidate, + run_dir: Path, + selected_tasks: list[str], + manifest: Path | None, + tasks: list[str], + all_tasks: bool, + skip_build: bool, + allow_dirty: bool, + timeout: int, + codex_auth_json: Path | None, + nmp_base_url: str, + anthropic_base_url: str, +) -> dict[str, object]: + """Run one candidate subprocess and return normalized metadata.""" + candidate_dir = run_dir / candidate.candidate_id + candidate_dir.mkdir(parents=True, exist_ok=True) + command = build_nat_runner_command( + candidate=candidate, + jobs_dir=candidate_dir, + manifest=manifest, + tasks=tasks, + all_tasks=all_tasks, + skip_build=skip_build, + allow_dirty=allow_dirty, + timeout=timeout, + codex_auth_json=codex_auth_json, + nmp_base_url=nmp_base_url, + anthropic_base_url=anthropic_base_url, + ) + _log( + "Candidate " + f"{index}/{total_candidates} START {candidate.candidate_id} " + f"(backend={candidate.backend}, model={candidate.model or 'default'}, tasks={len(selected_tasks)})" + ) + _log(f"Artifacts: {candidate_dir}") + _log(f"Command: {' '.join(command)}") + started_at = datetime.now(UTC).isoformat() + started_monotonic = time.monotonic() + completed = subprocess.run(command, check=False) + finished_at = datetime.now(UTC).isoformat() + duration_sec = time.monotonic() - started_monotonic + _log( + "Candidate " + f"{index}/{total_candidates} DONE {candidate.candidate_id} " + f"returncode={completed.returncode} elapsed={_format_seconds(duration_sec)}" + ) + return { + "candidate_id": candidate.candidate_id, + "agent_backend": candidate.backend, + "agent_model": candidate.model or "default", + "candidate_params": candidate.params, + "candidate_index": index, + "jobs_dir": str(candidate_dir), + "returncode": completed.returncode, + "started_at": started_at, + "finished_at": finished_at, + "duration_sec": round(duration_sec, 3), + } + + +def build_matrix_summary( + *, + run_dir: Path, + candidates: list[Candidate], + tasks: list[str], + run_metadata: list[dict[str, object]] | None = None, + manifest: Path | None = None, +) -> dict[str, object]: + """Build the canonical matrix summary payload.""" + rows: list[MatrixRow] = [] + for candidate in candidates: + rows.extend(_load_candidate_rows(run_dir=run_dir, candidate=candidate, tasks=tasks)) + + summaries = _rank_candidate_summaries(_summarize_candidates(candidates=candidates, tasks=tasks, rows=rows)) + row_payloads = [asdict(row) for row in rows] + summary_payloads = [asdict(summary) for summary in summaries] + generated_at = datetime.now(UTC).isoformat() + winner = summary_payloads[0]["candidate_id"] if summary_payloads else None + + return { + "schema_version": "1.0", + "generated_at": generated_at, + "run_dir": str(run_dir), + "manifest": str(manifest) if manifest is not None else None, + "tasks": tasks, + "winner": winner, + "ranking": summary_payloads, + "candidate_groups": _candidate_groups(summary_payloads), + "rows": row_payloads, + "run_metadata": run_metadata or [], + } + + +def write_reports(summary: dict[str, object], output_dir: Path) -> dict[str, str]: + """Write JSON, Markdown, and static HTML reports.""" + output_dir.mkdir(parents=True, exist_ok=True) + paths = { + "json": output_dir / REPORT_JSON, + "markdown": output_dir / REPORT_MARKDOWN, + "html": output_dir / REPORT_HTML, + } + paths["json"].write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + paths["markdown"].write_text(render_markdown(summary), encoding="utf-8") + paths["html"].write_text(render_html(summary), encoding="utf-8") + return {name: str(path) for name, path in paths.items()} + + +def render_markdown(summary: dict[str, object]) -> str: + """Render a compact Markdown benchmark report.""" + ranking = _list_of_dicts(summary.get("ranking")) + candidate_groups = _dict_of_list_of_dicts(summary.get("candidate_groups")) + rows = _list_of_dicts(summary.get("rows")) + tasks = _string_list(summary.get("tasks")) + lines = [ + "# Agent Matrix Benchmark", + "", + f"- Generated: `{summary.get('generated_at')}`", + f"- Run dir: `{summary.get('run_dir')}`", + f"- Winner: `{summary.get('winner') or 'n/a'}`", + "", + "## Ranking", + "", + "| Rank | Candidate | Backend | Model | Pass Rate | Passed | Tokens | Token Coverage | Input | Output | Cache Read | Cache Write | Runtime |", + "|---:|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for candidate in ranking: + lines.append( + "| {rank} | `{candidate_id}` | `{agent_backend}` | `{agent_model}` | {pass_rate:.3f} | " + "{passed_tasks}/{total_tasks} | {tokens} | {token_coverage:.0%} | {input_tokens} | {output_tokens} | " + "{cache_read_tokens} | {cache_creation_tokens} | {runtime} |".format( + rank=candidate.get("rank"), + candidate_id=candidate.get("candidate_id"), + agent_backend=candidate.get("agent_backend"), + agent_model=candidate.get("agent_model"), + pass_rate=_float_value(candidate.get("pass_rate")), + passed_tasks=candidate.get("passed_tasks"), + total_tasks=candidate.get("total_tasks"), + tokens=_format_number(candidate.get("total_tokens_sum")), + token_coverage=_float_value(candidate.get("token_metrics_coverage")), + input_tokens=_format_number(candidate.get("prompt_tokens_sum")), + output_tokens=_format_number(candidate.get("completion_tokens_sum")), + cache_read_tokens=_format_number(candidate.get("cache_read_tokens_sum")), + cache_creation_tokens=_format_number(candidate.get("cache_creation_tokens_sum")), + runtime=_format_seconds(candidate.get("runtime_sec_sum")), + ) + ) + lines.extend( + [ + "", + "Token bucket note: `n/a` means the backend did not emit that token bucket; `0` means it emitted the bucket with value zero.", + ] + ) + + if candidate_groups: + lines.extend(["", "## Agent Model Groups", ""]) + for backend in sorted(candidate_groups): + models = ", ".join( + f"`{candidate.get('agent_model')}` ({_format_params(candidate.get('candidate_params'))})" + for candidate in candidate_groups[backend] + ) + lines.append(f"- `{backend}`: {models}") + + lines.extend(["", "## Task Matrix", "", _render_markdown_matrix(tasks=tasks, rows=rows)]) + failures = [row for row in rows if row.get("status") != "passed"] + if failures: + lines.extend(["", "## Failures", ""]) + for row in failures: + result_path = row.get("result_path") or "" + lines.append(f"- `{row.get('candidate_id')}` / `{row.get('task')}`: {row.get('status')} `{result_path}`") + return "\n".join(lines) + "\n" + + +def render_html(summary: dict[str, object]) -> str: + """Render a self-contained browser-viewable benchmark report.""" + ranking = _list_of_dicts(summary.get("ranking")) + candidate_groups = _dict_of_list_of_dicts(summary.get("candidate_groups")) + rows = _list_of_dicts(summary.get("rows")) + tasks = _string_list(summary.get("tasks")) + winner = str(summary.get("winner") or "n/a") + run_dir = str(summary.get("run_dir") or "") + + cards = "\n".join(_render_candidate_card(candidate) for candidate in ranking) + group_cards = _render_candidate_groups(candidate_groups) + leaderboard = _render_leaderboard(ranking) + task_matrix = _render_html_task_matrix(tasks=tasks, ranking=ranking, rows=rows) + provenance_badges = _render_provenance_badges(rows) + token_chart = _render_bar_chart(ranking, value_key="total_tokens_sum", label="Total Tokens", css_class="tokens") + runtime_chart = _render_bar_chart( + ranking, value_key="runtime_sec_sum", label="Runtime Seconds", css_class="runtime" + ) + failures = "\n".join( + _render_failure_detail(row, run_dir=Path(run_dir)) for row in rows if row.get("status") != "passed" + ) + if not failures: + failures = '

No failures recorded.

' + + payload = html.escape(json.dumps(summary, indent=2), quote=False) + return f""" + + + + + Agent Matrix Benchmark + + + +
+
+
+

Agent Matrix Benchmark

+
+ Generated: {html.escape(str(summary.get("generated_at") or ""))} + Run dir: {html.escape(run_dir)} + {provenance_badges} +
+
+
+ Overall winner + {html.escape(winner)} +
+
+
+
+
+
{cards}
+ +

Candidate Leaderboard

+ {leaderboard} + +

Task Matrix

+ {_render_status_legend()} + {task_matrix} + +

Agents And Models

+ {group_cards} + +

Token And Runtime Comparison

+
+
{token_chart}
+
{runtime_chart}
+
+ +

Failure Details

+ {failures} + +

Provenance

+
{_render_provenance(rows)}
+ +
+ Raw summary JSON + 
{payload}
+
+
+
+ + + +""" + + +def main() -> int: + """CLI entrypoint.""" + parser = argparse.ArgumentParser(description="Run an agentic-use agent matrix benchmark.") + parser.add_argument("--config", type=Path, help="Python matrix config file defining MATRIX.") + parser.add_argument("tasks", nargs="*", metavar="TASK_OR_GLOB", help="Task names or glob patterns.") + parser.add_argument("--all", action="store_true", help="Include all agentic-use tasks.") + parser.add_argument( + "--manifest", type=Path, help="Task manifest file. Relative paths resolve from tests/agentic-use/." + ) + parser.add_argument( + "--agent", + action="append", + help=( + "Candidate backend[:model]. May be repeated. Use backend-only with --agent-model " + "to run a model collection under one backend." + ), + ) + parser.add_argument( + "--agent-model", + action="append", + help="Model candidate as backend:model. May be repeated, for example --agent-model codex:gpt-5.1.", + ) + parser.add_argument( + "--jobs-dir", + type=Path, + default=nat_runner.DEFAULT_JOBS_DIR / "agent-matrix", + help="Root directory for matrix artifacts.", + ) + parser.add_argument("--run-id", help="Optional run id. Defaults to a UTC timestamp.") + parser.add_argument("--skip-build", action="store_true", help="Forward --skip-build to nat_runner.py.") + parser.add_argument("--allow-dirty", action="store_true", help="Forward --allow-dirty to nat_runner.py.") + parser.add_argument("--timeout", type=int, help="Agent timeout in seconds.") + parser.add_argument( + "--parallel-candidates", + type=int, + default=None, + help=( + "Number of candidate backends to run concurrently. Each candidate still runs its selected tasks serially. " + "Values greater than 1 require --skip-build to avoid concurrent Docker tag rebuilds." + ), + ) + parser.add_argument("--nmp-base-url") + parser.add_argument( + "--anthropic-base-url", + default=None, + ) + parser.add_argument("--codex-auth-json", type=Path, help="Explicit Codex auth.json path for the codex backend.") + args = parser.parse_args() + + if args.config is not None and ( + args.all or args.manifest is not None or args.tasks or args.agent or args.agent_model + ): + print( + "ERROR: --config is authoritative for tasks and candidates; do not combine it with " + "--all, --manifest, TASK_OR_GLOB, --agent, or --agent-model.", + file=sys.stderr, + ) + return 1 + if args.config is None: + if args.all and args.manifest is not None: + print("ERROR: --all and --manifest are mutually exclusive.", file=sys.stderr) + return 1 + if args.tasks and args.manifest is not None: + print("ERROR: Provide either TASK_OR_GLOB args or --manifest, not both.", file=sys.stderr) + return 1 + if args.tasks and args.all: + print("ERROR: TASK_OR_GLOB args and --all are mutually exclusive.", file=sys.stderr) + return 1 + if not args.all and args.manifest is None and not args.tasks: + print("ERROR: No tasks specified. Use --config, --manifest, --all, or TASK_OR_GLOB args.", file=sys.stderr) + return 1 + + try: + if args.config is not None: + config = load_matrix_config(args.config) + candidates = [candidate.to_candidate() for candidate in config.candidates] + manifest = Path(config.manifest) if config.manifest is not None else None + tasks = config.tasks + all_tasks = config.all_tasks + timeout = args.timeout or config.defaults.timeout or nat_runner.DEFAULT_TIMEOUT + skip_build = args.skip_build or bool(config.defaults.skip_build) + allow_dirty = args.allow_dirty or bool(config.defaults.allow_dirty) + parallel_candidates = ( + args.parallel_candidates + if args.parallel_candidates is not None + else config.defaults.parallel_candidates + ) + if parallel_candidates is None: + parallel_candidates = 1 + nmp_base_url = args.nmp_base_url or config.defaults.nmp_base_url or nat_runner.DEFAULT_LOCAL_NMP_BASE_URL + anthropic_base_url = ( + args.anthropic_base_url + or config.defaults.anthropic_base_url + or os.environ.get("ANTHROPIC_BASE_URL", "https://inference-api.nvidia.com") + ) + else: + candidates = parse_candidates(args.agent, args.agent_model) + manifest = args.manifest + tasks = args.tasks + all_tasks = args.all + timeout = args.timeout or nat_runner.DEFAULT_TIMEOUT + skip_build = args.skip_build + allow_dirty = args.allow_dirty + parallel_candidates = args.parallel_candidates if args.parallel_candidates is not None else 1 + nmp_base_url = args.nmp_base_url or nat_runner.DEFAULT_LOCAL_NMP_BASE_URL + anthropic_base_url = args.anthropic_base_url or os.environ.get( + "ANTHROPIC_BASE_URL", "https://inference-api.nvidia.com" + ) + selected_tasks = resolve_selected_tasks(manifest=manifest, tasks=tasks, all_tasks=all_tasks) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + if parallel_candidates < 1: + print("ERROR: --parallel-candidates must be >= 1.", file=sys.stderr) + return 1 + if parallel_candidates > 1 and not skip_build: + print( + "ERROR: --parallel-candidates > 1 requires --skip-build. " + "Build the selected task images once, then rerun the matrix with --skip-build.", + file=sys.stderr, + ) + return 1 + + run_id = args.run_id or datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + run_dir = args.jobs_dir.expanduser().resolve() / run_id + run_dir.mkdir(parents=True, exist_ok=True) + + _log(f"Run id: {run_id}") + _log(f"Run dir: {run_dir}") + _log(f"Selected tasks ({len(selected_tasks)}): {', '.join(selected_tasks)}") + _log( + "Candidates " + f"({len(candidates)}): " + + ", ".join( + f"{candidate.candidate_id}[{candidate.backend}:{candidate.model or 'default'}]" for candidate in candidates + ) + ) + matrix_started = time.monotonic() + run_metadata = run_candidates( + candidates=candidates, + run_dir=run_dir, + selected_tasks=selected_tasks, + manifest=manifest, + tasks=tasks, + all_tasks=all_tasks, + skip_build=skip_build, + allow_dirty=allow_dirty, + timeout=timeout, + codex_auth_json=args.codex_auth_json.expanduser().resolve() if args.codex_auth_json else None, + nmp_base_url=nmp_base_url, + anthropic_base_url=anthropic_base_url, + parallel_candidates=parallel_candidates, + ) + summary = build_matrix_summary( + run_dir=run_dir, + candidates=candidates, + tasks=selected_tasks, + run_metadata=run_metadata, + manifest=manifest, + ) + reports = write_reports(summary, run_dir) + _log(f"Matrix run complete elapsed={_format_seconds(time.monotonic() - matrix_started)}") + _log("Wrote reports:") + for name, path in reports.items(): + _log(f" {name}: {path}") + return 0 + + +def _log(message: str) -> None: + timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + print(f"[agent_matrix {timestamp}] {message}", flush=True) + + +def _load_python_module(path: Path) -> ModuleType: + module_name = f"_agent_matrix_config_{abs(hash(path))}" + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise ValueError(f"Could not load matrix config: {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +def _flatten_candidate_params(params: dict[str, object], *, prefix: str = "") -> dict[str, object]: + flattened: dict[str, object] = {} + for key, value in params.items(): + if value in (None, {}, []): + continue + name = f"{prefix}.{key}" if prefix else key + if isinstance(value, dict): + flattened.update(_flatten_candidate_params(_object_dict(value), prefix=name)) + elif isinstance(value, list): + flattened[name] = ".".join(str(entry) for entry in value) + else: + flattened[name] = value + return flattened + + +def _candidate_id(*, backend: str, model: str | None, params: dict[str, object]) -> str: + parts = [backend] + if model is not None: + parts.append(model) + for key, value in sorted(_flatten_candidate_params(params).items()): + parts.extend([key, str(value)]) + raw = "-".join(parts) + sanitized = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw).strip("-") + return sanitized or backend + + +def _load_candidate_rows(*, run_dir: Path, candidate: Candidate, tasks: list[str]) -> list[MatrixRow]: + candidate_dir = run_dir / candidate.candidate_id + payloads = _latest_results_by_task(candidate_dir) + rows: list[MatrixRow] = [] + for task in tasks: + payload = payloads.get(task) + if payload is None: + rows.append( + MatrixRow( + candidate_id=candidate.candidate_id, + agent_backend=candidate.backend, + agent_model=candidate.model or "default", + candidate_params=candidate.params, + task=task, + status="missing", + passed=False, + reward=0.0, + runtime_sec=None, + total_tokens=None, + prompt_tokens=None, + completion_tokens=None, + cache_read_tokens=None, + cache_creation_tokens=None, + token_metrics_status="unavailable", + verifier_scores=None, + result_path=None, + output_dir=None, + failure_excerpt="No result.json was produced for this task.", + provenance={}, + ) + ) + continue + rows.append(_row_from_payload(candidate=candidate, task=task, payload=payload)) + return rows + + +def _latest_results_by_task(candidate_dir: Path) -> dict[str, dict[str, object]]: + latest: dict[str, tuple[float, dict[str, object]]] = {} + if not candidate_dir.exists(): + return {} + for result_path in candidate_dir.rglob("result.json"): + try: + payload = json.loads(result_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + if not isinstance(payload, dict): + continue + task = payload.get("task") + if not isinstance(task, str): + continue + normalized = dict(payload) + normalized["_result_path"] = str(result_path) + mtime = result_path.stat().st_mtime + current = latest.get(task) + if current is None or mtime > current[0]: + latest[task] = (mtime, normalized) + return {task: payload for task, (_mtime, payload) in latest.items()} + + +def _row_from_payload(*, candidate: Candidate, task: str, payload: dict[str, object]) -> MatrixRow: + metric_dict = _object_dict(payload.get("metrics")) + output_dir = _string_value(payload.get("output_dir")) + result_path = _string_value(payload.get("_result_path")) + passed = payload.get("passed") is True or _float_value(payload.get("reward")) >= 1.0 + status = _status_from_payload(payload=payload, passed=passed) + return MatrixRow( + candidate_id=_string_value(payload.get("candidate_id")) or candidate.candidate_id, + agent_backend=_string_value(payload.get("agent_backend")) or candidate.backend, + agent_model=_string_value(payload.get("agent_model")) or candidate.model or "default", + candidate_params=_object_dict(payload.get("candidate_params")) or candidate.params, + task=task, + status=status, + passed=passed, + reward=_float_value(payload.get("reward")), + runtime_sec=_optional_float(payload.get("runtime_sec")), + total_tokens=_optional_int(metric_dict.get("total_tokens")), + prompt_tokens=_optional_int(metric_dict.get("prompt_tokens")), + completion_tokens=_optional_int(metric_dict.get("completion_tokens")), + cache_read_tokens=_optional_int(metric_dict.get("cache_read_tokens")), + cache_creation_tokens=_optional_int(metric_dict.get("cache_creation_tokens")), + token_metrics_status=_string_value(metric_dict.get("token_metrics_status")) or "unavailable", + verifier_scores=_object_dict(payload.get("verifier_scores")) or None, + result_path=result_path, + output_dir=output_dir, + failure_excerpt=None if passed else _failure_excerpt(output_dir), + provenance=_object_dict(payload.get("provenance")), + ) + + +def _summarize_candidates( + *, + candidates: list[Candidate], + tasks: list[str], + rows: list[MatrixRow], +) -> list[CandidateSummary]: + summaries: list[CandidateSummary] = [] + by_candidate = {candidate.candidate_id: [] for candidate in candidates} + for row in rows: + by_candidate.setdefault(row.candidate_id, []).append(row) + + total_tasks = len(tasks) + for candidate in candidates: + candidate_rows = by_candidate.get(candidate.candidate_id, []) + passed_tasks = sum(1 for row in candidate_rows if row.passed) + token_values = [row.total_tokens for row in candidate_rows if row.total_tokens is not None] + prompt_values = [row.prompt_tokens for row in candidate_rows if row.prompt_tokens is not None] + completion_values = [row.completion_tokens for row in candidate_rows if row.completion_tokens is not None] + cache_read_values = [row.cache_read_tokens for row in candidate_rows if row.cache_read_tokens is not None] + cache_creation_values = [ + row.cache_creation_tokens for row in candidate_rows if row.cache_creation_tokens is not None + ] + runtime_values = [row.runtime_sec for row in candidate_rows if row.runtime_sec is not None] + runtime_complete = len(runtime_values) == total_tasks + summaries.append( + CandidateSummary( + candidate_id=candidate.candidate_id, + agent_backend=candidate.backend, + agent_model=candidate.model or "default", + candidate_params=candidate.params, + total_tasks=total_tasks, + passed_tasks=passed_tasks, + pass_rate=(passed_tasks / total_tasks) if total_tasks else 0.0, + total_tokens_sum=sum(token_values) if token_values else None, + prompt_tokens_sum=sum(prompt_values) if prompt_values else None, + completion_tokens_sum=sum(completion_values) if completion_values else None, + cache_read_tokens_sum=sum(cache_read_values) if cache_read_values else None, + cache_creation_tokens_sum=sum(cache_creation_values) if cache_creation_values else None, + token_metrics_coverage=(len(token_values) / total_tasks) if total_tasks else 0.0, + runtime_sec_sum=sum(runtime_values) if runtime_complete else None, + ) + ) + return summaries + + +def _rank_candidate_summaries(summaries: list[CandidateSummary]) -> list[CandidateSummary]: + def sort_key(summary: CandidateSummary) -> tuple[float, int, float, float, str]: + token_complete = summary.token_metrics_coverage >= 1.0 + runtime_value = summary.runtime_sec_sum if summary.runtime_sec_sum is not None else float("inf") + token_value = summary.total_tokens_sum if summary.total_tokens_sum is not None else float("inf") + return ( + -summary.pass_rate, + 0 if token_complete else 1, + float(token_value), + float(runtime_value), + summary.candidate_id, + ) + + ranked = sorted(summaries, key=sort_key) + return [ + CandidateSummary( + candidate_id=summary.candidate_id, + agent_backend=summary.agent_backend, + agent_model=summary.agent_model, + candidate_params=summary.candidate_params, + total_tasks=summary.total_tasks, + passed_tasks=summary.passed_tasks, + pass_rate=summary.pass_rate, + total_tokens_sum=summary.total_tokens_sum, + prompt_tokens_sum=summary.prompt_tokens_sum, + completion_tokens_sum=summary.completion_tokens_sum, + cache_read_tokens_sum=summary.cache_read_tokens_sum, + cache_creation_tokens_sum=summary.cache_creation_tokens_sum, + token_metrics_coverage=summary.token_metrics_coverage, + runtime_sec_sum=summary.runtime_sec_sum, + rank=index, + ) + for index, summary in enumerate(ranked, start=1) + ] + + +def _status_from_payload(*, payload: dict[str, object], passed: bool) -> str: + if passed: + return "passed" + agent = _string_value(payload.get("agent")) + verify = _string_value(payload.get("verify")) + if agent and agent != "ok": + return "agent_failed" + if verify and verify != "ok": + return "verify_failed" + return "failed" + + +def _failure_excerpt(output_dir: str | None) -> str | None: + if output_dir is None: + return None + root = Path(output_dir) + for relative in ("verifier/test-stdout.txt", "agent/nat_agent.stderr", "agent/nat_agent.log"): + path = root / relative + if not path.is_file(): + continue + text = path.read_text(encoding="utf-8", errors="replace").strip() + if text: + return text[-1600:] + return None + + +def _render_markdown_matrix(*, tasks: list[str], rows: list[dict[str, object]]) -> str: + candidates = sorted({str(row.get("candidate_id")) for row in rows}) + by_key = {(str(row.get("task")), str(row.get("candidate_id"))): row for row in rows} + lines = [ + "| Task | " + " | ".join(f"`{candidate}`" for candidate in candidates) + " |", + "|---|" + "|".join("---" for _candidate in candidates) + "|", + ] + for task in tasks: + cells = [] + for candidate in candidates: + row = by_key.get((task, candidate)) + cells.append(_status_symbol(str(row.get("status"))) if row else "MISSING") + lines.append(f"| `{task}` | " + " | ".join(cells) + " |") + return "\n".join(lines) + + +def _render_candidate_card(candidate: dict[str, object]) -> str: + pass_rate = _float_value(candidate.get("pass_rate")) + return """
+
+

{candidate_id}

+
{backend} / {model}
+
+
+
{pass_rate:.0%}
+
{passed}/{total} passed
+
+
+
""".format( + candidate_id=html.escape(str(candidate.get("candidate_id"))), + backend=html.escape(str(candidate.get("agent_backend"))), + model=html.escape(str(candidate.get("agent_model"))), + pass_rate=pass_rate, + pass_width=max(0.0, min(pass_rate * 100, 100.0)), + passed=html.escape(str(candidate.get("passed_tasks"))), + total=html.escape(str(candidate.get("total_tasks"))), + ) + + +def _render_leaderboard(candidates: list[dict[str, object]]) -> str: + frozen_headers = "".join( + _render_leaderboard_header(column_id, label, visible, frozen=True) + for column_id, label, visible in LEADERBOARD_COLUMNS + if column_id in LEADERBOARD_FROZEN_COLUMN_IDS + ) + scroll_headers = "".join( + _render_leaderboard_header(column_id, label, visible, frozen=False) + for column_id, label, visible in LEADERBOARD_COLUMNS + if column_id not in LEADERBOARD_FROZEN_COLUMN_IDS + ) + frozen_rows = "\n".join(_render_frozen_ranking_row(candidate) for candidate in candidates) + scroll_rows = "\n".join(_render_scroll_ranking_row(candidate) for candidate in candidates) + toggles = "".join( + """""".format( + column_id=html.escape(column_id), + checked="checked" if visible else "", + label=html.escape(label), + ) + for column_id, label, visible in LEADERBOARD_COLUMNS + if column_id not in LEADERBOARD_FROZEN_COLUMN_IDS + ) + return """
+
+

+ Scroll horizontally for wide comparisons. Drag header dividers to resize columns. + Token cells use n/a when the backend did not emit that bucket; 0 means the bucket was emitted with value zero. +

+
+ Columns +
{toggles}
+
+
+
+
+ + + + + + {frozen_headers} + {frozen_rows} +
+
+
+ + + + + + + + + + + + + + + + {scroll_headers} + {scroll_rows} +
+
+
+
""".format( + toggles=toggles, + frozen_headers=frozen_headers, + frozen_rows=frozen_rows, + scroll_headers=scroll_headers, + scroll_rows=scroll_rows, + ) + + +def _render_leaderboard_header(column_id: str, label: str, visible: bool, *, frozen: bool) -> str: + classes: list[str] = [] + min_width = 96 + if column_id == "rank": + classes.append("numeric") + min_width = 72 + elif column_id == "candidate": + classes.append("resizable") + min_width = 180 + elif column_id in {"backend", "model", "params"}: + classes.append("resizable") + min_width = 120 if column_id != "params" else 160 + elif column_id in {"tokens", "input", "output", "cache-read", "cache-write"}: + classes.extend(["numeric", "resizable"]) + min_width = 100 + elif column_id in {"pass-rate", "passed", "runtime"}: + classes.append("numeric") + class_attr = " ".join(classes) + hidden_class = "" if visible else " is-hidden" + handle = '' if "resizable" in classes else "" + data_frozen = "true" if frozen else "false" + return ( + '' + "{label}{handle}" + ).format( + column_id=html.escape(column_id), + data_frozen=data_frozen, + min_width=min_width, + classes=html.escape(class_attr), + hidden_class=hidden_class, + label=html.escape(label), + handle=handle, + ) + + +def _render_frozen_ranking_row(candidate: dict[str, object]) -> str: + return """ + {rank} + {candidate_id} +""".format( + rank=html.escape(str(candidate.get("rank"))), + candidate_id=html.escape(str(candidate.get("candidate_id"))), + ) + + +def _render_scroll_ranking_row(candidate: dict[str, object]) -> str: + return """ + {backend} + {model} +
{params}
+ {pass_rate:.3f} + {passed}/{total} + {tokens} + {token_coverage} + {input_tokens} + {output_tokens} + {cache_read_tokens} + {cache_creation_tokens} + {runtime} +""".format( + backend=html.escape(str(candidate.get("agent_backend"))), + model=html.escape(str(candidate.get("agent_model"))), + params=_render_param_chips(candidate.get("candidate_params")), + pass_rate=_float_value(candidate.get("pass_rate")), + passed=html.escape(str(candidate.get("passed_tasks"))), + total=html.escape(str(candidate.get("total_tasks"))), + tokens=_render_metric_cell(candidate.get("total_tokens_sum")), + token_coverage=html.escape(f"{_float_value(candidate.get('token_metrics_coverage')):.0%}"), + input_tokens=_render_metric_cell(candidate.get("prompt_tokens_sum")), + output_tokens=_render_metric_cell(candidate.get("completion_tokens_sum")), + cache_read_tokens=_render_metric_cell(candidate.get("cache_read_tokens_sum")), + cache_creation_tokens=_render_metric_cell(candidate.get("cache_creation_tokens_sum")), + runtime=html.escape(_format_seconds(candidate.get("runtime_sec_sum"))), + ) + + +def _render_candidate_groups(candidate_groups: dict[str, list[dict[str, object]]]) -> str: + if not candidate_groups: + return '

No candidate groups recorded.

' + sections = ['
'] + for backend in sorted(candidate_groups): + candidates = candidate_groups[backend] + rows = [] + for candidate in candidates: + rows.append( + """
+
{model}
+
{candidate}
+
{params}
+
{passed}/{total} ({pass_rate:.0%})
+
""".format( + model=html.escape(str(candidate.get("agent_model"))), + candidate=html.escape(str(candidate.get("candidate_id"))), + params=_render_param_chips(candidate.get("candidate_params")), + pass_rate=_float_value(candidate.get("pass_rate")), + passed=html.escape(str(candidate.get("passed_tasks"))), + total=html.escape(str(candidate.get("total_tasks"))), + ) + ) + sections.append( + """
+

{backend}

+ {rows} +
""".format(backend=html.escape(backend), rows="".join(rows)) + ) + sections.append("
") + return "\n".join(sections) + + +def _render_html_task_matrix( + *, tasks: list[str], ranking: list[dict[str, object]], rows: list[dict[str, object]] +) -> str: + candidates = [str(candidate.get("candidate_id")) for candidate in ranking] + by_key = {(str(row.get("task")), str(row.get("candidate_id"))): row for row in rows} + head = "".join(f"{html.escape(candidate)}" for candidate in candidates) + body_lines = [] + for task in tasks: + cells = [] + for candidate in candidates: + row = by_key.get((task, candidate)) + cells.append(_render_task_result_cell(row)) + body_lines.append(f"{html.escape(task)}{''.join(cells)}") + return ( + '
' + f'{head}' + f"{''.join(body_lines)}
Task
" + ) + + +def _render_task_result_cell(row: dict[str, object] | None) -> str: + status = str(row.get("status")) if row else "missing" + status_pill = '{symbol}'.format( + status=html.escape(status), + description=html.escape(_status_description(status)), + symbol=html.escape(_status_symbol(status)), + ) + return ( + '' + f'
{status_pill}
' + f"{_render_token_bucket_cell(row)}" + "" + ) + + +def _render_token_bucket_cell(row: dict[str, object] | None) -> str: + if row is None: + return 'missing' + if row.get("token_metrics_status") != "available": + return 'n/a' + bucket_labels = [ + ("Total", row.get("total_tokens")), + ("Input", row.get("prompt_tokens")), + ("Output", row.get("completion_tokens")), + ("Cache Read", row.get("cache_read_tokens")), + ("Cache Write", row.get("cache_creation_tokens")), + ] + items = "".join( + "
{label}
{value}
".format( + label=html.escape(label), + value=_render_metric_cell(value), + ) + for label, value in bucket_labels + ) + return f'
{items}
' + + +def _render_status_legend() -> str: + entries = [ + ( + "passed", + "PASS", + "Benchmark success", + "The agent ran, verification ran, and the deterministic Evaluator SDK checks passed.", + ), + ( + "verify_failed", + "VERIFY", + "Verifier failed", + "The agent completed, but the task-specific verifier rejected its output or artifacts.", + ), + ( + "agent_failed", + "AGENT", + "Agent failed", + "The agent phase failed before the task could be successfully verified.", + ), + ( + "failed", + "FAIL", + "Failed", + "The run failed without a more specific agent or verifier status.", + ), + ( + "missing", + "MISSING", + "No result", + "No result.json was produced for this candidate/task pair.", + ), + ] + return ( + '
Status Legend
' + + "".join( + """
+ {pill} + {title}{description} +
""".format( + css_class=html.escape(css_class), + pill=html.escape(pill), + title=html.escape(title), + description=html.escape(description), + ) + for css_class, pill, title, description in entries + ) + + "
" + ) + + +def _status_description(status: str) -> str: + return { + "passed": "The agent ran, verification ran, and the deterministic Evaluator SDK checks passed.", + "verify_failed": "The agent completed, but the task-specific verifier rejected its output or artifacts.", + "agent_failed": "The agent phase failed before the task could be successfully verified.", + "failed": "The run failed without a more specific agent or verifier status.", + "missing": "No result.json was produced for this candidate/task pair.", + }.get(status, status) + + +def _render_bar_chart(candidates: list[dict[str, object]], *, value_key: str, label: str, css_class: str) -> str: + values = [_optional_float(candidate.get(value_key)) for candidate in candidates] + max_value = max([value for value in values if value is not None], default=0.0) + rows = [f"

{html.escape(label)}

"] + for candidate, value in zip(candidates, values, strict=True): + width = 0.0 if not value or max_value <= 0 else (value / max_value) * 100 + rows.append( + """
+ {candidate} + + {value} +
""".format( + candidate=html.escape(str(candidate.get("candidate_id"))), + css_class=html.escape(css_class), + width=width, + value=html.escape(_format_metric_number(value)), + ) + ) + return "\n".join(rows) + + +def _render_failure_detail(row: dict[str, object], *, run_dir: Path) -> str: + output_dir = str(row.get("output_dir") or "") + href = _artifact_href(output_dir, run_dir=run_dir) + excerpt = html.escape(str(row.get("failure_excerpt") or "No failure excerpt available.")) + scores_payload = _object_dict(row.get("verifier_scores")) + diagnosis = _render_failure_diagnosis(row=row, scores_payload=scores_payload) + metric_outcome = _render_metric_outcome_summary(scores_payload) + scores = _render_verifier_scores(scores_payload) + status = str(row.get("status") or "failed") + return f"""
+
+

{html.escape(str(row.get("candidate_id")))} / {html.escape(str(row.get("task")))}

+
{html.escape(_status_symbol(status))} artifact directory
+
+ {diagnosis} + {metric_outcome} + {scores} +
+ Raw verifier log + 
{excerpt}
+
+
""" + + +def _render_verifier_scores(value: object) -> str: + scores_payload = _object_dict(value) + aggregate_scores = scores_payload.get("aggregate_scores") + if not isinstance(aggregate_scores, list) or not aggregate_scores: + return '

No structured Evaluator SDK scores were captured for this run.

' + + score_dicts = [_object_dict(score) for score in aggregate_scores if isinstance(score, dict)] + chips = [_render_score_chip(score) for score in score_dicts] + failed_chips = [_render_score_chip(score) for score in score_dicts if _is_failed_score(score)] + if not chips: + return '

No structured Evaluator SDK scores were captured for this run.

' + failed_section = "" + if failed_chips: + failed_section = ( + "
Failed or blocking metrics" + '
' + "".join(failed_chips) + "
" + ) + return ( + failed_section + + '
All metric scores
' + + "".join(chips) + + "
" + ) + + +def _render_failure_diagnosis(*, row: dict[str, object], scores_payload: dict[str, object]) -> str: + status = str(row.get("status") or "failed") + title = { + "verify_failed": "Verifier rejected the completed agent attempt.", + "agent_failed": "Agent execution failed before successful verification.", + "failed": "Run failed.", + "missing": "No result was produced.", + }.get(status, "Run did not pass.") + reason_parts = [] + task_success = _aggregate_score_mean(scores_payload, "task_success") + output_schema_valid = _aggregate_score_mean(scores_payload, "output_schema_valid") + verification_score = _aggregate_score_mean(scores_payload, "verification_score") + if task_success == 0.0: + reason_parts.append("Task success is false.") + if output_schema_valid == 0.0: + reason_parts.append("Output schema/contract is invalid.") + if verification_score is not None: + reason_parts.append(f"Verification score: {verification_score:.3f}.") + if not reason_parts: + reason_parts.append(_status_description(status)) + return '
{title}{reason}
'.format( + title=html.escape(title), + reason=html.escape(" ".join(reason_parts)), + ) + + +def _render_metric_outcome_summary(scores_payload: dict[str, object]) -> str: + task_success = _aggregate_score_mean(scores_payload, "task_success") + verification_score = _aggregate_score_mean(scores_payload, "verification_score") + output_schema_valid = _aggregate_score_mean(scores_payload, "output_schema_valid") + if task_success is None and verification_score is None and output_schema_valid is None: + return "" + cards = [ + _render_check_card("Task Success", _format_score_pass(task_success)), + _render_check_card( + "Verification Score", + "n/a" if verification_score is None else f"{verification_score:.3f}", + ), + _render_check_card("Output Contract", _format_score_pass(output_schema_valid)), + ] + return '
' + "".join(cards) + "
" + + +def _render_check_card(label: str, value: str) -> str: + return '
{label}{value}
'.format( + label=html.escape(label), + value=html.escape(value), + ) + + +def _render_score_chip(score: dict[str, object]) -> str: + name = str(score.get("name") or "unknown") + mean = _optional_float(score.get("mean")) + if mean is None: + css_class = "neutral" + label = "n/a" + else: + css_class = _score_chip_class(name, mean) + label = f"{mean:.3f}" + short_name = name.split(".", maxsplit=1)[-1] + return '{name}{value}'.format( + css_class=css_class, + name=html.escape(_humanize_metric_name(short_name)), + value=html.escape(label), + ) + + +def _is_failed_score(score: dict[str, object]) -> bool: + name = str(score.get("name") or "unknown") + mean = _optional_float(score.get("mean")) + return mean is not None and _score_chip_class(name, mean) == "bad" + + +def _score_chip_class(name: str, mean: float) -> str: + short_name = name.split(".", maxsplit=1)[-1] + if short_name in {"surface_violation_count", "legacy_surface_hit_count", "failed_command_count"}: + return "good" if mean == 0.0 else "bad" + if short_name in {"tool_call_count", "recovery_event_count"}: + return "neutral" + return "good" if mean >= 1.0 else "bad" + + +def _format_boolish(value: object) -> str: + if value is True: + return "Pass" + if value is False: + return "Fail" + if value is None: + return "n/a" + return str(value) + + +def _format_score_pass(value: float | None) -> str: + if value is None: + return "n/a" + if value >= 1.0: + return "Pass" + if value <= 0.0: + return "Fail" + return f"{value:.3f}" + + +def _aggregate_score_mean(scores_payload: dict[str, object], score_name: str) -> float | None: + aggregate_scores = scores_payload.get("aggregate_scores") + if not isinstance(aggregate_scores, list): + return None + for score in aggregate_scores: + score_dict = _object_dict(score) + name = str(score_dict.get("name") or "") + if name.endswith(f".{score_name}"): + return _optional_float(score_dict.get("mean")) + return None + + +def _humanize_metric_name(name: str) -> str: + return name.replace("_", " ").replace(".", " / ").title() + + +def _render_provenance(rows: list[dict[str, object]]) -> str: + observed: dict[str, set[str]] = {} + for row in rows: + provenance = row.get("provenance") + if not isinstance(provenance, dict): + continue + for key, value in provenance.items(): + if value is None: + continue + observed.setdefault(str(key), set()).add(str(value)) + if not observed: + return '

No provenance recorded.

' + lines = [""] + for key in sorted(observed): + values = ", ".join(sorted(observed[key])) + lines.append(f"") + lines.append("
FieldObserved Values
{html.escape(key)}{html.escape(values)}
") + return "\n".join(lines) + + +def _render_provenance_badges(rows: list[dict[str, object]]) -> str: + for row in rows: + provenance = _object_dict(row.get("provenance")) + if not provenance: + continue + if _is_truthy_provenance_value(provenance.get("commit_dirty")): + return ( + '' + "dirty worktree" + ) + return "" + + +def _is_truthy_provenance_value(value: object) -> bool: + if value is True: + return True + if isinstance(value, str): + return value.lower() == "true" + return False + + +def _artifact_href(output_dir: str, *, run_dir: Path) -> str: + if not output_dir: + return "#" + try: + return os.path.relpath(output_dir, run_dir) + except ValueError: + return output_dir + + +def _status_symbol(status: str) -> str: + return { + "passed": "PASS", + "failed": "FAIL", + "agent_failed": "AGENT", + "verify_failed": "VERIFY", + "missing": "MISSING", + }.get(status, status.upper()) + + +def _list_of_dicts(value: object) -> list[dict[str, object]]: + if not isinstance(value, list): + return [] + return [_object_dict(entry) for entry in value if isinstance(entry, dict)] + + +def _dict_of_list_of_dicts(value: object) -> dict[str, list[dict[str, object]]]: + if not isinstance(value, dict): + return {} + groups: dict[str, list[dict[str, object]]] = {} + for key, entries in value.items(): + if isinstance(entries, list): + groups[str(key)] = [_object_dict(entry) for entry in entries if isinstance(entry, dict)] + return groups + + +def _object_dict(value: object) -> dict[str, object]: + if not isinstance(value, dict): + return {} + return {str(key): item for key, item in value.items()} + + +def _string_list(value: object) -> list[str]: + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, str)] + + +def _candidate_groups(candidates: list[dict[str, object]]) -> dict[str, list[dict[str, object]]]: + groups: dict[str, list[dict[str, object]]] = {} + for candidate in candidates: + backend = str(candidate.get("agent_backend") or "unknown") + groups.setdefault(backend, []).append(candidate) + return groups + + +def _string_value(value: object) -> str | None: + return value if isinstance(value, str) else None + + +def _optional_int(value: object) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def _optional_float(value: object) -> float | None: + if isinstance(value, bool): + return None + if isinstance(value, int | float): + return float(value) + return None + + +def _float_value(value: object) -> float: + if isinstance(value, bool): + return 0.0 + if isinstance(value, int | float): + return float(value) + return 0.0 + + +def _format_number(value: object) -> str: + numeric = _optional_float(value) + if numeric is None: + return "-" + numeric = float(numeric) + if numeric >= 1_000_000: + return f"{numeric / 1_000_000:.1f}M" + if numeric >= 1_000: + return f"{numeric / 1_000:.1f}k" + if numeric.is_integer(): + return str(int(numeric)) + return f"{numeric:.1f}" + + +def _format_metric_number(value: object) -> str: + return "n/a" if _optional_float(value) is None else _format_number(value) + + +def _render_metric_cell(value: object) -> str: + if _optional_float(value) is None: + return 'n/a' + formatted = _format_number(value) + if formatted == "0": + return '0' + return html.escape(formatted) + + +def _format_seconds(value: object) -> str: + numeric = _optional_float(value) + if numeric is None: + return "-" + minutes = int(numeric) // 60 + seconds = int(numeric) % 60 + return f"{minutes}:{seconds:02d}" + + +def _format_params(value: object) -> str: + params = _object_dict(value) + flattened = _flatten_candidate_params(params) + if not flattened: + return "-" + return ", ".join(f"{key}={flattened[key]}" for key in sorted(flattened)) + + +def _render_param_chips(value: object) -> str: + params = _object_dict(value) + flattened = _flatten_candidate_params(params) + if not flattened: + return 'default' + return "".join( + f'{html.escape(str(key))}={html.escape(str(flattened[key]))}' + for key in sorted(flattened) + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/agentic-use/evaluator-standalone-sdk-agent-target/environment/Dockerfile b/tests/agentic-use/evaluator-standalone-sdk-agent-target/environment/Dockerfile new file mode 100644 index 00000000000..4fab70170a3 --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-agent-target/environment/Dockerfile @@ -0,0 +1 @@ +FROM nmp-agentic-base:latest diff --git a/tests/agentic-use/evaluator-standalone-sdk-agent-target/instruction.md b/tests/agentic-use/evaluator-standalone-sdk-agent-target/instruction.md new file mode 100644 index 00000000000..81293e204ab --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-agent-target/instruction.md @@ -0,0 +1,16 @@ +# Configure An Agent Target + +Use the local standalone Evaluator SDK to configure a mock agent endpoint as the evaluated target and run a one-row exact-match evaluation. + +Use only `packages/nemo_evaluator_sdk` and SDK-level APIs in your answer. Do not propose the `nemo` CLI, plugin SDK APIs, or any `services/*` implementation path. + +Your final answer must include: + +- The package/path `packages/nemo_evaluator_sdk`. +- The import or API symbols `Evaluator`, `run_sync`, `Agent`, `AgentFormat.GENERIC`, and `ExactMatchMetric`, including `Agent` from `nemo_evaluator_sdk.values.agents`. +- A directly runnable Python snippet that starts a mock HTTP endpoint returning `{"answer": "4", "trajectory": {"steps": [{"tool_calls": [{"name": "calculator"}]}]}}`. +- A generic `Agent` target configured with `body`, `response_path="$.answer"`, and `trajectory_path="$.trajectory"`. +- A dataset row asking `What is 2+2?` with expected answer `4`. +- A call to `Evaluator().run_sync(...)` using the `Agent` target. +- Output that includes a printed JSON object with keys `answer`, `exact_match`, `trajectory_tool_calls`, `candidate_agent_runtime`, and `candidate_agent_model`. +- In that printed JSON object, the extracted answer must be `4`, the exact-match score must be `1.0`, the trajectory must include the `calculator` tool call, and the metadata must identify the candidate agent runtime and model. diff --git a/tests/agentic-use/evaluator-standalone-sdk-agent-target/task.toml b/tests/agentic-use/evaluator-standalone-sdk-agent-target/task.toml new file mode 100644 index 00000000000..0aa92802918 --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-agent-target/task.toml @@ -0,0 +1,48 @@ +version = "1.0" + +[metadata] +author_name = "NMP Team" +difficulty = "medium" +category = "evaluator-agent-benchmark" +suite_id = "evaluator_agent_benchmark" +suite_version = "v0" +tags = ["evaluator", "agent-benchmark", "agent-target-configuration", "standalone-sdk"] + +[verifier] +timeout_sec = 60.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 +storage_mb = 10240 +gpus = 0 +allow_internet = false + +[evaluator.surface] +constraint = "standalone_sdk" +allowed = ["standalone_sdk"] +forbidden = ["cli", "plugin_sdk", "legacy_service"] + +[evaluator.expected] +required_terms = [ + "packages/nemo_evaluator_sdk", + "Evaluator", + "run_sync", + "Agent", + "nemo_evaluator_sdk.values.agents", + "AgentFormat.GENERIC", + "ExactMatchMetric", + "response_path", + "$.answer", + "trajectory_path", + "$.trajectory", + "candidate_agent_runtime", + "candidate_agent_model", +] + +[evaluator] +forbidden_patterns = ["services/", "nemo evaluation", "plugin SDK"] diff --git a/tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/task_metrics.py b/tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/task_metrics.py new file mode 100644 index 00000000000..93ddc882e3f --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/task_metrics.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Task-specific metrics for the standalone SDK agent-target task.""" + +import ast +import asyncio +import subprocess +from pathlib import Path + +from evaluator_agent_eval.artifacts import AgentArtifacts +from evaluator_agent_eval.task_config import load_agentic_use_task_config +from evaluator_agent_eval.task_metric_utils import ( + contains_all, + extract_fenced_python_code, + extract_json_object_from_stdout, + find_workspace_python_code, + object_dict, + run_python_code, + score_checks, +) +from nemo_evaluator_sdk.values.results import MetricResult, MetricScore + + +class AgentTargetConfigurationMetric: + """Verify the candidate configured an Evaluator SDK Agent target.""" + + @property + def type(self) -> str: + return "agent_eval/agent_target_configuration" + + def score_names(self) -> list[str]: + return [ + "task_success", + "verification_score", + "output_schema_valid", + "code_block_present", + "code_ran", + "required_terms_present", + "agent_target_terms_present", + "agent_target_used", + "extracted_answer", + "score_correct", + "trajectory_captured", + "candidate_metadata_present", + ] + + async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + final_text = str(item.get("output_text", "")) + artifacts = _artifacts_from_item(item) + code = _extract_python_code(final_text) or _extract_workspace_python_code(artifacts) + process = await asyncio.to_thread(_run_candidate_code, code) if code else None + stdout = process.stdout if process is not None else "" + output_payload = _extract_output_payload(stdout) + + text_to_check = f"{final_text}\n{code or ''}" + required_terms_present = contains_all(text_to_check, _required_terms(item)) + code_ran = process is not None and process.returncode == 0 + extracted_answer = output_payload.get("answer") == "4" if output_payload else False + score_correct = output_payload.get("exact_match") == 1.0 if output_payload else False + trajectory_tool_calls = output_payload.get("trajectory_tool_calls") if output_payload else None + trajectory_captured = "calculator" in _tool_call_names(trajectory_tool_calls) + agent_target_used = _uses_agent_as_run_sync_target(code) if code else False + candidate_metadata_present = bool( + output_payload + and output_payload.get("candidate_agent_runtime") + and output_payload.get("candidate_agent_model") + ) + agent_target_terms_present = all( + term in text_to_check + for term in ( + "Agent", + "AgentFormat.GENERIC", + "body", + "response_path", + "$.answer", + "trajectory_path", + "$.trajectory", + ) + ) + output_schema_valid = bool( + code and code_ran and extracted_answer and score_correct and trajectory_captured and agent_target_used + ) + task_success = bool( + item.get("final_answer_extracted") is True + and required_terms_present + and agent_target_used + and candidate_metadata_present + and output_schema_valid + ) + checks = [ + item.get("final_answer_extracted") is True, + required_terms_present, + code is not None, + code_ran, + agent_target_used, + extracted_answer, + score_correct, + trajectory_captured, + candidate_metadata_present, + ] + + return MetricResult( + scores=[ + MetricScore(name="task_success", value=float(task_success)), + MetricScore(name="verification_score", value=score_checks(checks)), + MetricScore(name="output_schema_valid", value=float(output_schema_valid)), + MetricScore(name="code_block_present", value=float(code is not None)), + MetricScore(name="code_ran", value=float(code_ran)), + MetricScore(name="required_terms_present", value=float(required_terms_present)), + MetricScore(name="agent_target_terms_present", value=float(agent_target_terms_present)), + MetricScore(name="agent_target_used", value=float(agent_target_used)), + MetricScore(name="extracted_answer", value=float(extracted_answer)), + MetricScore(name="score_correct", value=float(score_correct)), + MetricScore(name="trajectory_captured", value=float(trajectory_captured)), + MetricScore(name="candidate_metadata_present", value=float(candidate_metadata_present)), + ] + ) + + +def _extract_python_code(text: str) -> str | None: + return extract_fenced_python_code(text, predicate=_looks_like_agent_target_code) + + +def _artifacts_from_item(item: dict) -> AgentArtifacts: + agent_log_dir = item.get("agent_log_dir") + if not isinstance(agent_log_dir, str): + raise ValueError("agent_log_dir is required for agent-target metric") + workspace_dir = item.get("workspace_dir") + return AgentArtifacts.from_dir( + agent_log_dir, workspace_dir=workspace_dir if isinstance(workspace_dir, str) else None + ) + + +def _required_terms(item: dict) -> list[str]: + task_dir = item.get("task_dir") + if not isinstance(task_dir, str): + return [] + return load_agentic_use_task_config(Path(task_dir)).evaluator.expected.required_terms + + +def _extract_workspace_python_code(artifacts: AgentArtifacts) -> str | None: + return find_workspace_python_code( + artifacts, + preferred_names=["agent_target_exact_match.py", "solution.py"], + predicate=_looks_like_agent_target_code, + ) + + +def _run_candidate_code(code: str) -> subprocess.CompletedProcess[str]: + return run_python_code( + code, + filename="candidate_agent_target.py", + timeout=30, + timeout_stderr="candidate agent-target code timed out", + ) + + +def _looks_like_agent_target_code(code: str) -> bool: + return "Evaluator" in code and "Agent" in code and "run_sync" in code + + +def _uses_agent_as_run_sync_target(code: str) -> bool: + try: + tree = ast.parse(code) + except SyntaxError: + return False + + agent_names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Call) and _call_name(node.value) == "Agent": + if not _agent_call_has_required_paths(node.value): + continue + for target in node.targets: + if isinstance(target, ast.Name): + agent_names.add(target.id) + + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if _call_name(node) != "run_sync": + continue + for keyword in node.keywords: + if keyword.arg != "target": + continue + if isinstance(keyword.value, ast.Name) and keyword.value.id in agent_names: + return True + if isinstance(keyword.value, ast.Call) and _call_name(keyword.value) == "Agent": + return _agent_call_has_required_paths(keyword.value) + return False + + +def _agent_call_has_required_paths(call: ast.Call) -> bool: + keyword_values = { + keyword.arg: keyword.value.value + for keyword in call.keywords + if keyword.arg is not None and isinstance(keyword.value, ast.Constant) + } + return keyword_values.get("response_path") == "$.answer" and keyword_values.get("trajectory_path") == "$.trajectory" + + +def _call_name(call: ast.Call) -> str: + if isinstance(call.func, ast.Name): + return call.func.id + if isinstance(call.func, ast.Attribute): + return call.func.attr + return "" + + +def _extract_output_payload(stdout: str) -> dict[str, object] | None: + return extract_json_object_from_stdout(stdout) + + +def _tool_call_names(value: object) -> list[str]: + if not isinstance(value, list): + return [] + names: list[str] = [] + for entry in value: + if isinstance(entry, str): + names.append(entry) + elif isinstance(entry, dict): + name = object_dict(entry).get("name") + if isinstance(name, str): + names.append(name) + return names diff --git a/tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/test.sh b/tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/test.sh new file mode 100755 index 00000000000..e0ad48afe46 --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/bash +source /app/tests/agentic-use/shared/verify-tests.sh diff --git a/tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/test_outputs.py b/tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/test_outputs.py new file mode 100644 index 00000000000..b5fb2d19e21 --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-agent-target/tests/test_outputs.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Task-local verifier trampoline for the shared Evaluator benchmark verifier.""" + +from evaluator_agent_eval.pytest_verifier import verify_agent_attempt_scores_with_evaluator_sdk +from task_metrics import AgentTargetConfigurationMetric + + +def test_agent_attempt_scores_with_evaluator_sdk(): + verify_agent_attempt_scores_with_evaluator_sdk(task_specific_metrics=[AgentTargetConfigurationMetric()]) diff --git a/tests/agentic-use/evaluator-standalone-sdk-simple-exact-match/instruction.md b/tests/agentic-use/evaluator-standalone-sdk-simple-exact-match/instruction.md index d39a49781f2..6e82a53865a 100644 --- a/tests/agentic-use/evaluator-standalone-sdk-simple-exact-match/instruction.md +++ b/tests/agentic-use/evaluator-standalone-sdk-simple-exact-match/instruction.md @@ -19,5 +19,6 @@ Write your runnable solution to `workspace/solution.py` from the repo root. The - Select the appropriate SDK metric for exact-match scoring. - Use `Evaluator().run_sync(...)` to evaluate both rows. - Call `print_summary()` on the evaluation result. +- Ensure the printed SDK summary shows one matching row, one mismatching row, and an aggregate exact-match score of `0.5`. Your final answer should be a short summary only. diff --git a/tests/agentic-use/evaluator-standalone-sdk-simple-exact-match/tests/task_metrics.py b/tests/agentic-use/evaluator-standalone-sdk-simple-exact-match/tests/task_metrics.py index a3a4e104d1b..86877423ba6 100644 --- a/tests/agentic-use/evaluator-standalone-sdk-simple-exact-match/tests/task_metrics.py +++ b/tests/agentic-use/evaluator-standalone-sdk-simple-exact-match/tests/task_metrics.py @@ -7,10 +7,10 @@ import contextlib import io import subprocess -import sys from pathlib import Path from evaluator_agent_eval.task_config import load_agentic_use_task_config +from evaluator_agent_eval.task_metric_utils import contains_all, run_python_file, score_checks from nemo_evaluator_sdk import Evaluator, ExactMatchMetric from nemo_evaluator_sdk.values.results import MetricResult, MetricScore @@ -34,7 +34,7 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: code_ran = process is not None and process.returncode == 0 expected_summary = await asyncio.to_thread(_render_expected_summary) if code_ran else "" actual_summary = process.stdout if process is not None else "" - required_terms_present = code is not None and _contains_all( + required_terms_present = code is not None and contains_all( f"{final_text}\n{code}\n{actual_summary}", _required_terms(item), ) @@ -45,19 +45,15 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: task_success = bool( item.get("final_answer_extracted") is True and required_terms_present and output_schema_valid and code_ran ) - verification_score = ( - sum( - float(value) - for value in ( - item.get("final_answer_extracted") is True, - required_terms_present, - code is not None, - code_ran, - has_summary_call, - summary_matches, - ) - ) - / 6.0 + verification_score = score_checks( + [ + item.get("final_answer_extracted") is True, + required_terms_present, + code is not None, + code_ran, + has_summary_call, + summary_matches, + ] ) return MetricResult( scores=[ @@ -92,24 +88,13 @@ def _read_solution(path: Path | None) -> str | None: def _run_solution(path: Path | None) -> subprocess.CompletedProcess[str]: - if path is None: - return subprocess.CompletedProcess(args=[], returncode=1, stdout="", stderr="missing solution.py") - try: - return subprocess.run( - [sys.executable, str(path)], - check=False, - capture_output=True, - text=True, - cwd=_repo_root(path), - timeout=20, - ) - except subprocess.TimeoutExpired as exc: - return subprocess.CompletedProcess( - args=exc.cmd or [sys.executable, str(path)], - returncode=124, - stdout=_timeout_text(exc.stdout), - stderr=_timeout_text(exc.stderr, fallback="candidate exact-match code timed out"), - ) + return run_python_file( + path, + timeout=20, + cwd=_repo_root(path), + timeout_stderr="candidate exact-match code timed out", + missing_stderr="missing solution.py", + ) def _render_expected_summary() -> str: @@ -130,14 +115,6 @@ def _render_expected_summary() -> str: return output.getvalue() -def _timeout_text(value: str | bytes | None, *, fallback: str = "") -> str: - if isinstance(value, bytes): - return value.decode("utf-8", errors="replace") - if isinstance(value, str): - return value - return fallback - - def _repo_root(path: Path | None) -> Path: candidates: list[Path] = [] if path is not None: @@ -153,11 +130,6 @@ def _normalize_summary(text: str) -> str: return text.strip().replace("\r\n", "\n") -def _contains_all(text: str, terms: list[str]) -> bool: - lowered = text.lower() - return all(term.lower() in lowered for term in terms) - - def _required_terms(item: dict) -> list[str]: task_dir = _task_dir(item) if task_dir is None: diff --git a/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/environment/Dockerfile b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/environment/Dockerfile new file mode 100644 index 00000000000..4fab70170a3 --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/environment/Dockerfile @@ -0,0 +1 @@ +FROM nmp-agentic-base:latest diff --git a/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/instruction.md b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/instruction.md new file mode 100644 index 00000000000..7b37c2b8294 --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/instruction.md @@ -0,0 +1,17 @@ +# Add A Custom Surface-Adherence Metric + +Use the local standalone Evaluator SDK to draft and exercise a custom metric that scores whether observed Evaluator surfaces stay within a task's allowed surfaces. + +Use only `packages/nemo_evaluator_sdk` and SDK-level APIs in your answer. Do not propose the `nemo` CLI, plugin SDK APIs, or any `services/*` implementation path. + +Your final answer must include: + +- The package/path `packages/nemo_evaluator_sdk`. +- A directly runnable Python snippet defining a zero-argument metric class compatible with the SDK `Metric` protocol, including a `type` property. +- The API symbols `compute_scores`, `score_names`, `MetricResult`, and `MetricScore`. +- Score names `surface_adherence` and `surface_violation_count`. +- Logic that reads `observed_surfaces`, `allowed_surfaces`, and `forbidden_surfaces` from the dataset item. +- Treat `observed_surfaces` as the integration surfaces the evaluated path actually used, `allowed_surfaces` as the surfaces permitted for the task, and `forbidden_surfaces` as surfaces that should always count as violations. +- Define `surface_adherence` as a numeric score where `1.0` means all observed surfaces are allowed and no forbidden surfaces were used, and lower values indicate violations. Define `surface_violation_count` as the number of observed surfaces that are forbidden or outside the allowed set. +- A passing example where `observed_surfaces=["standalone_sdk"]` and `allowed_surfaces=["standalone_sdk"]` produces `surface_adherence=1.0` and `surface_violation_count=0`. +- A failing example where `observed_surfaces` includes `legacy_service` and `forbidden_surfaces=["legacy_service"]` produces `surface_adherence=0.0` or another penalty below `1.0`, with `surface_violation_count` greater than `0`. diff --git a/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/task.toml b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/task.toml new file mode 100644 index 00000000000..bdd5a7b76a0 --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/task.toml @@ -0,0 +1,47 @@ +version = "1.0" + +[metadata] +author_name = "NMP Team" +difficulty = "medium" +category = "evaluator-agent-benchmark" +suite_id = "evaluator_agent_benchmark" +suite_version = "v0" +tags = ["evaluator", "agent-benchmark", "sdk-metric-authoring", "standalone-sdk"] + +[verifier] +timeout_sec = 60.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 +storage_mb = 10240 +gpus = 0 +allow_internet = false + +[evaluator.surface] +constraint = "standalone_sdk" +allowed = ["standalone_sdk"] +forbidden = ["cli", "plugin_sdk", "legacy_service"] + +[evaluator.expected] +required_terms = [ + "packages/nemo_evaluator_sdk", + "Metric", + "type", + "compute_scores", + "score_names", + "MetricResult", + "MetricScore", + "surface_adherence", + "surface_violation_count", + "observed_surfaces", + "allowed_surfaces", + "forbidden_surfaces", +] + +[evaluator] +forbidden_patterns = ["services/", "nemo evaluation", "plugin SDK"] diff --git a/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/task_metrics.py b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/task_metrics.py new file mode 100644 index 00000000000..e451da68f63 --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/task_metrics.py @@ -0,0 +1,227 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Task-specific metrics for the standalone SDK surface-adherence metric task.""" + +import asyncio +import subprocess +from pathlib import Path + +from evaluator_agent_eval.artifacts import AgentArtifacts +from evaluator_agent_eval.task_config import load_agentic_use_task_config +from evaluator_agent_eval.task_metric_utils import ( + contains_all, + extract_fenced_python_code, + extract_marker_json_object, + find_workspace_python_code, + object_dict, + run_python_code, + score_checks, + string_list, +) +from nemo_evaluator_sdk.values.results import MetricResult, MetricScore + +HARNESS_RESULT_MARKER = "__SURFACE_METRIC_RESULT__=" + + +class SurfaceAdherenceMetricAuthoringMetric: + """Verify the candidate authored an SDK-compatible surface-adherence metric.""" + + @property + def type(self) -> str: + return "agent_eval/surface_adherence_metric_authoring" + + def score_names(self) -> list[str]: + return [ + "task_success", + "verification_score", + "output_schema_valid", + "code_block_present", + "code_ran", + "required_terms_present", + "protocol_compatible", + "type_property_valid", + "emits_expected_scores", + "pass_case_correct", + "fail_case_correct", + ] + + async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + final_text = str(item.get("output_text", "")) + artifacts = _artifacts_from_item(item) + code = _extract_python_code(final_text) or _extract_workspace_python_code(artifacts) + process = await asyncio.to_thread(_run_candidate_code_with_harness, code) if code else None + stdout = process.stdout if process is not None else "" + harness_result = _extract_harness_result(stdout) + + text_to_check = f"{final_text}\n{code or ''}" + required_terms_present = contains_all(text_to_check, _required_terms(item)) + code_ran = process is not None and process.returncode == 0 + score_names = string_list(harness_result.get("score_names")) if harness_result else [] + pass_scores = object_dict(harness_result.get("pass_scores")) if harness_result else {} + fail_scores = object_dict(harness_result.get("fail_scores")) if harness_result else {} + + protocol_compatible = harness_result.get("protocol_compatible") is True if harness_result else False + metric_type = harness_result.get("metric_type") if harness_result else None + type_property_valid = isinstance(metric_type, str) and bool(metric_type.strip()) + emits_expected_scores = "surface_adherence" in score_names and "surface_violation_count" in score_names + pass_case_correct = ( + pass_scores.get("surface_adherence") == 1.0 and pass_scores.get("surface_violation_count") == 0.0 + ) + fail_adherence = fail_scores.get("surface_adherence") + fail_violation_count = fail_scores.get("surface_violation_count") + fail_case_correct = ( + isinstance(fail_adherence, int | float) + and fail_adherence < 1.0 + and isinstance(fail_violation_count, int | float) + and fail_violation_count > 0.0 + ) + output_schema_valid = bool( + code + and code_ran + and protocol_compatible + and type_property_valid + and emits_expected_scores + and pass_case_correct + and fail_case_correct + ) + task_success = bool( + item.get("final_answer_extracted") is True and required_terms_present and output_schema_valid + ) + checks = [ + item.get("final_answer_extracted") is True, + required_terms_present, + code is not None, + code_ran, + protocol_compatible, + type_property_valid, + emits_expected_scores, + pass_case_correct, + fail_case_correct, + ] + + return MetricResult( + scores=[ + MetricScore(name="task_success", value=float(task_success)), + MetricScore(name="verification_score", value=score_checks(checks)), + MetricScore(name="output_schema_valid", value=float(output_schema_valid)), + MetricScore(name="code_block_present", value=float(code is not None)), + MetricScore(name="code_ran", value=float(code_ran)), + MetricScore(name="required_terms_present", value=float(required_terms_present)), + MetricScore(name="protocol_compatible", value=float(protocol_compatible)), + MetricScore(name="type_property_valid", value=float(type_property_valid)), + MetricScore(name="emits_expected_scores", value=float(emits_expected_scores)), + MetricScore(name="pass_case_correct", value=float(pass_case_correct)), + MetricScore(name="fail_case_correct", value=float(fail_case_correct)), + ] + ) + + +def _extract_python_code(text: str) -> str | None: + return extract_fenced_python_code(text, predicate=_looks_like_surface_metric_code) + + +def _artifacts_from_item(item: dict) -> AgentArtifacts: + agent_log_dir = item.get("agent_log_dir") + if not isinstance(agent_log_dir, str): + raise ValueError("agent_log_dir is required for surface-adherence metric") + workspace_dir = item.get("workspace_dir") + return AgentArtifacts.from_dir( + agent_log_dir, workspace_dir=workspace_dir if isinstance(workspace_dir, str) else None + ) + + +def _required_terms(item: dict) -> list[str]: + task_dir = item.get("task_dir") + if not isinstance(task_dir, str): + return [] + return load_agentic_use_task_config(Path(task_dir)).evaluator.expected.required_terms + + +def _extract_workspace_python_code(artifacts: AgentArtifacts) -> str | None: + return find_workspace_python_code( + artifacts, + preferred_names=["surface_adherence_metric.py", "solution.py"], + predicate=_looks_like_surface_metric_code, + ) + + +def _run_candidate_code_with_harness(code: str) -> subprocess.CompletedProcess[str]: + return run_python_code( + code, + filename="candidate_surface_adherence_metric.py", + timeout=20, + appended_code=_HARNESS_CODE, + timeout_stderr="candidate surface-adherence metric code timed out", + ) + + +def _looks_like_surface_metric_code(code: str) -> bool: + return "compute_scores" in code and "MetricResult" in code and "MetricScore" in code + + +def _extract_harness_result(stdout: str) -> dict[str, object] | None: + return extract_marker_json_object(stdout, marker=HARNESS_RESULT_MARKER) + + +_HARNESS_CODE = r""" +import asyncio +import inspect +import json + +from nemo_evaluator_sdk.metrics.base import Metric + + +def __scores_to_dict(metric_result): + return {score.name: float(score.value) for score in metric_result.scores} + + +async def __surface_metric_harness(): + metric_class = None + for value in list(globals().values()): + if not inspect.isclass(value): + continue + if value.__module__ != "__main__": + continue + if hasattr(value, "compute_scores") and hasattr(value, "score_names"): + metric_class = value + break + + if metric_class is None: + raise RuntimeError("No candidate metric class with compute_scores and score_names was found") + + metric = metric_class() + pass_result = await metric.compute_scores( + { + "observed_surfaces": ["standalone_sdk"], + "allowed_surfaces": ["standalone_sdk"], + "forbidden_surfaces": ["legacy_service"], + }, + {}, + ) + fail_result = await metric.compute_scores( + { + "observed_surfaces": ["standalone_sdk", "legacy_service"], + "allowed_surfaces": ["standalone_sdk"], + "forbidden_surfaces": ["legacy_service"], + }, + {}, + ) + print( + "__SURFACE_METRIC_RESULT__=" + + json.dumps( + { + "class_name": metric_class.__name__, + "protocol_compatible": isinstance(metric, Metric), + "metric_type": metric.type, + "score_names": list(metric.score_names()), + "pass_scores": __scores_to_dict(pass_result), + "fail_scores": __scores_to_dict(fail_result), + }, + sort_keys=True, + ) + ) + + +asyncio.run(__surface_metric_harness()) +""" diff --git a/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/test.sh b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/test.sh new file mode 100755 index 00000000000..e0ad48afe46 --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/bash +source /app/tests/agentic-use/shared/verify-tests.sh diff --git a/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/test_outputs.py b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/test_outputs.py new file mode 100644 index 00000000000..cb4e3a199b0 --- /dev/null +++ b/tests/agentic-use/evaluator-standalone-sdk-surface-adherence-metric/tests/test_outputs.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Task-local verifier trampoline for the shared Evaluator benchmark verifier.""" + +from evaluator_agent_eval.pytest_verifier import verify_agent_attempt_scores_with_evaluator_sdk +from task_metrics import SurfaceAdherenceMetricAuthoringMetric + + +def test_agent_attempt_scores_with_evaluator_sdk(): + verify_agent_attempt_scores_with_evaluator_sdk(task_specific_metrics=[SurfaceAdherenceMetricAuthoringMetric()]) diff --git a/tests/agentic-use/evaluator-standalone-sdk-surface-discovery/tests/task_metrics.py b/tests/agentic-use/evaluator-standalone-sdk-surface-discovery/tests/task_metrics.py index b3049957060..144b01161aa 100644 --- a/tests/agentic-use/evaluator-standalone-sdk-surface-discovery/tests/task_metrics.py +++ b/tests/agentic-use/evaluator-standalone-sdk-surface-discovery/tests/task_metrics.py @@ -8,6 +8,7 @@ from pathlib import Path from evaluator_agent_eval.task_config import load_agentic_use_task_config +from evaluator_agent_eval.task_metric_utils import contains_all, extract_fenced_python_code, score_checks from nemo_evaluator_sdk.values.results import MetricResult, MetricScore @@ -24,7 +25,7 @@ def score_names(self) -> list[str]: async def compute_scores(self, item: dict, sample: dict) -> MetricResult: final_text = str(item.get("output_text", "")) code = _extract_python_code(final_text) - required_terms_present = _contains_all(final_text, _required_terms(item)) + required_terms_present = contains_all(final_text, _required_terms(item)) imports_valid = bool(code) and _imports_sdk_symbols(code) dataset_valid = bool(code) and _contains_expected_dataset(code) metric_templates_valid = bool(code) and _uses_exact_match_templates(code) @@ -42,7 +43,7 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: metric_templates_valid, invocation_valid, ] - verification_score = sum(float(value) for value in checks) / len(checks) + verification_score = score_checks(checks) task_success = bool( item.get("final_answer_extracted") is True and required_terms_present and output_schema_valid ) @@ -57,25 +58,23 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: def _extract_python_code(text: str) -> str | None: - best_code: str | None = None - best_score = 0 - for match in re.finditer(r"```(?:python|py)\s*\n(?P.*?)```", text, re.DOTALL | re.IGNORECASE): - code = match.group("code").strip() - if "Evaluator" not in code or "ExactMatchMetric" not in code: - continue - score = sum( - [ - "dataset" in code, - "ExactMatchMetric(" in code, - "{{item.expected}}" in code, - "{{item.prediction}}" in code, - ".run_sync(" in code or ".run(" in code, - ] - ) - if score > best_score: - best_code = code - best_score = score - return best_code + return extract_fenced_python_code( + text, + predicate=lambda code: "Evaluator" in code and "ExactMatchMetric" in code, + scorer=_surface_discovery_code_score, + ) + + +def _surface_discovery_code_score(code: str) -> int: + return sum( + [ + "dataset" in code, + "ExactMatchMetric(" in code, + "{{item.expected}}" in code, + "{{item.prediction}}" in code, + ".run_sync(" in code or ".run(" in code, + ] + ) def _imports_sdk_symbols(code: str) -> bool: @@ -181,11 +180,6 @@ def _literal_value(node: ast.AST): return None -def _contains_all(text: str, terms: list[str]) -> bool: - lowered = text.lower() - return all(term.lower() in lowered for term in terms) - - def _required_terms(item: dict) -> list[str]: task_dir = item.get("task_dir") if not isinstance(task_dir, str): diff --git a/tests/agentic-use/manifests/evaluator_agent_benchmark_mvp.txt b/tests/agentic-use/manifests/evaluator_agent_benchmark_mvp.txt index 1d2a24e4911..9934fa8d2ec 100644 --- a/tests/agentic-use/manifests/evaluator_agent_benchmark_mvp.txt +++ b/tests/agentic-use/manifests/evaluator_agent_benchmark_mvp.txt @@ -1,3 +1,5 @@ -# Evaluator agent benchmark MVP: one task with Evaluator SDK verification. +# Evaluator agent benchmark MVP: standalone SDK tasks with Evaluator SDK verification. evaluator-standalone-sdk-surface-discovery evaluator-standalone-sdk-simple-exact-match +evaluator-standalone-sdk-agent-target +evaluator-standalone-sdk-surface-adherence-metric diff --git a/tests/agentic-use/matrices/evaluator_agent_benchmark_mvp.py b/tests/agentic-use/matrices/evaluator_agent_benchmark_mvp.py new file mode 100644 index 00000000000..60c13ff68ad --- /dev/null +++ b/tests/agentic-use/matrices/evaluator_agent_benchmark_mvp.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Default MVP matrix for Evaluator agent benchmark runs.""" + +from __future__ import annotations + +from agent_matrix_benchmark import ( + AgentMatrixConfig, + CandidateConfig, + ClaudeCodeParams, + CodexParams, + CursorAgentParams, + MatrixDefaults, +) + +MATRIX = AgentMatrixConfig( + manifest="manifests/evaluator_agent_benchmark_mvp.txt", + defaults=MatrixDefaults( + timeout=600, + skip_build=True, + allow_dirty=True, + ), + candidates=[ + CandidateConfig( + id="codex-gpt-5.5-high-standard", + backend="codex", + model="gpt-5.5", + params=CodexParams(intelligence="high", speed="standard"), + ), + CandidateConfig( + id="codex-gpt-5.5-high-fast", + backend="codex", + model="gpt-5.5", + params=CodexParams(intelligence="high", speed="fast"), + ), + CandidateConfig( + id="claude-code-sonnet-4-6", + backend="claude-code", + model="aws/anthropic/bedrock-claude-sonnet-4-6", + params=ClaudeCodeParams(permission_mode="bypassPermissions"), + ), + CandidateConfig( + id="cursor-agent-gpt-5.5-high", + backend="cursor-agent", + model="gpt-5.5-high", + params=CursorAgentParams(sandbox="disabled"), + ), + CandidateConfig( + id="cursor-agent-sonnet-4", + backend="cursor-agent", + model="sonnet-4", + params=CursorAgentParams(sandbox="disabled"), + ), + ], +) diff --git a/tests/agentic-use/nat_runner.py b/tests/agentic-use/nat_runner.py index d2e4279f37f..d38f264e88e 100755 --- a/tests/agentic-use/nat_runner.py +++ b/tests/agentic-use/nat_runner.py @@ -72,6 +72,7 @@ import fnmatch import json import os +import shlex import subprocess import sys import textwrap @@ -97,50 +98,6 @@ DOCKER_SOCKET_HOST_PATH = Path("/var/run/docker.sock") DOCKER_SOCKET_CONTAINER_PATH = "/var/run/docker.sock" PLACEHOLDER_SECRET_VALUES = {"null", "none"} -CODEX_STRUCTURED_OUTPUT_SCHEMA = json.dumps( - { - "type": "object", - "additionalProperties": False, - "properties": { - "final_summary": { - "type": "string", - "description": "A short final answer to show as the agent's response.", - }, - "output_files": { - "type": "array", - "description": "Files the runner should materialize under /app after Codex finishes.", - "items": { - "type": "object", - "additionalProperties": False, - "properties": { - "path": { - "type": "string", - "description": "Path relative to /app, for example workspace/solution.py.", - }, - "content": { - "type": "string", - "description": "Complete UTF-8 file contents.", - }, - }, - "required": ["path", "content"], - }, - }, - }, - "required": ["final_summary", "output_files"], - }, - indent=2, -) -CODEX_STRUCTURED_PROMPT_PREFIX = """\ -You are running in a read-only Codex benchmark backend. - -Inspect the repository as needed with read-only commands. Do not attempt to -write files directly. When the task asks you to create or modify files, return -those files in the structured output_files array instead. The benchmark runner -will materialize them under /app after Codex exits. - -Use paths relative to /app. For example, /app/workspace/solution.py should be -returned as {"path": "workspace/solution.py", "content": "..."}. -""" def _normalize_secret(value: str | None) -> str: @@ -155,6 +112,58 @@ def _secret_from_env(name: str) -> str: return _normalize_secret(os.environ.get(name)) +_CANDIDATE_PARAM_STRING_KEYS = frozenset( + { + "permission_mode", + "intelligence", + "speed", + "sandbox", + "mode", + } +) +_CANDIDATE_PARAM_NUMBER_KEYS = frozenset({"max_budget_usd"}) +_CANDIDATE_PARAM_OBJECT_KEYS = frozenset({"config"}) +_CANDIDATE_PARAM_KEYS = _CANDIDATE_PARAM_STRING_KEYS | _CANDIDATE_PARAM_NUMBER_KEYS | _CANDIDATE_PARAM_OBJECT_KEYS + + +def _coerce_candidate_number_param(key: str, value: object) -> int | float: + if type(value) in (int, float): + return value + if isinstance(value, str): + try: + return float(value) + except ValueError as exc: + raise ValueError(f"--candidate-params key {key!r} must be numeric") from exc + raise ValueError(f"--candidate-params key {key!r} must be numeric") + + +def _parse_candidate_params(value: str | None) -> dict[str, Any]: + if not value: + return {} + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError(f"--candidate-params must be a JSON object: {exc}") from exc + if not isinstance(parsed, dict): + raise ValueError("--candidate-params must be a JSON object") + unknown_keys = sorted(set(parsed) - _CANDIDATE_PARAM_KEYS) + if unknown_keys: + raise ValueError(f"--candidate-params includes unsupported key(s): {', '.join(unknown_keys)}") + normalized: dict[str, Any] = {} + for key, raw_value in parsed.items(): + if key in _CANDIDATE_PARAM_STRING_KEYS: + if not isinstance(raw_value, str): + raise ValueError(f"--candidate-params key {key!r} must be a string") + normalized[key] = raw_value + elif key in _CANDIDATE_PARAM_NUMBER_KEYS: + normalized[key] = _coerce_candidate_number_param(key, raw_value) + elif key in _CANDIDATE_PARAM_OBJECT_KEYS: + if not isinstance(raw_value, dict): + raise ValueError(f"--candidate-params key {key!r} must be a JSON object") + normalized[key] = dict(raw_value) + return normalized + + # --------------------------------------------------------------------------- # Docker helpers # --------------------------------------------------------------------------- @@ -720,7 +729,7 @@ def _build_aut_agent_cmd(instruction_container: str) -> list[str]: ] -def _build_claude_code_agent_cmd(instruction_container: str) -> list[str]: +def _build_claude_code_agent_cmd(instruction_container: str, agent_params: dict[str, Any]) -> list[str]: """Build the ``bash -c`` command that runs the claude-code backend. ``--output-format json`` wraps the agent's final response in a single @@ -730,6 +739,7 @@ def _build_claude_code_agent_cmd(instruction_container: str) -> list[str]: envelope and surfaces the same token shape we record for AUT, so the dual-baseline comparison is apples-to-apples. """ + extra_args = _claude_code_shell_args(agent_params) return [ "bash", "-c", @@ -739,9 +749,11 @@ def _build_claude_code_agent_cmd(instruction_container: str) -> list[str]: if [ -n "${{AGENT_MODEL:-}}" ]; then model_args=(--model "${{AGENT_MODEL}}") fi + extra_args=({extra_args}) set +e claude -p "$(cat {instruction_container})" \ "${{model_args[@]}}" \ + "${{extra_args[@]}}" \ --output-format json \ > /tmp/nat_agent.log 2> /tmp/nat_agent.stderr rc=$? @@ -753,22 +765,52 @@ def _build_claude_code_agent_cmd(instruction_container: str) -> list[str]: ] -def _build_codex_agent_cmd(instruction_container: str) -> list[str]: +def _claude_code_shell_args(agent_params: dict[str, Any]) -> str: + args: list[str] = [] + permission_mode = agent_params.get("permission_mode") + if isinstance(permission_mode, str): + args.extend(["--permission-mode", permission_mode]) + max_budget_usd = agent_params.get("max_budget_usd") + if isinstance(max_budget_usd, int | float): + args.extend(["--max-budget-usd", str(max_budget_usd)]) + return " ".join(shlex.quote(arg) for arg in args) + + +def _codex_config_shell_args(agent_params: dict[str, Any]) -> str: + config: dict[str, Any] = {} + intelligence = agent_params.get("intelligence") + if isinstance(intelligence, str): + config["model_reasoning_effort"] = intelligence + speed = agent_params.get("speed") + if speed == "fast": + config["service_tier"] = "fast" + explicit_config = agent_params.get("config") + if isinstance(explicit_config, dict): + config.update(explicit_config) + args: list[str] = [] + for key, value in sorted(config.items()): + args.append("-c") + args.append(f"{key}={json.dumps(value)}") + return " ".join(shlex.quote(arg) for arg in args) + + +def _build_codex_agent_cmd(instruction_container: str, agent_params: dict[str, Any]) -> list[str]: """Build the command that runs Codex CLI headlessly inside the task container.""" - return ["bash", "-c", _codex_agent_script(instruction_container)] + return ["bash", "-c", _codex_agent_script(instruction_container, agent_params)] -def _codex_agent_script(instruction_container: str) -> str: +def _codex_agent_script(instruction_container: str, agent_params: dict[str, Any]) -> str: + config_args = _codex_config_shell_args(agent_params) return ( CODEX_AGENT_SCRIPT_TEMPLATE_PATH.read_text(encoding="utf-8") - .replace("@@CODEX_STRUCTURED_OUTPUT_SCHEMA@@", repr(CODEX_STRUCTURED_OUTPUT_SCHEMA)) - .replace("@@INSTRUCTION_CONTAINER@@", repr(instruction_container)) - .replace("@@CODEX_STRUCTURED_PROMPT_PREFIX@@", repr(CODEX_STRUCTURED_PROMPT_PREFIX)) + .replace("@@INSTRUCTION_CONTAINER@@", shlex.quote(instruction_container)) + .replace("@@CODEX_CONFIG_ARGS@@", config_args) ) -def _build_cursor_agent_cmd(instruction_container: str) -> list[str]: +def _build_cursor_agent_cmd(instruction_container: str, agent_params: dict[str, Any]) -> list[str]: """Build the command that runs Cursor Agent headlessly inside the task container.""" + extra_args = _cursor_agent_shell_args(agent_params) return [ "bash", "-c", @@ -778,14 +820,15 @@ def _build_cursor_agent_cmd(instruction_container: str) -> list[str]: if [ -n "${{AGENT_MODEL:-}}" ]; then model_args=(--model "${{AGENT_MODEL}}") fi + extra_args=({extra_args}) set +e cursor-agent \ --print \ --output-format json \ - --sandbox disabled \ --trust \ --workspace /app \ "${{model_args[@]}}" \ + "${{extra_args[@]}}" \ "$(cat {instruction_container})" \ > /tmp/nat_agent.log 2> /tmp/nat_agent.stderr rc=$? @@ -798,6 +841,17 @@ def _build_cursor_agent_cmd(instruction_container: str) -> list[str]: ] +def _cursor_agent_shell_args(agent_params: dict[str, Any]) -> str: + args: list[str] = [] + if "sandbox" in agent_params: + sandbox = agent_params.get("sandbox") + args.extend(["--sandbox", sandbox if isinstance(sandbox, str) else "disabled"]) + mode = agent_params.get("mode") + if isinstance(mode, str): + args.extend(["--mode", mode]) + return " ".join(shlex.quote(arg) for arg in args) + + def run_agent_phase( task_dir: Path, image: str, @@ -808,6 +862,7 @@ def run_agent_phase( anthropic_base_url: str, nmp_base_url: str, agent_model: str | None, + agent_params: dict[str, Any], codex_auth_json: Path | None, timeout: int, agent_backend: str, @@ -888,6 +943,8 @@ def run_agent_phase( } if DOCKER_SOCKET_HOST_PATH.exists(): env["DOCKER_HOST"] = f"unix://{DOCKER_SOCKET_CONTAINER_PATH}" + if agent_params: + env["NAT_CANDIDATE_PARAMS"] = json.dumps(agent_params, sort_keys=True) nvidia_api_key = _normalize_secret(nvidia_api_key) anthropic_api_key = _normalize_secret(anthropic_api_key) if agent_backend in {"aut", "workflow"} and nvidia_api_key: @@ -960,11 +1017,11 @@ def run_agent_phase( agent_cmd = _build_aut_agent_cmd(instruction_container) elif agent_backend == "claude-code": env["ANTHROPIC_BASE_URL"] = anthropic_base_url - agent_cmd = _build_claude_code_agent_cmd(instruction_container) + agent_cmd = _build_claude_code_agent_cmd(instruction_container, agent_params) elif agent_backend == "codex": - agent_cmd = _build_codex_agent_cmd(instruction_container) + agent_cmd = _build_codex_agent_cmd(instruction_container, agent_params) elif agent_backend == "cursor-agent": - agent_cmd = _build_cursor_agent_cmd(instruction_container) + agent_cmd = _build_cursor_agent_cmd(instruction_container, agent_params) else: raise ValueError(f"Unsupported agent backend: {agent_backend}") @@ -1111,17 +1168,9 @@ class TokenMetrics(TypedDict): duration_ms: float | None -def _is_positive_int(value: object) -> bool: - """Token-budget gate predicate: a metric is "present" iff it is a - strictly positive int. - - Used at the ``has_tokens`` decision point to decide whether the - extracted agent log carried usable token accounting at all. Negative - or zero values are treated as missing — some backends emit ``0`` as a - placeholder for "field exists but no data" and we want those to fall - through to the unavailable path. - """ - return isinstance(value, int) and value > 0 +def _is_nonnegative_int(value: object) -> bool: + """Token-budget gate predicate for a parsed usage envelope.""" + return isinstance(value, int) and value >= 0 def _iter_agent_log_json_payloads(agent_log: str) -> list[dict[str, Any]]: @@ -1196,33 +1245,47 @@ def _extract_usage_metrics(agent_log: str) -> TokenMetrics: if not agent_log.strip(): return zero - def _bucket_from_usage(usage_obj: dict[str, Any]) -> tuple[int, int, int, int]: - """Return (input, output, cache_creation, cache_read) tokens for one usage dict.""" - input_tokens = usage_obj.get("input_tokens") - if not isinstance(input_tokens, int): - input_tokens = usage_obj.get("prompt_tokens") - if not isinstance(input_tokens, int): - input_tokens = 0 - output_tokens = usage_obj.get("output_tokens") - if not isinstance(output_tokens, int): - output_tokens = usage_obj.get("completion_tokens") - if not isinstance(output_tokens, int): - output_tokens = 0 + def _first_int(usage_obj: dict[str, Any], keys: tuple[str, ...]) -> tuple[int | None, bool]: + for key in keys: + value = usage_obj.get(key) + if isinstance(value, int): + return value, True + return None, False + + def _bucket_from_usage(usage_obj: dict[str, Any]) -> tuple[int | None, int | None, int | None, int | None, bool]: + """Return (input, output, cache_creation, cache_read, has_known_key).""" + input_tokens, has_input = _first_int(usage_obj, ("input_tokens", "prompt_tokens", "inputTokens")) + output_tokens, has_output = _first_int(usage_obj, ("output_tokens", "completion_tokens", "outputTokens")) # Anthropic raw shape (claude-code --output-format json). - cache_creation_tokens = usage_obj.get("cache_creation_input_tokens") - cache_read_tokens = usage_obj.get("cache_read_input_tokens") + cache_creation_tokens, has_cache_creation = _first_int( + usage_obj, + ("cache_creation_input_tokens", "cacheWriteTokens"), + ) + cache_read_tokens, has_cache_read = _first_int( + usage_obj, + ("cache_read_input_tokens", "cacheReadTokens", "cached_input_tokens"), + ) # LangChain normalised shape (AUT via langchain_anthropic / LiteLLM). details = usage_obj.get("input_token_details") if isinstance(details, dict): - if not isinstance(cache_creation_tokens, int): - cache_creation_tokens = details.get("cache_creation") - if not isinstance(cache_read_tokens, int): - cache_read_tokens = details.get("cache_read") - if not isinstance(cache_creation_tokens, int): - cache_creation_tokens = 0 - if not isinstance(cache_read_tokens, int): - cache_read_tokens = 0 - return input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens + if not has_cache_creation: + cache_creation_tokens, has_cache_creation = _first_int(details, ("cache_creation",)) + if not has_cache_read: + cache_read_tokens, has_cache_read = _first_int(details, ("cache_read",)) + # OpenAI-style usage reports surface total input tokens plus a + # cached-input subset under ``cached_input_tokens``. Convert that to + # the benchmark's bucketed shape: + # prompt_tokens=uncached input, cache_read_tokens=cached input. + if ( + "cached_input_tokens" in usage_obj + and has_input + and has_cache_read + and input_tokens is not None + and cache_read_tokens is not None + ): + input_tokens = max(input_tokens - cache_read_tokens, 0) + has_known_key = has_input or has_output or has_cache_creation or has_cache_read + return input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens, has_known_key def _looks_usage_bearing(d: dict[str, Any]) -> bool: """Heuristic: does this dict contain something we can extract usage from?""" @@ -1263,6 +1326,12 @@ def _looks_usage_bearing(d: dict[str, Any]) -> bool: "cache_read_tokens": 0, "n_assistant_messages": 0, } + bucket_presence = { + "input_tokens": False, + "output_tokens": False, + "cache_creation_tokens": False, + "cache_read_tokens": False, + } has_data = False # Path 1 (preferred when present): walk every message in messages[] and @@ -1289,12 +1358,22 @@ def _looks_usage_bearing(d: dict[str, Any]) -> bool: usage_obj = token_usage if not usage_obj: continue - input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens = _bucket_from_usage(usage_obj) - if input_tokens or output_tokens or cache_creation_tokens or cache_read_tokens: - sums["input_tokens"] += input_tokens - sums["output_tokens"] += output_tokens - sums["cache_creation_tokens"] += cache_creation_tokens - sums["cache_read_tokens"] += cache_read_tokens + input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens, has_known_key = _bucket_from_usage( + usage_obj + ) + if has_known_key: + if input_tokens is not None: + sums["input_tokens"] += input_tokens + bucket_presence["input_tokens"] = True + if output_tokens is not None: + sums["output_tokens"] += output_tokens + bucket_presence["output_tokens"] = True + if cache_creation_tokens is not None: + sums["cache_creation_tokens"] += cache_creation_tokens + bucket_presence["cache_creation_tokens"] = True + if cache_read_tokens is not None: + sums["cache_read_tokens"] += cache_read_tokens + bucket_presence["cache_read_tokens"] = True sums["n_assistant_messages"] += 1 has_data = True else: @@ -1320,14 +1399,26 @@ def _looks_usage_bearing(d: dict[str, Any]) -> bool: for key in ("usage", "usage_metadata"): usage_obj = candidate_payload.get(key) if isinstance(usage_obj, dict) and usage_obj: - input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens = _bucket_from_usage( - usage_obj - ) - if input_tokens or output_tokens or cache_creation_tokens or cache_read_tokens: - sums["input_tokens"] = input_tokens - sums["output_tokens"] = output_tokens - sums["cache_creation_tokens"] = cache_creation_tokens - sums["cache_read_tokens"] = cache_read_tokens + ( + input_tokens, + output_tokens, + cache_creation_tokens, + cache_read_tokens, + has_known_key, + ) = _bucket_from_usage(usage_obj) + if has_known_key: + if input_tokens is not None: + sums["input_tokens"] = input_tokens + bucket_presence["input_tokens"] = True + if output_tokens is not None: + sums["output_tokens"] = output_tokens + bucket_presence["output_tokens"] = True + if cache_creation_tokens is not None: + sums["cache_creation_tokens"] = cache_creation_tokens + bucket_presence["cache_creation_tokens"] = True + if cache_read_tokens is not None: + sums["cache_read_tokens"] = cache_read_tokens + bucket_presence["cache_read_tokens"] = True # ``num_turns`` (claude-code) is the natural N here. num_turns_value = candidate_payload.get("num_turns") sums["n_assistant_messages"] = num_turns_value if isinstance(num_turns_value, int) else 1 @@ -1347,13 +1438,19 @@ def _looks_usage_bearing(d: dict[str, Any]) -> bool: ) return zero - total = sums["input_tokens"] + sums["output_tokens"] + sums["cache_creation_tokens"] + sums["cache_read_tokens"] + total_components = [ + sums["input_tokens"] if bucket_presence["input_tokens"] else None, + sums["output_tokens"] if bucket_presence["output_tokens"] else None, + sums["cache_creation_tokens"] if bucket_presence["cache_creation_tokens"] else None, + sums["cache_read_tokens"] if bucket_presence["cache_read_tokens"] else None, + ] + total = sum(component for component in total_components if component is not None) out: TokenMetrics = { - "prompt_tokens": sums["input_tokens"], - "completion_tokens": sums["output_tokens"], - "total_tokens": total, - "cache_creation_tokens": sums["cache_creation_tokens"], - "cache_read_tokens": sums["cache_read_tokens"], + "prompt_tokens": sums["input_tokens"] if bucket_presence["input_tokens"] else None, + "completion_tokens": sums["output_tokens"] if bucket_presence["output_tokens"] else None, + "total_tokens": total if any(component is not None for component in total_components) else None, + "cache_creation_tokens": sums["cache_creation_tokens"] if bucket_presence["cache_creation_tokens"] else None, + "cache_read_tokens": sums["cache_read_tokens"] if bucket_presence["cache_read_tokens"] else None, "n_assistant_messages": sums["n_assistant_messages"], "cost_usd": None, "num_turns": None, @@ -1452,6 +1549,8 @@ def run_verify_phase( (str(tests_dir), "/tests"), (str(task_dir), "/task"), (str(workspace_dir), "/app/workspace"), + (str(SHARED_DIR), "/app/tests/agentic-use/shared:ro"), + (str(REPO_ROOT / "packages" / "nemo_evaluator_sdk" / "src"), "/app/packages/nemo_evaluator_sdk/src:ro"), (str(output_dir / "agent"), "/logs/agent"), (str(verifier_log_dir), "/logs/verifier"), # Persist service/database state across AGENT and VERIFY containers. @@ -1493,15 +1592,18 @@ def run_task( anthropic_base_url: str, nmp_base_url: str, agent_model: str | None, + agent_params: dict[str, Any] | None, codex_auth_json: Path | None, agent_timeout: int, skip_build: bool, + build_only: bool, skip_agent: bool, agent_backend: str, aut_agent_name: str, aut_agent_config: str | None, aut_seed_providers: bool, smoke_workspace: str | None, + candidate_id: str | None = None, provenance: dict[str, Any] | None = None, ) -> dict: """Run a single agentic-use eval task end to end.""" @@ -1525,6 +1627,10 @@ def run_task( "timestamp": ts, "output_dir": str(output_dir), "image": image_tag, + "agent_backend": agent_backend, + "agent_model": _agent_model_for_backend(agent_backend=agent_backend, agent_model=agent_model), + "candidate_params": agent_params or {}, + "candidate_id": candidate_id, "build": None, "agent": None, "verify": None, @@ -1544,6 +1650,7 @@ def run_task( "token_metrics_status": "unavailable", "token_metrics_note": None, }, + "verifier_scores": None, "provenance": dict(provenance) if provenance else None, } task_timeout = _task_agent_timeout(task_dir) @@ -1587,6 +1694,15 @@ def run_task( _write_result(output_dir, result) return result + if build_only: + result["agent"] = "skipped" + result["verify"] = "skipped" + result["passed"] = None + result["reward"] = None + result["runtime_sec"] = round(time.monotonic() - task_start_monotonic, 3) + _write_result(output_dir, result) + return result + # ------------------------------------------------------------------ # 2. AGENT # ------------------------------------------------------------------ @@ -1601,6 +1717,7 @@ def run_task( anthropic_base_url=anthropic_base_url, nmp_base_url=nmp_base_url, agent_model=agent_model, + agent_params=agent_params or {}, codex_auth_json=codex_auth_json, timeout=effective_agent_timeout, agent_backend=agent_backend, @@ -1627,7 +1744,7 @@ def run_task( ): result["metrics"][key] = usage_metrics.get(key) has_tokens = any( - _is_positive_int(result["metrics"].get(k)) + _is_nonnegative_int(result["metrics"].get(k)) for k in ("prompt_tokens", "completion_tokens", "total_tokens") ) if has_tokens: @@ -1677,6 +1794,7 @@ def run_task( result["verify"] = "ok" if verify_ok else "failed" result["passed"] = verify_ok result["reward"] = 1 if verify_ok else 0 + result["verifier_scores"] = _read_json_object(output_dir / "verifier" / "evaluator_scores.json") except Exception as e: print(f"[nat_runner] Verify phase error: {e}") result["verify"] = f"error: {e}" @@ -1698,6 +1816,17 @@ def _agent_model_for_backend( return agent_model or "unknown" +def _read_json_object(path: Path) -> dict[str, Any] | None: + if not path.exists(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + print(f"[nat_runner] WARN: could not read JSON object from {path}: {exc}") + return None + return payload if isinstance(payload, dict) else None + + def _agent_model_from_env(agent_backend: str) -> str | None: """Resolve the candidate-agent model from env, including legacy names.""" unified = os.environ.get("NAT_AGENT_MODEL") or os.environ.get("AGENT_MODEL") @@ -1811,6 +1940,11 @@ def main() -> int: action="store_true", help="Skip agent phase; only run the verifier (useful for debugging tests)", ) + parser.add_argument( + "--build-only", + action="store_true", + help="Build selected task images and skip agent/verifier phases.", + ) parser.add_argument( "--jobs-dir", type=Path, @@ -1868,6 +2002,16 @@ def main() -> int: help="Explicit path to a Codex auth.json file to mount read-only for " "--agent-backend=codex when OPENAI_API_KEY is unset.", ) + parser.add_argument( + "--candidate-id", + default=os.environ.get("NAT_CANDIDATE_ID"), + help="Optional stable candidate id to record in result.json for matrix benchmark aggregation.", + ) + parser.add_argument( + "--candidate-params", + default=os.environ.get("NAT_CANDIDATE_PARAMS"), + help="Optional JSON object of candidate-specific params to record and pass to backend command construction.", + ) parser.add_argument( "--aut-agent-name", default=os.environ.get("AUT_AGENT_NAME"), @@ -1988,13 +2132,18 @@ def main() -> int: if args.manifest is not None and args.tasks: print("ERROR: Provide either TASK_OR_GLOB args or --manifest, not both.", file=sys.stderr) return 1 + try: + candidate_params = _parse_candidate_params(args.candidate_params) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 # Validate API keys early nvidia_api_key = _secret_from_env("NVIDIA_API_KEY") anthropic_api_key = _secret_from_env("ANTHROPIC_API_KEY") or nvidia_api_key openai_api_key = _secret_from_env("OPENAI_API_KEY") cursor_api_key = _secret_from_env("CURSOR_API_KEY") - if not args.skip_agent: + if not args.skip_agent and not args.build_only: if args.agent_backend in {"aut", "workflow"} and not nvidia_api_key: print("ERROR: NVIDIA_API_KEY environment variable is required for this backend.", file=sys.stderr) print(" export NVIDIA_API_KEY= # from https://build.nvidia.com", file=sys.stderr) @@ -2139,19 +2288,25 @@ def main() -> int: anthropic_base_url=args.anthropic_base_url, nmp_base_url=args.nmp_base_url, agent_model=args.agent_model, + agent_params=candidate_params, codex_auth_json=args.codex_auth_json, agent_timeout=args.timeout, skip_build=args.skip_build, + build_only=args.build_only, skip_agent=args.skip_agent, agent_backend=args.agent_backend, aut_agent_name=args.aut_agent_name, aut_agent_config=args.aut_agent_config, aut_seed_providers=args.aut_seed_providers, smoke_workspace=args.smoke_workspace, + candidate_id=args.candidate_id, provenance=provenance, ) results.append(result) - if result.get("reward") != 1: + if args.build_only: + if result.get("build") not in {"ok", "skipped"}: + failed.append(task_name) + elif result.get("reward") != 1: failed.append(task_name) except Exception as e: print(f"[nat_runner] ERROR running task {task_name}: {e}", file=sys.stderr) @@ -2159,6 +2314,18 @@ def main() -> int: # Summary total = len(results) + if args.build_only: + build_ok = sum(1 for r in results if r.get("build") == "ok") + build_skipped = sum(1 for r in results if r.get("build") == "skipped") + build_failed = len(failed) + print(f"\n{'=' * 60}") + print(f"[nat_runner] BUILD SUMMARY: {build_ok + build_skipped}/{total} task images ready") + print(f"[nat_runner] built: {build_ok} skipped: {build_skipped} failed: {build_failed}") + if failed: + print(f"[nat_runner] FAILED builds: {', '.join(failed)}") + print(f"{'=' * 60}") + return 0 if not failed else 1 + passed = sum(1 for r in results if r.get("reward") == 1) token_values = [ r.get("metrics", {}).get("total_tokens") diff --git a/tests/agentic-use/scripts/codex_agent_runner.sh b/tests/agentic-use/scripts/codex_agent_runner.sh index 7f7d08588a2..b667fdbf3f6 100644 --- a/tests/agentic-use/scripts/codex_agent_runner.sh +++ b/tests/agentic-use/scripts/codex_agent_runner.sh @@ -7,88 +7,29 @@ model_args=() if [ -n "${AGENT_MODEL:-}" ]; then model_args=(--model "${AGENT_MODEL}") fi -export CODEX_HOME="${CODEX_HOME:-/home/harbor/.codex}" +config_args=(@@CODEX_CONFIG_ARGS@@) +export CODEX_HOME="/home/harbor/codex-benchmark-home" +rm -rf "$CODEX_HOME" +mkdir -p "$CODEX_HOME" if [ -f /tmp/codex_host_auth.json ]; then - mkdir -p "$CODEX_HOME" cp /tmp/codex_host_auth.json "$CODEX_HOME/auth.json" fi -python - <<'PY' -from pathlib import Path - -Path("/tmp/codex_output_schema.json").write_text(@@CODEX_STRUCTURED_OUTPUT_SCHEMA@@, encoding="utf-8") -instruction = Path(@@INSTRUCTION_CONTAINER@@).read_text(encoding="utf-8") -prompt = @@CODEX_STRUCTURED_PROMPT_PREFIX@@ + "\n\nTask instruction:\n" + instruction -Path("/tmp/codex_structured_prompt.md").write_text(prompt, encoding="utf-8") -PY +# Run from /app like the other direct agent backends. Only auth is optionally +# copied into a fresh container-local CODEX_HOME; host config is not reproduced. set +e codex exec \ + --ignore-user-config \ --skip-git-repo-check \ - --sandbox read-only \ + --sandbox danger-full-access \ --cd /app \ - --output-schema /tmp/codex_output_schema.json \ - --output-last-message /logs/agent/codex_structured_output.json \ + --output-last-message /logs/agent/final_message.txt \ --json \ "${model_args[@]}" \ - - < /tmp/codex_structured_prompt.md \ + "${config_args[@]}" \ + - < @@INSTRUCTION_CONTAINER@@ \ > /tmp/nat_agent.log 2> /tmp/nat_agent.stderr rc=$? set -e cp /tmp/nat_agent.log /logs/agent/nat_agent.log 2>/dev/null || true cp /tmp/nat_agent.stderr /logs/agent/nat_agent.stderr 2>/dev/null || true -if [ $rc -eq 0 ]; then - python - <<'PY' -import json -import sys -from pathlib import Path, PurePosixPath - -app_root = Path("/app") -payload_path = Path("/logs/agent/codex_structured_output.json") -try: - payload = json.loads(payload_path.read_text(encoding="utf-8")) -except Exception as exc: - print(f"Failed to parse Codex structured output: {exc}", file=sys.stderr) - sys.exit(1) - -final_summary = payload.get("final_summary") -if not isinstance(final_summary, str) or not final_summary.strip(): - print("Codex structured output missing non-empty final_summary", file=sys.stderr) - sys.exit(1) - -output_files = payload.get("output_files") -if not isinstance(output_files, list): - print("Codex structured output missing output_files list", file=sys.stderr) - sys.exit(1) - -for index, entry in enumerate(output_files): - if not isinstance(entry, dict): - print(f"output_files[{index}] must be an object", file=sys.stderr) - sys.exit(1) - rel = entry.get("path") - content = entry.get("content") - if not isinstance(rel, str) or not rel.strip(): - print(f"output_files[{index}].path must be a non-empty string", file=sys.stderr) - sys.exit(1) - if not isinstance(content, str): - print(f"output_files[{index}].content must be a string", file=sys.stderr) - sys.exit(1) - - rel_path = PurePosixPath(rel) - if rel_path.is_absolute() or not rel_path.parts: - print(f"output_files[{index}].path must be relative to /app: {rel}", file=sys.stderr) - sys.exit(1) - if any(part in {"", ".", ".."} for part in rel_path.parts): - print(f"output_files[{index}].path contains an unsafe path segment: {rel}", file=sys.stderr) - sys.exit(1) - if rel_path.parts[0] == ".git": - print(f"output_files[{index}].path may not target .git: {rel}", file=sys.stderr) - sys.exit(1) - - target = app_root.joinpath(*rel_path.parts) - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(content, encoding="utf-8") - -Path("/logs/agent/final_message.txt").write_text(final_summary.strip() + "\n", encoding="utf-8") -PY - rc=$? -fi exit $rc diff --git a/tests/agentic-use/shared/evaluator_agent_eval/factory.py b/tests/agentic-use/shared/evaluator_agent_eval/factory.py index cdac28cbcf4..ee7cdfedba2 100644 --- a/tests/agentic-use/shared/evaluator_agent_eval/factory.py +++ b/tests/agentic-use/shared/evaluator_agent_eval/factory.py @@ -14,7 +14,6 @@ AgentAttemptTrace, CapturedAgentAttempt, EvaluatorScoringRow, - TaskCheckResult, ) from evaluator_agent_eval.surfaces import detect_surfaces, evidence_from_artifacts from evaluator_agent_eval.task_config import AgenticUseTaskConfig, load_agentic_use_task_config @@ -84,7 +83,6 @@ def build_evaluator_scoring_row( task_dir: str | Path, attempt: CapturedAgentAttempt, artifacts: AgentArtifacts, - task_check_result: TaskCheckResult | None = None, task_config: AgenticUseTaskConfig | None = None, ) -> EvaluatorScoringRow: """Build an Evaluator SDK dataset row for one captured attempt.""" @@ -127,10 +125,6 @@ def build_evaluator_scoring_row( final_answer_extracted=attempt.output.final_answer_extracted, final_answer_source=attempt.output.final_answer_source, raw_log_paths=attempt.output.raw_log_paths, - task_success=task_check_result.task_success if task_check_result is not None else None, - verification_score=task_check_result.verification_score if task_check_result is not None else None, - output_schema_valid=task_check_result.output_schema_valid if task_check_result is not None else None, - verification_details=task_check_result.details if task_check_result is not None else {}, observed_surfaces=surface_detection.observed_surfaces, forbidden_surface_hits=surface_detection.forbidden_surface_hits, atif_trajectory_path=attempt.trace.atif_path if attempt.trace else None, diff --git a/tests/agentic-use/shared/evaluator_agent_eval/metrics/__init__.py b/tests/agentic-use/shared/evaluator_agent_eval/metrics/__init__.py index 0888d4bfd44..bcbe2885b66 100644 --- a/tests/agentic-use/shared/evaluator_agent_eval/metrics/__init__.py +++ b/tests/agentic-use/shared/evaluator_agent_eval/metrics/__init__.py @@ -5,12 +5,6 @@ from collections.abc import Sequence -from evaluator_agent_eval.metrics.outcome import ( - DeterministicTaskSuccessMetric, - OutputSchemaValidMetric, - SurfaceGatedSuccessMetric, - VerificationScoreMetric, -) from evaluator_agent_eval.metrics.surface import ( LEGACY_SURFACE, LEGACY_TEXT_PATTERNS, @@ -34,13 +28,9 @@ } __all__ = [ - "DeterministicTaskSuccessMetric", "LegacySurfaceAvoidanceMetric", - "OutputSchemaValidMetric", "SurfaceAdherenceMetric", - "SurfaceGatedSuccessMetric", "TrajectoryEvidenceMetric", - "VerificationScoreMetric", "default_agent_eval_metrics", ] diff --git a/tests/agentic-use/shared/evaluator_agent_eval/metrics/outcome.py b/tests/agentic-use/shared/evaluator_agent_eval/metrics/outcome.py deleted file mode 100644 index f4f36ad5b5f..00000000000 --- a/tests/agentic-use/shared/evaluator_agent_eval/metrics/outcome.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Outcome metrics for evaluator agent benchmark rows.""" - -import logging -from math import nan - -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore - -logger = logging.getLogger(__name__) - - -class DeterministicTaskSuccessMetric: - """Score task success from task-local deterministic checks.""" - - def __init__(self, success_key: str) -> None: - self.success_key = success_key - - @property - def type(self) -> str: - return "agent_eval/task_success" - - def score_names(self) -> list[str]: - return ["task_success"] - - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: - return MetricResult(scores=[MetricScore(name="task_success", value=float(item.get(self.success_key) is True))]) - - -class VerificationScoreMetric: - """Pass through a normalized task-specific verification score.""" - - def __init__(self, score_key: str) -> None: - self.score_key = score_key - - @property - def type(self) -> str: - return "agent_eval/verification_score" - - def score_names(self) -> list[str]: - return ["verification_score"] - - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: - value = item.get(self.score_key) - if isinstance(value, bool) or not isinstance(value, int | float) or value < 0.0 or value > 1.0: - return MetricResult(scores=[MetricScore(name="verification_score", value=nan)]) - return MetricResult(scores=[MetricScore(name="verification_score", value=float(value))]) - - -class OutputSchemaValidMetric: - """Score whether the task-specific checker accepted the output shape.""" - - def __init__(self, valid_key: str) -> None: - self.valid_key = valid_key - - @property - def type(self) -> str: - return "agent_eval/output_schema_valid" - - def score_names(self) -> list[str]: - return ["output_schema_valid"] - - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: - return MetricResult( - scores=[MetricScore(name="output_schema_valid", value=float(item.get(self.valid_key) is True))] - ) - - -class SurfaceGatedSuccessMetric: - """Score success only when task success and surface adherence both pass.""" - - def __init__( - self, - *, - success_key: str, - observed_surfaces_key: str, - allowed_surfaces_key: str, - forbidden_surfaces_key: str, - forbidden_surface_hits_key: str, - ) -> None: - self.success_key = success_key - self.observed_surfaces_key = observed_surfaces_key - self.allowed_surfaces_key = allowed_surfaces_key - self.forbidden_surfaces_key = forbidden_surfaces_key - self.forbidden_surface_hits_key = forbidden_surface_hits_key - - @property - def type(self) -> str: - return "agent_eval/surface_gated_success" - - def score_names(self) -> list[str]: - return ["surface_gated_success"] - - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: - try: - observed_surfaces = item[self.observed_surfaces_key] - allowed_surfaces = item[self.allowed_surfaces_key] - forbidden_surfaces = item[self.forbidden_surfaces_key] - forbidden_surface_hits = item[self.forbidden_surface_hits_key] - if ( - not isinstance(observed_surfaces, list) - or not isinstance(allowed_surfaces, list) - or not isinstance(forbidden_surfaces, list) - or not isinstance(forbidden_surface_hits, list) - or not all(isinstance(entry, str) for entry in observed_surfaces) - or not all(isinstance(entry, str) for entry in allowed_surfaces) - or not all(isinstance(entry, str) for entry in forbidden_surfaces) - or not all(isinstance(entry, str) for entry in forbidden_surface_hits) - ): - raise TypeError("Surface fields must be list[str]") - observed = set(observed_surfaces) - allowed = set(allowed_surfaces) - forbidden = set(forbidden_surfaces) - disallowed_observed = observed.difference(allowed) - forbidden_observed = observed.intersection(forbidden) - violation_count = len(disallowed_observed.union(forbidden_observed)) + len(forbidden_surface_hits) - adherence = 1.0 if violation_count == 0 else 0.0 - except (KeyError, TypeError) as exc: - logger.warning("Unable to compute surface-gated success score: %s", exc) - return MetricResult(scores=[MetricScore(name="surface_gated_success", value=nan)]) - success = item.get(self.success_key) is True and adherence == 1.0 - return MetricResult(scores=[MetricScore(name="surface_gated_success", value=float(success))]) diff --git a/tests/agentic-use/shared/evaluator_agent_eval/pytest_verifier.py b/tests/agentic-use/shared/evaluator_agent_eval/pytest_verifier.py index 40afa786464..feef35a2b96 100644 --- a/tests/agentic-use/shared/evaluator_agent_eval/pytest_verifier.py +++ b/tests/agentic-use/shared/evaluator_agent_eval/pytest_verifier.py @@ -3,6 +3,7 @@ """Shared pytest verifier for Evaluator agent benchmark tasks.""" +import json import os from pathlib import Path from typing import Sequence @@ -15,9 +16,9 @@ from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult AGENT_LOG_DIR = Path("/logs/agent") +VERIFIER_LOG_DIR = Path("/logs/verifier") TASK_DIR = Path(os.environ.get("AGENTIC_USE_TASK_DIR", "/task")) WORKSPACE_DIR = Path(os.environ.get("AGENTIC_USE_WORKSPACE_DIR", "/app/workspace")) - TASK_SUCCESS_SCORE = "task_success" VERIFICATION_SCORE = "verification_score" OUTPUT_SCHEMA_VALID_SCORE = "output_schema_valid" @@ -41,7 +42,12 @@ def verify_agent_attempt_scores_with_evaluator_sdk(*, task_specific_metrics: Seq artifacts=artifacts, task_config=task_config, ) - scored = score_evaluator_rows([scoring_row], additional_metrics=task_specific_metrics) + + scored = score_evaluator_rows( + [scoring_row], + additional_metrics=task_specific_metrics, + ) + _write_evaluator_scores(scored=scored) _assert_evaluator_scores(scored) @@ -56,7 +62,7 @@ def _assert_evaluator_scores(scored: BenchmarkEvaluationResult) -> None: if missing: raise AssertionError(f"Task-specific Evaluator metrics did not emit required scores: {', '.join(missing)}") - aggregate_scores = {score.name: score.mean for score in scored.aggregate_scores.scores} + aggregate_scores = _aggregate_scores(scored) assert all(value == 1.0 for value in score_values[TASK_SUCCESS_SCORE]) assert sum(score_values[VERIFICATION_SCORE]) / len(score_values[VERIFICATION_SCORE]) == 1.0 assert all(value == 1.0 for value in score_values[OUTPUT_SCHEMA_VALID_SCORE]) @@ -64,9 +70,7 @@ def _assert_evaluator_scores(scored: BenchmarkEvaluationResult) -> None: assert aggregate_scores["agent_eval/legacy_surface_avoidance.legacy_surface_avoidance"] == 1.0 -def _task_score_values( - scored: BenchmarkEvaluationResult, -) -> dict[str, list[float]]: +def _task_score_values(scored: BenchmarkEvaluationResult) -> dict[str, list[float]]: score_values: dict[str, list[float]] = { TASK_SUCCESS_SCORE: [], VERIFICATION_SCORE: [], @@ -78,5 +82,19 @@ def _task_score_values( for metric_scores in scored.row_scores[0].metrics.values(): for score in metric_scores: if score.name in score_values: - score_values[score.name].append(score.value) + score_values[score.name].append(float(score.value)) return score_values + + +def _aggregate_scores(scored: BenchmarkEvaluationResult) -> dict[str, float]: + return {str(score.name): float(score.mean) for score in scored.aggregate_scores.scores if score.mean is not None} + + +def _write_evaluator_scores(*, scored: BenchmarkEvaluationResult) -> None: + VERIFIER_LOG_DIR.mkdir(parents=True, exist_ok=True) + payload = { + "aggregate_scores": [ + {"name": name, "mean": value} for name, value in sorted(_aggregate_scores(scored).items()) + ], + } + (VERIFIER_LOG_DIR / "evaluator_scores.json").write_text(json.dumps(payload, indent=2), encoding="utf-8") diff --git a/tests/agentic-use/shared/evaluator_agent_eval/schemas.py b/tests/agentic-use/shared/evaluator_agent_eval/schemas.py index fed99b4c92f..37758688d8d 100644 --- a/tests/agentic-use/shared/evaluator_agent_eval/schemas.py +++ b/tests/agentic-use/shared/evaluator_agent_eval/schemas.py @@ -94,24 +94,6 @@ def _non_negative(cls, value: int) -> int: return value -class TaskCheckResult(BaseModel): - """Normalized result produced by a task-specific checker.""" - - model_config = ConfigDict(extra="forbid") - - task_success: bool - verification_score: float - output_schema_valid: bool - details: dict[str, object] = Field(default_factory=dict) - - @field_validator("verification_score") - @classmethod - def _score_in_unit_interval(cls, value: float) -> float: - if value < 0.0 or value > 1.0: - raise ValueError("verification_score must be between 0.0 and 1.0") - return value - - class EvaluatorScoringRow(BaseModel): """Evaluator-specific dataset row derived from one captured attempt. @@ -141,10 +123,6 @@ class EvaluatorScoringRow(BaseModel): final_answer_extracted: bool = False final_answer_source: str | None = None raw_log_paths: list[str] = Field(default_factory=list) - task_success: bool | None = None - verification_score: float | None = None - output_schema_valid: bool | None = None - verification_details: dict[str, object] = Field(default_factory=dict) observed_surfaces: list[SurfaceName] = Field(default_factory=list) forbidden_surface_hits: list[str] = Field(default_factory=list) atif_trajectory_path: str | None = None @@ -159,13 +137,6 @@ def _allowed_surfaces_not_empty(cls, value: list[SurfaceName]) -> list[SurfaceNa raise ValueError("allowed_surfaces must not be empty") return value - @field_validator("verification_score") - @classmethod - def _verification_score_in_unit_interval(cls, value: float | None) -> float | None: - if value is not None and (value < 0.0 or value > 1.0): - raise ValueError("verification_score must be between 0.0 and 1.0") - return value - def to_dataset_row(self) -> dict[str, object]: """Serialize as a JSON-compatible Evaluator SDK dataset row.""" return self.model_dump(mode="json") diff --git a/tests/agentic-use/shared/evaluator_agent_eval/task_metric_utils.py b/tests/agentic-use/shared/evaluator_agent_eval/task_metric_utils.py new file mode 100644 index 00000000000..e73f9aff886 --- /dev/null +++ b/tests/agentic-use/shared/evaluator_agent_eval/task_metric_utils.py @@ -0,0 +1,181 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers for task-local Evaluator SDK metrics. + +These helpers keep task-specific metric modules focused on their scoring policy. +They deliberately return plain Python values so callers can use them from SDK +``Metric`` implementations today and from enriched metric outputs later. +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +import tempfile +from collections.abc import Callable +from pathlib import Path + +from evaluator_agent_eval.artifacts import AgentArtifacts + +CodePredicate = Callable[[str], bool] +CodeScorer = Callable[[str], int] + + +def contains_all(text: str, terms: list[str]) -> bool: + lowered = text.lower() + return all(term.lower() in lowered for term in terms) + + +def object_dict(value: object) -> dict[str, object]: + if not isinstance(value, dict): + return {} + return {str(key): item for key, item in value.items()} + + +def string_list(value: object) -> list[str]: + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, str)] + + +def score_checks(checks: list[bool]) -> float: + if not checks: + return 0.0 + return sum(float(value) for value in checks) / len(checks) + + +def extract_fenced_python_code( + text: str, + *, + predicate: CodePredicate, + scorer: CodeScorer | None = None, +) -> str | None: + best_code: str | None = None + best_score = -1 + for match in re.finditer(r"```(?:python|py)\s*\n(?P.*?)```", text, re.DOTALL | re.IGNORECASE): + code = match.group("code").strip() + if not predicate(code): + continue + if scorer is None: + return code + score = scorer(code) + if score > best_score: + best_code = code + best_score = score + return best_code + + +def find_workspace_python_code( + artifacts: AgentArtifacts, + *, + preferred_names: list[str], + predicate: CodePredicate, +) -> str | None: + workspace_dir = artifacts.workspace_dir + if workspace_dir is None: + return None + + candidates = [artifacts.workspace_artifact(name) for name in preferred_names] + candidates.extend(sorted(path for path in workspace_dir.glob("*.py") if path.is_file())) + + seen: set[Path] = set() + for path in candidates: + if path is None or path in seen or not path.is_file(): + continue + seen.add(path) + code = path.read_text(encoding="utf-8", errors="replace").strip() + if predicate(code): + return code + return None + + +def run_python_code( + code: str, + *, + filename: str, + timeout: int, + appended_code: str = "", + cwd: Path | None = None, + timeout_stderr: str, +) -> subprocess.CompletedProcess[str]: + script_text = f"{code}\n\n{appended_code}\n" if appended_code else code + with tempfile.TemporaryDirectory() as tmp_dir: + script_path = Path(tmp_dir) / filename + script_path.write_text(script_text, encoding="utf-8") + return run_python_file( + script_path, + timeout=timeout, + cwd=cwd, + timeout_stderr=timeout_stderr, + missing_stderr=f"missing {filename}", + ) + + +def run_python_file( + path: Path | None, + *, + timeout: int, + cwd: Path | None = None, + timeout_stderr: str, + missing_stderr: str, +) -> subprocess.CompletedProcess[str]: + if path is None: + return subprocess.CompletedProcess(args=[], returncode=1, stdout="", stderr=missing_stderr) + command = [sys.executable, str(path)] + try: + return subprocess.run( + command, + check=False, + capture_output=True, + text=True, + cwd=cwd, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + return subprocess.CompletedProcess( + args=exc.cmd or command, + returncode=124, + stdout=timeout_text(exc.stdout), + stderr=timeout_text(exc.stderr, fallback=timeout_stderr), + ) + + +def extract_json_object_from_stdout(stdout: str) -> dict[str, object] | None: + lines = stdout.splitlines() + for start in range(len(lines)): + candidate = "\n".join(lines[start:]).strip() + if not candidate.startswith("{"): + continue + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + parsed = object_dict(value) + if parsed: + return parsed + return None + + +def extract_marker_json_object(stdout: str, *, marker: str) -> dict[str, object] | None: + for line in reversed(stdout.splitlines()): + if not line.startswith(marker): + continue + try: + value = json.loads(line.removeprefix(marker)) + except json.JSONDecodeError: + continue + parsed = object_dict(value) + if parsed: + return parsed + return None + + +def timeout_text(value: str | bytes | None, *, fallback: str = "") -> str: + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if isinstance(value, str): + return value + return fallback diff --git a/tests/agentic-use/tests/evaluator_agent_eval/test_construction.py b/tests/agentic-use/tests/evaluator_agent_eval/test_construction.py index ad6606713a1..403ba72f5a4 100644 --- a/tests/agentic-use/tests/evaluator_agent_eval/test_construction.py +++ b/tests/agentic-use/tests/evaluator_agent_eval/test_construction.py @@ -8,7 +8,6 @@ from evaluator_agent_eval.artifacts import AgentArtifacts from evaluator_agent_eval.factory import AgentRunMetadata, build_evaluator_scoring_row, capture_agent_attempt -from evaluator_agent_eval.schemas import TaskCheckResult def test_capture_agent_attempt_from_task_config_and_artifacts(evaluator_task_dir: Path, agent_log_dir: Path): @@ -50,7 +49,6 @@ def test_build_evaluator_scoring_row_from_task_config_and_artifacts(evaluator_ta task_dir=evaluator_task_dir, attempt=attempt, artifacts=artifacts, - task_check_result=TaskCheckResult(task_success=True, verification_score=1.0, output_schema_valid=True), ) assert scoring_row.task_id == "task" @@ -63,10 +61,6 @@ def test_build_evaluator_scoring_row_from_task_config_and_artifacts(evaluator_ta assert scoring_row.agent_model == "sonnet" assert scoring_row.final_answer_extracted is True assert scoring_row.final_answer_source == "final_message.txt" - assert scoring_row.task_success is True - assert scoring_row.verification_score == 1.0 - assert scoring_row.output_schema_valid is True - assert scoring_row.verification_details == {} assert scoring_row.observed_surfaces == ["standalone_sdk"] @@ -90,9 +84,6 @@ def test_build_evaluator_scoring_row_allows_sdk_task_metrics_to_populate_scores( artifacts=artifacts, ) - assert scoring_row.task_success is None - assert scoring_row.verification_score is None - assert scoring_row.output_schema_valid is None assert scoring_row.output_text == "Use packages/nemo_evaluator_sdk with Evaluator, run_sync, and ExactMatchMetric." @@ -137,13 +128,9 @@ def test_build_evaluator_scoring_row_fails_success_when_final_answer_missing( task_dir=evaluator_task_dir, attempt=attempt, artifacts=artifacts, - task_check_result=TaskCheckResult(task_success=False, verification_score=0.0, output_schema_valid=False), ) assert scoring_row.output_text == "" - assert scoring_row.task_success is False - assert scoring_row.verification_score == 0.0 - assert scoring_row.output_schema_valid is False def test_capture_agent_attempt_carries_runtime_metadata(evaluator_task_dir: Path, agent_log_dir: Path): @@ -194,7 +181,6 @@ def test_build_evaluator_scoring_row_validates_atif_and_populates_trajectory_sum task_dir=evaluator_task_dir, attempt=attempt, artifacts=artifacts, - task_check_result=TaskCheckResult(task_success=True, verification_score=1.0, output_schema_valid=True), ) assert scoring_row.atif_trajectory_path == str(agent_log_dir / "trajectory.json") @@ -226,7 +212,6 @@ def test_build_evaluator_scoring_row_ignores_raw_log_prompt_echo_for_surface_det task_dir=evaluator_task_dir, attempt=attempt, artifacts=artifacts, - task_check_result=TaskCheckResult(task_success=True, verification_score=1.0, output_schema_valid=True), ) assert scoring_row.observed_surfaces == ["standalone_sdk"] diff --git a/tests/agentic-use/tests/evaluator_agent_eval/test_metrics.py b/tests/agentic-use/tests/evaluator_agent_eval/test_metrics.py index 540fd08774c..c9981498efa 100644 --- a/tests/agentic-use/tests/evaluator_agent_eval/test_metrics.py +++ b/tests/agentic-use/tests/evaluator_agent_eval/test_metrics.py @@ -7,13 +7,9 @@ import pytest from evaluator_agent_eval.metrics import ( - DeterministicTaskSuccessMetric, LegacySurfaceAvoidanceMetric, - OutputSchemaValidMetric, SurfaceAdherenceMetric, - SurfaceGatedSuccessMetric, TrajectoryEvidenceMetric, - VerificationScoreMetric, default_agent_eval_metrics, ) from evaluator_agent_eval.schemas import EvaluatorScoringRow @@ -42,9 +38,6 @@ def _row(**overrides: object) -> EvaluatorScoringRow: "allowed_surfaces": ["standalone_sdk"], "forbidden_surfaces": ["legacy_service"], "output_text": "Used nemo_evaluator_sdk only.", - "task_success": True, - "verification_score": 1.0, - "output_schema_valid": True, "observed_surfaces": ["standalone_sdk"], "trajectory_summary": {"tool_call_count": 3, "failed_command_count": 1, "recovery_event_count": 1}, } @@ -56,14 +49,6 @@ def _scores_by_name(result) -> dict[str, float]: return {score.name: score.value for score in result.scores} -@pytest.mark.asyncio -async def test_task_success_metric_can_use_configured_success_key(): - metric = DeterministicTaskSuccessMetric(success_key="verifier_passed") - result = await metric.compute_scores({"verifier_passed": True}, {}) - - assert _scores_by_name(result)["task_success"] == 1.0 - - @pytest.mark.asyncio async def test_surface_adherence_penalizes_forbidden_surface_hits(): metric = SurfaceAdherenceMetric(**SURFACE_FIELD_KEYS) @@ -79,31 +64,6 @@ async def test_surface_adherence_penalizes_forbidden_surface_hits(): assert scores["surface_violation_count"] == 2.0 -@pytest.mark.asyncio -async def test_surface_gated_success_requires_task_success_and_adherence(): - metric = SurfaceGatedSuccessMetric(success_key="task_success", **SURFACE_FIELD_KEYS) - clean = await metric.compute_scores(_row().to_dataset_row(), {}) - violated = await metric.compute_scores( - _row(observed_surfaces=["standalone_sdk", "legacy_service"]).to_dataset_row(), {} - ) - failed = await metric.compute_scores(_row(task_success=False).to_dataset_row(), {}) - - assert _scores_by_name(clean)["surface_gated_success"] == 1.0 - assert _scores_by_name(violated)["surface_gated_success"] == 0.0 - assert _scores_by_name(failed)["surface_gated_success"] == 0.0 - - -@pytest.mark.asyncio -async def test_surface_gated_success_can_use_configured_success_key(): - metric = SurfaceGatedSuccessMetric(success_key="verifier_passed", **SURFACE_FIELD_KEYS) - result = await metric.compute_scores( - _row(task_success=False).to_dataset_row() | {"verifier_passed": True}, - {}, - ) - - assert _scores_by_name(result)["surface_gated_success"] == 1.0 - - @pytest.mark.asyncio async def test_legacy_surface_avoidance_scans_output_text(): metric = LegacySurfaceAvoidanceMetric( @@ -177,31 +137,6 @@ async def test_surface_adherence_returns_nan_for_malformed_surface_fields(): assert isnan(scores["surface_violation_count"]) -@pytest.mark.asyncio -async def test_surface_gated_success_returns_nan_for_malformed_surface_fields(): - metric = SurfaceGatedSuccessMetric(success_key="task_success", **SURFACE_FIELD_KEYS) - result = await metric.compute_scores( - _row().to_dataset_row() | {"forbidden_surface_hits": [1]}, - {}, - ) - - assert isnan(_scores_by_name(result)["surface_gated_success"]) - - -@pytest.mark.asyncio -async def test_surface_gated_success_logs_malformed_surface_error(caplog): - metric = SurfaceGatedSuccessMetric(success_key="task_success", **SURFACE_FIELD_KEYS) - - with caplog.at_level("WARNING", logger="evaluator_agent_eval.metrics.outcome"): - result = await metric.compute_scores( - _row().to_dataset_row() | {"forbidden_surface_hits": [1]}, - {}, - ) - - assert isnan(_scores_by_name(result)["surface_gated_success"]) - assert "Surface fields must be list[str]" in caplog.text - - @pytest.mark.asyncio async def test_legacy_surface_avoidance_returns_nan_for_malformed_fields(): metric = LegacySurfaceAvoidanceMetric( @@ -272,27 +207,3 @@ async def test_trajectory_metric_returns_nan_for_malformed_summary(): assert isnan(scores["tool_call_count"]) assert isnan(scores["failed_command_count"]) assert isnan(scores["recovery_event_count"]) - - -@pytest.mark.asyncio -async def test_verification_score_metric_passes_through_configured_score_key(): - metric = VerificationScoreMetric(score_key="score") - result = await metric.compute_scores({"score": 0.75}, {}) - - assert _scores_by_name(result)["verification_score"] == 0.75 - - -@pytest.mark.asyncio -async def test_verification_score_metric_returns_nan_for_bad_score(): - metric = VerificationScoreMetric(score_key="score") - result = await metric.compute_scores({"score": 1.5}, {}) - - assert isnan(_scores_by_name(result)["verification_score"]) - - -@pytest.mark.asyncio -async def test_output_schema_valid_metric_can_use_configured_key(): - metric = OutputSchemaValidMetric(valid_key="schema_ok") - result = await metric.compute_scores({"schema_ok": True}, {}) - - assert _scores_by_name(result)["output_schema_valid"] == 1.0 diff --git a/tests/agentic-use/tests/evaluator_agent_eval/test_runner.py b/tests/agentic-use/tests/evaluator_agent_eval/test_runner.py index 0a07c9ee0d1..2fad974850a 100644 --- a/tests/agentic-use/tests/evaluator_agent_eval/test_runner.py +++ b/tests/agentic-use/tests/evaluator_agent_eval/test_runner.py @@ -29,9 +29,6 @@ def _row(**overrides: object) -> EvaluatorScoringRow: "allowed_surfaces": ["standalone_sdk"], "forbidden_surfaces": ["legacy_service"], "output_text": "Used nemo_evaluator_sdk only.", - "task_success": True, - "verification_score": 1.0, - "output_schema_valid": True, "observed_surfaces": ["standalone_sdk"], "trajectory_summary": {"tool_call_count": 3, "failed_command_count": 1, "recovery_event_count": 1}, } @@ -40,9 +37,7 @@ def _row(**overrides: object) -> EvaluatorScoringRow: def test_score_evaluator_rows_uses_evaluator_sdk(): - result = score_evaluator_rows( - [_row(), _row(task_success=False, verification_score=0.25, output_schema_valid=False)] - ) + result = score_evaluator_rows([_row(), _row(output_text="Still used nemo_evaluator_sdk.")]) assert len(result.row_scores) == 2 aggregate_names = {score.name for score in result.aggregate_scores.scores} diff --git a/tests/agentic-use/tests/evaluator_agent_eval/test_schemas.py b/tests/agentic-use/tests/evaluator_agent_eval/test_schemas.py index a6190923a9c..44f586a1600 100644 --- a/tests/agentic-use/tests/evaluator_agent_eval/test_schemas.py +++ b/tests/agentic-use/tests/evaluator_agent_eval/test_schemas.py @@ -10,7 +10,6 @@ AgentAttemptOutput, CapturedAgentAttempt, EvaluatorScoringRow, - TaskCheckResult, TrajectorySummary, ) @@ -70,16 +69,3 @@ def test_schema_keeps_candidate_model(): def test_trajectory_summary_rejects_negative_counts(): with pytest.raises(ValueError, match="non-negative"): TrajectorySummary(tool_call_count=-1) - - -def test_task_check_result_requires_unit_interval_score(): - with pytest.raises(ValueError, match="verification_score"): - TaskCheckResult(task_success=False, verification_score=1.1, output_schema_valid=True) - - -def test_scoring_row_carries_normalized_check_result_fields(): - row = _scoring_row(task_success=True, verification_score=0.75, output_schema_valid=True) - - assert row.task_success is True - assert row.verification_score == 0.75 - assert row.output_schema_valid is True diff --git a/tests/agentic-use/tests/evaluator_agent_eval/test_task_config.py b/tests/agentic-use/tests/evaluator_agent_eval/test_task_config.py index 98b5899bd44..87266d97756 100644 --- a/tests/agentic-use/tests/evaluator_agent_eval/test_task_config.py +++ b/tests/agentic-use/tests/evaluator_agent_eval/test_task_config.py @@ -53,6 +53,8 @@ def test_manifest_tasks_have_evaluator_config_and_verifier(): assert task_names == [ "evaluator-standalone-sdk-surface-discovery", "evaluator-standalone-sdk-simple-exact-match", + "evaluator-standalone-sdk-agent-target", + "evaluator-standalone-sdk-surface-adherence-metric", ] for task_name in task_names: task_dir = agentic_use_dir / task_name diff --git a/tests/agentic-use/tests/evaluator_agent_eval/test_task_metrics.py b/tests/agentic-use/tests/evaluator_agent_eval/test_task_metrics.py index 17deec4bece..b075203d9d7 100644 --- a/tests/agentic-use/tests/evaluator_agent_eval/test_task_metrics.py +++ b/tests/agentic-use/tests/evaluator_agent_eval/test_task_metrics.py @@ -12,6 +12,7 @@ from evaluator_agent_eval.runner import score_evaluator_rows from evaluator_agent_eval.schemas import CapturedAgentAttempt from evaluator_agent_eval.task_config import AgenticUseTaskConfig, load_agentic_use_task_config +from evaluator_agent_eval.task_metric_utils import extract_marker_json_object from nemo_evaluator_sdk.metrics.base import Metric @@ -378,3 +379,255 @@ def _score_task_metric( if score.name in score_values: score_values[score.name].append(score.value) return score_values + + +def test_agent_target_metric_requires_runnable_agent_target_code(tmp_path: Path, agent_log_dir: Path): + task_dir = tmp_path / "task" + task_dir.mkdir() + (task_dir / "task.toml").write_text( + """ +version = "1.0" + +[evaluator.surface] +constraint = "standalone_sdk" +allowed = ["standalone_sdk"] +forbidden = ["cli", "plugin_sdk", "legacy_service"] + +[evaluator.expected] +required_terms = [ + "packages/nemo_evaluator_sdk", + "Evaluator", + "run_sync", + "Agent", + "nemo_evaluator_sdk.values.agents", + "AgentFormat.GENERIC", + "ExactMatchMetric", + "response_path", + "$.answer", + "trajectory_path", + "$.trajectory", + "candidate_agent_runtime", + "candidate_agent_model", +] +""".strip(), + encoding="utf-8", + ) + (task_dir / "instruction.md").write_text("Configure an agent target.", encoding="utf-8") + (agent_log_dir / "final_message.txt").write_text( + """ +Use packages/nemo_evaluator_sdk with Evaluator.run_sync, Agent, AgentFormat.GENERIC, and ExactMatchMetric. +The metadata identifies candidate_agent_runtime and candidate_agent_model. + +```python +from http.server import BaseHTTPRequestHandler, HTTPServer +from threading import Thread +import json + +from nemo_evaluator_sdk import Evaluator, ExactMatchMetric +from nemo_evaluator_sdk.enums import AgentFormat +from nemo_evaluator_sdk.values.agents import Agent + + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + self.rfile.read(int(self.headers.get("content-length", "0"))) + body = {"answer": "4", "trajectory": {"steps": [{"tool_calls": [{"name": "calculator"}]}]}} + payload = json.dumps(body).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + +server = HTTPServer(("127.0.0.1", 0), Handler) +thread = Thread(target=server.serve_forever, daemon=True) +thread.start() +try: + agent = Agent( + url=f"http://127.0.0.1:{server.server_port}", + name="mock-calculator-agent", + format=AgentFormat.GENERIC, + body={"prompt": "{{ prompt }}"}, + response_path="$.answer", + trajectory_path="$.trajectory", + ) + rows = [ + { + "prompt": "What is 2+2?", + "expected": "4", + "candidate_agent_runtime": "mock-agent", + "candidate_agent_model": "mock-calculator-v1", + } + ] + result = Evaluator().run_sync( + metrics=[ExactMatchMetric(reference="{{ item.expected }}")], + dataset=rows, + target=agent, + prompt_template="{{ item.prompt }}", + ) + payload = { + "answer": result.row_scores[0].sample["output_text"], + "exact_match": result.row_scores[0].metrics["exact-match"][0].value, + "trajectory_tool_calls": result.row_scores[0].sample["trajectory"]["steps"][0]["tool_calls"], + "candidate_agent_runtime": rows[0]["candidate_agent_runtime"], + "candidate_agent_model": rows[0]["candidate_agent_model"], + } + print(json.dumps(payload, indent=2)) +finally: + server.shutdown() +``` +""".strip(), + encoding="utf-8", + ) + artifacts = AgentArtifacts.from_dir(agent_log_dir) + task_config = load_agentic_use_task_config(task_dir) + attempt = capture_agent_attempt(task_dir=task_dir, artifacts=artifacts) + module = _load_task_metrics("evaluator-standalone-sdk-agent-target") + + scores = _score_task_metric( + metric=module.AgentTargetConfigurationMetric(), + task_dir=task_dir, + artifacts=artifacts, + task_config=task_config, + attempt=attempt, + ) + + assert scores["task_success"] == [1.0] + assert scores["verification_score"] == [1.0] + assert scores["output_schema_valid"] == [1.0] + + +def test_agent_target_metric_reads_workspace_python_artifact(tmp_path: Path, agent_log_dir: Path): + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + (workspace_dir / "agent_target_exact_match.py").write_text( + "from nemo_evaluator_sdk import Evaluator\n" + "from nemo_evaluator_sdk.values.agents import Agent\n" + "Evaluator().run_sync(target=Agent(response_path='$.answer', trajectory_path='$.trajectory'))\n", + encoding="utf-8", + ) + artifacts = AgentArtifacts.from_dir(agent_log_dir, workspace_dir=workspace_dir) + module = _load_task_metrics("evaluator-standalone-sdk-agent-target") + + code = module._extract_workspace_python_code(artifacts) + + assert code is not None + assert "agent_target_exact_match.py" not in code + assert "Evaluator().run_sync" in code + + +def test_surface_adherence_metric_authoring_runs_candidate_metric(tmp_path: Path, agent_log_dir: Path): + task_dir = tmp_path / "task" + task_dir.mkdir() + (task_dir / "task.toml").write_text( + """ +version = "1.0" + +[evaluator.surface] +constraint = "standalone_sdk" +allowed = ["standalone_sdk"] +forbidden = ["cli", "plugin_sdk", "legacy_service"] + +[evaluator.expected] +required_terms = [ + "packages/nemo_evaluator_sdk", + "Metric", + "type", + "compute_scores", + "score_names", + "MetricResult", + "MetricScore", + "surface_adherence", + "surface_violation_count", + "observed_surfaces", + "allowed_surfaces", + "forbidden_surfaces", +] +""".strip(), + encoding="utf-8", + ) + (task_dir / "instruction.md").write_text("Author a surface-adherence metric.", encoding="utf-8") + (agent_log_dir / "final_message.txt").write_text( + """ +Use packages/nemo_evaluator_sdk and implement a Metric-compatible class with compute_scores and score_names. + +```python +from nemo_evaluator_sdk.values.results import MetricResult, MetricScore + + +class SurfaceAdherenceMetric: + @property + def type(self) -> str: + return "agent_eval/surface_adherence" + + def score_names(self) -> list[str]: + return ["surface_adherence", "surface_violation_count"] + + async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + observed_surfaces = set(item["observed_surfaces"]) + allowed_surfaces = set(item["allowed_surfaces"]) + forbidden_surfaces = set(item["forbidden_surfaces"]) + violations = (observed_surfaces - allowed_surfaces) | (observed_surfaces & forbidden_surfaces) + adherence = 1.0 if not violations else 0.0 + return MetricResult( + scores=[ + MetricScore(name="surface_adherence", value=adherence), + MetricScore(name="surface_violation_count", value=float(len(violations))), + ] + ) +``` +""".strip(), + encoding="utf-8", + ) + artifacts = AgentArtifacts.from_dir(agent_log_dir) + task_config = load_agentic_use_task_config(task_dir) + attempt = capture_agent_attempt(task_dir=task_dir, artifacts=artifacts) + module = _load_task_metrics("evaluator-standalone-sdk-surface-adherence-metric") + + scores = _score_task_metric( + metric=module.SurfaceAdherenceMetricAuthoringMetric(), + task_dir=task_dir, + artifacts=artifacts, + task_config=task_config, + attempt=attempt, + ) + + assert scores["task_success"] == [1.0] + assert scores["verification_score"] == [1.0] + assert scores["output_schema_valid"] == [1.0] + + +def test_surface_adherence_metric_reads_workspace_python_artifact(tmp_path: Path, agent_log_dir: Path): + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + (workspace_dir / "surface_adherence_metric.py").write_text( + "from nemo_evaluator_sdk.values.results import MetricResult, MetricScore\n" + "class SurfaceAdherenceMetric:\n" + " async def compute_scores(self, item, sample):\n" + " return MetricResult(scores=[MetricScore(name='surface_adherence', value=1.0)])\n", + encoding="utf-8", + ) + artifacts = AgentArtifacts.from_dir(agent_log_dir, workspace_dir=workspace_dir) + module = _load_task_metrics("evaluator-standalone-sdk-surface-adherence-metric") + + code = module._extract_workspace_python_code(artifacts) + + assert code is not None + assert "compute_scores" in code + assert "MetricScore" in code + + +def test_extract_marker_json_object_skips_trailing_malformed_marker(): + stdout = "\n".join( + [ + "prefix", + '__RESULT__={"score": 1}', + '__RESULT__={"truncated"', + ] + ) + + assert extract_marker_json_object(stdout, marker="__RESULT__=") == {"score": 1} diff --git a/tests/agentic-use/tests/test_agent_matrix_benchmark.py b/tests/agentic-use/tests/test_agent_matrix_benchmark.py new file mode 100644 index 00000000000..ce6ce7dbf48 --- /dev/null +++ b/tests/agentic-use/tests/test_agent_matrix_benchmark.py @@ -0,0 +1,564 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the agent_matrix_benchmark orchestration/reporting layer.""" + +from __future__ import annotations + +import json +import subprocess +import threading +from pathlib import Path +from typing import cast + +import pytest +from agent_matrix_benchmark import ( + AgentMatrixConfig, + Candidate, + CandidateConfig, + CodexParams, + _render_leaderboard_header, + build_matrix_summary, + build_nat_runner_command, + load_matrix_config, + main, + parse_candidate, + parse_candidates, + render_html, + render_markdown, + run_candidates, + write_reports, +) + + +class TestCandidateParsing: + def test_parses_backend_without_model(self) -> None: + candidate = parse_candidate("codex") + + assert candidate == Candidate(backend="codex", model=None, candidate_id="codex") + + def test_parses_backend_with_model(self) -> None: + candidate = parse_candidate("codex:gpt-5.1") + + assert candidate == Candidate(backend="codex", model="gpt-5.1", candidate_id="codex-gpt-5.1") + + def test_sanitizes_model_for_candidate_id(self) -> None: + candidate = parse_candidate("cursor-agent:vendor/model name") + + assert candidate.candidate_id == "cursor-agent-vendor-model-name" + + def test_defaults_to_direct_agent_backends(self) -> None: + candidates = parse_candidates(None) + + assert [candidate.backend for candidate in candidates] == ["codex", "claude-code", "cursor-agent"] + + def test_expands_agent_model_specs_under_selected_backends(self) -> None: + candidates = parse_candidates( + ["codex", "claude-code"], + ["codex:gpt-5.1", "codex:gpt-5.2", "claude-code:sonnet"], + ) + + assert candidates == [ + Candidate(backend="codex", model="gpt-5.1", candidate_id="codex-gpt-5.1"), + Candidate(backend="codex", model="gpt-5.2", candidate_id="codex-gpt-5.2"), + Candidate(backend="claude-code", model="sonnet", candidate_id="claude-code-sonnet"), + ] + + def test_keeps_backend_default_when_no_sub_models_are_specified(self) -> None: + candidates = parse_candidates(["cursor-agent"], ["codex:gpt-5.1"]) + + assert candidates == [ + Candidate(backend="codex", model="gpt-5.1", candidate_id="codex-gpt-5.1"), + Candidate(backend="cursor-agent", model=None, candidate_id="cursor-agent"), + ] + + def test_rejects_agent_model_spec_without_model(self) -> None: + with pytest.raises(ValueError, match="must use backend:model"): + parse_candidates(None, ["codex"]) + + def test_rejects_unknown_backend(self) -> None: + with pytest.raises(ValueError, match="Unsupported"): + parse_candidate("aut") + + def test_rejects_duplicate_candidate_ids(self) -> None: + with pytest.raises(ValueError, match="Duplicate"): + parse_candidates(["codex", "codex"]) + + +class TestPythonMatrixConfig: + def test_candidate_config_builds_typed_candidate_params(self) -> None: + candidate = CandidateConfig( + backend="codex", + model="gpt-5.5", + params=CodexParams(intelligence="high", speed="fast"), + ).to_candidate() + + assert candidate == Candidate( + backend="codex", + model="gpt-5.5", + candidate_id="codex-gpt-5.5-intelligence-high-speed-fast", + params={"intelligence": "high", "speed": "fast", "config": {}}, + ) + + def test_load_matrix_config_from_python_file(self, tmp_path: Path) -> None: + config_file = tmp_path / "matrix.py" + config_file.write_text( + """ +from agent_matrix_benchmark import AgentMatrixConfig, CandidateConfig, CodexParams + +MATRIX = AgentMatrixConfig( + manifest="manifests/evaluator_agent_benchmark_mvp.txt", + candidates=[ + CandidateConfig( + id="codex-fast", + backend="codex", + model="gpt-5.5", + params=CodexParams(intelligence="high", speed="fast"), + ) + ], +) +""", + encoding="utf-8", + ) + + config = load_matrix_config(config_file) + + assert isinstance(config, AgentMatrixConfig) + assert config.manifest == "manifests/evaluator_agent_benchmark_mvp.txt" + assert config.candidates[0].to_candidate().candidate_id == "codex-fast" + + def test_config_rejects_backend_param_mismatch(self) -> None: + with pytest.raises(ValueError, match="ClaudeCodeParams"): + CandidateConfig(backend="claude-code", model="sonnet", params=CodexParams(intelligence="high")) + + +def test_leaderboard_resize_handle_only_renders_for_resizable_columns() -> None: + assert "resize-handle" in _render_leaderboard_header("candidate", "Candidate", True, frozen=False) + assert "resize-handle" not in _render_leaderboard_header("passed", "Passed", True, frozen=False) + + +def test_build_nat_runner_command_forwards_matrix_flags(tmp_path: Path) -> None: + command = build_nat_runner_command( + candidate=Candidate( + backend="codex", + model="gpt-5.1", + candidate_id="codex-gpt-5.1", + params={"intelligence": "high", "speed": "fast"}, + ), + jobs_dir=tmp_path / "jobs" / "codex-gpt-5.1", + manifest=Path("manifests/evaluator_agent_benchmark_mvp.txt"), + tasks=[], + all_tasks=False, + skip_build=True, + allow_dirty=True, + timeout=123, + codex_auth_json=Path("/tmp/auth.json"), + nmp_base_url="http://localhost:8080", + anthropic_base_url="https://anthropic.example", + python_executable="python-test", + ) + + assert command[:2] == ["python-test", str(Path(__file__).parents[1] / "nat_runner.py")] + assert command[command.index("--agent-backend") + 1] == "codex" + assert command[command.index("--candidate-id") + 1] == "codex-gpt-5.1" + assert command[command.index("--agent-model") + 1] == "gpt-5.1" + assert json.loads(command[command.index("--candidate-params") + 1]) == { + "intelligence": "high", + "speed": "fast", + } + assert command[command.index("--timeout") + 1] == "123" + assert command[command.index("--codex-auth-json") + 1] == "/tmp/auth.json" + assert "--skip-build" in command + assert "--allow-dirty" in command + assert command[command.index("--manifest") + 1] == "manifests/evaluator_agent_benchmark_mvp.txt" + + +def test_build_nat_runner_command_uses_task_args_when_manifest_omitted(tmp_path: Path) -> None: + command = build_nat_runner_command( + candidate=Candidate(backend="claude-code", model=None, candidate_id="claude-code"), + jobs_dir=tmp_path / "jobs" / "claude-code", + manifest=None, + tasks=["task-a", "task-b"], + all_tasks=False, + skip_build=False, + allow_dirty=False, + timeout=600, + codex_auth_json=None, + nmp_base_url="http://localhost:8080", + anthropic_base_url="https://anthropic.example", + python_executable="python-test", + ) + + assert "--manifest" not in command + assert command[-2:] == ["task-a", "task-b"] + assert command[command.index("--anthropic-base-url") + 1] == "https://anthropic.example" + + +def test_main_rejects_empty_task_selection(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr("sys.argv", ["agent_matrix_benchmark.py", "--agent", "codex"]) + + assert main() == 1 + + captured = capsys.readouterr() + assert "No tasks specified" in captured.err + + +def test_main_rejects_parallel_candidates_without_skip_build( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr( + "sys.argv", + [ + "agent_matrix_benchmark.py", + "--parallel-candidates", + "2", + "--agent", + "codex", + "evaluator-standalone-sdk-surface-discovery", + ], + ) + + assert main() == 1 + + captured = capsys.readouterr() + assert "--parallel-candidates > 1 requires --skip-build" in captured.err + + +def test_main_rejects_zero_parallel_candidates( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr( + "sys.argv", + [ + "agent_matrix_benchmark.py", + "--parallel-candidates", + "0", + "--agent", + "codex", + "evaluator-standalone-sdk-surface-discovery", + ], + ) + + assert main() == 1 + + captured = capsys.readouterr() + assert "--parallel-candidates must be >= 1" in captured.err + + +def test_run_candidates_parallelizes_across_candidates(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + active = 0 + max_active = 0 + lock = threading.Lock() + two_active = threading.Event() + + def fake_run(command: list[str], *, check: bool) -> subprocess.CompletedProcess[str]: + nonlocal active, max_active + with lock: + active += 1 + max_active = max(max_active, active) + if active == 2: + two_active.set() + two_active.wait(timeout=1) + with lock: + active -= 1 + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr("agent_matrix_benchmark.subprocess.run", fake_run) + + metadata = run_candidates( + candidates=[ + Candidate(backend="codex", model=None, candidate_id="codex"), + Candidate(backend="cursor-agent", model=None, candidate_id="cursor-agent"), + ], + run_dir=tmp_path, + selected_tasks=["task-a"], + manifest=None, + tasks=["task-a"], + all_tasks=False, + skip_build=True, + allow_dirty=True, + timeout=30, + codex_auth_json=None, + nmp_base_url="http://localhost:8080", + anthropic_base_url="https://anthropic.example", + parallel_candidates=2, + ) + + assert max_active == 2 + assert [entry["candidate_id"] for entry in metadata] == ["codex", "cursor-agent"] + + +class TestAggregationAndReports: + def test_build_matrix_summary_counts_missing_tasks_and_ranks_candidates(self, tmp_path: Path) -> None: + run_dir = tmp_path / "matrix" + _write_result( + run_dir / "codex" / "run-a" / "result.json", + task="task-a", + candidate_id="codex", + backend="codex", + model="default", + passed=True, + total_tokens=100, + runtime_sec=10.0, + ) + _write_result( + run_dir / "codex" / "run-b" / "result.json", + task="task-b", + candidate_id="codex", + backend="codex", + model="default", + passed=False, + total_tokens=40, + runtime_sec=20.0, + verifier_text="assertion failed", + ) + _write_result( + run_dir / "cursor-agent" / "run-a" / "result.json", + task="task-a", + candidate_id="cursor-agent", + backend="cursor-agent", + model="default", + passed=True, + total_tokens=80, + runtime_sec=12.0, + ) + _write_result( + run_dir / "cursor-agent" / "run-b" / "result.json", + task="task-b", + candidate_id="cursor-agent", + backend="cursor-agent", + model="default", + passed=False, + total_tokens=30, + runtime_sec=15.0, + ) + _write_result( + run_dir / "claude-code" / "run-a" / "result.json", + task="task-a", + candidate_id="claude-code", + backend="claude-code", + model="default", + passed=True, + total_tokens=None, + runtime_sec=8.0, + ) + + summary = build_matrix_summary( + run_dir=run_dir, + candidates=[ + Candidate(backend="codex", model=None, candidate_id="codex"), + Candidate(backend="cursor-agent", model=None, candidate_id="cursor-agent"), + Candidate(backend="claude-code", model=None, candidate_id="claude-code"), + ], + tasks=["task-a", "task-b"], + ) + + ranking = cast(list[dict[str, object]], summary["ranking"]) + assert [candidate["candidate_id"] for candidate in ranking] == ["cursor-agent", "codex", "claude-code"] + assert ranking[0]["pass_rate"] == 0.5 + assert ranking[0]["total_tokens_sum"] == 110 + assert ranking[0]["prompt_tokens_sum"] == 110 + assert ranking[0]["completion_tokens_sum"] == 0 + assert ranking[0]["cache_read_tokens_sum"] == 0 + assert ranking[0]["cache_creation_tokens_sum"] == 0 + candidate_groups = cast(dict[str, list[dict[str, object]]], summary["candidate_groups"]) + assert [candidate["candidate_id"] for candidate in candidate_groups["cursor-agent"]] == ["cursor-agent"] + rows = cast(list[dict[str, object]], summary["rows"]) + missing = [row for row in rows if row["candidate_id"] == "claude-code" and row["task"] == "task-b"] + assert missing[0]["status"] == "missing" + + def test_candidate_token_sums_include_partial_usage_with_coverage(self, tmp_path: Path) -> None: + run_dir = tmp_path / "matrix" + _write_result( + run_dir / "cursor-agent" / "run-a" / "result.json", + task="task-a", + candidate_id="cursor-agent", + backend="cursor-agent", + model="default", + passed=True, + total_tokens=140, + runtime_sec=12.0, + ) + _write_result( + run_dir / "cursor-agent" / "run-b" / "result.json", + task="task-b", + candidate_id="cursor-agent", + backend="cursor-agent", + model="default", + passed=False, + total_tokens=None, + runtime_sec=15.0, + ) + + summary = build_matrix_summary( + run_dir=run_dir, + candidates=[Candidate(backend="cursor-agent", model=None, candidate_id="cursor-agent")], + tasks=["task-a", "task-b"], + ) + + ranking = cast(list[dict[str, object]], summary["ranking"]) + assert ranking[0]["total_tokens_sum"] == 140 + assert ranking[0]["prompt_tokens_sum"] == 140 + assert ranking[0]["completion_tokens_sum"] == 0 + assert ranking[0]["cache_read_tokens_sum"] == 0 + assert ranking[0]["cache_creation_tokens_sum"] == 0 + assert ranking[0]["token_metrics_coverage"] == 0.5 + + def test_report_renderers_emit_json_markdown_and_static_html(self, tmp_path: Path) -> None: + summary = { + "schema_version": "1.0", + "generated_at": "2026-05-12T00:00:00+00:00", + "run_dir": str(tmp_path), + "manifest": "manifests/demo.txt", + "tasks": ["task-a"], + "winner": "codex", + "ranking": [ + { + "rank": 1, + "candidate_id": "codex", + "agent_backend": "codex", + "agent_model": "default", + "total_tasks": 1, + "passed_tasks": 1, + "pass_rate": 1.0, + "total_tokens_sum": 100, + "prompt_tokens_sum": 90, + "completion_tokens_sum": 10, + "cache_read_tokens_sum": 0, + "cache_creation_tokens_sum": None, + "token_metrics_coverage": 1.0, + "runtime_sec_sum": 9.0, + } + ], + "candidate_groups": { + "codex": [ + { + "rank": 1, + "candidate_id": "codex", + "agent_backend": "codex", + "agent_model": "default", + "total_tasks": 1, + "passed_tasks": 1, + "pass_rate": 1.0, + "total_tokens_sum": 100, + "prompt_tokens_sum": 90, + "completion_tokens_sum": 10, + "cache_read_tokens_sum": 0, + "cache_creation_tokens_sum": None, + "token_metrics_coverage": 1.0, + "runtime_sec_sum": 9.0, + } + ] + }, + "rows": [ + { + "candidate_id": "codex", + "agent_backend": "codex", + "agent_model": "default", + "task": "task-a", + "status": "verify_failed", + "passed": False, + "reward": 0.0, + "runtime_sec": 9.0, + "total_tokens": 100, + "prompt_tokens": 90, + "completion_tokens": 10, + "cache_read_tokens": 0, + "cache_creation_tokens": None, + "token_metrics_status": "available", + "verifier_scores": { + "aggregate_scores": [ + {"name": "agent_eval/task_success.task_success", "mean": 0.0}, + {"name": "agent_eval/verification_score.verification_score", "mean": 0.75}, + {"name": "agent_eval/output_schema_valid.output_schema_valid", "mean": 0.0}, + {"name": "agent_eval/surface_adherence.surface_adherence", "mean": 1.0}, + ], + }, + "result_path": str(tmp_path / "result.json"), + "output_dir": str(tmp_path), + "failure_excerpt": "", + "provenance": {"commit_short": "abc123", "commit_dirty": True}, + } + ], + "run_metadata": [], + } + + reports = write_reports(summary, tmp_path) + markdown = render_markdown(summary) + rendered_html = render_html(summary) + + assert Path(reports["json"]).exists() + assert json.loads(Path(reports["json"]).read_text())["winner"] == "codex" + assert "| Rank | Candidate |" in markdown + assert "Cache Read" in markdown + assert "Task Matrix" in markdown + assert "Agent Matrix Benchmark" in rendered_html + assert "Columns" in rendered_html + assert 'data-role="candidate-leaderboard"' in rendered_html + assert "Agents And Models" in rendered_html + assert 'class="task-result-cell"' in rendered_html + assert 'class="token-buckets"' in rendered_html + assert "dirty worktree" in rendered_html + assert "n/a" in rendered_html + assert "This backend did not emit this token bucket." in rendered_html + assert "This backend emitted this token bucket with value zero." in rendered_html + assert "Verifier rejected the completed agent attempt" in rendered_html + assert "Output Contract" in rendered_html + assert "Verification Score" in rendered_html + assert "Failed or blocking metrics" in rendered_html + assert "All metric scores" in rendered_html + assert "Benchmark success" in rendered_html + assert "Verifier failed" in rendered_html + assert "Agent failed" in rendered_html + assert "task_success" in rendered_html + assert "surface_adherence" in rendered_html + assert "<script>alert" in rendered_html + assert "script src=" not in rendered_html + assert "http://" not in rendered_html + assert Path(reports["html"]).read_text() == rendered_html + + +def _write_result( + path: Path, + *, + task: str, + candidate_id: str, + backend: str, + model: str, + passed: bool, + total_tokens: int | None, + runtime_sec: float, + verifier_text: str | None = None, +) -> None: + output_dir = path.parent + output_dir.mkdir(parents=True, exist_ok=True) + if verifier_text is not None: + verifier_dir = output_dir / "verifier" + verifier_dir.mkdir() + (verifier_dir / "test-stdout.txt").write_text(verifier_text, encoding="utf-8") + path.write_text( + json.dumps( + { + "task": task, + "candidate_id": candidate_id, + "agent_backend": backend, + "agent_model": model, + "output_dir": str(output_dir), + "agent": "ok", + "verify": "ok" if passed else "failed", + "passed": passed, + "reward": 1 if passed else 0, + "runtime_sec": runtime_sec, + "metrics": { + "total_tokens": total_tokens, + "prompt_tokens": total_tokens, + "completion_tokens": 0 if total_tokens is not None else None, + "cache_read_tokens": 0 if total_tokens is not None else None, + "cache_creation_tokens": 0 if total_tokens is not None else None, + "token_metrics_status": "available" if total_tokens is not None else "unavailable", + }, + "provenance": {"commit_short": "abc123", "commit_dirty": False}, + } + ), + encoding="utf-8", + ) diff --git a/tests/agentic-use/tests/test_nat_runner_config.py b/tests/agentic-use/tests/test_nat_runner_config.py index d2eae358ad9..f94e25e4f4e 100644 --- a/tests/agentic-use/tests/test_nat_runner_config.py +++ b/tests/agentic-use/tests/test_nat_runner_config.py @@ -3,8 +3,10 @@ """Tests for _prepare_aut_config_for_runtime IGW routing logic.""" +import json import subprocess from pathlib import Path +from typing import cast import nat_runner import pytest @@ -17,9 +19,12 @@ _build_claude_code_agent_cmd, _build_codex_agent_cmd, _build_cursor_agent_cmd, + _extract_usage_metrics, _normalize_secret, _prepare_aut_config_for_runtime, run_agent_phase, + run_task, + run_verify_phase, ) @@ -126,6 +131,102 @@ def test_non_openai_llm_not_injected(self, tmp_path: Path) -> None: assert "base_url" not in cfg["llms"]["agent"] +def test_parse_candidate_params_normalizes_supported_values() -> None: + params = nat_runner._parse_candidate_params( + json.dumps( + { + "intelligence": "high", + "speed": "fast", + "max_budget_usd": "1.25", + "config": {"model_reasoning_summary": "auto"}, + } + ) + ) + + assert params == { + "intelligence": "high", + "speed": "fast", + "max_budget_usd": 1.25, + "config": {"model_reasoning_summary": "auto"}, + } + + +def test_parse_candidate_params_rejects_unsupported_keys() -> None: + with pytest.raises(ValueError, match="unsupported key"): + nat_runner._parse_candidate_params('{"ignored": true}') + + +def test_parse_candidate_params_rejects_wrong_types() -> None: + with pytest.raises(ValueError, match="'sandbox' must be a string"): + nat_runner._parse_candidate_params('{"sandbox": false}') + + +class TestUsageMetrics: + def test_extracts_cursor_camel_case_usage(self) -> None: + metrics = _extract_usage_metrics( + json.dumps( + { + "type": "result", + "usage": { + "inputTokens": 100, + "outputTokens": 20, + "cacheReadTokens": 300, + "cacheWriteTokens": 4, + }, + } + ) + ) + + assert metrics["prompt_tokens"] == 100 + assert metrics["completion_tokens"] == 20 + assert metrics["cache_read_tokens"] == 300 + assert metrics["cache_creation_tokens"] == 4 + assert metrics["total_tokens"] == 424 + + def test_extracts_zero_usage_when_usage_envelope_is_present(self) -> None: + metrics = _extract_usage_metrics( + json.dumps( + { + "type": "result", + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + "num_turns": 1, + } + ) + ) + + assert metrics["prompt_tokens"] == 0 + assert metrics["completion_tokens"] == 0 + assert metrics["cache_read_tokens"] == 0 + assert metrics["cache_creation_tokens"] == 0 + assert metrics["total_tokens"] == 0 + assert metrics["num_turns"] == 1 + + def test_extracts_codex_cached_input_tokens(self) -> None: + metrics = _extract_usage_metrics( + json.dumps( + { + "type": "result", + "usage": { + "input_tokens": 195742, + "output_tokens": 4012, + "cached_input_tokens": 125312, + }, + } + ) + ) + + assert metrics["prompt_tokens"] == 70430 + assert metrics["completion_tokens"] == 4012 + assert metrics["cache_read_tokens"] == 125312 + assert metrics["cache_creation_tokens"] is None + assert metrics["total_tokens"] == 199754 + + class TestAgentBackends: """Verify backend-specific command construction and metadata helpers.""" @@ -147,7 +248,9 @@ def test_normalize_secret(self, value: str | None, expected: str) -> None: assert _normalize_secret(value) == expected def test_build_claude_code_agent_cmd_uses_unified_agent_model(self) -> None: - cmd = _build_claude_code_agent_cmd("/tmp/instruction.md") + cmd = _build_claude_code_agent_cmd( + "/tmp/instruction.md", {"permission_mode": "bypassPermissions", "max_budget_usd": 1.5} + ) script = cmd[2] assert cmd[:2] == ["bash", "-c"] @@ -155,6 +258,8 @@ def test_build_claude_code_agent_cmd_uses_unified_agent_model(self) -> None: assert "AGENT_MODEL" in script assert "CLAUDE_MODEL" not in script assert "--output-format json" in script + assert "--permission-mode bypassPermissions" in script + assert "--max-budget-usd 1.5" in script def test_build_aut_agent_cmd_recreates_existing_agent_when_config_is_present(self) -> None: cmd = _build_aut_agent_cmd("/tmp/instruction.md") @@ -182,34 +287,41 @@ def test_build_aut_agent_cmd_has_valid_shell_syntax(self) -> None: subprocess.run(["bash", "-n"], input=script, text=True, check=True) def test_build_codex_agent_cmd_runs_headlessly_with_artifacts(self) -> None: - cmd = _build_codex_agent_cmd("/tmp/instruction.md") + cmd = _build_codex_agent_cmd("/tmp/instruction.md", {"intelligence": "high", "speed": "fast"}) script = cmd[2] assert cmd[:2] == ["bash", "-c"] assert nat_runner.CODEX_AGENT_SCRIPT_TEMPLATE_PATH.is_file() assert "codex exec" in script - assert "--sandbox read-only" in script - assert "--output-schema /tmp/codex_output_schema.json" in script - assert "--output-last-message /logs/agent/codex_structured_output.json" in script - assert "codex_structured_prompt.md" in script - assert "output_files" in script + assert "--sandbox danger-full-access" in script + assert "--ignore-user-config" in script + assert "--cd /app" in script + assert "/tmp/codex-task" not in script + assert "--output-schema" not in script + assert "--output-last-message /logs/agent/final_message.txt" in script + assert "codex_structured_prompt.md" not in script + assert "output_files" not in script assert "final_message.txt" in script assert "--dangerously-bypass-approvals-and-sandbox" not in script - assert "--output-last-message /logs/agent/final_message.txt" not in script assert "--json" in script assert "AGENT_MODEL" in script assert "CODEX_HOME" in script + assert 'CODEX_HOME="/home/harbor/codex-benchmark-home"' in script + assert 'rm -rf "$CODEX_HOME"' in script + assert "/home/harbor/.codex" not in script assert "/tmp/codex_host_auth.json" in script assert "CODEX_MODEL" not in script - assert "/tmp/instruction.md" in script - assert "@@CODEX_STRUCTURED_OUTPUT_SCHEMA@@" not in script + assert "-c 'model_reasoning_effort=\"high\"'" in script + assert "-c 'service_tier=\"fast\"'" in script + assert "- < /tmp/instruction.md" in script assert "@@INSTRUCTION_CONTAINER@@" not in script + assert "@@CODEX_CONFIG_ARGS@@" not in script def test_codex_agent_script_template_has_valid_shell_syntax(self) -> None: subprocess.run(["bash", "-n", str(nat_runner.CODEX_AGENT_SCRIPT_TEMPLATE_PATH)], check=True) def test_build_cursor_agent_cmd_runs_headlessly_with_artifacts(self) -> None: - cmd = _build_cursor_agent_cmd("/tmp/instruction.md") + cmd = _build_cursor_agent_cmd("/tmp/instruction.md", {"sandbox": "enabled", "mode": "plan"}) script = cmd[2] assert cmd[:2] == ["bash", "-c"] @@ -217,12 +329,21 @@ def test_build_cursor_agent_cmd_runs_headlessly_with_artifacts(self) -> None: assert "--print" in script assert "--output-format json" in script assert "--force" not in script - assert "--sandbox disabled" in script + assert "--sandbox enabled" in script + assert "--mode plan" in script assert "--workspace /app" in script assert "AGENT_MODEL" in script assert "CURSOR_MODEL" not in script assert "/tmp/instruction.md" in script + def test_build_cursor_agent_cmd_preserves_default_sandbox_when_unset(self) -> None: + cmd = _build_cursor_agent_cmd("/tmp/instruction.md", {}) + script = cmd[2] + + assert "cursor-agent" in script + assert "--sandbox" not in script + assert "--workspace /app" in script + @pytest.mark.parametrize( ("agent_backend", "expected"), [ @@ -306,8 +427,8 @@ def test_run_agent_phase_scopes_direct_agent_secrets(self, monkeypatch: pytest.M def fake_docker_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess: env = kwargs["env"] assert isinstance(env, dict) - captured_env.update(env) - return subprocess.CompletedProcess(args=args, returncode=0, stdout="", stderr="") + captured_env.update(cast(dict[str, str], env)) + return subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") monkeypatch.setattr(nat_runner, "_docker_run", fake_docker_run) @@ -320,6 +441,7 @@ def fake_docker_run(*args: object, **kwargs: object) -> subprocess.CompletedProc anthropic_base_url="https://anthropic.example", nmp_base_url="http://localhost:8080", agent_model=None, + agent_params={}, codex_auth_json=None, timeout=10, agent_backend="codex", @@ -357,8 +479,8 @@ def test_run_agent_phase_mounts_explicit_codex_auth_read_only( def fake_docker_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess: mounts = kwargs["mounts"] assert isinstance(mounts, list) - captured_mounts.extend(mounts) - return subprocess.CompletedProcess(args=args, returncode=0, stdout="", stderr="") + captured_mounts.extend(cast(list[tuple[str, str]], mounts)) + return subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") monkeypatch.setattr(nat_runner, "_docker_run", fake_docker_run) @@ -371,6 +493,7 @@ def fake_docker_run(*args: object, **kwargs: object) -> subprocess.CompletedProc anthropic_base_url="https://anthropic.example", nmp_base_url="http://localhost:8080", agent_model=None, + agent_params={}, codex_auth_json=codex_auth, timeout=10, agent_backend="codex", @@ -385,6 +508,136 @@ def fake_docker_run(*args: object, **kwargs: object) -> subprocess.CompletedProc assert (str(workspace_dir), "/app/workspace") in captured_mounts assert (str(codex_home), "/tmp/codex_host_home") not in captured_mounts + def test_run_verify_phase_mounts_shared_evaluator_harness( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + task_dir = tmp_path / "task" + tests_dir = task_dir / "tests" + tests_dir.mkdir(parents=True) + (tests_dir / "test_outputs.py").write_text("def test_ok(): pass\n") + output_dir = tmp_path / "out" + (output_dir / "agent").mkdir(parents=True) + state_dir = tmp_path / "state" + workspace_dir = tmp_path / "workspace" + captured_mounts: list[tuple[str, str]] = [] + + def fake_docker_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess: + mounts = kwargs["mounts"] + assert isinstance(mounts, list) + captured_mounts.extend(cast(list[tuple[str, str]], mounts)) + return subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + + monkeypatch.setattr(nat_runner, "_docker_run", fake_docker_run) + + passed, _stdout = run_verify_phase( + task_dir, + "task-image", + output_dir, + nmp_base_url="http://localhost:8080", + state_dir=state_dir, + workspace_dir=workspace_dir, + agent_backend="codex", + agent_model="default", + ) + + assert passed is True + assert (str(nat_runner.SHARED_DIR), "/app/tests/agentic-use/shared:ro") in captured_mounts + assert ( + str(nat_runner.REPO_ROOT / "packages" / "nemo_evaluator_sdk" / "src"), + "/app/packages/nemo_evaluator_sdk/src:ro", + ) in captured_mounts + + def test_run_task_records_agent_metadata(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + task_dir = tmp_path / "tasks" / "task-a" + task_dir.mkdir(parents=True) + (task_dir / "instruction.md").write_text("Do the thing.") + jobs_dir = tmp_path / "jobs" + + monkeypatch.setattr(nat_runner, "TASKS_DIR", tmp_path / "tasks") + monkeypatch.setattr(nat_runner, "_docker_image_exists", lambda _image: True) + monkeypatch.setattr(nat_runner, "run_verify_phase", lambda *args, **kwargs: (True, "ok")) + + result = run_task( + "task-a", + jobs_dir=jobs_dir, + nvidia_api_key="", + anthropic_api_key="", + anthropic_base_url="https://anthropic.example", + nmp_base_url="http://localhost:8080", + agent_model="gpt-test", + agent_params={"intelligence": "high", "speed": "fast"}, + codex_auth_json=None, + agent_timeout=10, + skip_build=True, + build_only=False, + skip_agent=True, + agent_backend="codex", + aut_agent_name="nemo-agent", + aut_agent_config=None, + aut_seed_providers=True, + smoke_workspace=None, + candidate_id="codex-gpt-test", + provenance={"commit_short": "abc123"}, + ) + + assert result["agent_backend"] == "codex" + assert result["agent_model"] == "gpt-test" + assert result["candidate_params"] == {"intelligence": "high", "speed": "fast"} + assert result["candidate_id"] == "codex-gpt-test" + result_file = next(jobs_dir.rglob("result.json")) + payload = json.loads(result_file.read_text()) + assert payload["agent_backend"] == "codex" + assert payload["agent_model"] == "gpt-test" + assert payload["candidate_params"] == {"intelligence": "high", "speed": "fast"} + assert payload["candidate_id"] == "codex-gpt-test" + + def test_run_task_build_only_skips_agent_and_verify(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + task_dir = tmp_path / "tasks" / "task-a" + task_dir.mkdir(parents=True) + (task_dir / "instruction.md").write_text("Do the thing.") + jobs_dir = tmp_path / "jobs" + built_tags: list[str] = [] + + monkeypatch.setattr(nat_runner, "TASKS_DIR", tmp_path / "tasks") + monkeypatch.setattr(nat_runner, "build_task_image", lambda _task_dir, tag: built_tags.append(tag)) + monkeypatch.setattr( + nat_runner, + "run_agent_phase", + lambda *args, **kwargs: pytest.fail("agent phase should be skipped"), + ) + monkeypatch.setattr( + nat_runner, + "run_verify_phase", + lambda *args, **kwargs: pytest.fail("verify phase should be skipped"), + ) + + result = run_task( + "task-a", + jobs_dir=jobs_dir, + nvidia_api_key="", + anthropic_api_key="", + anthropic_base_url="https://anthropic.example", + nmp_base_url="http://localhost:8080", + agent_model=None, + agent_params=None, + codex_auth_json=None, + agent_timeout=10, + skip_build=False, + build_only=True, + skip_agent=False, + agent_backend="codex", + aut_agent_name="nemo-agent", + aut_agent_config=None, + aut_seed_providers=True, + smoke_workspace=None, + ) + + assert built_tags == ["nmp-nat-task-a:latest"] + assert result["build"] == "ok" + assert result["agent"] == "skipped" + assert result["verify"] == "skipped" + assert result["reward"] is None + def test_run_agent_phase_normalizes_platform_agent_secrets( self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -407,8 +660,8 @@ def test_run_agent_phase_normalizes_platform_agent_secrets( def fake_docker_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess: env = kwargs["env"] assert isinstance(env, dict) - captured_env.update(env) - return subprocess.CompletedProcess(args=args, returncode=0, stdout="", stderr="") + captured_env.update(cast(dict[str, str], env)) + return subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") monkeypatch.setattr(nat_runner, "_docker_run", fake_docker_run) @@ -421,6 +674,7 @@ def fake_docker_run(*args: object, **kwargs: object) -> subprocess.CompletedProc anthropic_base_url="https://anthropic.example", nmp_base_url="http://localhost:8080", agent_model=None, + agent_params={}, codex_auth_json=None, timeout=10, agent_backend="aut", @@ -495,3 +749,45 @@ def test_rejects_aut_without_agent_config(self, monkeypatch, capsys): result = nat_runner.main() assert result == 1 assert "requires --aut-agent-config" in capsys.readouterr().err + + +class TestBuildOnlySummary: + def test_main_reports_build_summary_in_build_only_mode( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + task_dir = tmp_path / "tasks" / "task-a" + task_dir.mkdir(parents=True) + (task_dir / "instruction.md").write_text("Do the thing.") + jobs_dir = tmp_path / "jobs" + + monkeypatch.setattr(nat_runner, "TASKS_DIR", tmp_path / "tasks") + monkeypatch.setattr(nat_runner, "build_task_image", lambda _task_dir, _tag: None) + monkeypatch.setattr(nat_runner, "_capture_image_digest", lambda _image: "sha256:test") + monkeypatch.setattr( + nat_runner, + "_capture_repo_provenance", + lambda: { + "commit_short": "abc123", + "branch": "test", + "commit_dirty": False, + }, + ) + monkeypatch.setattr( + "sys.argv", + [ + "nat_runner.py", + "task-a", + "--agent-backend", + "codex", + "--build-only", + "--jobs-dir", + str(jobs_dir), + ], + ) + + result = nat_runner.main() + + captured = capsys.readouterr() + assert result == 0 + assert "BUILD SUMMARY: 1/1 task images ready" in captured.out + assert "tasks passed" not in captured.out