diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/.gitignore b/packages/nemo_evaluator_sdk/examples/run_agent_eval/.gitignore new file mode 100644 index 0000000000..a1d688cf03 --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/.gitignore @@ -0,0 +1,2 @@ +# Generated run bundles from this example. +run-agent-eval-output/ diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/README.md b/packages/nemo_evaluator_sdk/examples/run_agent_eval/README.md new file mode 100644 index 0000000000..3698220cd3 --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/README.md @@ -0,0 +1,228 @@ +# run_agent_eval — workflow runtime over the trials API + +A minimal, self-contained example that runs agent-eval tasks through a +**workflow runtime** and scores them with the trials-based +`nemo_evaluator_sdk.agent_eval` SDK, following the standard shape: load task → +run agent → score → gate, plus offline rescoring. + +It offers two paths: + +- a **toy path** (the default) whose agent is a tiny bundled script + (`mini_agent.py`), so it runs **end to end with no external infrastructure**; +- a **real path** (`--agentic-task `) whose `platform_runtime.py` glue + runs an actual `tests/agentic-use` task through Docker + `nat run` + a pytest + verifier (see [Running a real `tests/agentic-use` + task](#running-a-real-testsagentic-use-task)). + +## What it demonstrates + +The SDK stays responsible for generating trials and scoring them; the example +owns the CI-policy glue (the pass/fail gate, the pipeline wrapper, the +Harbor-style verifier, and the run layout). Components and the building blocks +they use (SDK unless noted *example-local*): + +| Component | File | Role / building blocks | +|---|---|---| +| `WorkflowAgentRuntime` | `workflow_runtime.py` | `AgentTaskRunner` (toy host-subprocess agent): `resolve_trial_status`, `standard_evidence_descriptors` (SDK) + `prepare_run_layout` (example-local `layout.py`) | +| `TrialJsonSerde` | `workflow_runtime.py` | `AgentTrialSerde`: offline read/write of a stored `AgentEvalTrial` | +| Tasks + metrics | `workflow_runtime.py` | `AgentPhaseSuccessMetric`, `EvidencePresenceMetric`, a task-authored `OutputContainsMetric` | +| CLI harness | `run_agent_eval.py` | `AgentEvaluator` (SDK) via the example-local `AgentEvalPipeline` (`pipeline.py`) + gate (`gating.py`) | +| `mini_agent.py` | `mini_agent.py` | default agent — a dependency-free stand-in workflow | +| `NatWorkflowRuntime` | `platform_runtime.py` | `AgentTaskRunner` (real Docker + `nat run`), plus `agentic_task_from_dir`, `ensure_task_image`, `VerifierRewardMetric`; verify via example-local `verify.py` | +| `NatAutRuntime` | `aut_runtime.py` | `AgentTaskRunner` (deployed agent-under-test); `usage.py` extracts token usage from its logs | + +## Run it + +From the repository root: + +```bash +# Online: run the workflow agent on every example task, score, and gate. +python -m packages.nemo_evaluator_sdk.examples.run_agent_eval.run_agent_eval --task all + +# A single task. +python -m packages.nemo_evaluator_sdk.examples.run_agent_eval.run_agent_eval --task write-report + +# List available tasks. +python -m packages.nemo_evaluator_sdk.examples.run_agent_eval.run_agent_eval --list-tasks +``` + +Each online run writes a bundle to `run-agent-eval-output/` +(`tasks.jsonl`, `trials.jsonl`, `scores.jsonl`, `summary.json`, `gate.json`, +`report.html`) plus per-task evidence under `evidence/workflow//`. + +Example output: + +```text +run_id: agent-eval-20260620195929 +tasks: 2 trials: 2 + agent_phase_success.agent_phase_success: 2/2 true + evidence_presence.evidence_present: 2/2 true + output_contains.output_contains: 2/2 true +output_dir: .../run-agent-eval-output +gate: .../run-agent-eval-output/gate.json +``` + +> The metrics here emit booleans, which aggregate as `nan` in the numeric +> `summary.scores` view (that view is for range scores); the example summarizes +> them as a true-rate instead, and the gate reads `agent_phase_success` as the +> per-task pass signal. + +### Offline rescoring (no agent execution) + +Re-score already-captured trials through the same pipeline. Pass either a full +run bundle (reads `trials.jsonl`) or a single runtime run dir (reads +`trial.json` via `TrialJsonSerde`): + +```bash +# Re-score an entire prior bundle. +python -m packages.nemo_evaluator_sdk.examples.run_agent_eval.run_agent_eval \ + --rescore-dir run-agent-eval-output --output-dir /tmp/rescore + +# Re-score one captured run dir via the serde. +python -m packages.nemo_evaluator_sdk.examples.run_agent_eval.run_agent_eval \ + --rescore-dir run-agent-eval-output/evidence/workflow/write-report --output-dir /tmp/rescore-one +``` + +## Execution flow + +```mermaid +flowchart TD + cli["run_agent_eval.py (CLI)"] --> pipe["AgentEvalPipeline"] + pipe -->|online: run_tasks| rt["WorkflowAgentRuntime (AgentTaskRunner)"] + rt --> layout["prepare_run_layout"] + rt --> proc["launch workflow command (mini_agent.py)"] + proc --> trial["AgentEvalTrial (resolve_trial_status + standard_evidence_descriptors)"] + pipe -->|offline: score_trials| serde["TrialJsonSerde / trials.jsonl"] + serde --> trial + trial --> eval["AgentEvaluator: score metrics"] + eval --> gate["evaluate_gate → gate.json"] + eval --> bundle["persist bundle + dashboard"] +``` + +## Plugging in a real workflow + +`WorkflowAgentRuntime` launches `config.command` per task, substituting the +tokens `{instruction}`, `{workspace}`, and `{input_json}`. The default runs the +bundled `mini_agent.py`; supply your own command to drive a real agent: + +```python +from nemo_evaluator_sdk.examples.run_agent_eval.workflow_runtime import ( + WorkflowAgentRuntime, + WorkflowRuntimeConfig, +) + +runtime = WorkflowAgentRuntime( + WorkflowRuntimeConfig( + command=["nat", "run", "--config", "workflow.yml", "--input", "{instruction}"], + agent_model="my-model", + ) +) +``` + +Any executable that reads the task input, writes results into `{workspace}`, +prints a final answer to stdout, and exits non-zero on failure works unchanged. + +## Running a real `tests/agentic-use` task + +`platform_runtime.py` adds the NeMo-Platform-specific glue the toy path +deliberately omits, so the example can drive a **real** agentic-use task through +the same three phases as `nat_runner`: + +1. **BUILD** — `ensure_task_image` builds the task's `environment/Dockerfile` + (or `environment.yaml`) into `nmp-nat-:latest`. +2. **AGENT** — `NatWorkflowRuntime` runs `nat run --config_file workflow.yml` + inside that image via the SDK's `DockerEnvironmentProvider`, capturing + `nat_agent.log` + `trajectory.json`. +3. **VERIFY** *(optional)* — runs the task's `tests/test_outputs.py` under pytest + in the same image; the reward is scored by `VerifierRewardMetric`. + +Each run records the same token/runtime measurements `nat_runner` writes into +`result.json["metrics"]`: `usage.py` is the port of `nat_runner`'s +`_extract_usage_metrics`, and `build_trial_from_artifacts` stamps +`prompt_tokens`/`completion_tokens`/`total_tokens`/`cache_*` + `runtime_sec` onto +each trial so the summary, gate, and dashboard aggregates populate. + +### Two backends (`--backend`) + +`nat_runner` supports a task-local `nat run` **workflow** backend and a deployed +**agent-under-test (AUT)** backend; this example mirrors both. + +The **workflow** backend runs a task-local `nat run`: + +```bash +python -m packages.nemo_evaluator_sdk.examples.run_agent_eval.run_agent_eval \ + --agentic-task entities-basic-cli-easy --backend workflow --verify \ + --agent-model meta/llama-3.3-70b-instruct --output-dir ~/rae-real +``` + +The **AUT** backend (`aut_runtime.py`) runs the full `nat_runner` lifecycle +inside the task image: create the agent from `--aut-agent-config` → seed +inference providers (`providers.yaml`) → deploy → wait → health-check → invoke +via `nat_trace_export.py invoke-aut`. + +### A task-capable AUT agent (verified green) + +The deployed agent must have the tools its task needs. `nemo-mcp` currently +exposes only **workspace** operations (`create_workspace`, `list_workspaces`, +`delete_workspace`), so `aut_agent.workspace.example.yml` is matched to +`workspace-basic-mcp` — a task those MCP tools fully cover — and scores a green +verifier reward end to end: + +```bash +INFERENCE_NVIDIA_API_KEY= \ +python -m packages.nemo_evaluator_sdk.examples.run_agent_eval.run_agent_eval \ + --agentic-task workspace-basic-mcp --backend aut \ + --aut-agent-name run-agent-eval-workspace \ + --aut-agent-config packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_agent.workspace.example.yml \ + --agent-model vm-opus-nemotron-random --verify --output-dir ~/rae-workspace +``` + +```text +run_id: agent-eval-20260620211345 +tasks: 1 trials: 1 + agent_phase_success.agent_phase_success: 1/1 true + agentic_use_verifier_reward.verifier_reward: mean=1.000 + runtime_sec: 87.9 across 1/1 trials +gate: ~/rae-workspace/gate.json (gate_passed: true, pass_rate=1.000) +``` + +> **Model routing note.** `providers.yaml` seeds `vm-opus-nemotron-random`, a +> 50/50 random-routing virtual model. The config sets +> `use_native_tool_calling: false` so the ReAct agent emits text-based steps with +> non-empty content; some routed backends reject the empty-content assistant +> messages that native tool-calling produces. Token totals stay `null` for the +> AUT backend because this platform's invoke endpoint streams only the answer +> text, not per-message `usage` — `runtime_sec` is still recorded. +> +> **Prerequisites (real path):** a working Docker daemon, the +> `nmp-agentic-base:latest` base image, and `NVIDIA_API_KEY`. The AUT backend +> additionally needs an inference model the platform **inference gateway** +> exposes as a valid model entity: either set `INFERENCE_NVIDIA_API_KEY` so +> `providers.yaml` seeds its virtual model and pass that as `--agent-model`, or +> point `--agent-model` at a discovered IGW model-entity name. A provider model +> id with slashes/dots (e.g. `meta/llama-3.3-70b-instruct`) is rejected by the +> gateway's OpenAI route — same model-routing setup `nat_runner` requires. Real +> runs fail fast with a clear message and exit non-zero; the toy path needs none +> of this. + +| Flag | Purpose | +|---|---| +| `--agentic-task ` | Run `tests/agentic-use/` end to end. | +| `--backend {workflow,aut}` | Task-local `nat run` vs. deployed agent-under-test. | +| `--aut-agent-name` | Name of the deployed AUT (aut backend). | +| `--aut-agent-config` | NAT config for the AUT; created/recreated if needed. | +| `--no-seed-providers` | Skip seeding `providers.yaml` (aut backend). | +| `--skip-build` | Skip BUILD; require `nmp-nat-:latest` to already exist. | +| `--verify` | Run the pytest VERIFY phase and add `VerifierRewardMetric`. | +| `--nmp-base-url` | Platform URL injected into the workflow/agent + containers. | +| `--agent-model` | Model for the workflow / AUT agent. | + +## Files + +- `run_agent_eval.py` — CLI harness (toy online + offline rescore + real task + gate). +- `workflow_runtime.py` — toy runtime adapter, trial serde, example tasks, metric. +- `platform_runtime.py` — NeMo-Platform glue + workflow backend (BUILD/AGENT/VERIFY). +- `aut_runtime.py` — AUT backend: create/seed/deploy/invoke a deployed agent. +- `usage.py` — token-usage extraction ported from `nat_runner._extract_usage_metrics`. +- `aut_agent.workspace.example.yml` — task-capable AUT config matched to `workspace-basic-mcp` (verified green). +- `mini_agent.py` — dependency-free stand-in agent (the default toy workflow). diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_agent.workspace.example.yml b/packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_agent.workspace.example.yml new file mode 100644 index 0000000000..e116ad0574 --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_agent.workspace.example.yml @@ -0,0 +1,52 @@ +# Task-capable agent-under-test (AUT) config for the `workspace-basic-mcp` task. +# +# The deployed agent must have the tools its task needs. `nemo-mcp` currently +# exposes only workspace operations (create_workspace, list_workspaces, +# delete_workspace), and `workspace-basic-mcp` asks the agent to create and list a +# workspace -- so this MCP react agent scores a green verifier reward end to end. +# +# Usage (build the task image on first run, then invoke + verify): +# python -m packages.nemo_evaluator_sdk.examples.run_agent_eval.run_agent_eval \ +# --agentic-task workspace-basic-mcp --backend aut \ +# --aut-agent-name run-agent-eval-workspace \ +# --aut-agent-config packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_agent.workspace.example.yml \ +# --agent-model vm-opus-nemotron-random --verify --output-dir ~/rae-workspace +# +# Prerequisites: Docker, the `nmp-agentic-base:latest` base image, NVIDIA_API_KEY, +# and an inference-gateway model entity (set INFERENCE_NVIDIA_API_KEY so +# providers.yaml seeds `vm-opus-nemotron-random`, then pass it as --agent-model). +# +# The LLM `base_url` is rewritten to the platform inference gateway by +# `prepare_aut_config_for_runtime` (inject_gateway_url) at runtime. + +llms: + primary_llm: + _type: openai + api_key: not-used + model_name: meta/llama-3.3-70b-instruct + temperature: 0.0 + # Cap below the smallest routed model's limit (e.g. opus-4-5 caps at 64k). + max_tokens: 8192 + +function_groups: + nemo_tools: + _type: mcp_client + server: + transport: stdio + command: /app/.venv/bin/nemo-mcp + args: + - "--base-url" + - "http://localhost:8080" + tool_call_timeout: 120 + +workflow: + _type: react_agent + llm_name: primary_llm + tool_names: + - nemo_tools + verbose: true + # Text-based ReAct (no native tool_calls) keeps every assistant message + # non-empty, so it works across whichever backend the gateway routes to + # (some models reject the empty-content messages native tool-calling emits). + use_native_tool_calling: false + parse_agent_response_max_retries: 3 diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_runtime.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_runtime.py new file mode 100644 index 0000000000..36869a9f11 --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_runtime.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""AUT (agent-under-test) backend — the canonical ``nat_runner`` execution path. + +Instead of a task-local ``nat run`` workflow, it drives a **deployed platform +agent**: inside the task image it creates the agent from ``--aut-agent-config`` +(if needed), optionally seeds inference providers, deploys + health-checks it, +then invokes it via ``nat_trace_export.py invoke-aut``. The agent is user- +supplied (``--aut-agent-name`` + ``--aut-agent-config``); see +``aut_agent.workspace.example.yml`` for a task-capable starting point. +""" + +from __future__ import annotations + +import os +import textwrap +from collections.abc import Sequence +from dataclasses import dataclass, field +from pathlib import Path + +import yaml +from nemo_evaluator_sdk.agent_eval.runtimes.environment import AgentEnvironmentProvider, EnvRunSpec +from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial + +from .platform_runtime import ( + DEFAULT_LOCAL_NMP_BASE_URL, + DEFAULT_TIMEOUT_SEC, + INSTRUCTION_CONTAINER_PATH, + NAT_TRACE_EXPORT_SCRIPT_CONTAINER_PATH, + REPO_ROOT, + AgenticRunLayout, + PlatformDockerEnvironmentProvider, + base_container_env, + docker_socket_mounts, + resolve_run_layout, + run_agent_then_verify, + task_agent_timeout_sec, +) + +RUNTIME_NAME = "aut" +AUT_CONFIG_CONTAINER_PATH = "/tmp/aut_agent.yml" + + +@dataclass(frozen=True) +class AutConfig: + """Configuration for :class:`NatAutRuntime`.""" + + aut_agent_name: str + aut_agent_config: Path | None = None + aut_seed_providers: bool = True + aut_health_wait_seconds: int = int(os.environ.get("NAT_AUT_HEALTH_WAIT_SECONDS", "60")) + agent_model: str | None = None + nmp_base_url: str = DEFAULT_LOCAL_NMP_BASE_URL + nvidia_api_key: str | None = None + inference_nvidia_api_key: str | None = None + anthropic_api_key: str | None = None + timeout_sec: int = DEFAULT_TIMEOUT_SEC + run_verify: bool = False + docker_extra_args: list[str] = field(default_factory=list) + + +def build_aut_agent_cmd(instruction_container: str) -> list[str]: + """``bash -c`` command that creates/deploys/invokes the agent-under-test.""" + return [ + "bash", + "-c", + textwrap.dedent(f"""\ + set -euo pipefail + EFFECTIVE_AUT_AGENT_CONFIG="${{AUT_AGENT_CONFIG}}" + if [ -n "${{AUT_AGENT_CONFIG}}" ] && [ -n "${{NVIDIA_API_KEY:-}}" -o -n "${{ANTHROPIC_API_KEY:-}}" ]; then + /app/.venv/bin/python -c "from pathlib import Path; import os; text = Path(os.environ['AUT_AGENT_CONFIG']).read_text(); text = text.replace('\\${{NVIDIA_API_KEY}}', os.environ.get('NVIDIA_API_KEY', '')); text = text.replace('\\${{ANTHROPIC_API_KEY}}', os.environ.get('ANTHROPIC_API_KEY', '')); Path('/tmp/aut_agent.resolved.yml').write_text(text)" + EFFECTIVE_AUT_AGENT_CONFIG="/tmp/aut_agent.resolved.yml" + fi + if /app/.venv/bin/nemo agents get "${{AUT_AGENT_NAME}}" >/tmp/aut_get_before.log 2>&1; then + if [ -n "${{EFFECTIVE_AUT_AGENT_CONFIG}}" ]; then + echo "AUT '${{AUT_AGENT_NAME}}' already exists; recreating from AUT_AGENT_CONFIG." + /app/.venv/bin/nemo agents undeploy --agent "${{AUT_AGENT_NAME}}" >/tmp/aut_undeploy_before_recreate.log 2>&1 || true + /app/.venv/bin/nemo agents delete "${{AUT_AGENT_NAME}}" >/tmp/aut_delete_before_recreate.log 2>&1 || true + /app/.venv/bin/nemo agents create --name "${{AUT_AGENT_NAME}}" --agent-config "${{EFFECTIVE_AUT_AGENT_CONFIG}}" >/tmp/aut_create.log 2>&1 + else + echo "AUT '${{AUT_AGENT_NAME}}' already exists." + fi + else + if [ -z "${{EFFECTIVE_AUT_AGENT_CONFIG}}" ]; then + echo "AUT agent '${{AUT_AGENT_NAME}}' not found and no AUT_AGENT_CONFIG was provided." >&2 + echo "Set --aut-agent-config so the runner can create the agent." >&2 + cp /tmp/aut_get_before.log /logs/agent/aut_get_before.log 2>/dev/null || true + exit 1 + fi + /app/.venv/bin/nemo agents create --name "${{AUT_AGENT_NAME}}" --agent-config "${{EFFECTIVE_AUT_AGENT_CONFIG}}" >/tmp/aut_create.log 2>&1 + fi + if [ "${{AUT_SEED_PROVIDERS:-1}}" = "1" ]; then + /app/.venv/bin/python /app/tests/agentic-use/seed_providers.py \\ + --manifest /app/tests/agentic-use/providers.yaml \\ + --base-url "${{NMP_BASE_URL:-http://localhost:8080}}" \\ + 2>&1 | tee /tmp/aut_provider_seed.log + fi + collect_aut_diagnostics() {{ + set +e + /app/.venv/bin/nemo agents deployments list >/tmp/aut_deployments.list.json 2>&1 + cp /tmp/aut_deployments.list.json /logs/agent/aut_deployments.list.json 2>/dev/null || true + cp /tmp/aut_create.log /logs/agent/aut_create.log 2>/dev/null || true + cp /tmp/aut_get_before.log /logs/agent/aut_get_before.log 2>/dev/null || true + cp /tmp/aut_provider_seed.log /logs/agent/aut_provider_seed.log 2>/dev/null || true + cp /tmp/nmp-api.log /logs/agent/nmp-api.log 2>/dev/null || true + return 0 + }} + cleanup() {{ + cleanup_rc=$? + set +e + collect_aut_diagnostics + /app/.venv/bin/nemo agents undeploy --agent "${{AUT_AGENT_NAME}}" >/tmp/aut_undeploy_after.log 2>&1 || true + cp /tmp/aut_undeploy_after.log /logs/agent/aut_undeploy_after.log 2>/dev/null || true + cp /tmp/nat_agent.log /logs/agent/nat_agent.log 2>/dev/null || true + exit "$cleanup_rc" + }} + trap cleanup EXIT + /app/.venv/bin/nemo agents undeploy --agent "${{AUT_AGENT_NAME}}" >/tmp/aut_undeploy.log 2>&1 || true + /app/.venv/bin/nemo agents deploy --agent "${{AUT_AGENT_NAME}}" + /app/.venv/bin/nemo agents deployments wait --agent "${{AUT_AGENT_NAME}}" + dep_endpoint=$( + /app/.venv/bin/nemo agents deployments list 2>/dev/null | /app/.venv/bin/python -c "import json,sys; data=json.load(sys.stdin).get('data', []); match=next((d.get('endpoint') for d in data if d.get('agent') == '${{AUT_AGENT_NAME}}' and d.get('status') == 'running' and d.get('endpoint')), ''); print(match)" + ) + if [ -z "$dep_endpoint" ]; then + echo "No running AUT deployment endpoint found for agent '${{AUT_AGENT_NAME}}'." >&2 + exit 1 + fi + health_wait="${{AUT_HEALTH_WAIT_SECONDS:-60}}" + aut_healthy=0 + for i in $(seq 1 "$health_wait"); do + if curl -sf "$dep_endpoint/health" >/dev/null 2>&1; then + aut_healthy=1 + break + fi + sleep 1 + done + if [ "$aut_healthy" -ne 1 ]; then + echo "AUT deployment did not become healthy within $health_wait seconds: $dep_endpoint" >&2 + exit 1 + fi + set +e + /app/.venv/bin/python {NAT_TRACE_EXPORT_SCRIPT_CONTAINER_PATH} invoke-aut \\ + --endpoint "$dep_endpoint" \\ + --instruction {instruction_container} \\ + --output-dir /logs/agent \\ + --timeout "${{AUT_INVOKE_HTTP_TIMEOUT:-600}}" \\ + 2>&1 | tee /tmp/nat_agent.log + rc=${{PIPESTATUS[0]}} + set -e + if [ $rc -ne 0 ]; then + collect_aut_diagnostics + fi + exit $rc + """), + ] + + +def prepare_aut_config_for_runtime( + config_path: Path, + output_dir: Path, + *, + nat_model: str | None = None, + nmp_base_url: str = DEFAULT_LOCAL_NMP_BASE_URL, + workspace: str = "default", +) -> Path: + """Prepare an AUT agent config for IGW-routed container runtime.""" + from nemo_agents_plugin.utils import inject_gateway_url + + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + if nat_model: + for llm_cfg in config.get("llms", {}).values(): + if isinstance(llm_cfg, dict) and llm_cfg.get("_type") in ("openai", "nim"): + llm_cfg["model_name"] = nat_model + break + config = inject_gateway_url(config, workspace, base_url=nmp_base_url) + rewritten = output_dir / "aut.runtime.yml" + rewritten.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False), encoding="utf-8") + return rewritten + + +class NatAutRuntime: + """Run agentic-use tasks via a deployed platform agent-under-test (an ``AgentTaskRunner``).""" + + def __init__(self, config: AutConfig, *, environment: AgentEnvironmentProvider | None = None) -> None: + if not config.aut_agent_name: + raise ValueError("NatAutRuntime requires aut_agent_name") + self.config = config + self.environment = environment or PlatformDockerEnvironmentProvider() + + async def run_tasks( + self, + tasks: Sequence[AgentEvalTask], + config: AgentEvalRunConfig | None = None, + ) -> Sequence[AgentEvalTrial]: + return [await self._run_task(task, config) for task in tasks] + + async def _run_task(self, task: AgentEvalTask, config: AgentEvalRunConfig | None) -> AgentEvalTrial: + layout = resolve_run_layout(task, config) + agent_model = self.config.agent_model or "unknown" + handle = await self.environment.prepare(task, config) + return await run_agent_then_verify( + handle, + task=task, + layout=layout, + spec=self._agent_run_spec(task, layout), + runtime_name=RUNTIME_NAME, + agent_model=agent_model, + run_verify=self.config.run_verify, + nmp_base_url=self.config.nmp_base_url, + verify_timeout_sec=self.config.timeout_sec + 120, + docker_extra_args=list(self.config.docker_extra_args), + ) + + def _agent_run_spec(self, task: AgentEvalTask, layout: AgenticRunLayout) -> EnvRunSpec: + task_dir = Path(str(task.metadata["task_dir"])) + task_timeout = task_agent_timeout_sec(task_dir) or 0 + timeout_sec = max(self.config.timeout_sec, task_timeout) + + env = base_container_env(self.config.nmp_base_url, timeout_sec=timeout_sec) + if self.config.nvidia_api_key: + env["NVIDIA_API_KEY"] = self.config.nvidia_api_key + if self.config.anthropic_api_key: + env["ANTHROPIC_API_KEY"] = self.config.anthropic_api_key + if self.config.aut_seed_providers and self.config.inference_nvidia_api_key: + env["INFERENCE_NVIDIA_API_KEY"] = self.config.inference_nvidia_api_key + if self.config.agent_model: + env["NAT_MODEL"] = self.config.agent_model + env["AUT_AGENT_NAME"] = self.config.aut_agent_name + env["AUT_SEED_PROVIDERS"] = "1" if self.config.aut_seed_providers else "0" + env["AUT_HEALTH_WAIT_SECONDS"] = str(self.config.aut_health_wait_seconds) + + mounts: list[tuple[str, str]] = [ + (str(layout.instruction_path), INSTRUCTION_CONTAINER_PATH), + (str(layout.agent_log_dir), "/logs/agent"), + (str(layout.workspace_dir), "/app/workspace"), + (str(layout.state_dir), "/data"), + ] + + if self.config.aut_agent_config is not None: + aut_config_path = Path(self.config.aut_agent_config) + if not aut_config_path.is_absolute(): + aut_config_path = (REPO_ROOT / aut_config_path).resolve() + if not aut_config_path.exists(): + raise FileNotFoundError(f"AUT config not found: {aut_config_path}") + aut_config_host = prepare_aut_config_for_runtime( + aut_config_path, + layout.agent_log_dir, + nat_model=self.config.agent_model, + nmp_base_url=self.config.nmp_base_url, + ) + env["AUT_AGENT_CONFIG"] = AUT_CONFIG_CONTAINER_PATH + mounts.append((str(aut_config_host), AUT_CONFIG_CONTAINER_PATH)) + else: + env["AUT_AGENT_CONFIG"] = "" + + mounts += docker_socket_mounts() + + return EnvRunSpec( + command=build_aut_agent_cmd(INSTRUCTION_CONTAINER_PATH), + env=env, + mounts=mounts, + timeout=timeout_sec + 120, + extra_args=list(self.config.docker_extra_args), + ) + + +__all__ = ["AutConfig", "NatAutRuntime", "build_aut_agent_cmd", "prepare_aut_config_for_runtime"] diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/build_spec.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/build_spec.py new file mode 100644 index 0000000000..c7cf1132d7 --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/build_spec.py @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Declarative *build* spec for agent-eval task images (example-local glue). + +Resolves a task's ``environment.yaml`` (or a Dockerfile escape hatch) into a +:class:`BuildPlan` and builds the image. This is the example's prepare-task +(build) concern, paired with the SDK's run-time environment abstraction +(``nemo_evaluator_sdk.agent_eval.runtimes.environment``). + +``yaml`` is imported lazily so the module stays importable without PyYAML. + +Spec shape (``environment.yaml`` in the task dir):: + + environment: + image: nemo-platform-agentic-base:2026.06 + profile: evaluator-platform + dependencies: + python: [pytest, nemo-evaluator-sdk] + setup: [seed-providers, create-workspace] + +Escape hatch:: + + environment: + dockerfile: environment/Dockerfile +""" + +from __future__ import annotations + +import shlex +import subprocess +from dataclasses import dataclass, field +from pathlib import Path + +ENVIRONMENT_SPEC_FILENAME = "environment.yaml" +DEFAULT_DOCKERFILE_RELPATH = "environment/Dockerfile" + + +@dataclass(frozen=True) +class BuildSpec: + """Declarative build inputs for one task (or a Dockerfile escape hatch).""" + + image: str | None = None + profile: str | None = None + python_dependencies: list[str] = field(default_factory=list) + setup: list[str] = field(default_factory=list) + dockerfile: Path | None = None + + def __post_init__(self) -> None: + if self.dockerfile is None and self.image is None: + raise ValueError("build spec requires either 'image' or 'dockerfile'") + + +def load_build_spec(task_dir: str | Path) -> BuildSpec: + """Load a task's build spec. + + Resolution order: ``environment.yaml`` (declarative spec, preferred), then + ``environment/Dockerfile`` (backward-compatible escape hatch). + """ + root = Path(task_dir) + spec_path = root / ENVIRONMENT_SPEC_FILENAME + if spec_path.is_file(): + import yaml # optional dependency; imported only when a spec is read + + return _parse_spec(yaml.safe_load(spec_path.read_text(encoding="utf-8")) or {}, root) + + dockerfile = root / DEFAULT_DOCKERFILE_RELPATH + if dockerfile.is_file(): + return BuildSpec(dockerfile=dockerfile) + + raise FileNotFoundError( + f"No build spec for task {root}: expected {ENVIRONMENT_SPEC_FILENAME} or {DEFAULT_DOCKERFILE_RELPATH}" + ) + + +def _parse_spec(payload: dict, task_dir: Path) -> BuildSpec: + data = payload.get("environment", payload) if isinstance(payload, dict) else {} + if not isinstance(data, dict): + raise ValueError(f"Invalid build spec in {task_dir}: expected a mapping") + + dockerfile_value = data.get("dockerfile") + dockerfile = None + if dockerfile_value: + dockerfile = Path(dockerfile_value) + if not dockerfile.is_absolute(): + dockerfile = (task_dir / dockerfile).resolve() + if not dockerfile.is_file(): + raise FileNotFoundError(f"environment.dockerfile not found: {dockerfile}") + + dependencies = data.get("dependencies") or {} + python_deps = dependencies.get("python") if isinstance(dependencies, dict) else None + + return BuildSpec( + image=data.get("image"), + profile=data.get("profile"), + python_dependencies=_str_list(python_deps, "dependencies.python", task_dir), + setup=_str_list(data.get("setup"), "setup", task_dir), + dockerfile=dockerfile, + ) + + +def _str_list(value: object, field_name: str, task_dir: Path) -> list[str]: + """Coerce a YAML value to a list[str], rejecting wrong shapes loudly. + + Guards against the silent ``list("pytest") -> ['p', 'y', ...]`` trap: a bare + string (or any non-list) for a list-valued field is a spec error, not a + character sequence. + """ + if value is None: + return [] + if not isinstance(value, list): + raise ValueError(f"Invalid build spec in {task_dir}: '{field_name}' must be a list of strings") + result: list[str] = [] + for item in value: + if not isinstance(item, str): + raise ValueError(f"Invalid build spec in {task_dir}: '{field_name}' must be a list of strings") + result.append(item) + return result + + +@dataclass(frozen=True) +class BuildPlan: + """A resolved, executable Docker build for one task.""" + + image_tag: str + dockerfile: Path + context_dir: Path + generated: bool + base_image: str | None = None + setup: list[str] = field(default_factory=list) + + +def plan_task_build( + task_dir: str | Path, + image_tag: str, + *, + spec: BuildSpec | None = None, + generated_dir: Path | None = None, +) -> BuildPlan: + """Resolve a task's build spec into a concrete :class:`BuildPlan`. + + For the Dockerfile escape hatch the existing Dockerfile/context is used; for + an ``image``-based spec a minimal derived Dockerfile is written under + ``generated_dir`` (defaults to ``/.agentic-build``). + """ + root = Path(task_dir) + spec = spec or load_build_spec(root) + + if spec.dockerfile is not None: + return BuildPlan( + image_tag=image_tag, + dockerfile=spec.dockerfile, + context_dir=spec.dockerfile.parent, + generated=False, + setup=list(spec.setup), + ) + + # image-based spec: generate a tiny derived Dockerfile. + context_dir = generated_dir if generated_dir is not None else (root / ".agentic-build") + context_dir.mkdir(parents=True, exist_ok=True) + dockerfile = context_dir / "Dockerfile" + dockerfile.write_text(render_derived_dockerfile(spec), encoding="utf-8") + return BuildPlan( + image_tag=image_tag, + dockerfile=dockerfile, + context_dir=context_dir, + generated=True, + base_image=spec.image, + setup=list(spec.setup), + ) + + +def execute_build_plan(plan: BuildPlan) -> None: + """Build the Docker image described by ``plan`` via the ``docker`` CLI.""" + cmd = ["docker", "build", "-f", str(plan.dockerfile), "-t", plan.image_tag, str(plan.context_dir)] + print(f"[agent-eval-runtime] $ {' '.join(cmd)}") + subprocess.run(cmd, check=True) + + +# Note: '<'/'>' are intentionally allowed — pip version pins (``numpy<2``, +# ``pytest>=8``) use them and a Dockerfile ``RUN`` arg doesn't shell-redirect. +_DOCKERFILE_UNSAFE_CHARS = set('\n\r"`$;&|') + + +def _reject_unsafe(values: list[str], field_name: str) -> None: + """Reject tokens that would break out of the generated Dockerfile line. + + Inputs come from a task's ``environment.yaml``. Even though that file is + author-controlled, embedding shell metacharacters into a generated ``RUN``/ + ``LABEL`` line is almost always a mistake; fail loudly instead of silently + emitting an injectable Dockerfile. + """ + for value in values: + if _DOCKERFILE_UNSAFE_CHARS & set(value): + raise ValueError(f"Unsafe character in build spec '{field_name}' entry: {value!r}") + + +def render_derived_dockerfile(spec: BuildSpec) -> str: + """Render a minimal derived Dockerfile from an image-based spec.""" + if spec.image is None: + raise ValueError("cannot render a derived Dockerfile without a base image") + _reject_unsafe(spec.python_dependencies, "dependencies.python") + _reject_unsafe(spec.setup, "setup") + lines = [f"FROM {spec.image}"] + if spec.profile: + lines.append(f"LABEL com.nvidia.agentic.profile={spec.profile}") + if spec.python_dependencies: + # Quote each dep so version pins (numpy<2, pytest>=8) aren't treated as + # shell redirection by the shell-form RUN instruction. + deps = " ".join(shlex.quote(dep) for dep in spec.python_dependencies) + lines.append(f"RUN pip install --no-cache-dir {deps}") + if spec.setup: + # Setup steps are recorded for traceability only (not executed here). + lines.append(f'LABEL com.nvidia.agentic.setup="{",".join(spec.setup)}"') + return "\n".join(lines) + "\n" diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/gating.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/gating.py new file mode 100644 index 0000000000..557c2e5a06 --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/gating.py @@ -0,0 +1,357 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Deterministic candidate-vs-baseline gate over a run (example-local CI policy). + +This lives in the example, not the SDK: turning scores into a pass/fail decision +is a CI policy concern, while the SDK stays responsible for generating trials and +running scorers/metrics. Adds the pass-rate/token/runtime-tie-breaker gate on top +of the persisted run bundle. Note ``pass_rate`` here is a per-task pass/fail count +against a reward threshold — deliberately different from +:class:`~nemo_evaluator_sdk.agent_eval.results.AgentEvalSummary`'s mean-per-output. +Token/runtime are read via +:class:`~nemo_evaluator_sdk.agent_eval.metrics.TrialMeasurements`. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from nemo_evaluator_sdk.agent_eval.metrics import TrialMeasurements +from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult +from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore +from pydantic import BaseModel + +# Metric outputs, in priority order, that represent a task's pass/reward signal. +DEFAULT_REWARD_OUTPUTS: tuple[str, ...] = ("verifier_reward", "agent_phase_success") + + +@dataclass(frozen=True) +class GateThresholds: + """Knobs controlling the candidate gate (defaults are the strict CI policy).""" + + min_pass_rate: float = 1.0 + require_token_metrics: bool = False + max_pass_rate_drop: float = 0.0 + max_token_regression_pct: float = 0.0 + max_runtime_regression_pct: float = 0.0 + + +@dataclass +class GateCheck: + name: str + passed: bool + details: str + + +@dataclass +class GateReport: + gate_passed: bool + summary: dict[str, Any] + checks: list[GateCheck] = field(default_factory=list) + + def to_payload(self) -> dict[str, Any]: + return { + "gate_passed": self.gate_passed, + "summary": self.summary, + "checks": [asdict(check) for check in self.checks], + } + + +def evaluate_gate( + result: AgentEvalResult, + *, + thresholds: GateThresholds | None = None, + baseline_summary: dict[str, Any] | None = None, + reward_outputs: tuple[str, ...] = DEFAULT_REWARD_OUTPUTS, +) -> GateReport: + """Summarize a run and apply gate checks, optionally against a baseline.""" + thresholds = thresholds or GateThresholds() + summary = summarize_run(result, reward_outputs=reward_outputs) + checks = run_gate_checks(summary, thresholds=thresholds, baseline_summary=baseline_summary) + return GateReport(gate_passed=all(check.passed for check in checks), summary=summary, checks=checks) + + +def write_gate_report(report: GateReport, output_dir: str | Path, *, filename: str = "gate.json") -> Path: + """Persist the gate report alongside the run bundle.""" + path = Path(output_dir) + path.mkdir(parents=True, exist_ok=True) + gate_path = path / filename + gate_path.write_text(json.dumps(report.to_payload(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + return gate_path + + +def load_baseline_summary(path: str | Path) -> dict[str, Any]: + """Load + normalize a baseline summary (raw summary or a prior gate.json).""" + source = Path(path) + payload = json.loads(source.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"Baseline summary must be a JSON object: {source}") + summary = payload.get("summary") if isinstance(payload.get("summary"), dict) else payload + _validate_baseline_summary(summary, source) + return summary + + +def summarize_run( + result: AgentEvalResult, + *, + reward_outputs: tuple[str, ...] = DEFAULT_REWARD_OUTPUTS, +) -> dict[str, Any]: + """Aggregate pass-rate, token, and runtime for one run. + + Token/runtime are read via :class:`TrialMeasurements`; the reward used for + pass-rate prefers a scored metric output (``reward_outputs``) and falls back + to the trial's recorded reward. + """ + trials_by_task = {trial.task_id: trial for trial in result.trials} + reward_by_task = _rewards_by_task(result.scores, reward_outputs) + task_ids = sorted({task.id for task in result.tasks} | set(trials_by_task)) + + passed = 0 + token_sum = 0 + token_count = 0 + token_unavailable: list[str] = [] + runtime_sum = 0.0 + runtime_count = 0 + runtime_unavailable: list[str] = [] + + for task_id in task_ids: + trial = trials_by_task.get(task_id) + measurements = TrialMeasurements.from_metadata(trial.metadata if trial is not None else {}) + + reward_value = reward_by_task.get(task_id) + if reward_value is None: + reward_value = measurements.reward if measurements.reward is not None else 0.0 + if reward_value >= 1.0: + passed += 1 + + if measurements.total_tokens is not None: + token_sum += measurements.total_tokens + token_count += 1 + else: + token_unavailable.append(task_id) + + if measurements.runtime_sec is not None: + runtime_sum += measurements.runtime_sec + runtime_count += 1 + else: + runtime_unavailable.append(task_id) + + total = len(task_ids) + return { + "run_id": result.run_id, + "benchmark": result.benchmark, + "total_tasks": total, + "passed_tasks": passed, + "pass_rate": (passed / total) if total else 0.0, + "task_names": task_ids, + "total_tokens_sum": token_sum if token_count else None, + "avg_total_tokens": (token_sum / token_count) if token_count else None, + "token_metrics_coverage": (token_count / total) if total else 0.0, + "token_metrics_available_tasks": token_count, + "token_metrics_unavailable_tasks": sorted(token_unavailable), + "runtime_sec_sum": runtime_sum if runtime_count else None, + "avg_runtime_sec": (runtime_sum / runtime_count) if runtime_count else None, + "runtime_metrics_coverage": (runtime_count / total) if total else 0.0, + "runtime_metrics_available_tasks": runtime_count, + "runtime_metrics_unavailable_tasks": sorted(runtime_unavailable), + } + + +def run_gate_checks( + summary: dict[str, Any], + *, + thresholds: GateThresholds, + baseline_summary: dict[str, Any] | None = None, +) -> list[GateCheck]: + """Apply absolute + relative (vs baseline) gate checks to a summary.""" + checks: list[GateCheck] = [] + total_tasks = int(summary["total_tasks"]) + pass_rate = float(summary["pass_rate"]) + + checks.append(GateCheck("non_empty_result_set", total_tasks > 0, f"total_tasks={total_tasks}")) + checks.append( + GateCheck( + "min_pass_rate", + pass_rate >= thresholds.min_pass_rate, + f"pass_rate={pass_rate:.3f}, min_pass_rate={thresholds.min_pass_rate:.3f}", + ) + ) + + if thresholds.require_token_metrics: + token_coverage = float(summary["token_metrics_coverage"]) + runtime_coverage = float(summary["runtime_metrics_coverage"]) + checks.append( + GateCheck( + "token_metrics_available_for_all_tasks", + token_coverage == 1.0, + f"token_metrics_coverage={token_coverage:.3f}", + ) + ) + checks.append( + GateCheck( + "runtime_metrics_available_for_all_tasks", + runtime_coverage == 1.0, + f"runtime_metrics_coverage={runtime_coverage:.3f}", + ) + ) + + if baseline_summary is not None: + checks.extend(_baseline_checks(summary, baseline_summary, thresholds)) + + return checks + + +def _baseline_checks( + summary: dict[str, Any], + baseline_summary: dict[str, Any], + thresholds: GateThresholds, +) -> list[GateCheck]: + checks: list[GateCheck] = [] + pass_rate = float(summary["pass_rate"]) + total_tokens_sum = summary["total_tokens_sum"] + runtime_sec_sum = summary["runtime_sec_sum"] + + # Regression checks only make sense when both runs measured the same tasks. + baseline_tasks = baseline_summary.get("task_names") + candidate_tasks = summary.get("task_names") + task_sets_comparable = True + if isinstance(baseline_tasks, list) and isinstance(candidate_tasks, list): + comparable = sorted(baseline_tasks) == sorted(candidate_tasks) + task_sets_comparable = comparable + checks.append( + GateCheck( + "baseline_candidate_task_sets_match", + comparable, + ( + f"both runs measured {len(candidate_tasks)} tasks" + if comparable + else f"baseline={sorted(baseline_tasks)} candidate={sorted(candidate_tasks)}; " + "regression checks short-circuited" + ), + ) + ) + else: + checks.append( + GateCheck( + "baseline_candidate_task_sets_match", + True, + "task_names not present on baseline and/or candidate; skipping equality guard", + ) + ) + + if not task_sets_comparable: + return checks + + baseline_pass_rate = float(baseline_summary.get("pass_rate", 0.0)) + checks.append( + GateCheck( + "no_pass_rate_regression_vs_baseline", + pass_rate >= baseline_pass_rate - thresholds.max_pass_rate_drop, + f"pass_rate={pass_rate:.3f}, baseline={baseline_pass_rate:.3f}, max_drop={thresholds.max_pass_rate_drop:.3f}", + ) + ) + + baseline_tokens = baseline_summary.get("total_tokens_sum") + if isinstance(total_tokens_sum, int) and isinstance(baseline_tokens, int): + max_allowed = baseline_tokens * (1.0 + thresholds.max_token_regression_pct / 100.0) + checks.append( + GateCheck( + "tokens_not_worse_than_baseline", + total_tokens_sum <= max_allowed, + f"total_tokens_sum={total_tokens_sum}, baseline={baseline_tokens}, " + f"max_regression_pct={thresholds.max_token_regression_pct:.2f}", + ) + ) + else: + checks.append( + GateCheck( + "tokens_not_worse_than_baseline", + False, + "Missing token totals for candidate or baseline; cannot run deterministic token comparison.", + ) + ) + + # Runtime is only a tie-breaker when token totals match exactly. + baseline_runtime = baseline_summary.get("runtime_sec_sum") + tokens_tied = ( + isinstance(total_tokens_sum, int) and isinstance(baseline_tokens, int) and total_tokens_sum == baseline_tokens + ) + if not tokens_tied: + checks.append( + GateCheck( + "runtime_tie_breaker_not_worse_than_baseline", + True, + "Not applicable (token totals differ from baseline).", + ) + ) + elif isinstance(runtime_sec_sum, int | float) and isinstance(baseline_runtime, int | float): + max_allowed_runtime = float(baseline_runtime) * (1.0 + thresholds.max_runtime_regression_pct / 100.0) + checks.append( + GateCheck( + "runtime_tie_breaker_not_worse_than_baseline", + float(runtime_sec_sum) <= max_allowed_runtime, + f"runtime_sec_sum={float(runtime_sec_sum):.3f}, baseline={float(baseline_runtime):.3f}, " + f"max_regression_pct={thresholds.max_runtime_regression_pct:.2f}", + ) + ) + else: + checks.append( + GateCheck( + "runtime_tie_breaker_not_worse_than_baseline", + False, + "Token totals tied with baseline but runtime totals missing; cannot run tie-breaker.", + ) + ) + + return checks + + +def _rewards_by_task(scores: list[AgentEvalTaskScore], reward_outputs: tuple[str, ...]) -> dict[str, float]: + rewards: dict[str, float] = {} + for score in scores: + if score.status == AgentEvalScoreStatus.FAILED: + continue + for output_name in reward_outputs: + value = _numeric_output(score, output_name) + if value is not None: + # Highest-priority output wins; don't overwrite with later metrics. + rewards.setdefault(score.task_id, value) + break + return rewards + + +def _numeric_output(score: AgentEvalTaskScore, name: str) -> float | None: + for output in score.outputs: + if output.name == name: + return _reward_value(output.value) + return None + + +def _reward_value(value: Any) -> float | None: + # A boolean reward signal (e.g. agent_phase_success) maps to 1.0/0.0. + if isinstance(value, bool): + return 1.0 if value else 0.0 + if isinstance(value, int | float): + return float(value) + if isinstance(value, BaseModel): + root = getattr(value, "root", None) + if isinstance(root, bool): + return 1.0 if root else 0.0 + if isinstance(root, int | float): + return float(root) + return None + + +def _validate_baseline_summary(summary: dict[str, Any], source: Path) -> None: + missing = [key for key in ("pass_rate", "total_tokens_sum", "runtime_sec_sum") if key not in summary] + if missing: + raise ValueError( + f"Baseline summary {source} is missing required key(s): {', '.join(missing)}. " + "Expected a raw summary object or a gate.json with a `summary`." + ) + if not isinstance(summary.get("pass_rate"), int | float): + raise ValueError(f"Baseline summary {source} has invalid `pass_rate`; expected a number.") diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/layout.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/layout.py new file mode 100644 index 0000000000..5c858cb037 --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/layout.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Generic on-disk layout for a single agent-eval task run. + +A run produces an agent-log dir and a workspace dir under a run dir, plus a +written instruction file. Callers that need extra directories (e.g. preserved +platform state) add them on top of :class:`RunLayout`. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class RunLayout: + """Filesystem layout for one task run.""" + + run_dir: Path + agent_log_dir: Path + workspace_dir: Path + instruction_path: Path + + +def resolve_run_dir(output_dir: str | Path | None, default_factory: Callable[[], Path]) -> Path: + """Resolve the run dir to an absolute path. + + An explicit ``output_dir`` must be made absolute: run-dir subpaths are used as + Docker bind-mount sources, and Docker treats a relative ``-v`` source as a + (slash-free) named volume rather than a host directory. + """ + if output_dir is not None: + return Path(output_dir).resolve() + return default_factory() + + +def prepare_run_layout( + run_dir: str | Path, + instruction_text: str, + *, + agent_subdir: str = "agent", + workspace_subdir: str = "workspace", + instruction_name: str = "instruction.md", +) -> RunLayout: + """Create the agent/workspace dirs under ``run_dir`` and write the instruction.""" + run_dir = Path(run_dir) + agent_log_dir = run_dir / agent_subdir + workspace_dir = run_dir / workspace_subdir + agent_log_dir.mkdir(parents=True, exist_ok=True) + workspace_dir.mkdir(parents=True, exist_ok=True) + + instruction_path = agent_log_dir / instruction_name + instruction_path.write_text(instruction_text, encoding="utf-8") + + return RunLayout( + run_dir=run_dir, + agent_log_dir=agent_log_dir, + workspace_dir=workspace_dir, + instruction_path=instruction_path, + ) diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/mini_agent.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/mini_agent.py new file mode 100644 index 0000000000..91cbc95b43 --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/mini_agent.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A tiny, dependency-free stand-in agent CLI used as the default "workflow". + +Stands in for whatever real agent a workflow backend would launch (``nat run``, +``codex exec``, ...). The contract ``WorkflowAgentRuntime`` relies on: read the +task-input JSON (``--input-json``), do the work inside ``--workspace`` (here: +write the requested file), print a final answer to stdout, exit non-zero on +failure. Any executable honoring this contract drops in unchanged. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser(description="Toy file-writing agent.") + parser.add_argument("--instruction", type=Path, required=True, help="Path to the human-readable instruction.") + parser.add_argument("--workspace", type=Path, required=True, help="Directory the agent may write into.") + parser.add_argument("--input-json", type=Path, required=True, help="Path to the structured task inputs.") + args = parser.parse_args() + + spec = json.loads(args.input_json.read_text(encoding="utf-8")) if args.input_json.exists() else {} + create_file = spec.get("create_file") + content = spec.get("content", "") + + if not isinstance(create_file, str) or not create_file: + print("No 'create_file' directive found in task inputs; nothing to do.", file=sys.stderr) + return 2 + + args.workspace.mkdir(parents=True, exist_ok=True) + target = args.workspace / create_file + target.write_text(content, encoding="utf-8") + + print(f"Created {create_file} ({len(content.encode('utf-8'))} bytes) in the workspace. Task complete.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py new file mode 100644 index 0000000000..67111f285b --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Example-local pipeline: AgentEvaluator run + deterministic gate. + +Thin convenience wrapper used by this example only (not SDK API): it runs the +SDK's :class:`~nemo_evaluator_sdk.agent_eval.evaluator.AgentEvaluator` and applies +the example-local gate. Two seams keep it backend-agnostic: + +* **verify-enable is inverted to data**: callers pass ``extra_metrics`` to append + (e.g. a verifier-reward metric). The pipeline never introspects a runtime's + config to decide what to score. +* **environment prep is an injected hook**: ``prepare_task`` (e.g. "build the task + image") runs per task before execution, so Docker/build specifics live in the + caller, not here. +""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from pathlib import Path + +from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator +from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult +from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTarget, AgentEvalTrial +from nemo_evaluator_sdk.metrics.protocol import Metric + +from .gating import GateThresholds, evaluate_gate, load_baseline_summary, write_gate_report + + +@dataclass(frozen=True) +class PipelineConfig: + """Run-level knobs shared by the online and offline paths.""" + + parallelism: int = 1 + write_dashboard: bool = True + write_gate: bool = True + gate_thresholds: GateThresholds | None = None + baseline_summary_path: Path | None = None + + +class AgentEvalPipeline: + """Run tasks through ``AgentEvaluator`` (online or offline) and apply the gate.""" + + def __init__( + self, + *, + config: PipelineConfig | None = None, + extra_metrics: Sequence[Metric] = (), + ) -> None: + self.config = config or PipelineConfig() + self._extra_metrics = list(extra_metrics) + + async def run_tasks( + self, + tasks: Sequence[AgentEvalTask], + *, + target: AgentEvalTarget, + benchmark: dict[str, object] | None = None, + output_dir: Path | None = None, + run_id: str | None = None, + prepare_task: Callable[[AgentEvalTask], None] | None = None, + ) -> AgentEvalResult: + """Online path: optionally prep each task, run the target, score, gate.""" + prepared = [self._with_extra_metrics(task) for task in tasks] + if prepare_task is not None: + for task in prepared: + prepare_task(task) + + result = await AgentEvaluator().run( + tasks=prepared, + target=target, + config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark), + ) + self._maybe_write_gate(result) + return result + + async def score_trials( + self, + tasks: Sequence[AgentEvalTask], + *, + trials: Sequence[AgentEvalTrial], + benchmark: dict[str, object] | None = None, + output_dir: Path | None = None, + run_id: str | None = None, + ) -> AgentEvalResult: + """Offline path: score already-captured trials (no agent execution).""" + prepared = [self._with_extra_metrics(task) for task in tasks] + result = await AgentEvaluator().run( + tasks=prepared, + trials=list(trials), + config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark), + ) + self._maybe_write_gate(result) + return result + + def _run_config( + self, + *, + output_dir: Path | None, + run_id: str | None, + benchmark: dict[str, object] | None, + ) -> AgentEvalRunConfig: + return AgentEvalRunConfig( + output_dir=output_dir, + run_id=run_id, + parallelism=self.config.parallelism, + write_dashboard=self.config.write_dashboard, + benchmark=dict(benchmark or {}), + ) + + def _with_extra_metrics(self, task: AgentEvalTask) -> AgentEvalTask: + """Append injected metrics, honoring task-authored metrics and avoiding duplicate types.""" + if not self._extra_metrics: + return task + metrics: list[Metric] = list(task.metrics) + existing_types = {metric.type for metric in metrics} + appended = [metric for metric in self._extra_metrics if metric.type not in existing_types] + if not appended: + return task + return task.model_copy(update={"metrics": metrics + appended}) + + def _maybe_write_gate(self, result: AgentEvalResult) -> None: + if not (self.config.write_gate and result.output_dir is not None): + return + baseline = ( + load_baseline_summary(self.config.baseline_summary_path) + if self.config.baseline_summary_path is not None + else None + ) + report = evaluate_gate(result, thresholds=self.config.gate_thresholds, baseline_summary=baseline) + write_gate_report(report, result.output_dir) + + +__all__ = [ + "AgentEvalPipeline", + "GateThresholds", + "PipelineConfig", +] diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py new file mode 100644 index 0000000000..8c97094d75 --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py @@ -0,0 +1,631 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""NeMo-Platform glue that lets this example run a real ``tests/agentic-use`` task. + +Generic logic lives in ``nemo_evaluator_sdk.agent_eval``; this module holds only +the agentic-use-specific pieces: :func:`agentic_task_from_dir` (load a task from +``instruction.md`` + ``task.toml``), :func:`ensure_task_image` (BUILD), +:class:`NatWorkflowRuntime` (AGENT via ``nat run`` + optional pytest VERIFY, +shaped through the shared :func:`run_agent_then_verify`), and +:class:`VerifierRewardMetric` (scores the pytest reward). + +Running a real task requires Docker, the ``nmp-agentic-base:latest`` image, a +running NeMo Platform, and ``NVIDIA_API_KEY`` — see this example's README. +""" + +from __future__ import annotations + +import subprocess +import textwrap +import time +import tomllib +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml +from nemo_evaluator_sdk.agent_eval.runtimes.environment import ( + AgentEnvironmentHandle, + AgentEnvironmentProvider, + DockerEnvironmentProvider, + EnvRunSpec, +) +from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask +from nemo_evaluator_sdk.agent_eval.trials import ( + AgentEvalTrial, + AgentEvalTrialStatus, + AgentOutput, + resolve_trial_status, + standard_evidence_descriptors, +) +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor + +from .build_spec import execute_build_plan, plan_task_build +from .layout import prepare_run_layout, resolve_run_dir +from .usage import agent_log_has_workflow_error, extract_usage_metrics +from .verify import ( + VerifierOutcome, + apply_verify_to_metadata, + collect_verifier_outcome, + skipped_outcome, +) + +REPO_ROOT = Path(__file__).resolve().parents[4] +AGENTIC_USE_DIR = REPO_ROOT / "tests" / "agentic-use" +SHARED_DIR = AGENTIC_USE_DIR / "shared" +EVALUATOR_SDK_SRC = REPO_ROOT / "packages" / "nemo_evaluator_sdk" / "src" + +RUNTIME_NAME = "workflow" +DEFAULT_TIMEOUT_SEC = 600 +DEFAULT_LOCAL_NMP_BASE_URL = "http://localhost:8080" +FILES_STORAGE_CONFIG = '{"type":"local","path":"/data/files_storage"}' +PLATFORM_CONFIG_PATH = "/app/packages/nmp_platform/config/local.yaml" +NAT_TRACE_EXPORT_SCRIPT_CONTAINER_PATH = "/app/tests/agentic-use/scripts/nat_trace_export.py" +INSTRUCTION_CONTAINER_PATH = "/tmp/nat_instruction.md" +WORKFLOW_CONTAINER_PATH = "/tmp/nat_workflow.yml" +DOCKER_SOCKET_HOST_PATH = Path("/var/run/docker.sock") +DOCKER_SOCKET_CONTAINER_PATH = "/var/run/docker.sock" + + +# --------------------------------------------------------------------------- # +# Configuration +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class NatWorkflowConfig: + """Configuration for :class:`NatWorkflowRuntime`.""" + + nmp_base_url: str = DEFAULT_LOCAL_NMP_BASE_URL + nvidia_api_key: str | None = None + agent_model: str | None = None + timeout_sec: int = DEFAULT_TIMEOUT_SEC + run_verify: bool = False + docker_extra_args: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- # +# Run layout + image tag +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class AgenticRunLayout: + """Run layout extending the SDK's generic one with a platform ``state_dir``.""" + + run_dir: Path + agent_log_dir: Path + workspace_dir: Path + state_dir: Path + instruction_path: Path + + +def task_image_tag(task_id: str) -> str: + return f"nmp-nat-{task_id}:latest" + + +def resolve_run_layout(task: AgentEvalTask, config: AgentEvalRunConfig | None) -> AgenticRunLayout: + """Resolve/create the on-disk layout for one task run.""" + output_dir = config.output_dir if config is not None else None + run_dir = resolve_run_dir(output_dir, lambda: Path.cwd() / "nat-jobs" / task.id) / task.id + base = prepare_run_layout(run_dir, str(task.inputs.get("instruction") or task.intent)) + state_dir = base.run_dir / "state" + state_dir.mkdir(parents=True, exist_ok=True) + return AgenticRunLayout( + run_dir=base.run_dir, + agent_log_dir=base.agent_log_dir, + workspace_dir=base.workspace_dir, + state_dir=state_dir, + instruction_path=base.instruction_path, + ) + + +class PlatformDockerEnvironmentProvider(DockerEnvironmentProvider): + """Docker provider defaulting each task to ``nmp-nat-:latest``.""" + + def __init__(self, *, image_tag_fn: Callable[[str], str] = task_image_tag) -> None: + super().__init__(image_tag_fn=image_tag_fn) + + +# --------------------------------------------------------------------------- # +# Verifier reward metric (compatibility shim for the pytest reward) +# --------------------------------------------------------------------------- # +class VerifierRewardMetric: + """Score the pytest verifier reward stamped on trial metadata.""" + + @property + def type(self) -> str: + return "agentic_use_verifier_reward" + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score("verifier_reward")] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + metadata = input.candidate.metadata + reward = metadata.get("reward") + if reward is None: + reward = 1.0 if metadata.get("passed") else 0.0 + return MetricResult(outputs=[MetricOutput(name="verifier_reward", value=float(reward))]) + + +# --------------------------------------------------------------------------- # +# Task loader +# --------------------------------------------------------------------------- # +def load_task_toml(task_dir: Path) -> dict[str, Any]: + task_toml = task_dir / "task.toml" + if not task_toml.exists(): + return {} + try: + with task_toml.open("rb") as handle: + data = tomllib.load(handle) + except (OSError, tomllib.TOMLDecodeError): + return {} + return data if isinstance(data, dict) else {} + + +def task_agent_timeout_sec(task_dir: Path) -> int | None: + agent = load_task_toml(task_dir).get("agent") + if not isinstance(agent, dict): + return None + timeout_value = agent.get("timeout_sec") + if isinstance(timeout_value, int | float) and timeout_value > 0: + return int(timeout_value) + return None + + +def agentic_task_from_dir(task_dir: str | Path, *, tasks_root: Path | None = None) -> AgentEvalTask: + """Build an ``AgentEvalTask`` from an agentic-use task directory. + + ``inputs`` carries only agent-facing material (``instruction``); runtime + materialization (``task_dir``) lives in ``metadata`` so it can't leak into a + metric scoring row. Metrics default to ``[AgentPhaseSuccessMetric()]``. + """ + from nemo_evaluator_sdk.agent_eval.metrics import AgentPhaseSuccessMetric + + root = Path(tasks_root or AGENTIC_USE_DIR) + task_path = Path(task_dir) + if not task_path.is_absolute(): + task_path = (root / task_path).resolve() + + instruction_path = task_path / "instruction.md" + if not instruction_path.exists(): + raise FileNotFoundError(f"instruction.md not found in {task_path}") + instruction = instruction_path.read_text(encoding="utf-8") + + return AgentEvalTask( + id=task_path.name, + intent=instruction, + inputs={"instruction": instruction}, + metrics=[AgentPhaseSuccessMetric()], + metadata={ + "benchmark": "agentic-use", + "task_toml": load_task_toml(task_path), + "instruction_path": str(instruction_path), + "task_dir": str(task_path), + }, + ) + + +# --------------------------------------------------------------------------- # +# BUILD phase +# --------------------------------------------------------------------------- # +def ensure_task_image(task: AgentEvalTask, *, skip_build: bool = False) -> str: + """Build (or verify the presence of) the task's Docker image; return its tag.""" + image_tag = task_image_tag(task.id) + task_dir = Path(str(task.metadata["task_dir"])) + if skip_build: + exists = ( + subprocess.run(["docker", "image", "inspect", image_tag], capture_output=True, check=False).returncode == 0 + ) + if not exists: + raise RuntimeError(f"--skip-build set but image {image_tag!r} is not available locally; build it first.") + return image_tag + execute_build_plan(plan_task_build(task_dir, image_tag)) + return image_tag + + +# --------------------------------------------------------------------------- # +# AGENT phase: command + workflow prep + container env +# --------------------------------------------------------------------------- # +def build_workflow_agent_cmd(workflow_container: str, instruction_container: str) -> list[str]: + """``bash -c`` command that runs ``nat run`` and exports the trajectory.""" + return [ + "bash", + "-c", + textwrap.dedent(f"""\ + /app/.venv/bin/nat run \\ + --config_file {workflow_container} \\ + --input "$(cat {instruction_container})" \\ + 2>&1 | tee /tmp/nat_agent.log + EXIT=${{PIPESTATUS[0]}} + cp /tmp/nat_agent.log /logs/agent/nat_agent.log 2>/dev/null || true + if [ -f /logs/agent/intermediate_steps.jsonl ]; then + /app/.venv/bin/python {NAT_TRACE_EXPORT_SCRIPT_CONTAINER_PATH} convert-jsonl \\ + --input /logs/agent/intermediate_steps.jsonl \\ + --output /logs/agent/trajectory.json \\ + >> /tmp/nat_agent.log 2>&1 + cp /tmp/nat_agent.log /logs/agent/nat_agent.log 2>/dev/null || true + fi + exit $EXIT + """), + ] + + +def prepare_workflow_for_runtime( + workflow_path: Path, + output_dir: Path, + nmp_base_url: str, + *, + nat_model: str | None = None, +) -> Path: + """Rewrite a task ``workflow.yml`` for container execution + trajectory export.""" + text = workflow_path.read_text(encoding="utf-8") + text = text.replace("http://localhost:8080", nmp_base_url) + if nat_model: + text = text.replace( + "model_name: nvidia/llama-3.1-nemotron-70b-instruct", + f"model_name: {nat_model}", + 1, + ) + if ("_type: mcp_client" in text or "_type: per_user_mcp_client" in text) and ( + "\nfunction_groups:\n" not in text and "\nfunctions:\n" in text + ): + text = text.replace("\nfunctions:\n", "\nfunction_groups:\n", 1) + + config = yaml.safe_load(text) + if not isinstance(config, dict): + raise ValueError(f"Workflow config must be a mapping: {workflow_path}") + general = config.setdefault("general", {}) + telemetry = general.setdefault("telemetry", {}) + tracing = telemetry.setdefault("tracing", {}) + tracing["agentic_use_file_trace"] = { + "_type": "file", + "output_path": "/logs/agent/intermediate_steps.jsonl", + "project": "agentic-use", + "mode": "overwrite", + "cleanup_on_init": True, + } + + rewritten = output_dir / "workflow.runtime.yml" + rewritten.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False), encoding="utf-8") + return rewritten + + +def base_container_env(nmp_base_url: str, *, timeout_sec: int) -> dict[str, str]: + env = { + "NMP_BASE_URL": nmp_base_url, + "AGENTIC_USE_WORKSPACE_DIR": "/app/workspace", + "DATABASE_DIALECT": "sqlite", + "DATABASE_PATH": "/data/nmp-platform.db", + "NMP_FILES_DEFAULT_STORAGE_CONFIG": FILES_STORAGE_CONFIG, + "NMP_CONFIG_FILE_PATH": PLATFORM_CONFIG_PATH, + "NEMO_AGENTS_GATEWAY_READ_TIMEOUT": str(timeout_sec), + "NEMO_AGENTS_INVOKE_TIMEOUT": str(timeout_sec), + } + if DOCKER_SOCKET_HOST_PATH.exists(): + env["DOCKER_HOST"] = f"unix://{DOCKER_SOCKET_CONTAINER_PATH}" + return env + + +def docker_socket_mounts() -> list[tuple[str, str]]: + """Bind-mount the host Docker socket into the container when it exists.""" + if DOCKER_SOCKET_HOST_PATH.exists(): + return [(str(DOCKER_SOCKET_HOST_PATH), DOCKER_SOCKET_CONTAINER_PATH)] + return [] + + +# --------------------------------------------------------------------------- # +# VERIFY phase +# --------------------------------------------------------------------------- # +def verifier_log_dir(layout: AgenticRunLayout) -> Path: + return layout.run_dir / "verifier" + + +def build_verify_run_spec( + task_dir: Path, + layout: AgenticRunLayout, + *, + nmp_base_url: str, + agent_model: str, + agent_backend: str = RUNTIME_NAME, + timeout_sec: int | None = None, + extra_args: list[str] | None = None, +) -> EnvRunSpec | None: + """Build the verifier ``EnvRunSpec`` (pytest ``test_outputs.py``), or ``None``.""" + tests_dir = task_dir / "tests" + if not (tests_dir / "test_outputs.py").exists(): + return None + + log_dir = verifier_log_dir(layout) + log_dir.mkdir(parents=True, exist_ok=True) + layout.workspace_dir.mkdir(parents=True, exist_ok=True) + + verify_cmd = [ + "bash", + "-c", + textwrap.dedent("""\ + export PYTHONPATH="/app/tests/agentic-use/shared:/app/packages/nemo_evaluator_sdk/src:${PYTHONPATH}" + export NAT_AGENT=1 + /app/.venv/bin/python -m pytest /tests/test_outputs.py -rA -v 2>&1 | tee /logs/verifier/test-stdout.txt + EXIT=${PIPESTATUS[0]} + if [ $EXIT -eq 0 ]; then echo 1; else echo 0; fi > /logs/verifier/reward.txt + exit $EXIT + """), + ] + + env = base_container_env(nmp_base_url, timeout_sec=timeout_sec or DEFAULT_TIMEOUT_SEC) + env.update( + { + "NAT_AGENT": "1", + "NAT_AGENT_BACKEND": agent_backend, + "NAT_AGENT_MODEL": agent_model, + "AGENTIC_USE_TASK_DIR": "/task", + } + ) + + mounts = [ + (str(tests_dir), "/tests"), + (str(task_dir), "/task"), + (str(layout.workspace_dir), "/app/workspace"), + (str(SHARED_DIR), "/app/tests/agentic-use/shared:ro"), + (str(EVALUATOR_SDK_SRC), "/app/packages/nemo_evaluator_sdk/src:ro"), + (str(layout.agent_log_dir), "/logs/agent"), + (str(log_dir), "/logs/verifier"), + (str(layout.state_dir), "/data"), + *docker_socket_mounts(), + ] + + return EnvRunSpec( + command=verify_cmd, env=env, mounts=mounts, timeout=timeout_sec, extra_args=list(extra_args or []) + ) + + +async def maybe_run_verify( + handle: AgentEnvironmentHandle, + *, + enabled: bool, + task_dir: Path, + layout: AgenticRunLayout, + nmp_base_url: str, + agent_model: str, + agent_backend: str = RUNTIME_NAME, + timeout_sec: int | None = None, + extra_args: list[str] | None = None, +) -> VerifierOutcome: + """Run the verifier through ``handle`` when enabled and a verifier exists.""" + if not enabled: + return skipped_outcome() + spec = build_verify_run_spec( + task_dir, + layout, + nmp_base_url=nmp_base_url, + agent_model=agent_model, + agent_backend=agent_backend, + timeout_sec=timeout_sec, + extra_args=extra_args, + ) + if spec is None: + return skipped_outcome() + result = await handle.run_verifier(spec) + return collect_verifier_outcome(ok=result.ok, exit_code=result.exit_code, log_dir=verifier_log_dir(layout)) + + +# --------------------------------------------------------------------------- # +# Trial construction from live artifacts +# --------------------------------------------------------------------------- # +def build_trial_from_artifacts( + *, + task: AgentEvalTask, + layout: AgenticRunLayout, + runtime_name: str, + agent_model: str, + exit_code: int, + agent_ok: bool, + runtime_sec: float, +) -> AgentEvalTrial: + """Shape an ``AgentEvalTrial`` from on-disk agent artifacts. + + Token usage is parsed from the agent log with the ``nat_runner`` extractor so + the SDK summary's token/runtime aggregates populate exactly as they do in + ``result.json["metrics"]``. + """ + log_text = _read_agent_log(layout.agent_log_dir) + usage = extract_usage_metrics(log_text) + trace_path = layout.agent_log_dir / "trajectory.json" + descriptors = standard_evidence_descriptors( + logs_dir=layout.agent_log_dir, + final_state_dir=layout.workspace_dir, + trace_path=trace_path if trace_path.exists() else None, + verifier_logs_dir=verifier_log_dir(layout), + primary_log="nat_agent.log", + ) + descriptors["state"] = EvidenceDescriptor( + kind="filesystem", + format="dir", + ref=str(layout.state_dir), + metadata={"role": "platform_state", "extension": "nemo-platform"}, + ) + + output_text = log_text.strip() or ("" if agent_ok else "(agent phase failed)") + metadata: dict[str, Any] = { + "agent_runtime": runtime_name, + "agent_model": agent_model, + "agent_ok": agent_ok, + "exit_code": exit_code, + "runtime_sec": runtime_sec, + "run_dir": str(layout.run_dir), + "agent_log_dir": str(layout.agent_log_dir), + "workspace_dir": str(layout.workspace_dir), + "state_dir": str(layout.state_dir), + "generated": True, + # Token measurements (same keys nat_runner writes into result.json["metrics"]). + **{key: value for key, value in usage.items() if value is not None}, + } + return AgentEvalTrial( + id=f"{task.id}:{runtime_name}", + task_id=task.id, + status=resolve_trial_status(agent_ok), + output=AgentOutput( + output_text=output_text, + metadata={"runtime": runtime_name, "agent_model": agent_model}, + ), + evidence=CandidateEvidence(descriptors=descriptors, metadata={"runtime": runtime_name}), + metadata=metadata, + ) + + +def _read_agent_log(agent_log_dir: Path) -> str: + log_path = agent_log_dir / "nat_agent.log" + if log_path.is_file(): + return log_path.read_text(encoding="utf-8", errors="replace") + return "" + + +async def run_agent_then_verify( + handle: AgentEnvironmentHandle, + *, + task: AgentEvalTask, + layout: AgenticRunLayout, + spec: EnvRunSpec, + runtime_name: str, + agent_model: str, + run_verify: bool, + nmp_base_url: str, + verify_timeout_sec: int, + docker_extra_args: list[str], +) -> AgentEvalTrial: + """Shared AGENT → VERIFY → trial flow for the Docker-backed runtimes. + + Runs the agent ``spec`` through ``handle``, flips success on a logged + ``workflow_error``, optionally runs the pytest verifier, then shapes a trial + (promoting it to ``COMPLETED`` when the verifier passes). + """ + started = time.monotonic() + try: + result = await handle.run_agent(spec) + agent_ok = result.ok + log_text = _read_agent_log(layout.agent_log_dir) + if agent_ok and log_text and agent_log_has_workflow_error(log_text): + agent_ok = False + verify_outcome = await maybe_run_verify( + handle, + enabled=run_verify and agent_ok, + task_dir=Path(str(task.metadata["task_dir"])), + layout=layout, + nmp_base_url=nmp_base_url, + agent_model=agent_model, + agent_backend=runtime_name, + timeout_sec=verify_timeout_sec, + extra_args=docker_extra_args, + ) + finally: + await handle.close() + runtime_sec = time.monotonic() - started + + trial = build_trial_from_artifacts( + task=task, + layout=layout, + runtime_name=runtime_name, + agent_model=agent_model, + exit_code=result.exit_code, + agent_ok=agent_ok, + runtime_sec=runtime_sec, + ) + apply_verify_to_metadata(trial.metadata, verify_outcome) + if verify_outcome.ran and verify_outcome.passed and trial.status != AgentEvalTrialStatus.COMPLETED: + trial = trial.model_copy(update={"status": AgentEvalTrialStatus.COMPLETED}) + return trial + + +# --------------------------------------------------------------------------- # +# Runtime +# --------------------------------------------------------------------------- # +class NatWorkflowRuntime: + """Run agentic-use tasks via ``nat run`` inside the task image (an ``AgentTaskRunner``).""" + + def __init__( + self, + config: NatWorkflowConfig | None = None, + *, + environment: AgentEnvironmentProvider | None = None, + ) -> None: + self.config = config or NatWorkflowConfig() + self.environment = environment or PlatformDockerEnvironmentProvider() + + async def run_tasks( + self, + tasks: Sequence[AgentEvalTask], + config: AgentEvalRunConfig | None = None, + ) -> Sequence[AgentEvalTrial]: + trials: list[AgentEvalTrial] = [] + for task in tasks: + trials.append(await self._run_task(task, config)) + return trials + + async def _run_task(self, task: AgentEvalTask, config: AgentEvalRunConfig | None) -> AgentEvalTrial: + layout = resolve_run_layout(task, config) + task_dir = Path(str(task.metadata["task_dir"])) + agent_model = self.config.agent_model or "unknown" + handle = await self.environment.prepare(task, config) + return await run_agent_then_verify( + handle, + task=task, + layout=layout, + spec=self._agent_run_spec(task_dir, layout), + runtime_name=RUNTIME_NAME, + agent_model=agent_model, + run_verify=self.config.run_verify, + nmp_base_url=self.config.nmp_base_url, + verify_timeout_sec=self.config.timeout_sec + 120, + docker_extra_args=list(self.config.docker_extra_args), + ) + + def _agent_run_spec(self, task_dir: Path, layout: AgenticRunLayout) -> EnvRunSpec: + workflow_path = task_dir / "workflow.yml" + if not workflow_path.exists(): + raise FileNotFoundError(f"workflow.yml not found in {task_dir}") + + task_timeout = task_agent_timeout_sec(task_dir) or 0 + timeout_sec = max(self.config.timeout_sec, task_timeout) + workflow_host = prepare_workflow_for_runtime( + workflow_path, + layout.agent_log_dir, + self.config.nmp_base_url, + nat_model=self.config.agent_model, + ) + + env = base_container_env(self.config.nmp_base_url, timeout_sec=timeout_sec) + if self.config.nvidia_api_key: + env["NVIDIA_API_KEY"] = self.config.nvidia_api_key + if self.config.agent_model: + env["NAT_MODEL"] = self.config.agent_model + + mounts = [ + (str(layout.instruction_path), INSTRUCTION_CONTAINER_PATH), + (str(layout.agent_log_dir), "/logs/agent"), + (str(layout.workspace_dir), "/app/workspace"), + (str(workflow_host), WORKFLOW_CONTAINER_PATH), + (str(layout.state_dir), "/data"), + *docker_socket_mounts(), + ] + + return EnvRunSpec( + command=build_workflow_agent_cmd(WORKFLOW_CONTAINER_PATH, INSTRUCTION_CONTAINER_PATH), + env=env, + mounts=mounts, + timeout=timeout_sec, + extra_args=list(self.config.docker_extra_args), + ) + + +__all__ = [ + "AGENTIC_USE_DIR", + "AgenticRunLayout", + "NatWorkflowConfig", + "NatWorkflowRuntime", + "PlatformDockerEnvironmentProvider", + "VerifierRewardMetric", + "agentic_task_from_dir", + "ensure_task_image", + "run_agent_then_verify", + "task_image_tag", +] diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py new file mode 100644 index 0000000000..3d564eae6e --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py @@ -0,0 +1,277 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run agent-eval tasks through a workflow runtime, using the trials-based SDK. + +It drives the example-local ``AgentEvalPipeline`` two ways: + +* **online** — generate trials by running :class:`WorkflowAgentRuntime` + (the agent), score them, and apply the deterministic gate; or +* **offline** — re-score the ``trials.jsonl`` of a prior run with no agent + execution (``--rescore-dir``). + +Run it as a module from the repository root:: + + python -m packages.nemo_evaluator_sdk.examples.run_agent_eval.run_agent_eval --task all +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import os +from pathlib import Path + +if __package__ in {None, ""}: + raise SystemExit( + "Run this example as a module from the repository root:\n" + " python -m packages.nemo_evaluator_sdk.examples.run_agent_eval.run_agent_eval --task all" + ) + +from nemo_evaluator_sdk.agent_eval.metrics import TrialMeasurements +from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult +from nemo_evaluator_sdk.metrics.protocol import Metric + +from .aut_runtime import AutConfig, NatAutRuntime +from .gating import GateThresholds +from .pipeline import AgentEvalPipeline, PipelineConfig +from .platform_runtime import ( + NatWorkflowConfig, + NatWorkflowRuntime, + VerifierRewardMetric, + agentic_task_from_dir, + ensure_task_image, +) +from .workflow_runtime import ( + WorkflowAgentRuntime, + WorkflowRuntimeConfig, + example_tasks, + load_stored_trials, + tasks_by_id, +) + +DEFAULT_OUTPUT_DIR = Path(__file__).resolve().parent / "run-agent-eval-output" + + +def _configure_logging() -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s:%(name)s:%(message)s") + logging.getLogger("httpx").setLevel(logging.WARNING) + + +def _pipeline(min_pass_rate: float, *, extra_metrics: tuple[Metric, ...] = ()) -> AgentEvalPipeline: + return AgentEvalPipeline( + config=PipelineConfig( + parallelism=2, + write_dashboard=True, + write_gate=True, + gate_thresholds=GateThresholds(min_pass_rate=min_pass_rate), + ), + extra_metrics=extra_metrics, + ) + + +async def run_online(task_names: list[str], *, output_dir: Path, min_pass_rate: float) -> AgentEvalResult: + tasks = tasks_by_id(task_names) + runtime = WorkflowAgentRuntime(WorkflowRuntimeConfig()) + return await _pipeline(min_pass_rate).run_tasks( + tasks, + target=runtime, + benchmark={"benchmark": "run-agent-eval", "mode": "online"}, + output_dir=output_dir, + ) + + +async def run_agentic_task( + task_name: str, + *, + output_dir: Path, + min_pass_rate: float, + nmp_base_url: str, + agent_model: str | None, + skip_build: bool, + verify: bool, + backend: str, + aut_agent_name: str | None, + aut_agent_config: Path | None, + seed_providers: bool, +) -> AgentEvalResult: + """Run a real ``tests/agentic-use`` task: BUILD → AGENT → VERIFY → score → gate. + + ``backend='workflow'`` runs the task-local ``nat run`` workflow; ``backend='aut'`` + drives a deployed platform agent-under-test (the canonical ``nat_runner`` path). + """ + task = agentic_task_from_dir(task_name) + runtime: NatWorkflowRuntime | NatAutRuntime + if backend == "aut": + if not aut_agent_name: + raise ValueError("--backend aut requires --aut-agent-name") + runtime = NatAutRuntime( + AutConfig( + aut_agent_name=aut_agent_name, + aut_agent_config=aut_agent_config, + aut_seed_providers=seed_providers, + agent_model=agent_model, + nmp_base_url=nmp_base_url, + nvidia_api_key=os.environ.get("NVIDIA_API_KEY"), + inference_nvidia_api_key=os.environ.get("INFERENCE_NVIDIA_API_KEY"), + anthropic_api_key=os.environ.get("ANTHROPIC_API_KEY"), + run_verify=verify, + ), + ) + else: + runtime = NatWorkflowRuntime( + NatWorkflowConfig( + nmp_base_url=nmp_base_url, + nvidia_api_key=os.environ.get("NVIDIA_API_KEY"), + agent_model=agent_model, + run_verify=verify, + ), + ) + extra_metrics: tuple[Metric, ...] = (VerifierRewardMetric(),) if verify else () + return await _pipeline(min_pass_rate, extra_metrics=extra_metrics).run_tasks( + [task], + target=runtime, + benchmark={"benchmark": "agentic-use", "task": task_name, "backend": backend}, + output_dir=output_dir, + prepare_task=lambda t: ensure_task_image(t, skip_build=skip_build), + ) + + +async def rescore(rescore_dirs: list[Path], *, output_dir: Path, min_pass_rate: float) -> AgentEvalResult: + trials = [trial for run_dir in rescore_dirs for trial in load_stored_trials(run_dir)] + needed = {trial.task_id for trial in trials} + tasks = [task for task in example_tasks() if task.id in needed] + return await _pipeline(min_pass_rate).score_trials( + tasks, + trials=trials, + benchmark={"benchmark": "run-agent-eval", "mode": "offline"}, + output_dir=output_dir, + ) + + +def _print_result(result: AgentEvalResult) -> None: + print(f"run_id: {result.run_id}") + print(f"tasks: {result.summary.task_count} trials: {result.summary.trial_count}") + for metric_type, true_count, total in _boolean_true_rates(result): + print(f" {metric_type}: {true_count}/{total} true") + for score in result.summary.scores.scores: + if score.mean is not None: + print(f" {score.name}: mean={score.mean:.3f}") + _print_measurements(result) + if result.output_dir is not None: + print(f"output_dir: {result.output_dir}") + print(f"gate: {result.output_dir / 'gate.json'}") + + +def _print_measurements(result: AgentEvalResult) -> None: + """Print token/runtime totals (the same measurements nat_runner records).""" + measurements = [TrialMeasurements.from_metadata(trial.metadata) for trial in result.trials] + total_tokens = [m.total_tokens for m in measurements if m.total_tokens is not None] + runtimes = [m.runtime_sec for m in measurements if m.runtime_sec is not None] + if total_tokens: + print(f" total_tokens: {sum(total_tokens)} across {len(total_tokens)}/{len(measurements)} trials") + if runtimes: + print(f" runtime_sec: {sum(runtimes):.1f} across {len(runtimes)}/{len(measurements)} trials") + + +def _boolean_true_rates(result: AgentEvalResult) -> list[tuple[str, int, int]]: + """Tally True/total per metric output for the boolean signals this example emits.""" + tallies: dict[str, list[int]] = {} + for score in result.scores: + for output in score.outputs: + if isinstance(output.value, bool): + tally = tallies.setdefault(f"{score.metric_type}.{output.name}", [0, 0]) + tally[0] += int(output.value) + tally[1] += 1 + return [(name, true_count, total) for name, (true_count, total) in sorted(tallies.items())] + + +async def _main() -> int: + parser = argparse.ArgumentParser(description="Run agent-eval tasks through a workflow runtime (trials API).") + parser.add_argument( + "--task", + default="all", + help="Example task id to run, or 'all' (default). Available: " + ", ".join(task.id for task in example_tasks()), + ) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR, help="Run bundle output directory.") + parser.add_argument( + "--rescore-dir", + type=Path, + action="append", + default=None, + help="Re-score the trials.jsonl of a prior run dir offline (repeatable); skips agent execution.", + ) + parser.add_argument("--min-pass-rate", type=float, default=1.0, help="Gate threshold for the pass rate.") + parser.add_argument("--list-tasks", action="store_true", help="List available example tasks and exit.") + parser.add_argument( + "--agentic-task", + default=None, + help="Run a real tests/agentic-use/ task end to end via the NAT workflow runtime " + "(requires Docker, nmp-agentic-base, and a running NeMo Platform).", + ) + parser.add_argument( + "--backend", + choices=("workflow", "aut"), + default="workflow", + help="Agentic-task backend: 'workflow' (task-local nat run) or 'aut' (deployed agent-under-test).", + ) + parser.add_argument("--aut-agent-name", default=None, help="Name of the deployed agent-under-test (aut backend).") + parser.add_argument( + "--aut-agent-config", + type=Path, + default=None, + help="Path to the AUT agent NAT config; created/recreated on the platform if needed.", + ) + parser.add_argument( + "--no-seed-providers", + action="store_true", + help="Skip seeding inference providers from providers.yaml (aut backend).", + ) + parser.add_argument("--skip-build", action="store_true", help="Skip the BUILD phase (image must exist).") + parser.add_argument("--verify", action="store_true", help="Run the pytest VERIFY phase for the agentic task.") + parser.add_argument("--nmp-base-url", default=os.environ.get("NMP_BASE_URL", "http://localhost:8080")) + parser.add_argument("--agent-model", default=os.environ.get("NAT_AGENT_MODEL"), help="Model for the agent.") + args = parser.parse_args() + + if args.agentic_task and args.backend == "aut" and not args.aut_agent_name: + parser.error("--backend aut requires --aut-agent-name") + + if args.list_tasks: + for task in example_tasks(): + print(f"{task.id}: {task.intent}") + return 0 + + _configure_logging() + + if args.agentic_task: + try: + result = await run_agentic_task( + args.agentic_task, + output_dir=args.output_dir, + min_pass_rate=args.min_pass_rate, + nmp_base_url=args.nmp_base_url, + agent_model=args.agent_model, + skip_build=args.skip_build, + verify=args.verify, + backend=args.backend, + aut_agent_name=args.aut_agent_name, + aut_agent_config=args.aut_agent_config, + seed_providers=not args.no_seed_providers, + ) + except (RuntimeError, FileNotFoundError, OSError) as exc: + print(f"agentic-task run failed: {exc}") + print("Real tasks need Docker, the nmp-agentic-base image, and a running NeMo Platform (see README).") + return 1 + elif args.rescore_dir: + result = await rescore(args.rescore_dir, output_dir=args.output_dir, min_pass_rate=args.min_pass_rate) + else: + task_names = [task.id for task in example_tasks()] if args.task == "all" else [args.task] + result = await run_online(task_names, output_dir=args.output_dir, min_pass_rate=args.min_pass_rate) + + _print_result(result) + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(_main())) diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/usage.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/usage.py new file mode 100644 index 0000000000..fe64f1c4bd --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/usage.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Token-usage extraction ported from ``tests/agentic-use/nat_runner.py``. + +Reports the same token/runtime measurements ``nat_runner`` writes into +``result.json["metrics"]`` (the keys the SDK's ``TrialMeasurements`` reads): +``prompt_tokens``/``completion_tokens``/``cache_creation_tokens``/ +``cache_read_tokens`` plus their ``total_tokens`` sum, and ``duration_ms`` +(the ``runtime_sec`` fallback). Buckets follow Anthropic's prompt-caching shape +so AUT (``nemo agents invoke``) and other backends are comparable. +""" + +from __future__ import annotations + +import json +from typing import Any, TypedDict + + +class TokenMetrics(TypedDict): + """Token usage metrics returned by :func:`extract_usage_metrics`.""" + + prompt_tokens: int | None + completion_tokens: int | None + total_tokens: int | None + cache_creation_tokens: int | None + cache_read_tokens: int | None + duration_ms: float | None + + +def iter_agent_log_json_payloads(agent_log: str) -> list[dict[str, Any]]: + """Return JSON dict payloads embedded in an agent log, newest-first after the full log.""" + candidates = [agent_log.strip()] + lines = [ln.strip() for ln in agent_log.splitlines() if ln.strip()] + if lines: + candidates.append(lines[-1]) + candidates.extend(reversed(lines)) + + payloads: list[dict[str, Any]] = [] + seen: set[str] = set() + for candidate in candidates: + if not candidate or candidate in seen: + continue + seen.add(candidate) + try: + parsed = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + payloads.append(parsed) + return payloads + + +def agent_log_has_workflow_error(agent_log: str) -> bool: + """Detect AUT workflow errors returned as successful HTTP JSON payloads.""" + return any(payload.get("code") == "workflow_error" for payload in iter_agent_log_json_payloads(agent_log)) + + +def _first_int(usage_obj: dict[str, Any], keys: tuple[str, ...]) -> tuple[int | None, bool]: + for key in keys: + value = usage_obj.get(key) + if isinstance(value, int): + return value, True + return None, False + + +def _bucket_from_usage(usage_obj: dict[str, Any]) -> tuple[int | None, int | None, int | None, int | None, bool]: + """Return ``(input, output, cache_creation, cache_read, has_known_key)``.""" + input_tokens, has_input = _first_int(usage_obj, ("input_tokens", "prompt_tokens", "inputTokens")) + output_tokens, has_output = _first_int(usage_obj, ("output_tokens", "completion_tokens", "outputTokens")) + cache_creation_tokens, has_cache_creation = _first_int( + usage_obj, ("cache_creation_input_tokens", "cacheWriteTokens") + ) + cache_read_tokens, has_cache_read = _first_int( + usage_obj, ("cache_read_input_tokens", "cacheReadTokens", "cached_input_tokens") + ) + details = usage_obj.get("input_token_details") + if isinstance(details, dict): + if not has_cache_creation: + cache_creation_tokens, has_cache_creation = _first_int(details, ("cache_creation",)) + if not has_cache_read: + cache_read_tokens, has_cache_read = _first_int(details, ("cache_read",)) + if ( + "cached_input_tokens" in usage_obj + and has_input + and has_cache_read + and input_tokens is not None + and cache_read_tokens is not None + ): + input_tokens = max(input_tokens - cache_read_tokens, 0) + has_known_key = has_input or has_output or has_cache_creation or has_cache_read + return input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens, has_known_key + + +def _looks_usage_bearing(d: dict[str, Any]) -> bool: + """Heuristic: does this dict contain something we can extract usage from?""" + if "messages" in d: + return True + return any( + isinstance(d.get(key), dict) for key in ("usage", "usage_metadata", "response_metadata", "data", "metrics") + ) + + +def extract_usage_metrics(agent_log: str) -> TokenMetrics: + """Extract token usage metrics from an agent log. + + Aggregates across **all** assistant turns when the payload exposes a + ``messages[]`` array (the AUT shape from ``nemo agents invoke``); falls back + to a flat top-level ``usage`` block otherwise. Returns all-``None`` when no + known usage shape is present (e.g. plain ``nat run`` text logs). + """ + zero: TokenMetrics = { + "prompt_tokens": None, + "completion_tokens": None, + "total_tokens": None, + "cache_creation_tokens": None, + "cache_read_tokens": None, + "duration_ms": None, + } + if not agent_log.strip(): + return zero + + payload: dict[str, Any] | None = None + fallback_payload: dict[str, Any] | None = None + for parsed in iter_agent_log_json_payloads(agent_log): + if _looks_usage_bearing(parsed): + payload = parsed + break + if fallback_payload is None: + fallback_payload = parsed + if payload is None: + payload = fallback_payload + if not payload: + return zero + + payload_candidates: list[dict[str, Any]] = [payload] + nested_data = payload.get("data") + if isinstance(nested_data, dict): + payload_candidates.append(nested_data) + + sums = {"input_tokens": 0, "output_tokens": 0, "cache_creation_tokens": 0, "cache_read_tokens": 0} + bucket_presence = dict.fromkeys(sums, False) + has_data = False + + def _accumulate(usage_obj: dict[str, Any], *, replace: bool) -> bool: + nonlocal has_data + input_tokens, output_tokens, cache_creation, cache_read, has_known_key = _bucket_from_usage(usage_obj) + if not has_known_key: + return False + for key, value in ( + ("input_tokens", input_tokens), + ("output_tokens", output_tokens), + ("cache_creation_tokens", cache_creation), + ("cache_read_tokens", cache_read), + ): + if value is not None: + sums[key] = value if replace else sums[key] + value + bucket_presence[key] = True + has_data = True + return True + + # Path 1 (preferred): walk every message in messages[] and accumulate. + for candidate_payload in payload_candidates: + msgs = candidate_payload.get("messages") + if not isinstance(msgs, list) or not msgs: + continue + for msg in msgs: + if not isinstance(msg, dict): + continue + usage_obj: dict[str, Any] | None = None + for key in ("usage_metadata", "usage"): + value = msg.get(key) + if isinstance(value, dict) and value: + usage_obj = value + break + if usage_obj is None: + response_metadata = msg.get("response_metadata") + if isinstance(response_metadata, dict): + token_usage = response_metadata.get("token_usage") + if isinstance(token_usage, dict) and token_usage: + usage_obj = token_usage + if usage_obj: + _accumulate(usage_obj, replace=False) + if has_data: + break + + # Path 2 (fallback): flat top-level ``usage``/``usage_metadata``. + if not has_data: + for candidate_payload in payload_candidates: + for key in ("usage", "usage_metadata"): + usage_obj = candidate_payload.get(key) + if isinstance(usage_obj, dict) and usage_obj and _accumulate(usage_obj, replace=True): + break + if has_data: + break + + if not has_data: + return zero + + present = {key: sums[key] if bucket_presence[key] else None for key in sums} + components = [value for value in present.values() if value is not None] + out: TokenMetrics = { + "prompt_tokens": present["input_tokens"], + "completion_tokens": present["output_tokens"], + "total_tokens": sum(components) if components else None, + "cache_creation_tokens": present["cache_creation_tokens"], + "cache_read_tokens": present["cache_read_tokens"], + "duration_ms": None, + } + for candidate_payload in payload_candidates: + duration_ms = candidate_payload.get("duration_ms") + if isinstance(duration_ms, int | float) and out["duration_ms"] is None: + out["duration_ms"] = float(duration_ms) + return out + + +__all__ = ["TokenMetrics", "agent_log_has_workflow_error", "extract_usage_metrics", "iter_agent_log_json_payloads"] diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/verify.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/verify.py new file mode 100644 index 0000000000..73dace8b8d --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/verify.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Harbor-style verifier-phase mechanic: collect a reward + stamp trial metadata. + +Example-local glue (not SDK API). It encodes the Harbor/agentic-task verifier +convention: the caller runs its verifier through an environment handle, then uses +:func:`collect_verifier_outcome` to read the ``reward.txt``/``test-stdout.txt`` +files from the log dir and :func:`apply_verify_to_metadata` to stamp the outcome +onto a trial so a reward metric can score it. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class VerifierOutcome: + """Result of a verifier phase for one task.""" + + ran: bool + passed: bool + reward: int + exit_code: int + stdout: str + verifier_log_dir: Path | None + + +def skipped_outcome() -> VerifierOutcome: + """Outcome representing a verifier that did not run.""" + return VerifierOutcome(ran=False, passed=False, reward=0, exit_code=0, stdout="", verifier_log_dir=None) + + +def collect_verifier_outcome( + *, + ok: bool, + exit_code: int, + log_dir: str | Path, + reward_filename: str = "reward.txt", + stdout_filename: str = "test-stdout.txt", +) -> VerifierOutcome: + """Build a :class:`VerifierOutcome` from a verifier run's log dir. + + Reads ``reward.txt`` (``1``/``0``) when present; otherwise derives the reward + from ``ok`` and writes the file so reruns are stable. Reads ``test-stdout.txt`` + when present. + """ + log_dir = Path(log_dir) + passed = ok + + stdout = "" + stdout_path = log_dir / stdout_filename + if stdout_path.is_file(): + stdout = stdout_path.read_text(encoding="utf-8", errors="replace") + + reward_path = log_dir / reward_filename + if reward_path.is_file(): + reward = 1 if reward_path.read_text(encoding="utf-8").strip() == "1" else 0 + # reward.txt is the verifier's explicit verdict; keep passed consistent + # with it so metadata can't end up with reward=1 but passed=False. + passed = reward == 1 + else: + reward = 1 if passed else 0 + reward_path.parent.mkdir(parents=True, exist_ok=True) + reward_path.write_text("1\n" if passed else "0\n", encoding="utf-8") + + return VerifierOutcome( + ran=True, + passed=passed, + reward=reward, + exit_code=exit_code, + stdout=stdout, + verifier_log_dir=log_dir, + ) + + +def apply_verify_to_metadata(metadata: dict[str, Any], outcome: VerifierOutcome) -> None: + """Stamp verifier reward/status onto trial metadata for scoring + gating.""" + if not outcome.ran: + metadata.setdefault("verify_status", "skipped") + return + metadata["verify_status"] = "ok" if outcome.passed else "failed" + metadata["passed"] = outcome.passed + metadata["reward"] = outcome.reward + metadata["verifier_log_dir"] = str(outcome.verifier_log_dir) if outcome.verifier_log_dir else None diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py new file mode 100644 index 0000000000..0906e4afdc --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py @@ -0,0 +1,297 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Workflow-runtime adapter over the trials-based agent-eval SDK. + +The three pieces an integrator supplies on top of the promoted SDK generics: +:class:`WorkflowAgentRuntime` (an ``AgentTaskRunner`` that launches a per-task +command and shapes an :class:`AgentEvalTrial`), :class:`TrialJsonSerde` (the +``AgentTrialSerde`` for offline rescoring), and :func:`example_tasks` (self- +contained tasks wired to reusable + task-authored metrics). + +The default command runs the bundled :mod:`mini_agent` so the example runs +end-to-end with no external infrastructure; point ``command`` at a real agent +(``nat run``, ``codex exec``, ...) honoring the same CLI contract to swap it in. +""" + +from __future__ import annotations + +import asyncio +import json +import subprocess +import sys +import time +from collections.abc import Sequence +from dataclasses import dataclass, field +from pathlib import Path + +from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask +from nemo_evaluator_sdk.agent_eval.trials import ( + AgentEvalTrial, + AgentEvalTrialStatus, + AgentOutput, + resolve_trial_status, + standard_evidence_descriptors, +) +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from nemo_evaluator_sdk.values.evidence import CandidateEvidence + +from .layout import prepare_run_layout + +RUNTIME_NAME = "workflow" +MINI_AGENT = Path(__file__).resolve().parent / "mini_agent.py" +DEFAULT_TIMEOUT_S = 120 + +# Command tokens substituted per task before launch. +_INSTRUCTION_TOKEN = "{instruction}" +_WORKSPACE_TOKEN = "{workspace}" +_INPUT_JSON_TOKEN = "{input_json}" + + +def _default_workflow_command() -> list[str]: + """Run the bundled toy agent via the current interpreter (no external deps).""" + return [ + sys.executable, + str(MINI_AGENT), + "--instruction", + _INSTRUCTION_TOKEN, + "--workspace", + _WORKSPACE_TOKEN, + "--input-json", + _INPUT_JSON_TOKEN, + ] + + +@dataclass(frozen=True) +class WorkflowRuntimeConfig: + """Configuration for :class:`WorkflowAgentRuntime`.""" + + command: list[str] = field(default_factory=_default_workflow_command) + timeout_s: int = DEFAULT_TIMEOUT_S + agent_model: str = "mini-agent" + + +class WorkflowAgentRuntime: + """Run agent-eval tasks via a per-task workflow command (an ``AgentTaskRunner``).""" + + def __init__(self, config: WorkflowRuntimeConfig | None = None) -> None: + self.config = config or WorkflowRuntimeConfig() + + async def run_tasks( + self, + tasks: Sequence[AgentEvalTask], + config: AgentEvalRunConfig | None = None, + ) -> Sequence[AgentEvalTrial]: + resolved = config or AgentEvalRunConfig() + semaphore = asyncio.Semaphore(resolved.parallelism) + + async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: + async with semaphore: + return await self._run_task(index, task, resolved) + + return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) + + async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> AgentEvalTrial: + run_dir = self._run_dir(index, task, config) + instruction = str(task.inputs.get("instruction") or task.intent) + layout = prepare_run_layout(run_dir, instruction) + + input_json_path = layout.agent_log_dir / "task_input.json" + input_json_path.write_text(json.dumps(task.inputs, indent=2), encoding="utf-8") + + command = self._format_command(layout.instruction_path, layout.workspace_dir, input_json_path) + started = time.monotonic() + process = None + try: + process = await asyncio.create_subprocess_exec( + *command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=self.config.timeout_s) + except Exception as exc: # noqa: BLE001 - any launch/timeout failure is a trial-production failure. + # wait_for cancels communicate() on timeout but leaves the child running; kill it. + if process is not None: + try: + process.kill() + await process.wait() + except Exception: + pass + return self._failed_trial(task, layout.run_dir, exc) + runtime_sec = round(time.monotonic() - started, 3) + + stdout_text = stdout.decode("utf-8", errors="replace") + stderr_text = stderr.decode("utf-8", errors="replace") + (layout.agent_log_dir / "stdout.txt").write_text(stdout_text, encoding="utf-8") + (layout.agent_log_dir / "stderr.txt").write_text(stderr_text, encoding="utf-8") + + agent_ok = process.returncode == 0 + descriptors = standard_evidence_descriptors( + logs_dir=layout.agent_log_dir, + final_state_dir=layout.workspace_dir, + ) + trial = AgentEvalTrial( + id=f"{task.id}:{RUNTIME_NAME}", + task_id=task.id, + status=resolve_trial_status(agent_ok), + output=AgentOutput( + output_text=stdout_text.strip(), + metadata={"runtime": RUNTIME_NAME, "agent_model": self.config.agent_model}, + ), + evidence=CandidateEvidence(descriptors=descriptors, metadata={"runtime": RUNTIME_NAME}), + metadata={ + "runtime": RUNTIME_NAME, + "agent_model": self.config.agent_model, + "agent_ok": agent_ok, + "exit_code": process.returncode, + "runtime_sec": runtime_sec, + "run_dir": str(layout.run_dir), + "generated": True, + }, + ) + # Persist the trial next to its evidence so a single run dir can be + # re-scored offline via TrialJsonSerde. + TrialJsonSerde(layout.run_dir).write(trial) + return trial + + def _failed_trial(self, task: AgentEvalTask, run_dir: Path, exc: Exception) -> AgentEvalTrial: + return AgentEvalTrial( + id=f"{task.id}:{RUNTIME_NAME}", + task_id=task.id, + status=AgentEvalTrialStatus.FAILED, + output=None, + metadata={ + "runtime": RUNTIME_NAME, + "agent_ok": False, + "run_dir": str(run_dir), + "error_type": exc.__class__.__name__, + "error": str(exc), + "generated": True, + }, + ) + + def _format_command(self, instruction_path: Path, workspace_dir: Path, input_json: Path) -> list[str]: + substitutions = { + _INSTRUCTION_TOKEN: str(instruction_path), + _WORKSPACE_TOKEN: str(workspace_dir), + _INPUT_JSON_TOKEN: str(input_json), + } + return [substitutions.get(token, token) for token in self.config.command] + + def _run_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: + root = (config.output_dir or Path.cwd()) / "evidence" / RUNTIME_NAME + return root / (_safe_name(task.id) or f"task-{index}") + + +class TrialJsonSerde: + """Read/write one stored trial as ``/trial.json`` (an ``AgentTrialSerde``).""" + + def __init__(self, run_dir: str | Path) -> None: + self._path = Path(run_dir) / "trial.json" + + def read(self) -> AgentEvalTrial: + return AgentEvalTrial.model_validate_json(self._path.read_text(encoding="utf-8")) + + def write(self, trial: AgentEvalTrial) -> None: + self._path.parent.mkdir(parents=True, exist_ok=True) + self._path.write_text(trial.model_dump_json(indent=2), encoding="utf-8") + + +def load_stored_trials(run_dir: str | Path) -> list[AgentEvalTrial]: + """Load stored trial(s) from a run dir for offline rescoring. + + Accepts either a full run bundle (``trials.jsonl``) or a single runtime run + dir holding one ``trial.json`` (read via :class:`TrialJsonSerde`). + """ + run_dir = Path(run_dir) + jsonl = run_dir / "trials.jsonl" + if jsonl.exists(): + trials = [ + AgentEvalTrial.model_validate_json(line) for line in jsonl.read_text(encoding="utf-8").splitlines() if line + ] + if not trials: + raise ValueError(f"trials.jsonl under {run_dir!r} is empty") + return trials + if (run_dir / "trial.json").exists(): + return [TrialJsonSerde(run_dir).read()] + raise FileNotFoundError(f"no trials.jsonl or trial.json found under {run_dir!r}") + + +class OutputContainsMetric: + """Task-authored metric: emit ``True`` when the agent output contains ``expected``.""" + + def __init__(self, expected: str, *, output_name: str = "output_contains") -> None: + self._expected = expected + self._output_name = output_name + + @property + def type(self) -> str: + return "output_contains" + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.boolean(self._output_name)] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + text = input.candidate.output_text or "" + present = self._expected in text + return MetricResult(outputs=[MetricOutput(name=self._output_name, value=present)]) + + +def example_tasks() -> list[AgentEvalTask]: + """Two self-contained file-writing tasks wired to reusable + authored metrics.""" + from nemo_evaluator_sdk.agent_eval.metrics import AgentPhaseSuccessMetric, EvidencePresenceMetric + + def build(task_id: str, intent: str, create_file: str, content: str) -> AgentEvalTask: + return AgentEvalTask( + id=task_id, + intent=intent, + inputs={ + "instruction": f"{intent}\n\nWrite the file {create_file!r} into your workspace, then report.", + "create_file": create_file, + "content": content, + }, + metrics=[ + AgentPhaseSuccessMetric(), + EvidencePresenceMetric(), + OutputContainsMetric(create_file), + ], + ) + + return [ + build( + "write-report", + "Produce a status report file.", + "report.txt", + "status: green\nsummary: all systems nominal\n", + ), + build( + "write-notes", + "Capture a short notes file.", + "notes.md", + "# Notes\n\n- agent-eval trials example\n", + ), + ] + + +def tasks_by_id(task_names: Sequence[str]) -> list[AgentEvalTask]: + """Select example tasks by id, preserving the requested order.""" + catalog = {task.id: task for task in example_tasks()} + unknown = [name for name in task_names if name not in catalog] + if unknown: + raise ValueError(f"unknown task(s): {sorted(unknown)}; available: {sorted(catalog)}") + return [catalog[name] for name in task_names] + + +def _safe_name(value: str) -> str: + return "".join(char if char.isalnum() or char in "._-" else "-" for char in value).strip(".-")[:120] + + +__all__ = [ + "OutputContainsMetric", + "TrialJsonSerde", + "WorkflowAgentRuntime", + "WorkflowRuntimeConfig", + "example_tasks", + "load_stored_trials", + "tasks_by_id", +] diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/metrics.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/metrics.py new file mode 100644 index 0000000000..cf5512f8d2 --- /dev/null +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/metrics.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reusable agent-eval metrics and the typed view over trial measurements. + +Two complementary pieces, both keyed off ``AgentEvalTrial``: + +* Metrics (scorers) — ``AgentPhaseSuccessMetric`` reads the agent-phase outcome + stamped on trial metadata; ``EvidencePresenceMetric`` is a genuine + *metric-over-evidence* that scores by inspecting ``candidate.evidence`` (a + filesystem evidence handle) rather than trusting a verifier's stamped reward. +* ``TrialMeasurements`` — the single documented place that names the loose + metadata keys gating/reporting read, applying the fallbacks (``duration_ms`` → + ``runtime_sec``, ``passed`` → ``reward``). +""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from typing import Any + +from nemo_evaluator_sdk.agent_eval.trials import EVIDENCE_FINAL_STATE +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from pydantic import BaseModel, ConfigDict + +logger = logging.getLogger(__name__) + +# Token-measurement keys carried on trial metadata (and in result.json["metrics"]). +TOKEN_KEYS: tuple[str, ...] = ( + "prompt_tokens", + "completion_tokens", + "total_tokens", + "cache_creation_tokens", + "cache_read_tokens", +) + + +class AgentPhaseSuccessMetric: + """Emit ``True`` when the agent phase exited successfully, else ``False``. + + The metric ``type`` is overridable via the ``metric_type`` class attribute so + callers can namespace it; the output name stays ``agent_phase_success`` (which + gating reads as a reward signal — ``True``/``False`` coerces to ``1.0``/``0.0``). + """ + + metric_type: str = "agent_phase_success" + + @property + def type(self) -> str: + return self.metric_type + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.boolean("agent_phase_success")] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + # Only an explicit boolean counts as success; a stray truthy string + # (e.g. "false") must not mark a failed trial as passed. + raw_agent_ok = input.candidate.metadata.get("agent_ok") + agent_ok = raw_agent_ok if isinstance(raw_agent_ok, bool) else False + return MetricResult(outputs=[MetricOutput(name="agent_phase_success", value=agent_ok)]) + + +class EvidencePresenceMetric: + """Emit ``True`` when a named filesystem evidence directory exists (and is non-empty). + + Reads ``candidate.evidence`` directly — the canonical metric-over-evidence + pattern — so the result reflects what the agent actually produced on disk, + not a reward stamped into metadata by a verifier. + """ + + def __init__( + self, + *, + evidence_name: str = EVIDENCE_FINAL_STATE, + output_name: str = "evidence_present", + require_non_empty: bool = True, + ) -> None: + self._evidence_name = evidence_name + self._output_name = output_name + self._require_non_empty = require_non_empty + + @property + def type(self) -> str: + return "evidence_presence" + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.boolean(self._output_name)] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + present = False + evidence = input.candidate.evidence + if evidence is not None and evidence.get(self._evidence_name) is not None: + try: + handle = await evidence.filesystem(self._evidence_name) + if await handle.exists(): + present = bool(await handle.iter_paths(recursive=True)) if self._require_non_empty else True + except (KeyError, ValueError) as exc: + logger.warning( + "EvidencePresenceMetric scored False: could not resolve evidence %r for output %r: %s", + self._evidence_name, + self._output_name, + exc, + ) + return MetricResult(outputs=[MetricOutput(name=self._output_name, value=present)]) + + +class TrialMeasurements(BaseModel): + """Numeric measurements projected from trial metadata. + + Reporting/gating consume it via :meth:`from_metadata`; producers keep writing + the same keys onto ``AgentEvalTrial.metadata``. + """ + + model_config = ConfigDict(extra="forbid") + + prompt_tokens: int | None = None + completion_tokens: int | None = None + total_tokens: int | None = None + cache_creation_tokens: int | None = None + cache_read_tokens: int | None = None + runtime_sec: float | None = None + reward: float | None = None + passed: bool | None = None + + @classmethod + def from_metadata(cls, metadata: Mapping[str, Any] | None) -> TrialMeasurements: + """Project loose trial metadata onto the typed contract. + + Applies the historical fallbacks so callers don't re-implement them: + ``runtime_sec`` falls back to ``duration_ms / 1000``; ``reward`` falls + back to ``1.0``/``0.0`` derived from ``passed`` when no explicit reward + is recorded. + """ + metadata = metadata or {} + + tokens = {key: _as_int(metadata.get(key)) for key in TOKEN_KEYS} + passed = metadata.get("passed") + passed = bool(passed) if isinstance(passed, bool) else None + + return cls( + **tokens, + runtime_sec=_runtime_sec(metadata), + reward=_reward(metadata, passed), + passed=passed, + ) + + +def _as_int(value: Any) -> int | None: + # bool is an int subclass; never treat True/False as a token count. + if isinstance(value, bool): + return None + return value if isinstance(value, int) else None + + +def _runtime_sec(metadata: Mapping[str, Any]) -> float | None: + runtime_sec = metadata.get("runtime_sec") + if isinstance(runtime_sec, int | float) and not isinstance(runtime_sec, bool): + return float(runtime_sec) + duration_ms = metadata.get("duration_ms") + if isinstance(duration_ms, int | float) and not isinstance(duration_ms, bool): + return float(duration_ms) / 1000.0 + return None + + +def _reward(metadata: Mapping[str, Any], passed: bool | None) -> float | None: + reward = metadata.get("reward") + if reward is not None: + try: + return float(reward) + except (TypeError, ValueError): + return None + if passed is not None: + return 1.0 if passed else 0.0 + return None diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/environment.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/environment.py new file mode 100644 index 0000000000..69396e58f6 --- /dev/null +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/environment.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Process/filesystem environment boundary for agent-eval runtimes. + +Sits *below* :class:`AgentTaskRunner` so a runtime needn't know whether the +agent/verifier run under Docker, locally, or another filesystem-backed sandbox. +It is a process/filesystem abstraction: :class:`EnvRunSpec`'s ``mounts``/ +``extra_args`` are filesystem hints that non-filesystem providers may ignore. +Handles route both roles through a single :meth:`AbstractEnvironmentHandle.run`. +""" + +from __future__ import annotations + +import abc +import asyncio +import os +import re +import subprocess +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Literal, Protocol, runtime_checkable + +from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask + +EnvRole = Literal["agent", "verifier"] +_SENSITIVE_MARKERS = ("KEY", "TOKEN", "SECRET", "PASSWORD") + + +def _redact_for_logging(cmd: list[str]) -> str: + """Scrub secret-looking values (``KEY=…`` tokens and ``--flag value`` pairs).""" + out: list[str] = [] + redact_next = False + for token in cmd: + if redact_next: + out.append("***REDACTED***") + redact_next = False + elif "=" in token: + left, right = token.split("=", 1) + sensitive = any(m in left.upper() for m in _SENSITIVE_MARKERS) + out.append(f"{left}=***REDACTED***" if sensitive else f"{left}={right}") + else: + normalized = token.lstrip("-").replace("-", "_").upper() + if token.startswith("-") and any(m in normalized for m in _SENSITIVE_MARKERS): + redact_next = True + out.append(token) + return " ".join(out) + + +def default_image_tag(task_id: str) -> str: + """Default task → image-tag mapping (callers may inject their own). + + Sanitizes ``task_id`` to a valid Docker image name so ids with spaces or + other unsupported characters don't fail the build/run. + """ + safe = re.sub(r"[^a-z0-9_.-]+", "-", task_id.lower()).strip(".-") + return f"{safe or 'task'}:latest" + + +@dataclass(frozen=True) +class EnvCommandResult: + """Outcome of running a single command inside a prepared environment.""" + + exit_code: int + timed_out: bool = False + + @property + def ok(self) -> bool: + return self.exit_code == 0 and not self.timed_out + + +@dataclass +class EnvRunSpec: + """How to execute one command inside an environment handle. + + ``mounts``/``extra_args`` are filesystem-environment hints (e.g. Docker bind + mounts and extra CLI args). Non-filesystem providers may ignore them. + """ + + command: list[str] + env: dict[str, str] = field(default_factory=dict) + mounts: list[tuple[str, str]] = field(default_factory=list) + workdir: str | None = None + timeout: int | None = None + extra_args: list[str] = field(default_factory=list) + + +@runtime_checkable +class AgentEnvironmentHandle(Protocol): + """A prepared, single-task environment that can run agent/verifier commands.""" + + async def run_agent(self, spec: EnvRunSpec) -> EnvCommandResult: ... + + async def run_verifier(self, spec: EnvRunSpec) -> EnvCommandResult: ... + + async def close(self) -> None: ... + + +@runtime_checkable +class AgentEnvironmentProvider(Protocol): + """Creates per-task environment handles. Pluggable: Docker now, others later.""" + + async def prepare( + self, + task: AgentEvalTask, + config: AgentEvalRunConfig | None = None, + ) -> AgentEnvironmentHandle: ... + + +class AbstractEnvironmentHandle(abc.ABC): + """Base handle that routes both roles through a single :meth:`run`. + + Concrete handles implement :meth:`run`; ``run_agent``/``run_verifier`` are + role-specialized wrappers so the duplicated phase methods don't have to be + reimplemented per backend. + """ + + @abc.abstractmethod + async def run(self, spec: EnvRunSpec, role: EnvRole) -> EnvCommandResult: ... + + async def run_agent(self, spec: EnvRunSpec) -> EnvCommandResult: + return await self.run(spec, "agent") + + async def run_verifier(self, spec: EnvRunSpec) -> EnvCommandResult: + return await self.run(spec, "verifier") + + async def close(self) -> None: + return None + + +def _docker_run(image: str, spec: EnvRunSpec) -> EnvCommandResult: + """Run ``spec.command`` in a one-shot ``docker run --rm`` container. + + Shells out to the ``docker`` CLI (stdlib ``subprocess`` only), so no + ``agent-runtimes`` extra is needed — just a ``docker`` binary at call time. + """ + cmd = ["docker", "run", "--rm"] + if spec.workdir: + cmd += ["-w", spec.workdir] + for key, value in spec.env.items(): + cmd += ["-e", f"{key}={value}"] + for host_path, container_path in spec.mounts: + cmd += ["-v", f"{host_path}:{container_path}"] + cmd += spec.extra_args + os.environ.get("DOCKER_EXTRA_ARGS", "").split() + cmd += [image, *spec.command] + + print(f"[agent-eval-runtime] $ {_redact_for_logging(cmd)}") + try: + result = subprocess.run(cmd, check=False, text=True, timeout=spec.timeout) + except subprocess.TimeoutExpired: + return EnvCommandResult(exit_code=124, timed_out=True) + return EnvCommandResult(exit_code=result.returncode) + + +class DockerEnvironmentHandle(AbstractEnvironmentHandle): + """Docker-backed environment handle bound to one task image.""" + + def __init__(self, image: str) -> None: + self.image = image + + async def run(self, spec: EnvRunSpec, role: EnvRole = "agent") -> EnvCommandResult: + del role # Docker runs both roles identically against the same image. + return await asyncio.to_thread(_docker_run, self.image, spec) + + +class DockerEnvironmentProvider: + """Default provider that maps each task to its built Docker image.""" + + def __init__(self, *, image_tag_fn: Callable[[str], str] = default_image_tag) -> None: + self._image_tag_fn = image_tag_fn + + async def prepare( + self, + task: AgentEvalTask, + config: AgentEvalRunConfig | None = None, + ) -> DockerEnvironmentHandle: + del config + return DockerEnvironmentHandle(self._image_tag_fn(task.id)) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py index 497c44575a..187b431782 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py @@ -1,19 +1,31 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Trial artifacts and the runtime interface that produces them.""" +"""Trial artifacts, the runtime/serde interfaces that produce them, and the +runtime-agnostic helpers for shaping trials from artifacts (status mapping + +the standard evidence-key builder).""" from __future__ import annotations from collections.abc import Sequence from enum import Enum +from pathlib import Path from typing import Any, Protocol, runtime_checkable from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask from nemo_evaluator_sdk.values import Agent, Model -from nemo_evaluator_sdk.values.evidence import CandidateEvidence +from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +# Well-known evidence keys produced by ``standard_evidence_descriptors``. Harness +# code may import these to tag evidence consistently; callers may still add +# arbitrary extension keys alongside them. +EVIDENCE_INITIAL_STATE = "initial_state" +EVIDENCE_TRACE = "trace" +EVIDENCE_LOGS = "logs" +EVIDENCE_FINAL_STATE = "final_state" +EVIDENCE_VERIFIER_LOGS = "verifier_logs" + class AgentEvalTrialStatus(str, Enum): """Lifecycle status for a trial: completed, failed, or partial.""" @@ -88,4 +100,91 @@ async def run_tasks( ) -> Sequence[AgentEvalTrial]: ... +@runtime_checkable +class AgentTrialSerde(Protocol): + """Read/write a single stored trial artifact as an :class:`AgentEvalTrial`. + + The offline counterpart to :class:`AgentTaskRunner`: instead of *executing* an + agent it adapts a stored artifact (a run dir/file) to and from a trial, so prior + runs can be re-scored. The SDK ships only the protocol; concrete codecs (which + know a particular on-disk layout) live with their producers. + """ + + def read(self) -> AgentEvalTrial: ... + + def write(self, trial: AgentEvalTrial) -> None: ... + + AgentEvalTarget = Model | Agent | AgentTaskRunner + + +def resolve_trial_status(agent_ok: bool) -> AgentEvalTrialStatus: + """Map an agent-phase outcome to a *scorable* trial status. + + ``AgentEvaluator`` excludes ``FAILED`` trials from scoring, so an + executed-but-unsuccessful agent uses ``PARTIAL`` (still scored as ``0`` for + pass-rate gating); ``FAILED`` is reserved for trial-*production* failures, + which a runtime surfaces by raising rather than emitting an unscorable trial. + """ + return AgentEvalTrialStatus.COMPLETED if agent_ok else AgentEvalTrialStatus.PARTIAL + + +def standard_evidence_descriptors( + *, + logs_dir: str | Path, + final_state_dir: str | Path, + trace_path: str | Path | None = None, + initial_state_ref: str | None = None, + verifier_logs_dir: str | Path | None = None, + primary_log: str | None = None, +) -> dict[str, EvidenceDescriptor]: + """Build the documented evidence map for an agent-eval trial. + + Standard keys: ``initial_state`` (task input filesystem, when staged), + ``trace`` (trajectory, ATIF-normalized when available), ``logs`` (agent log + dir), ``final_state`` (workspace), and ``verifier_logs`` (only when present). + Callers may add their own extension keys to the returned mapping. + """ + descriptors: dict[str, EvidenceDescriptor] = {} + + if initial_state_ref: + descriptors[EVIDENCE_INITIAL_STATE] = EvidenceDescriptor( + kind="filesystem", + format="dir", + ref=str(initial_state_ref), + metadata={"role": EVIDENCE_INITIAL_STATE}, + ) + + if trace_path is not None: + trace_name = Path(trace_path).name.lower() + is_atif = trace_name.startswith("atif") or ".atif." in trace_name + descriptors[EVIDENCE_TRACE] = EvidenceDescriptor( + kind="trace", + format="atif" if is_atif else "json", + ref=str(trace_path), + ) + + logs_metadata = {"primary_log": primary_log} if primary_log else {} + descriptors[EVIDENCE_LOGS] = EvidenceDescriptor( + kind="logs", + format="dir", + ref=str(logs_dir), + metadata=logs_metadata, + ) + + descriptors[EVIDENCE_FINAL_STATE] = EvidenceDescriptor( + kind="filesystem", + format="dir", + ref=str(final_state_dir), + metadata={"role": EVIDENCE_FINAL_STATE}, + ) + + if verifier_logs_dir is not None and Path(verifier_logs_dir).exists(): + descriptors[EVIDENCE_VERIFIER_LOGS] = EvidenceDescriptor( + kind="logs", + format="dir", + ref=str(verifier_logs_dir), + metadata={"role": "verifier"}, + ) + + return descriptors diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_environment.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_environment.py new file mode 100644 index 0000000000..f8716adeff --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_environment.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the SDK environment boundary (process/filesystem execution seam).""" + +from __future__ import annotations + +import subprocess + +import pytest +from nemo_evaluator_sdk.agent_eval.runtimes import environment as env_mod +from nemo_evaluator_sdk.agent_eval.runtimes.environment import ( + DockerEnvironmentHandle, + DockerEnvironmentProvider, + EnvCommandResult, + EnvRunSpec, + default_image_tag, +) +from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask + + +@pytest.mark.asyncio +async def test_docker_handle_routes_roles_through_single_run(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, list[str]]] = [] + + def fake_docker_run(image: str, spec: EnvRunSpec) -> EnvCommandResult: + calls.append((image, spec.command)) + return EnvCommandResult(exit_code=0) + + monkeypatch.setattr(env_mod, "_docker_run", fake_docker_run) + + handle = DockerEnvironmentHandle("img:latest") + spec = EnvRunSpec(command=["echo", "hi"]) + assert (await handle.run_agent(spec)).ok + assert (await handle.run_verifier(spec)).ok + assert calls == [("img:latest", ["echo", "hi"]), ("img:latest", ["echo", "hi"])] + + +@pytest.mark.asyncio +async def test_docker_handle_reports_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + monkeypatch.setattr(env_mod.subprocess, "run", fake_run) + result = await DockerEnvironmentHandle("img").run(EnvRunSpec(command=["sleep"]), "agent") + assert result.timed_out and result.exit_code == 124 and not result.ok + + +@pytest.mark.asyncio +async def test_provider_uses_injected_image_tag_fn() -> None: + assert default_image_tag("t") == "t:latest" + provider = DockerEnvironmentProvider(image_tag_fn=lambda task_id: f"custom-{task_id}") + handle = await provider.prepare(AgentEvalTask(id="demo", intent="x", inputs={})) + assert isinstance(handle, DockerEnvironmentHandle) + assert handle.image == "custom-demo" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_import_hygiene.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_import_hygiene.py new file mode 100644 index 0000000000..651f4f7995 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_import_hygiene.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Guardrail: the agent_eval package must stay free of NeMo-Platform imports. + +The SDK is consumed by NeMo-Platform adapters, never the reverse. This test +fails if any module under ``agent_eval`` imports a platform-specific package, +which keeps the promoted generics from leaking coupling into the SDK. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import nemo_evaluator_sdk.agent_eval as agent_eval + +# agent_eval is an implicit namespace package (no __init__.py), so resolve its +# directory via __path__ rather than __file__ (which is None for namespaces). +AGENT_EVAL_ROOT = Path(next(iter(agent_eval.__path__))).resolve() + +# Import statements that would couple the SDK to the platform / adapter. +_FORBIDDEN = re.compile( + r"^\s*(?:from|import)\s+" + r"(nemo_platform|nmp_[A-Za-z0-9_]+|nat_runner|runtimes(?:\.|\s|$)|evaluator_agent_eval)", + re.MULTILINE, +) + + +def test_agent_eval_has_no_platform_imports() -> None: + offenders: list[str] = [] + for path in sorted(AGENT_EVAL_ROOT.rglob("*.py")): + text = path.read_text(encoding="utf-8") + for match in _FORBIDDEN.finditer(text): + line_no = text.count("\n", 0, match.start()) + 1 + offenders.append(f"{path.relative_to(AGENT_EVAL_ROOT)}:{line_no}: {match.group(0).strip()}") + + assert not offenders, "agent_eval must not import NeMo-Platform packages:\n" + "\n".join(offenders) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_metrics.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_metrics.py new file mode 100644 index 0000000000..5a29c98170 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_metrics.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the reusable agent-eval metrics and the TrialMeasurements contract.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from nemo_evaluator_sdk.agent_eval.metrics import ( + AgentPhaseSuccessMetric, + EvidencePresenceMetric, + TrialMeasurements, +) +from nemo_evaluator_sdk.agent_eval.trials import standard_evidence_descriptors +from nemo_evaluator_sdk.metrics.protocol import CandidateOutput, DatasetRow, MetricInput +from nemo_evaluator_sdk.values.evidence import CandidateEvidence + + +@pytest.mark.asyncio +async def test_agent_phase_success_metric_reads_metadata_and_namespaces_type() -> None: + metric = AgentPhaseSuccessMetric() + assert metric.type == "agent_phase_success" + ok = await metric.compute_scores( + MetricInput(row=DatasetRow(data={}), candidate=CandidateOutput(metadata={"agent_ok": True})) + ) + assert ok.outputs[0].value is True + + class Namespaced(AgentPhaseSuccessMetric): + metric_type = "agentic_use_agent_phase" + + assert Namespaced().type == "agentic_use_agent_phase" + + +@pytest.mark.asyncio +async def test_evidence_presence_metric_scores_over_evidence(tmp_path: Path) -> None: + final_state = tmp_path / "workspace" + final_state.mkdir() + (final_state / "result.txt").write_text("done", encoding="utf-8") + evidence = CandidateEvidence( + descriptors=standard_evidence_descriptors(logs_dir=tmp_path / "agent", final_state_dir=final_state) + ) + + metric = EvidencePresenceMetric() + present = await metric.compute_scores( + MetricInput(row=DatasetRow(data={}), candidate=CandidateOutput(evidence=evidence)) + ) + assert present.outputs[0].value is True + + # Empty workspace -> non-empty requirement fails; no evidence -> False. + (final_state / "result.txt").unlink() + empty = await metric.compute_scores( + MetricInput(row=DatasetRow(data={}), candidate=CandidateOutput(evidence=evidence)) + ) + assert empty.outputs[0].value is False + missing = await metric.compute_scores(MetricInput(row=DatasetRow(data={}), candidate=CandidateOutput())) + assert missing.outputs[0].value is False + + +def test_from_metadata_reads_tokens_runtime_reward() -> None: + measurements = TrialMeasurements.from_metadata( + { + "total_tokens": 120, + "prompt_tokens": 80, + "completion_tokens": 40, + "runtime_sec": 4.5, + "reward": 1, + "passed": True, + } + ) + assert measurements.total_tokens == 120 + assert measurements.runtime_sec == 4.5 + assert measurements.reward == 1.0 + assert measurements.passed is True + + +def test_from_metadata_applies_fallbacks_and_ignores_bad_types() -> None: + # duration_ms -> runtime_sec, passed -> reward, bool is not a token count. + measurements = TrialMeasurements.from_metadata({"duration_ms": 2500, "passed": False, "total_tokens": True}) + assert measurements.runtime_sec == 2.5 + assert measurements.reward == 0.0 + assert measurements.total_tokens is None + + empty = TrialMeasurements.from_metadata(None) + assert empty.reward is None and empty.runtime_sec is None diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_profbench.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_profbench.py deleted file mode 100644 index 8fb94229e7..0000000000 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_profbench.py +++ /dev/null @@ -1,492 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import importlib -import json -import re -from pathlib import Path -from typing import Any - -import pytest -from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator -from nemo_evaluator_sdk.agent_eval.runtimes.codex import runtime as codex_runtime -from nemo_evaluator_sdk.values import Model - -profbench = importlib.import_module("packages.nemo_evaluator_sdk.examples.profbench.profbench") -profbench_runner = importlib.import_module("packages.nemo_evaluator_sdk.examples.profbench.runner") - - -class _FakeUrlopenResponse: - def __init__(self, body: str) -> None: - self._body = body - self.headers = {"ETag": "test-etag", "x-repo-commit": "test-commit"} - - def __enter__(self) -> _FakeUrlopenResponse: - return self - - def __exit__(self, *args: object) -> None: - return None - - def read(self) -> bytes: - return self._body.encode("utf-8") - - -def _write_profbench_fixture(path: Path) -> Path: - row = { - "task_id": "pb-1", - "domain": "Chemistry PhD", - "prompt": "Explain the result.", - "o3_response": "Response A", - "r1-0528_response": "Response B", - "grok4_response": "Response C", - "rubrics": [ - { - "criterion_description": "Includes the main mechanism.", - "criterion_weight": "Critical", - "criterion_type": ["Correctness", "Reasoning"], - "o3_fulfilment": True, - "r1-0528_fulfilment": False, - "grok4_fulfilment": True, - }, - { - "criterion_description": "Mentions the limitation.", - "criterion_weight": "Major", - "criterion_type": "Completeness", - "o3_fulfilment": False, - "r1-0528_fulfilment": True, - "grok4_fulfilment": True, - }, - ], - } - path.write_text(json.dumps(row) + "\n", encoding="utf-8") - return path - - -def _stub_remote_profbench_source(monkeypatch: pytest.MonkeyPatch, body: str) -> str: - remote_source = "https://example.test/profbench/test.jsonl" - - def fake_urlopen(request: Any, timeout: int) -> _FakeUrlopenResponse: - assert request.full_url == remote_source - assert timeout == 60 - return _FakeUrlopenResponse(body) - - monkeypatch.setattr(profbench, "urlopen", fake_urlopen) - return remote_source - - -def test_load_profbench_expands_tasks_trials_and_line_index(tmp_path: Path) -> None: - fixture = _write_profbench_fixture(tmp_path / "profbench.jsonl") - - benchmark = profbench.load_profbench(fixture) - - assert benchmark.metadata["record_count"] == 1 - assert [trial.metadata["model_id"] for trial in benchmark.trials] == ["o3", "r1-0528", "grok4"] - - metric = benchmark.tasks[0].metrics[0] - assert isinstance(metric, profbench.ProfBenchRubricMetric) - assert [criterion.points for criterion in metric.criteria] == [ - profbench.PROFBENCH_WEIGHT_POINTS["Critical"], - profbench.PROFBENCH_WEIGHT_POINTS["Major"], - ] - assert [criterion.id for criterion in metric.criteria] == ["pb-1:criterion-1", "pb-1:criterion-2"] - assert metric.criteria[0].line_number == 1 - assert metric.criteria[0].json_path == "$.rubrics[0]" - - -def test_profbench_baseline_scoring_creates_traceable_criterion_scores(tmp_path: Path) -> None: - benchmark = profbench.load_profbench(_write_profbench_fixture(tmp_path / "profbench.jsonl")) - - result = AgentEvaluator().run_sync(tasks=benchmark.tasks, trials=benchmark.trials) - o3_score = next(row for row in result.scores if row.trial_id == "pb-1:o3") - details_output = next(output for output in o3_score.outputs if output.name == profbench.PROFBENCH_DETAILS_OUTPUT) - details = profbench.profbench_details(details_output) - assert details is not None - - assert details.score == 4 / 7 - assert details.earned_points == 4 - assert details.max_points == 7 - - failed = [criterion for criterion in details.criterion_scores if not criterion.fulfilled] - assert len(failed) == 1 - assert failed[0].points == 3 - assert failed[0].metadata["score_source"] == "dataset_label" - assert failed[0].evidence[0].line == 1 - assert failed[0].evidence[0].json_path == "$.rubrics[1]" - assert failed[0].evidence[0].href().startswith("file://") - assert "#L1" not in failed[0].evidence[0].href() - assert all(criterion.judge_reason is None for criterion in details.criterion_scores) - assert {criterion.metadata["score_source"] for criterion in details.criterion_scores} == {"dataset_label"} - - -def test_evidence_locator_local_file_href_omits_dead_line_fragment(tmp_path: Path) -> None: - evidence_file = tmp_path / "profbench-dataset.jsonl" - evidence_file.write_text("{}\n", encoding="utf-8") - - locator = profbench.EvidenceLocator(kind="profbench", uri=str(evidence_file), line=1, json_path="$.rubrics[0]") - - assert locator.href() == evidence_file.as_uri() - assert locator.href(base_dir=tmp_path) == "profbench-dataset.jsonl" - - -def test_profbench_live_judge_mode_scores_recorded_trials_without_cached_labels(tmp_path: Path) -> None: - class FakeJudge: - def __init__(self) -> None: - self.requests: list[Any] = [] - - async def judge(self, request: Any) -> Any: - self.requests.append(request) - return profbench.ProfBenchJudgeDecision( - fulfilled=request.criterion_id.endswith("criterion-1"), - reason=f"judged {request.criterion_id}", - ) - - judge = FakeJudge() - benchmark = profbench.load_profbench( - _write_profbench_fixture(tmp_path / "profbench.jsonl"), - judge=judge, - include_cached_fulfilments=False, - ) - - assert all("profbench_fulfilments" not in trial.metadata for trial in benchmark.trials) - - result = AgentEvaluator().run_sync(tasks=benchmark.tasks, trials=benchmark.trials) - o3_score = next(row for row in result.scores if row.trial_id == "pb-1:o3") - details_output = next(output for output in o3_score.outputs if output.name == profbench.PROFBENCH_DETAILS_OUTPUT) - details = profbench.profbench_details(details_output) - assert details is not None - - assert details.score == 4 / 7 - assert len(judge.requests) == 6 - assert {criterion.metadata["score_source"] for criterion in details.criterion_scores} == {"judge"} - assert [criterion.judge_reason for criterion in details.criterion_scores] == [ - "judged pb-1:criterion-1", - "judged pb-1:criterion-2", - ] - - -def test_agent_evaluator_scores_loaded_profbench_baselines(tmp_path: Path) -> None: - benchmark = profbench.load_profbench(_write_profbench_fixture(tmp_path / "profbench.jsonl")) - - result = AgentEvaluator().run_sync(tasks=benchmark.tasks, trials=benchmark.trials) - - assert result.summary.task_count == 1 - assert result.summary.trial_count == 3 - profbench_score = next( - score - for score in result.summary.scores.scores - if score.name == f"{profbench.PROFBENCH_METRIC_TYPE}.{profbench.PROFBENCH_METRIC_ID}" - ) - assert profbench_score.mean == 2 / 3 - - -def test_profbench_dashboard_renders_rubric_report(tmp_path: Path) -> None: - benchmark = profbench.load_profbench(_write_profbench_fixture(tmp_path / "profbench.jsonl")) - result = AgentEvaluator().run_sync(tasks=benchmark.tasks, trials=benchmark.trials) - - html = profbench.render_profbench_dashboard(result, evidence_base_dir=tmp_path) - - assert "ProfBench Agent Eval Report" in html - assert "Task Details" in html - assert "criterion-2" in html - assert "Chemistry PhD" in html - assert "dataset_label" in html - assert 'href="profbench.jsonl"' in html - assert "profbench.jsonl#L1" not in html - - report_path = profbench.write_profbench_dashboard(result, tmp_path / "report.html") - assert report_path.read_text(encoding="utf-8") == html - - -def test_profbench_example_writes_sdk_and_profbench_dashboards(tmp_path: Path) -> None: - benchmark = profbench.load_profbench(_write_profbench_fixture(tmp_path / "profbench.jsonl")) - result = AgentEvaluator().run_sync(tasks=benchmark.tasks, trials=benchmark.trials) - - sdk_path, default_path = profbench.write_example_dashboards(result, tmp_path) - - assert "Agent Eval Report" in sdk_path.read_text(encoding="utf-8") - assert "ProfBench Agent Eval Report" in default_path.read_text(encoding="utf-8") - assert not (tmp_path / "profbench-report.html").exists() - - -def test_profbench_run_instance_id_has_expected_format() -> None: - run_instance_id = profbench_runner._new_profbench_run_instance_id() - - assert re.fullmatch(r"\d{8}_\d{6}_\d{5}_[0-9a-f]{6}", run_instance_id) - - -def test_profbench_output_root_precedence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - env_root = tmp_path / "env-root" - cli_root = tmp_path / "cli-root" - - monkeypatch.setenv("NEMO_EVALUATOR_PROFBENCH_OUTPUT_DIR", str(env_root)) - - assert profbench_runner._resolve_profbench_output_root(cli_root) == cli_root - assert profbench_runner._resolve_profbench_output_root() == env_root - - monkeypatch.delenv("NEMO_EVALUATOR_PROFBENCH_OUTPUT_DIR") - assert profbench_runner._resolve_profbench_output_root() == profbench_runner.DEFAULT_OUTPUT_DIR - - -def test_profbench_output_dir_uses_run_then_mode_tree(tmp_path: Path) -> None: - run_instance_id = "20260604_154749_70985_82f7dd" - - assert profbench_runner._profbench_output_dir(tmp_path, run_instance_id, "baseline") == ( - tmp_path / run_instance_id / "baseline" - ) - assert profbench_runner._profbench_output_dir(tmp_path, run_instance_id, "live-candidate") == ( - tmp_path / run_instance_id / "live-candidate" - ) - assert profbench_runner._profbench_output_dir(tmp_path, run_instance_id, "live-judge") == ( - tmp_path / run_instance_id / "live-judge" - ) - - -def test_remote_profbench_source_is_saved_as_local_evidence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - fixture = _write_profbench_fixture(tmp_path / "profbench.jsonl") - remote_source = _stub_remote_profbench_source(monkeypatch, fixture.read_text(encoding="utf-8")) - evidence_dir = tmp_path / "run" / "evidence" - - benchmark = profbench.load_profbench(remote_source, limit=1, evidence_dir=evidence_dir) - - dataset_path = evidence_dir / "profbench-dataset.jsonl" - assert dataset_path.read_text(encoding="utf-8") == fixture.read_text(encoding="utf-8") - assert benchmark.metadata["source"] == str(dataset_path.resolve()) - assert benchmark.metadata["source_file"] == str(dataset_path.resolve()) - assert benchmark.metadata["remote_source"] == remote_source - assert benchmark.metadata["etag"] == "test-etag" - assert benchmark.metadata["resolved_commit"] == "test-commit" - assert benchmark.tasks[0].metadata["source_uri"] == str(dataset_path.resolve()) - assert benchmark.trials[0].evidence is not None - assert benchmark.trials[0].evidence.descriptors["source"].ref == str(dataset_path.resolve()) - - result = AgentEvaluator().run_sync(tasks=benchmark.tasks, trials=benchmark.trials) - failed_score = next(row for row in result.scores if row.trial_id == "pb-1:o3") - details_output = next( - output for output in failed_score.outputs if output.name == profbench.PROFBENCH_DETAILS_OUTPUT - ) - details = profbench.profbench_details(details_output) - assert details is not None - failed = next(criterion for criterion in details.criterion_scores if not criterion.fulfilled) - assert failed.evidence[0].href().startswith("file://") - assert not failed.evidence[0].href().startswith("https://") - - -def test_profbench_model_helpers_use_shared_defaults_with_optional_name_override() -> None: - evaluated_model = profbench_runner._evaluated_model() - overridden_model = profbench_runner._evaluated_model("custom-model") - judge_model = profbench_runner._judge_model() - - assert evaluated_model.url == profbench_runner.DEFAULT_MODEL_URL - assert evaluated_model.name == profbench_runner.DEFAULT_MODEL_NAME - assert overridden_model.name == "custom-model" - assert judge_model.url == profbench_runner.DEFAULT_MODEL_URL - assert judge_model.name == profbench_runner.DEFAULT_MODEL_NAME - - -def test_profbench_live_candidate_target_selects_codex_runtime(tmp_path: Path) -> None: - target, params, score_source, effective_runtime = profbench_runner._live_candidate_target( - agent=profbench_runner.AgentChoice.CODEX, - agent_model="gpt-5", - runtime=codex_runtime.RuntimeChoice.LOCAL, - output_dir=tmp_path / "live-candidate", - env={"OPENAI_API_KEY": "sk-test-key"}, - ) - - assert isinstance(target, codex_runtime.CodexCliAgentRuntime) - assert target._model == "gpt-5" - assert target._work_root == tmp_path / "live-candidate" / "evidence" / "codex" - assert params is None - assert score_source == "codex_cli_candidate_and_live_judge" - assert effective_runtime == codex_runtime.EffectiveCodexRuntime.LOCAL_CLI - - -@pytest.mark.asyncio -async def test_profbench_run_examples_reuses_one_run_folder_for_enabled_modes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - run_instance_id = "20260604_154749_70985_82f7dd" - calls: list[tuple[str, int | None, Path, str]] = [] - - async def fake_run_mode( - mode: profbench_runner.ProfBenchMode, - *, - limit: int | None, - output_root: str | Path | None, - run_instance_id: str | None, - agent: profbench_runner.AgentChoice = profbench_runner.AgentChoice.MODEL, - agent_model: str | None = None, - runtime: codex_runtime.RuntimeChoice = codex_runtime.RuntimeChoice.DOCKER, - ) -> None: - assert output_root is not None - assert run_instance_id is not None - if mode is profbench_runner.ProfBenchMode.LIVE_CANDIDATE: - assert agent == profbench_runner.AgentChoice.CODEX - assert agent_model == "gpt-5" - assert runtime == codex_runtime.RuntimeChoice.LOCAL - calls.append((mode.value, limit, Path(output_root), run_instance_id)) - - monkeypatch.setattr(profbench_runner, "run_profbench_mode", fake_run_mode) - - await profbench_runner.run_examples( - limit=1, - run_live_judge=True, - run_live_candidate=True, - output_root=tmp_path, - run_instance_id=run_instance_id, - agent=profbench_runner.AgentChoice.CODEX, - agent_model="gpt-5", - runtime=codex_runtime.RuntimeChoice.LOCAL, - ) - - assert calls == [ - ("baseline", 1, tmp_path, run_instance_id), - ("live-judge", 1, tmp_path, run_instance_id), - ("live-candidate", 1, tmp_path, run_instance_id), - ] - assert (tmp_path / run_instance_id).is_dir() - assert not (tmp_path / run_instance_id / "baseline").exists() - assert not (tmp_path / run_instance_id / "live-judge").exists() - assert not (tmp_path / run_instance_id / "live-candidate").exists() - - -@pytest.mark.asyncio -async def test_profbench_baseline_example_writes_run_then_mode_output( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - fixture = _write_profbench_fixture(tmp_path / "profbench.jsonl") - remote_source = _stub_remote_profbench_source(monkeypatch, fixture.read_text(encoding="utf-8")) - run_instance_id = "20260601_175657_75909_573ed6" - monkeypatch.setattr(profbench_runner, "_profbench_source", lambda: remote_source) - - await profbench_runner.run_profbench_mode( - profbench_runner.ProfBenchMode.BASELINE, - limit=1, - output_root=tmp_path, - run_instance_id=run_instance_id, - ) - - output_dir = tmp_path / run_instance_id / "baseline" - evidence_dir = output_dir / "evidence" - assert (output_dir / "summary.json").is_file() - assert (output_dir / "report.html").is_file() - assert (evidence_dir / "profbench-dataset.jsonl").is_file() - assert not (output_dir / "profbench-report.html").exists() - assert not (tmp_path / run_instance_id / "evidence" / "profbench-dataset.jsonl").exists() - - run_payload = json.loads((output_dir / "run.json").read_text(encoding="utf-8")) - assert run_payload["run_id"] == f"{run_instance_id}-baseline" - assert run_payload["output_dir"] == str(output_dir) - assert run_payload["artifacts"]["scores"] == "scores.jsonl" - - -@pytest.mark.asyncio -async def test_profbench_live_judge_example_writes_mode_evidence_artifacts( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - fixture = _write_profbench_fixture(tmp_path / "profbench.jsonl") - remote_source = _stub_remote_profbench_source(monkeypatch, fixture.read_text(encoding="utf-8")) - run_instance_id = "20260601_175657_75909_573ed6" - - class FakeProfBenchModelJudge: - def __init__(self, model: Model) -> None: - self.model = model - - async def judge(self, request: Any) -> Any: - return profbench.ProfBenchJudgeDecision( - fulfilled=request.criterion_id.endswith("criterion-1"), - reason=f"judged {request.criterion_id}", - ) - - monkeypatch.setattr(profbench_runner, "_profbench_source", lambda: remote_source) - monkeypatch.setattr(profbench_runner, "ProfBenchModelJudge", FakeProfBenchModelJudge) - - await profbench_runner.run_profbench_mode( - profbench_runner.ProfBenchMode.LIVE_JUDGE, - limit=1, - output_root=tmp_path, - run_instance_id=run_instance_id, - ) - - output_dir = tmp_path / run_instance_id / "live-judge" - evidence_dir = output_dir / "evidence" - judge_artifacts = sorted(evidence_dir.glob("judge-*.json")) - assert (evidence_dir / "profbench-dataset.jsonl").is_file() - assert judge_artifacts - assert not list((tmp_path / run_instance_id / "evidence").glob("judge-*.json")) - - score_payloads = [ - json.loads(line) for line in (output_dir / "scores.jsonl").read_text(encoding="utf-8").splitlines() - ] - first_judge_uri = next( - locator["uri"] - for score in score_payloads - for output in score["outputs"] - if output["name"] == profbench.PROFBENCH_DETAILS_OUTPUT - for criterion in output["value"]["criterion_scores"] - for locator in criterion["evidence"] - if locator["kind"] == "judge" - ) - assert first_judge_uri.startswith(str(evidence_dir.resolve())) - assert Path(first_judge_uri).is_file() - assert profbench.EvidenceLocator(kind="judge", uri=first_judge_uri, line=1).href().startswith("file://") - - -def test_profbench_judge_parser_accepts_clean_and_embedded_structured_json() -> None: - clean = profbench._parse_judge_decision('{"fulfilled": true, "reason": "matched"}') - assert clean.fulfilled is True - assert clean.reason == "matched" - - embedded = profbench._parse_judge_decision('```json\n{"fulfilled": false, "reason": "missing"}\n```') - assert embedded.fulfilled is False - assert embedded.reason == "missing" - - -def test_profbench_judge_parser_conservatively_scores_unparseable_output() -> None: - decision = profbench._parse_judge_decision( - r"\boxed{\begin{aligned}&\text{Liouville equation instead of judge JSON}\end{aligned}}" - ) - assert decision.fulfilled is False - assert "treating criterion as unfulfilled" in decision.reason - - missing_field = profbench._parse_judge_decision('{"reason": "missing explicit boolean"}') - assert missing_field.fulfilled is False - assert "treating criterion as unfulfilled" in missing_field.reason - - -@pytest.mark.asyncio -async def test_profbench_model_judge_uses_short_structured_params() -> None: - captured: dict[str, Any] = {} - - async def fake_inference( - model: Model, - request: dict[str, Any], - max_retries: int | None, - **kwargs: Any, - ) -> dict[str, Any]: - del model, max_retries, kwargs - captured.update(request) - return {"choices": [{"message": {"role": "assistant", "content": '{"fulfilled": true, "reason": "ok"}'}}]} - - judge = profbench.ProfBenchModelJudge( - model=Model(url="https://model.test/v1/chat/completions", name="judge-model"), - inference_fn=fake_inference, - ) - - decision = await judge.judge( - profbench.ProfBenchJudgeRequest( - task_id="pb-1", - prompt="Task prompt", - response="Candidate response", - criterion_id="pb-1:criterion-1", - criterion_description="Criterion text", - weight_name="Minor", - ) - ) - - assert decision.fulfilled is True - assert captured["temperature"] == 0.0 - assert captured["max_tokens"] == 256 - guided_json = captured["extra_body"]["nvext"]["guided_json"] - assert guided_json["required"] == ["fulfilled", "reason"] diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_trials.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_trials.py index 43e5b5f315..12d81abeac 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_trials.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_trials.py @@ -1,8 +1,15 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from pathlib import Path + import pytest -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus +from nemo_evaluator_sdk.agent_eval.trials import ( + AgentEvalTrial, + AgentEvalTrialStatus, + resolve_trial_status, + standard_evidence_descriptors, +) def test_trial_accepts_mapping_shaped_evidence_and_serializes_descriptors() -> None: @@ -47,3 +54,29 @@ def test_trial_accepts_mapping_shaped_evidence_and_serializes_descriptors() -> N def test_completed_trial_requires_output() -> None: with pytest.raises(ValueError, match="completed trial requires output"): AgentEvalTrial(id="trial-1", task_id="task-1", status=AgentEvalTrialStatus.COMPLETED) + + +def test_resolve_trial_status_maps_ran_but_failed_to_partial() -> None: + assert resolve_trial_status(True) == AgentEvalTrialStatus.COMPLETED + # A ran-but-unsuccessful agent stays scorable (PARTIAL), not dropped (FAILED). + assert resolve_trial_status(False) == AgentEvalTrialStatus.PARTIAL + + +def test_standard_evidence_descriptors_builds_documented_keys(tmp_path: Path) -> None: + verifier_dir = tmp_path / "verifier" + verifier_dir.mkdir() + descriptors = standard_evidence_descriptors( + logs_dir=tmp_path / "agent", + final_state_dir=tmp_path / "workspace", + trace_path=tmp_path / "atif-trace.json", + initial_state_ref="s3://inputs", + verifier_logs_dir=verifier_dir, + primary_log="agent.log", + ) + assert set(descriptors) == {"initial_state", "trace", "logs", "final_state", "verifier_logs"} + assert descriptors["trace"].format == "atif" + assert descriptors["logs"].metadata == {"primary_log": "agent.log"} + + # A missing verifier dir is omitted; trace is optional. + minimal = standard_evidence_descriptors(logs_dir=tmp_path / "a", final_state_dir=tmp_path / "w") + assert set(minimal) == {"logs", "final_state"} diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py new file mode 100644 index 0000000000..de60099921 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reusable agent-eval metrics and the typed view over trial measurements. + +Two complementary pieces, both keyed off ``AgentEvalTrial``: + +* Metrics (scorers) — ``AgentPhaseSuccessMetric`` reads the agent-phase outcome + stamped on trial metadata; ``EvidencePresenceMetric`` is a genuine + *metric-over-evidence* that scores by inspecting ``candidate.evidence`` (a + filesystem evidence handle) rather than trusting a verifier's stamped reward. +* ``TrialMeasurements`` — the single documented place that names the loose + metadata keys gating/reporting read, applying the fallbacks (``duration_ms`` → + ``runtime_sec``, ``passed`` → ``reward``). +""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from typing import Any + +from nemo_platform.beta.evaluator.agent_eval.trials import EVIDENCE_FINAL_STATE +from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from pydantic import BaseModel, ConfigDict + +logger = logging.getLogger(__name__) + +# Token-measurement keys carried on trial metadata (and in result.json["metrics"]). +TOKEN_KEYS: tuple[str, ...] = ( + "prompt_tokens", + "completion_tokens", + "total_tokens", + "cache_creation_tokens", + "cache_read_tokens", +) + + +class AgentPhaseSuccessMetric: + """Emit ``True`` when the agent phase exited successfully, else ``False``. + + The metric ``type`` is overridable via the ``metric_type`` class attribute so + callers can namespace it; the output name stays ``agent_phase_success`` (which + gating reads as a reward signal — ``True``/``False`` coerces to ``1.0``/``0.0``). + """ + + metric_type: str = "agent_phase_success" + + @property + def type(self) -> str: + return self.metric_type + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.boolean("agent_phase_success")] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + # Only an explicit boolean counts as success; a stray truthy string + # (e.g. "false") must not mark a failed trial as passed. + raw_agent_ok = input.candidate.metadata.get("agent_ok") + agent_ok = raw_agent_ok if isinstance(raw_agent_ok, bool) else False + return MetricResult(outputs=[MetricOutput(name="agent_phase_success", value=agent_ok)]) + + +class EvidencePresenceMetric: + """Emit ``True`` when a named filesystem evidence directory exists (and is non-empty). + + Reads ``candidate.evidence`` directly — the canonical metric-over-evidence + pattern — so the result reflects what the agent actually produced on disk, + not a reward stamped into metadata by a verifier. + """ + + def __init__( + self, + *, + evidence_name: str = EVIDENCE_FINAL_STATE, + output_name: str = "evidence_present", + require_non_empty: bool = True, + ) -> None: + self._evidence_name = evidence_name + self._output_name = output_name + self._require_non_empty = require_non_empty + + @property + def type(self) -> str: + return "evidence_presence" + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.boolean(self._output_name)] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + present = False + evidence = input.candidate.evidence + if evidence is not None and evidence.get(self._evidence_name) is not None: + try: + handle = await evidence.filesystem(self._evidence_name) + if await handle.exists(): + present = bool(await handle.iter_paths(recursive=True)) if self._require_non_empty else True + except (KeyError, ValueError) as exc: + logger.warning( + "EvidencePresenceMetric scored False: could not resolve evidence %r for output %r: %s", + self._evidence_name, + self._output_name, + exc, + ) + return MetricResult(outputs=[MetricOutput(name=self._output_name, value=present)]) + + +class TrialMeasurements(BaseModel): + """Numeric measurements projected from trial metadata. + + Reporting/gating consume it via :meth:`from_metadata`; producers keep writing + the same keys onto ``AgentEvalTrial.metadata``. + """ + + model_config = ConfigDict(extra="forbid") + + prompt_tokens: int | None = None + completion_tokens: int | None = None + total_tokens: int | None = None + cache_creation_tokens: int | None = None + cache_read_tokens: int | None = None + runtime_sec: float | None = None + reward: float | None = None + passed: bool | None = None + + @classmethod + def from_metadata(cls, metadata: Mapping[str, Any] | None) -> TrialMeasurements: + """Project loose trial metadata onto the typed contract. + + Applies the historical fallbacks so callers don't re-implement them: + ``runtime_sec`` falls back to ``duration_ms / 1000``; ``reward`` falls + back to ``1.0``/``0.0`` derived from ``passed`` when no explicit reward + is recorded. + """ + metadata = metadata or {} + + tokens = {key: _as_int(metadata.get(key)) for key in TOKEN_KEYS} + passed = metadata.get("passed") + passed = bool(passed) if isinstance(passed, bool) else None + + return cls( + **tokens, + runtime_sec=_runtime_sec(metadata), + reward=_reward(metadata, passed), + passed=passed, + ) + + +def _as_int(value: Any) -> int | None: + # bool is an int subclass; never treat True/False as a token count. + if isinstance(value, bool): + return None + return value if isinstance(value, int) else None + + +def _runtime_sec(metadata: Mapping[str, Any]) -> float | None: + runtime_sec = metadata.get("runtime_sec") + if isinstance(runtime_sec, int | float) and not isinstance(runtime_sec, bool): + return float(runtime_sec) + duration_ms = metadata.get("duration_ms") + if isinstance(duration_ms, int | float) and not isinstance(duration_ms, bool): + return float(duration_ms) / 1000.0 + return None + + +def _reward(metadata: Mapping[str, Any], passed: bool | None) -> float | None: + reward = metadata.get("reward") + if reward is not None: + try: + return float(reward) + except (TypeError, ValueError): + return None + if passed is not None: + return 1.0 if passed else 0.0 + return None diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/environment.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/environment.py new file mode 100644 index 0000000000..94b50eb890 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/environment.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Process/filesystem environment boundary for agent-eval runtimes. + +Sits *below* :class:`AgentTaskRunner` so a runtime needn't know whether the +agent/verifier run under Docker, locally, or another filesystem-backed sandbox. +It is a process/filesystem abstraction: :class:`EnvRunSpec`'s ``mounts``/ +``extra_args`` are filesystem hints that non-filesystem providers may ignore. +Handles route both roles through a single :meth:`AbstractEnvironmentHandle.run`. +""" + +from __future__ import annotations + +import abc +import asyncio +import os +import re +import subprocess +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Literal, Protocol, runtime_checkable + +from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask + +EnvRole = Literal["agent", "verifier"] +_SENSITIVE_MARKERS = ("KEY", "TOKEN", "SECRET", "PASSWORD") + + +def _redact_for_logging(cmd: list[str]) -> str: + """Scrub secret-looking values (``KEY=…`` tokens and ``--flag value`` pairs).""" + out: list[str] = [] + redact_next = False + for token in cmd: + if redact_next: + out.append("***REDACTED***") + redact_next = False + elif "=" in token: + left, right = token.split("=", 1) + sensitive = any(m in left.upper() for m in _SENSITIVE_MARKERS) + out.append(f"{left}=***REDACTED***" if sensitive else f"{left}={right}") + else: + normalized = token.lstrip("-").replace("-", "_").upper() + if token.startswith("-") and any(m in normalized for m in _SENSITIVE_MARKERS): + redact_next = True + out.append(token) + return " ".join(out) + + +def default_image_tag(task_id: str) -> str: + """Default task → image-tag mapping (callers may inject their own). + + Sanitizes ``task_id`` to a valid Docker image name so ids with spaces or + other unsupported characters don't fail the build/run. + """ + safe = re.sub(r"[^a-z0-9_.-]+", "-", task_id.lower()).strip(".-") + return f"{safe or 'task'}:latest" + + +@dataclass(frozen=True) +class EnvCommandResult: + """Outcome of running a single command inside a prepared environment.""" + + exit_code: int + timed_out: bool = False + + @property + def ok(self) -> bool: + return self.exit_code == 0 and not self.timed_out + + +@dataclass +class EnvRunSpec: + """How to execute one command inside an environment handle. + + ``mounts``/``extra_args`` are filesystem-environment hints (e.g. Docker bind + mounts and extra CLI args). Non-filesystem providers may ignore them. + """ + + command: list[str] + env: dict[str, str] = field(default_factory=dict) + mounts: list[tuple[str, str]] = field(default_factory=list) + workdir: str | None = None + timeout: int | None = None + extra_args: list[str] = field(default_factory=list) + + +@runtime_checkable +class AgentEnvironmentHandle(Protocol): + """A prepared, single-task environment that can run agent/verifier commands.""" + + async def run_agent(self, spec: EnvRunSpec) -> EnvCommandResult: ... + + async def run_verifier(self, spec: EnvRunSpec) -> EnvCommandResult: ... + + async def close(self) -> None: ... + + +@runtime_checkable +class AgentEnvironmentProvider(Protocol): + """Creates per-task environment handles. Pluggable: Docker now, others later.""" + + async def prepare( + self, + task: AgentEvalTask, + config: AgentEvalRunConfig | None = None, + ) -> AgentEnvironmentHandle: ... + + +class AbstractEnvironmentHandle(abc.ABC): + """Base handle that routes both roles through a single :meth:`run`. + + Concrete handles implement :meth:`run`; ``run_agent``/``run_verifier`` are + role-specialized wrappers so the duplicated phase methods don't have to be + reimplemented per backend. + """ + + @abc.abstractmethod + async def run(self, spec: EnvRunSpec, role: EnvRole) -> EnvCommandResult: ... + + async def run_agent(self, spec: EnvRunSpec) -> EnvCommandResult: + return await self.run(spec, "agent") + + async def run_verifier(self, spec: EnvRunSpec) -> EnvCommandResult: + return await self.run(spec, "verifier") + + async def close(self) -> None: + return None + + +def _docker_run(image: str, spec: EnvRunSpec) -> EnvCommandResult: + """Run ``spec.command`` in a one-shot ``docker run --rm`` container. + + Shells out to the ``docker`` CLI (stdlib ``subprocess`` only), so no + ``agent-runtimes`` extra is needed — just a ``docker`` binary at call time. + """ + cmd = ["docker", "run", "--rm"] + if spec.workdir: + cmd += ["-w", spec.workdir] + for key, value in spec.env.items(): + cmd += ["-e", f"{key}={value}"] + for host_path, container_path in spec.mounts: + cmd += ["-v", f"{host_path}:{container_path}"] + cmd += spec.extra_args + os.environ.get("DOCKER_EXTRA_ARGS", "").split() + cmd += [image, *spec.command] + + print(f"[agent-eval-runtime] $ {_redact_for_logging(cmd)}") + try: + result = subprocess.run(cmd, check=False, text=True, timeout=spec.timeout) + except subprocess.TimeoutExpired: + return EnvCommandResult(exit_code=124, timed_out=True) + return EnvCommandResult(exit_code=result.returncode) + + +class DockerEnvironmentHandle(AbstractEnvironmentHandle): + """Docker-backed environment handle bound to one task image.""" + + def __init__(self, image: str) -> None: + self.image = image + + async def run(self, spec: EnvRunSpec, role: EnvRole = "agent") -> EnvCommandResult: + del role # Docker runs both roles identically against the same image. + return await asyncio.to_thread(_docker_run, self.image, spec) + + +class DockerEnvironmentProvider: + """Default provider that maps each task to its built Docker image.""" + + def __init__(self, *, image_tag_fn: Callable[[str], str] = default_image_tag) -> None: + self._image_tag_fn = image_tag_fn + + async def prepare( + self, + task: AgentEvalTask, + config: AgentEvalRunConfig | None = None, + ) -> DockerEnvironmentHandle: + del config + return DockerEnvironmentHandle(self._image_tag_fn(task.id)) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py index 154b288675..9aa05a1584 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py @@ -1,19 +1,31 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Trial artifacts and the runtime interface that produces them.""" +"""Trial artifacts, the runtime/serde interfaces that produce them, and the +runtime-agnostic helpers for shaping trials from artifacts (status mapping + +the standard evidence-key builder).""" from __future__ import annotations from collections.abc import Sequence from enum import Enum +from pathlib import Path from typing import Any, Protocol, runtime_checkable from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask from nemo_platform.beta.evaluator.values import Agent, Model -from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence +from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +# Well-known evidence keys produced by ``standard_evidence_descriptors``. Harness +# code may import these to tag evidence consistently; callers may still add +# arbitrary extension keys alongside them. +EVIDENCE_INITIAL_STATE = "initial_state" +EVIDENCE_TRACE = "trace" +EVIDENCE_LOGS = "logs" +EVIDENCE_FINAL_STATE = "final_state" +EVIDENCE_VERIFIER_LOGS = "verifier_logs" + class AgentEvalTrialStatus(str, Enum): """Lifecycle status for a trial: completed, failed, or partial.""" @@ -88,4 +100,91 @@ async def run_tasks( ) -> Sequence[AgentEvalTrial]: ... +@runtime_checkable +class AgentTrialSerde(Protocol): + """Read/write a single stored trial artifact as an :class:`AgentEvalTrial`. + + The offline counterpart to :class:`AgentTaskRunner`: instead of *executing* an + agent it adapts a stored artifact (a run dir/file) to and from a trial, so prior + runs can be re-scored. The SDK ships only the protocol; concrete codecs (which + know a particular on-disk layout) live with their producers. + """ + + def read(self) -> AgentEvalTrial: ... + + def write(self, trial: AgentEvalTrial) -> None: ... + + AgentEvalTarget = Model | Agent | AgentTaskRunner + + +def resolve_trial_status(agent_ok: bool) -> AgentEvalTrialStatus: + """Map an agent-phase outcome to a *scorable* trial status. + + ``AgentEvaluator`` excludes ``FAILED`` trials from scoring, so an + executed-but-unsuccessful agent uses ``PARTIAL`` (still scored as ``0`` for + pass-rate gating); ``FAILED`` is reserved for trial-*production* failures, + which a runtime surfaces by raising rather than emitting an unscorable trial. + """ + return AgentEvalTrialStatus.COMPLETED if agent_ok else AgentEvalTrialStatus.PARTIAL + + +def standard_evidence_descriptors( + *, + logs_dir: str | Path, + final_state_dir: str | Path, + trace_path: str | Path | None = None, + initial_state_ref: str | None = None, + verifier_logs_dir: str | Path | None = None, + primary_log: str | None = None, +) -> dict[str, EvidenceDescriptor]: + """Build the documented evidence map for an agent-eval trial. + + Standard keys: ``initial_state`` (task input filesystem, when staged), + ``trace`` (trajectory, ATIF-normalized when available), ``logs`` (agent log + dir), ``final_state`` (workspace), and ``verifier_logs`` (only when present). + Callers may add their own extension keys to the returned mapping. + """ + descriptors: dict[str, EvidenceDescriptor] = {} + + if initial_state_ref: + descriptors[EVIDENCE_INITIAL_STATE] = EvidenceDescriptor( + kind="filesystem", + format="dir", + ref=str(initial_state_ref), + metadata={"role": EVIDENCE_INITIAL_STATE}, + ) + + if trace_path is not None: + trace_name = Path(trace_path).name.lower() + is_atif = trace_name.startswith("atif") or ".atif." in trace_name + descriptors[EVIDENCE_TRACE] = EvidenceDescriptor( + kind="trace", + format="atif" if is_atif else "json", + ref=str(trace_path), + ) + + logs_metadata = {"primary_log": primary_log} if primary_log else {} + descriptors[EVIDENCE_LOGS] = EvidenceDescriptor( + kind="logs", + format="dir", + ref=str(logs_dir), + metadata=logs_metadata, + ) + + descriptors[EVIDENCE_FINAL_STATE] = EvidenceDescriptor( + kind="filesystem", + format="dir", + ref=str(final_state_dir), + metadata={"role": EVIDENCE_FINAL_STATE}, + ) + + if verifier_logs_dir is not None and Path(verifier_logs_dir).exists(): + descriptors[EVIDENCE_VERIFIER_LOGS] = EvidenceDescriptor( + kind="logs", + format="dir", + ref=str(verifier_logs_dir), + metadata={"role": "verifier"}, + ) + + return descriptors