From 26cf4cfa62e708aea284cc85759b86c6ca73030f Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Thu, 23 Jul 2026 18:05:16 -0700 Subject: [PATCH 1/7] docs: harden examples for end-to-end use Signed-off-by: Zhongxuan Wang --- examples/README.md | 2 +- examples/code_review_agent/README.md | 31 ++++-- examples/code_review_agent/config.py | 9 -- .../skills/code-review/README.md | 8 -- .../skills/code-review/SKILL.md | 14 +++ examples/harbor/calculator/.gitignore | 6 + examples/harbor/calculator/README.md | 14 ++- examples/harbor/prepare_swebench.sh | 9 +- examples/harbor/swebench/README.md | 2 +- .../swebench/sample-artifacts/README.md | 6 + examples/notebooks/02_variations.ipynb | 104 +++++++++++------- examples/notebooks/README.md | 27 +++-- tests/integrations/test_harbor_runner.py | 21 ++++ tests/python/test_code_review_example.py | 40 +++++-- tests/python/test_native_sdk.py | 2 +- tests/python/test_notebook_examples.py | 18 +++ 16 files changed, 212 insertions(+), 101 deletions(-) delete mode 100644 examples/code_review_agent/skills/code-review/README.md create mode 100644 examples/code_review_agent/skills/code-review/SKILL.md create mode 100644 examples/harbor/calculator/.gitignore diff --git a/examples/README.md b/examples/README.md index 41cbcd4b0..f89f10f7b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -29,7 +29,7 @@ application-facing Python SDK contract: - resolving relative workspace and skill paths with `base_dir`; - running maintained Hermes Agent, Codex, Claude, and Deep Agents adapters through the Python SDK. -Start with: +Complete the [code-review setup](code_review_agent/README.md#set-up), then run: ```bash just build-all diff --git a/examples/code_review_agent/README.md b/examples/code_review_agent/README.md index 53bac57a0..7de50e7bd 100644 --- a/examples/code_review_agent/README.md +++ b/examples/code_review_agent/README.md @@ -21,12 +21,19 @@ just build-all ``` The default variant uses Hermes Agent with an NVIDIA-hosted model. Follow the -[Hermes Agent quick start](../../README.md#quick-start-hermes-agent) to install -Hermes Agent, then set `NVIDIA_API_KEY` and `ADAPTER_PYTHON` as described there. +[Hermes Agent adapter setup](../../adapters/hermes/README.md) with Python +3.11-3.13, then set `NVIDIA_API_KEY`. If Hermes Agent is installed in a +separate environment, select its interpreter only for the Hermes command: -The config also demonstrates a harness-native GitHub MCP server. Set -`GITHUB_MCP_URL` when you want to use that server; the review prompt below does -not require it. +```bash +ADAPTER_PYTHON="$PWD/.tmp/hermes-venv/bin/python" \ + .venv/bin/python -m examples.code_review_agent \ + --input "Reply with exactly: fabric works" +``` + +Do not export a Hermes-only `ADAPTER_PYTHON` globally before running the Codex, +Claude, or Deep Agents variants. Those variants use the project interpreter +unless you select another interpreter explicitly. ## Inspect the plan @@ -67,7 +74,11 @@ Add `--relay` to any variant to enable the Relay ATOF and ATIF configuration: Relay requirements depend on the selected adapter. The Codex and Claude adapters require an external `nemo-relay` CLI in the supported `0.6.x` range; -the Python package named `nemo-relay` does not install that command. +the Python package named `nemo-relay` does not install that command. Hermes +Agent and Deep Agents require the Relay Python package in their selected +adapter environment. Refer to the +[installation guide](../../docs/getting-started/install.mdx#install-nemo-relay) +for the current compatibility requirements. ```bash .venv/bin/python -m examples.code_review_agent \ @@ -89,6 +100,7 @@ application-owned composition: from examples.code_review_agent import ( BASE_DIR, hermes_config, + with_fabric_managed_github_mcp, with_opensandbox, with_relay, ) @@ -96,8 +108,9 @@ from examples.code_review_agent import ( config = hermes_config() relay_config = with_relay(config) sandbox_config = with_opensandbox(config) +github_config = with_fabric_managed_github_mcp(config) ``` -Each function returns a deep copy. `config`, `relay_config`, and -`sandbox_config` can therefore be planned or run independently with -`base_dir=BASE_DIR`. +Each function returns a deep copy. The four configs can therefore be planned or +run independently with `base_dir=BASE_DIR`. Set `GITHUB_MCP_URL` before running +`github_config`; the default smoke does not configure or contact that server. diff --git a/examples/code_review_agent/config.py b/examples/code_review_agent/config.py index 127f545cd..7d584a84e 100644 --- a/examples/code_review_agent/config.py +++ b/examples/code_review_agent/config.py @@ -58,12 +58,6 @@ def base_config() -> FabricConfig: telemetry=TelemetryConfig(), ) config.add_skill_path(SKILL_PATH) - config.add_mcp_server( - "github", - transport="streamable-http", - url="${GITHUB_MCP_URL}", - exposure="harness_native", - ) return config @@ -122,7 +116,6 @@ def codex_config() -> FabricConfig: workspace=WORKSPACE, artifacts="./artifacts/codex", ) - config.remove_mcp_server("github") config.remove_skill_path(SKILL_PATH) return config @@ -149,7 +142,6 @@ def deepagents_config() -> FabricConfig: workspace=WORKSPACE, artifacts="./artifacts/deepagents", ) - config.remove_mcp_server("github") config.remove_skill_path(SKILL_PATH) return config @@ -188,7 +180,6 @@ def claude_config() -> FabricConfig: workspace=WORKSPACE, artifacts="./artifacts/claude", ) - config.remove_mcp_server("github") config.remove_skill_path(SKILL_PATH) return config diff --git a/examples/code_review_agent/skills/code-review/README.md b/examples/code_review_agent/skills/code-review/README.md deleted file mode 100644 index e48509a37..000000000 --- a/examples/code_review_agent/skills/code-review/README.md +++ /dev/null @@ -1,8 +0,0 @@ - - -# Code Review Skill - -Minimal skill directory used by the code-review example. diff --git a/examples/code_review_agent/skills/code-review/SKILL.md b/examples/code_review_agent/skills/code-review/SKILL.md new file mode 100644 index 000000000..8b63e213f --- /dev/null +++ b/examples/code_review_agent/skills/code-review/SKILL.md @@ -0,0 +1,14 @@ +--- +name: code-review +description: Review Python changes for correctness risks and missing test coverage. +--- + +# Code Review + +1. Read the changed implementation and its nearest tests. +2. Identify correctness, error-handling, and compatibility risks before style issues. +3. Cite the smallest relevant file and line range for each finding. +4. Suggest the narrowest correction and a focused regression test. + +If no correctness issue is present, say so directly and identify any remaining +test-coverage uncertainty. diff --git a/examples/harbor/calculator/.gitignore b/examples/harbor/calculator/.gitignore new file mode 100644 index 000000000..0d3673d3e --- /dev/null +++ b/examples/harbor/calculator/.gitignore @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +runs/ +task/environment/vendor/ +task/environment/.vendor.*/ diff --git a/examples/harbor/calculator/README.md b/examples/harbor/calculator/README.md index bd54a5675..26d48dbd2 100644 --- a/examples/harbor/calculator/README.md +++ b/examples/harbor/calculator/README.md @@ -85,7 +85,7 @@ uv run --extra runtime --extra harbor harbor run \ --ak fabric_adapter_id=nvidia.fabric.hermes \ --ak fabric_config_base_dir=/opt/fabric-calculator \ --ak fabric_workspace=/app \ - --ak 'fabric_harness_settings={"cwd":"/app","base_url":"https://integrate.api.nvidia.com/v1","max_iterations":20}' \ + --ak 'fabric_harness_settings={"base_url":"https://integrate.api.nvidia.com/v1","max_iterations":20}' \ --ae "NVIDIA_API_KEY=$NVIDIA_API_KEY" \ --job-name fabric-hermes \ --jobs-dir "$RUNS_DIR" \ @@ -110,7 +110,7 @@ uv run --extra runtime --extra harbor harbor run \ --ak fabric_config_base_dir=/opt/fabric-calculator \ --ak fabric_workspace=/app \ --ak fabric_telemetry=relay \ - --ak 'fabric_harness_settings={"cwd":"/app","base_url":"https://integrate.api.nvidia.com/v1","max_iterations":4,"terminal_timeout":120}' \ + --ak 'fabric_harness_settings={"base_url":"https://integrate.api.nvidia.com/v1","max_iterations":4,"terminal_timeout":120}' \ --ae "NVIDIA_API_KEY=$NVIDIA_API_KEY" \ --job-name fabric-hermes-relay \ --jobs-dir "$RUNS_DIR" \ @@ -135,8 +135,9 @@ find "$RUNS_DIR/fabric-hermes-relay" \ ## 4. Claude The Claude Agent SDK supplies its compatible Claude Code executable. Harbor -passes the API key into the task environment; NeMo Fabric forwards only the model's -configured credential variable to Claude. +passes the API key into the task environment. NeMo Fabric forwards the supported +Claude authentication variables selected by the adapter; this command uses +`ANTHROPIC_API_KEY`. ```bash : "${ANTHROPIC_API_KEY:?Export ANTHROPIC_API_KEY before running Claude}" @@ -173,8 +174,9 @@ uv run --extra runtime --extra harbor harbor view "$RUNS_DIR" ``` Check NeMo Fabric status, harness and adapter identity, runtime and invocation IDs, -artifacts, telemetry, Harbor exceptions, and reward. A successful smoke run has -NeMo Fabric status `succeeded` and Harbor mean reward `1.0`. +artifacts, telemetry, Harbor exceptions, and reward. A successful calculator run +has one completed trial, zero errored trials, NeMo Fabric status `succeeded`, and +Harbor mean reward `1.0`. After the runs, remove the generated build-context copy: diff --git a/examples/harbor/prepare_swebench.sh b/examples/harbor/prepare_swebench.sh index 8ba135ba6..5930013d3 100755 --- a/examples/harbor/prepare_swebench.sh +++ b/examples/harbor/prepare_swebench.sh @@ -19,9 +19,10 @@ repo_root="$(git rev-parse --show-toplevel)" bundle_dir="$repo_root/examples/harbor/swebench" wheelhouse="$bundle_dir/.wheelhouse" relay_root="$bundle_dir/.relay" +relay_cli_version="0.6.0" -if [[ "$(uname -m)" != "x86_64" ]]; then - echo "This SWE-Bench example currently targets x86_64 task images." >&2 +if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then + echo "This SWE-Bench example currently requires an x86_64 Linux host." >&2 exit 1 fi @@ -46,14 +47,14 @@ docker run --rm \ # TEMP: Keep only this block if PyPI wheels land before Relay CLI distribution. relay_version="$({ "$relay_root/bin/nemo-relay" --version 2>/dev/null || true; })" -if [[ "$relay_version" != "nemo-relay 0.5.0" ]]; then +if [[ "$relay_version" != "nemo-relay $relay_cli_version" ]]; then docker run --rm \ --user "$(id -u):$(id -g)" \ -e CARGO_HOME=/tmp/cargo \ -e CARGO_TARGET_DIR=/tmp/target \ -v "$relay_root:/out" \ rust:1.94-bullseye \ - cargo install nemo-relay-cli --version 0.5.0 --locked --force --root /out + cargo install nemo-relay-cli --version "$relay_cli_version" --locked --force --root /out fi # TEMP: Record the source-built meta-wheel path for installation in the task. diff --git a/examples/harbor/swebench/README.md b/examples/harbor/swebench/README.md index cbf6be436..8102eb019 100644 --- a/examples/harbor/swebench/README.md +++ b/examples/harbor/swebench/README.md @@ -42,7 +42,7 @@ export RUNS_DIR="$PWD/.tmp/harbor/fabric-swebench" ``` The preparation script builds the native runtime in a manylinux2014 container, -builds the standalone Relay 0.5.0 CLI in Debian Bullseye, and writes the exact +builds the standalone Relay 0.6.0 CLI in Debian Bullseye, and writes the exact task-local package requirement to `.fabric-package`. The generated files are ignored by Git. The uploaded bundle contains adapter descriptors, the example MCP server, generated wheels, and Relay; it does not contain a persisted diff --git a/examples/harbor/swebench/sample-artifacts/README.md b/examples/harbor/swebench/sample-artifacts/README.md index c182ccedb..bf46bb1e9 100644 --- a/examples/harbor/swebench/sample-artifacts/README.md +++ b/examples/harbor/swebench/sample-artifacts/README.md @@ -20,3 +20,9 @@ Each harness directory contains: Use these samples to compare Harbor's canonical trajectory with Relay's direct ATOF and ATIF output before running the example yourself. + +The checked-in trajectories record Relay 0.5.0 and the earlier +`nvidia/nemotron-3-nano` model identifier because they are historical evidence +from the original runs. New runs use the compatible Relay CLI version pinned by +[`prepare_swebench.sh`](../../prepare_swebench.sh) and the model selected in the +current walkthrough; do not treat sample metadata as installation guidance. diff --git a/examples/notebooks/02_variations.ipynb b/examples/notebooks/02_variations.ipynb index 991fb3795..9c367b818 100644 --- a/examples/notebooks/02_variations.ipynb +++ b/examples/notebooks/02_variations.ipynb @@ -31,9 +31,10 @@ "connects NeMo Fabric to a harness; the **runtime** is NeMo Fabric's own start/invoke/\n", "stop lifecycle, which does not change when you switch harnesses.\n", "\n", - "Where a harness's prerequisites are present, the cells below run it;\n", + "Where a harness's visible prerequisites are present, the cells below run it;\n", "otherwise they inspect the resolved config with `plan()` and say what to\n", - "provide — so the notebook always executes top to bottom." + "provide. Attempted-run failures are collected so every variant is tried, then\n", + "raised after the remaining configuration examples execute." ] }, { @@ -112,11 +113,13 @@ "from examples.code_review_agent import base_config, BASE_DIR\n", "\n", "# A harness adapter runs in whatever Python environment has that adapter\n", - "# installed. The Hermes adapter lives in the repo's separate Hermes venv; the\n", - "# Codex, Claude, and Deep Agents adapters live in this notebook's interpreter.\n", + "# installed. Hermes can use a dedicated venv; Codex, Claude, and Deep Agents\n", + "# use this notebook's interpreter in this example.\n", "FABRIC_PY = sys.executable\n", "_hermes_python = REPO_ROOT / \".tmp\" / \"hermes-venv\" / \"bin\" / \"python\"\n", - "HERMES_PY = str(_hermes_python) if _hermes_python.exists() else os.environ.get(\"ADAPTER_PYTHON\")\n", + "if not _hermes_python.is_file():\n", + " _hermes_python = None\n", + "HERMES_PY = os.environ.get(\"ADAPTER_PYTHON\") or (str(_hermes_python) if _hermes_python else None)\n", "\n", "WORKSPACE = \"./repos/my-service\" # relative to BASE_DIR\n", "INSTRUCTION = \"You are a concise code reviewer. Point out correctness bugs and risks.\"\n", @@ -149,9 +152,9 @@ "- **Input schema** — the Codex adapter takes `text`; the others take `chat`.\n", "- **Instruction delivery** — Hermes Agent, Deep Agents, and Claude take the system\n", " prompt via `harness.settings`; Codex takes its instruction from the input.\n", - "- **Capabilities** — `base_config` ships a code-review skill and a GitHub MCP\n", - " server, but not every harness accepts every capability, so `for_harness`\n", - " drops them here. That difference across harnesses is exactly why the\n", + "- **Capabilities** — `base_config` ships a code-review skill, but not every\n", + " harness accepts the same capabilities, so `for_harness` drops it here. That\n", + " difference across harnesses is exactly why the\n", " example provides per-harness builders; Group 2 composes capabilities back." ] }, @@ -170,12 +173,11 @@ " values it forces to differ are changed here.\n", " \"\"\"\n", " cfg = base_config()\n", - " # base_config ships a code-review skill and a GitHub MCP server. Not every\n", - " # harness accepts every capability (which is exactly why the example provides\n", - " # per-harness builders), so drop them for an apples-to-apples comparison.\n", + " # base_config ships a code-review skill. Not every harness accepts every\n", + " # capability (which is exactly why the example provides per-harness\n", + " # builders), so drop it for an apples-to-apples comparison.\n", " # Group 2 composes capabilities back on.\n", " cfg.remove_skill_path(\"./skills/code-review\")\n", - " cfg.remove_mcp_server(\"github\")\n", " cfg.harness = HarnessConfig(\n", " adapter_id=harness[\"adapter_id\"], resolution=\"preinstalled\",\n", " settings={**harness[\"settings\"], \"python\": harness[\"python\"]},\n", @@ -244,6 +246,7 @@ "source": [ "PROMPT = \"In one sentence, what does the Python expression sum(v) / len(v) compute, and name one risk?\"\n", "\n", + "run_failures = []\n", "for harness in HARNESSES:\n", " cfg = for_harness(harness)\n", " plan = fabric.plan(cfg, base_dir=BASE_DIR)\n", @@ -261,9 +264,12 @@ " result = await fabric.run(cfg, base_dir=BASE_DIR, input=PROMPT)\n", " print(f\" RAN -> {result.status}\")\n", " print(f\" reply: {oneline(getattr(result.output, 'response', result.output))}\")\n", + " if result.status != \"succeeded\":\n", + " run_failures.append(f\"{harness['name']}: {result.error or result.status}\")\n", " except Exception as error:\n", " print(f\" run failed ({type(error).__name__}: {oneline(error, 120)}).\")\n", " print(f\" To run it, provide: {harness['needs']}\")\n", + " run_failures.append(f\"{harness['name']}: {type(error).__name__}: {error}\")\n", " print()" ] }, @@ -275,9 +281,9 @@ "## Group 2 — Agent configuration variation\n", "\n", "The other kind of variation keeps the harness fixed and composes on the\n", - "config. Starting from the maintained `base_config()` (which already carries\n", - "a code-review skill and a GitHub MCP server), you strip or add skills and MCP\n", - "servers and swap the model — all on a **copy**, leaving the base untouched." + "config. Starting from the maintained `base_config()` (which carries a\n", + "code-review skill), you strip or add skills and an optional MCP server and swap\n", + "the model — all on a **copy**, leaving the base untouched." ] }, { @@ -300,15 +306,17 @@ "# Strip capabilities on a copy.\n", "stripped = base.model_copy(deep=True)\n", "stripped.remove_skill_path(\"./skills/code-review\")\n", - "stripped.remove_mcp_server(\"github\")\n", "print(\"stripped :\", summarize(stripped))\n", "\n", - "# Compose on another copy: add a skill and MCP server, swap the model.\n", - "recomposed = base.model_copy(deep=True)\n", - "recomposed.add_skill_path(\"./skills/style-guide\")\n", - "recomposed.add_mcp_server(\n", - " \"docs\", transport=\"streamable-http\", url=\"${DOCS_MCP_URL}\", exposure=\"fabric_managed\"\n", - ")\n", + "# Compose on another copy: add the real example skill, optionally add the\n", + "# configured GitHub MCP server, and swap the model.\n", + "recomposed = stripped.model_copy(deep=True)\n", + "recomposed.add_skill_path(\"./skills/code-review\")\n", + "if github_mcp_url := os.environ.get(\"GITHUB_MCP_URL\"):\n", + " recomposed.add_mcp_server(\n", + " \"github\", transport=\"streamable-http\",\n", + " url=github_mcp_url, exposure=\"fabric_managed\",\n", + " )\n", "recomposed.models[\"default\"] = ModelConfig(\n", " provider=\"nvidia\", model=\"nvidia/nemotron-3-super-49b-a5b\", api_key_env=\"NVIDIA_API_KEY\"\n", ")\n", @@ -366,26 +374,38 @@ " ),\n", " ),\n", " )\n", - " try:\n", - " result = await fabric.run(traced, base_dir=BASE_DIR, input=PROMPT)\n", - " except Exception as error:\n", - " print(f\"Telemetry run failed ({type(error).__name__}: {oneline(error, 120)}).\")\n", - " print(\"To emit traces, provide:\", deep[\"needs\"])\n", - " else:\n", - " print(\"run status :\", result.status)\n", - " print(\"telemetry refs :\", [t.provider for t in result.telemetry] or \"(none)\")\n", - " traces = sorted(p for p in relay_dir.rglob(\"*\") if p.is_file())\n", - " print(\"trace files :\", len(traces))\n", - " for path in traces:\n", - " print(\" \", path.relative_to(REPO_ROOT))\n", - "\n", - " # Print the full Relay trace: every ATOF event.\n", - " for path in traces:\n", - " print(f\"\\n===== {path.relative_to(REPO_ROOT)} =====\")\n", - " lines = [line for line in path.read_text().splitlines() if line.strip()]\n", - " for i, line in enumerate(lines, 1):\n", - " print(f\"\\n--- event {i}/{len(lines)} ---\")\n", - " print(json.dumps(json.loads(line), indent=2))" + " result = await fabric.run(traced, base_dir=BASE_DIR, input=PROMPT)\n", + " if result.status != \"succeeded\":\n", + " raise RuntimeError(f\"Deep Agents Relay run failed: {result.error}\")\n", + " print(\"run status :\", result.status)\n", + " telemetry_providers = [t.provider for t in result.telemetry]\n", + " if \"relay\" not in telemetry_providers:\n", + " raise RuntimeError(\"Deep Agents Relay run returned no Relay telemetry reference\")\n", + " print(\"telemetry refs :\", telemetry_providers)\n", + " traces = sorted(p for p in relay_dir.rglob(\"*\") if p.is_file())\n", + " atof_paths = sorted(relay_dir.rglob(\"events.atof.jsonl\"))\n", + " if not atof_paths:\n", + " raise RuntimeError(\"Deep Agents Relay run produced no ATOF trace\")\n", + " for atof_path in atof_paths:\n", + " atof_lines = [line for line in atof_path.read_text().splitlines() if line.strip()]\n", + " if not atof_lines:\n", + " raise RuntimeError(f\"Deep Agents Relay run produced an empty ATOF trace: {atof_path}\")\n", + " for line in atof_lines:\n", + " json.loads(line)\n", + " print(\"trace files :\", len(traces))\n", + " for path in traces:\n", + " print(\" \", path.relative_to(REPO_ROOT))\n", + "\n", + " # Print the full Relay trace: every ATOF event.\n", + " for path in traces:\n", + " print(f\"\\n===== {path.relative_to(REPO_ROOT)} =====\")\n", + " lines = [line for line in path.read_text().splitlines() if line.strip()]\n", + " for i, line in enumerate(lines, 1):\n", + " print(f\"\\n--- event {i}/{len(lines)} ---\")\n", + " print(json.dumps(json.loads(line), indent=2))\n", + "\n", + "if run_failures:\n", + " raise RuntimeError(\"Harness variation failures:\\n- \" + \"\\n- \".join(run_failures))" ] }, { diff --git a/examples/notebooks/README.md b/examples/notebooks/README.md index f76990bbc..8f29026a4 100644 --- a/examples/notebooks/README.md +++ b/examples/notebooks/README.md @@ -19,22 +19,29 @@ variations notebook shows advanced composition against a real, maintained agent. ## Prerequisites - Build the SDK and native extension from the repo root: `just build-all`. This - alone is enough to run both notebooks top to bottom. + alone is enough to execute both notebooks top to bottom with unavailable live + harnesses skipped. - To actually *run* a harness (rather than just inspect its config), that harness's adapter and credentials must be present: - - **Hermes Agent** (both notebooks): install Hermes Agent in its own environment (the repo - README's [Hermes Agent quick start](../../README.md#quick-start-hermes-agent)) - and set `NVIDIA_API_KEY`. The setup cell auto-detects `.tmp/hermes-venv`. + - **Hermes Agent** (both notebooks): follow the + [Hermes Agent adapter setup](../../adapters/hermes/README.md) with Python + 3.11-3.13 and set `NVIDIA_API_KEY`. The setup cells auto-detect + `.tmp/hermes-venv`. - **Deep Agents, Codex, Claude** (variations notebook): the matching adapter installed in the NeMo Fabric environment, plus that harness's credentials - (`NVIDIA_API_KEY` for Deep Agents; an authenticated `codex` and - `OPENAI_API_KEY` for Codex; `ANTHROPIC_API_KEY` for Claude). + (`NVIDIA_API_KEY` for Deep Agents; an existing ChatGPT or provisioned API-key + login for Codex; `ANTHROPIC_API_KEY` for the documented Claude run). + Relay-enabled Hermes Agent and Deep Agents runs also need the `nemo-relay` + Python package in the selected adapter environment. - `NVIDIA_API_KEY` is loaded from a gitignored `.env` at the repo root if present. -Every live cell is guarded. With only `just build-all` done, both notebooks -still execute end to end: the variations notebook runs each harness whose -prerequisites are met and, for the rest, inspects the resolved config with -`plan()` and prints exactly what to provide to run it for real. +Every live cell checks the prerequisites it can inspect. Missing key-based +credentials skip their harness and still show its resolved plan. Codex +authentication is validated only after its adapter starts, so an attempted +Codex authentication failure is collected as an execution failure. After every +variant is attempted, the notebook raises on any attempted-run failure. The +Relay cell also requires a succeeded result, a Relay telemetry reference, and a +nonempty, parseable ATOF trace. ## Launch diff --git a/tests/integrations/test_harbor_runner.py b/tests/integrations/test_harbor_runner.py index 602871b22..c64c2800a 100644 --- a/tests/integrations/test_harbor_runner.py +++ b/tests/integrations/test_harbor_runner.py @@ -17,6 +17,7 @@ CALCULATOR_FABRIC_ROOT = CALCULATOR_ROOT / "task" / "environment" / "fabric" SWEBENCH_ROOT = ROOT / "examples" / "harbor" / "swebench" SWEBENCH_README = SWEBENCH_ROOT / "README.md" +SWEBENCH_PREPARE = ROOT / "examples" / "harbor" / "prepare_swebench.sh" SWEBENCH_MCP_CONFIG = SWEBENCH_ROOT / "mcp" / "repo-inspector.mcp.json" INTEGRATION_README = ROOT / "examples" / "harbor" / "README.md" SDK_INTEGRATION_README = ( @@ -331,6 +332,26 @@ def test_harbor_calculator_documents_explicit_cli_commands(): assert value in swebench +def test_harbor_generated_paths_are_ignored(): + ignore = (CALCULATOR_ROOT / ".gitignore").read_text(encoding="utf-8") + + assert "runs/" in ignore + assert "task/environment/vendor/" in ignore + assert "task/environment/.vendor.*/" in ignore + + +def test_swebench_bootstrap_pins_a_supported_relay_cli(): + from nemo_fabric_adapters.common.relay_gateway import RELAY_MINIMUM_VERSION + + prepare = SWEBENCH_PREPARE.read_text(encoding="utf-8") + minimum = ".".join(str(part) for part in RELAY_MINIMUM_VERSION) + + assert f'relay_cli_version="{minimum}"' in prepare + assert '"nemo-relay $relay_cli_version"' in prepare + assert 'nemo-relay-cli --version "$relay_cli_version" --locked' in prepare + assert '"$(uname -s)" != "Linux"' in prepare + + def test_harbor_calculator_setup_and_solution_fail_fast(): dockerfile = CALCULATOR_DOCKERFILE.read_text(encoding="utf-8") solution = CALCULATOR_SOLUTION.read_text(encoding="utf-8") diff --git a/tests/python/test_code_review_example.py b/tests/python/test_code_review_example.py index 1237ab834..9e0627745 100644 --- a/tests/python/test_code_review_example.py +++ b/tests/python/test_code_review_example.py @@ -14,6 +14,7 @@ from examples.code_review_agent import base_config from examples.code_review_agent import claude_config from examples.code_review_agent import codex_config +from examples.code_review_agent import deepagents_config from examples.code_review_agent import hermes_config from examples.code_review_agent import with_fabric_managed_github_mcp from examples.code_review_agent import with_native_otel @@ -31,8 +32,9 @@ def test_variant_builders_return_independent_complete_configs(): hermes = hermes_config() codex = codex_config() claude = claude_config() + deepagents = deepagents_config() - for config in (base, hermes, codex, claude): + for config in (base, hermes, codex, claude, deepagents): assert isinstance(config, FabricConfig) assert config.metadata.name == "code-review-agent" assert config.environment is not None @@ -51,8 +53,13 @@ def test_variant_builders_return_independent_complete_configs(): assert claude.models["default"].api_key_env == "ANTHROPIC_API_KEY" assert claude.mcp is None assert claude.skills is None - assert base.mcp is not None + assert deepagents.harness.adapter_id == "nvidia.fabric.langchain.deepagents" + assert deepagents.mcp is None + assert deepagents.skills is None + assert base.mcp is None assert base.skills is not None + skill_path = BASE_DIR / base.skills.paths[0] + assert (skill_path / "SKILL.md").is_file() def test_capability_and_telemetry_variants_do_not_mutate_their_input(): @@ -70,8 +77,7 @@ def test_capability_and_telemetry_variants_do_not_mutate_their_input(): assert base.telemetry.providers == {} assert base.environment is not None assert base.environment.provider == "local" - assert base.mcp is not None - assert base.mcp.servers["github"].exposure == "harness_native" + assert base.mcp is None assert all(variant is not base for variant in variants) assert variants[0].mcp is not None assert variants[0].mcp.servers["github"].exposure == "fabric_managed" @@ -86,7 +92,12 @@ def test_capability_and_telemetry_variants_do_not_mutate_their_input(): def test_variants_plan_from_complete_configs(): client = Fabric() - for config in (hermes_config(), codex_config(), claude_config()): + for config in ( + hermes_config(), + codex_config(), + claude_config(), + deepagents_config(), + ): plan = client.plan(config, base_dir=BASE_DIR) assert plan.base_dir == BASE_DIR assert plan.agent_name == "code-review-agent" @@ -94,11 +105,20 @@ def test_variants_plan_from_complete_configs(): def test_example_entrypoint_plans_without_starting_a_runtime(): - cases = ( - ([], "nvidia.fabric.hermes", False), - (["--variant", "codex"], "nvidia.fabric.codex", False), - (["--variant", "claude"], "nvidia.fabric.claude", False), - (["--relay"], "nvidia.fabric.hermes", True), + variants = ( + ("hermes", "nvidia.fabric.hermes"), + ("codex", "nvidia.fabric.codex"), + ("claude", "nvidia.fabric.claude"), + ("deepagents", "nvidia.fabric.langchain.deepagents"), + ) + cases = tuple( + ( + ["--variant", variant, *(["--relay"] if relay_enabled else [])], + adapter_id, + relay_enabled, + ) + for variant, adapter_id in variants + for relay_enabled in (False, True) ) for options, adapter_id, relay_enabled in cases: diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index cda9e6be2..e30d1a872 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -78,7 +78,7 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: plan["adapter_descriptor"]["descriptor"]["adapter_id"] == "nvidia.fabric.hermes" ) - assert plan["capability_plan"]["native"]["mcp_servers"]["github"] + assert "mcp_servers" not in plan["capability_plan"]["native"] assert plan["capability_plan"]["native"]["skill_paths"] minimal = FabricConfig.from_mapping( diff --git a/tests/python/test_notebook_examples.py b/tests/python/test_notebook_examples.py index bfa072528..cbf489c07 100644 --- a/tests/python/test_notebook_examples.py +++ b/tests/python/test_notebook_examples.py @@ -61,3 +61,21 @@ def test_variations_notebook_harnesses_plan_with_current_adapters(): assert plans["Codex"].config.harness.settings["sandbox"] == "workspace-write" assert plans["Codex"].config.harness.settings["reasoning_effort"] == "high" assert plans["Codex"].config.runtime.input_schema == "text" + + +def test_variations_notebook_uses_runnable_capabilities_and_checks_relay_status(): + notebook = json.loads(VARIATIONS_NOTEBOOK.read_text(encoding="utf-8")) + source = "\n".join( + "".join(cell["source"]) + for cell in notebook["cells"] + if cell["cell_type"] == "code" + ) + + assert "./skills/style-guide" not in source + assert "${DOCS_MCP_URL}" not in source + assert 'recomposed.add_skill_path("./skills/code-review")' in source + assert 'if result.status != "succeeded":' in source + assert "run_failures.append" in source + assert "Harness variation failures" in source + assert "returned no Relay telemetry reference" in source + assert "produced an empty ATOF trace" in source From 57c6f6c391e9268db882aebb2db9108f5a13a35e Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Thu, 23 Jul 2026 18:36:01 -0700 Subject: [PATCH 2/7] docs: show notebook run failures inline Signed-off-by: Zhongxuan Wang --- examples/notebooks/02_variations.ipynb | 16 +++++++++++----- tests/python/test_notebook_examples.py | 2 ++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/examples/notebooks/02_variations.ipynb b/examples/notebooks/02_variations.ipynb index 9c367b818..4f9fb84ae 100644 --- a/examples/notebooks/02_variations.ipynb +++ b/examples/notebooks/02_variations.ipynb @@ -69,6 +69,7 @@ "outputs": [], "source": [ "import importlib.util\n", + "import json\n", "import os\n", "import shutil\n", "import sys\n", @@ -234,7 +235,11 @@ "\n", "def oneline(text, limit=200):\n", " s = \" \".join(str(text).split())\n", - " return s if len(s) <= limit else s[:limit] + \" ...\"\n" + " return s if len(s) <= limit else s[:limit] + \" ...\"\n", + "\n", + "\n", + "def failure_detail(result):\n", + " return result.error.to_mapping() if result.error else {\"status\": result.status}\n" ] }, { @@ -265,7 +270,9 @@ " print(f\" RAN -> {result.status}\")\n", " print(f\" reply: {oneline(getattr(result.output, 'response', result.output))}\")\n", " if result.status != \"succeeded\":\n", - " run_failures.append(f\"{harness['name']}: {result.error or result.status}\")\n", + " detail = failure_detail(result)\n", + " print(\" error:\", json.dumps(detail, indent=2))\n", + " run_failures.append(f\"{harness['name']}: {json.dumps(detail, sort_keys=True)}\")\n", " except Exception as error:\n", " print(f\" run failed ({type(error).__name__}: {oneline(error, 120)}).\")\n", " print(f\" To run it, provide: {harness['needs']}\")\n", @@ -349,8 +356,6 @@ "metadata": {}, "outputs": [], "source": [ - "import json\n", - "\n", "relay_dir = REPO_ROOT / \"examples\" / \"notebooks\" / \"artifacts\" / \"relay\"\n", "deep = HARNESSES[1] # Deep Agents\n", "reason = blocker(deep)\n", @@ -376,7 +381,8 @@ " )\n", " result = await fabric.run(traced, base_dir=BASE_DIR, input=PROMPT)\n", " if result.status != \"succeeded\":\n", - " raise RuntimeError(f\"Deep Agents Relay run failed: {result.error}\")\n", + " detail = failure_detail(result)\n", + " raise RuntimeError(f\"Deep Agents Relay run failed: {json.dumps(detail, sort_keys=True)}\")\n", " print(\"run status :\", result.status)\n", " telemetry_providers = [t.provider for t in result.telemetry]\n", " if \"relay\" not in telemetry_providers:\n", diff --git a/tests/python/test_notebook_examples.py b/tests/python/test_notebook_examples.py index cbf489c07..0e119aa82 100644 --- a/tests/python/test_notebook_examples.py +++ b/tests/python/test_notebook_examples.py @@ -75,6 +75,8 @@ def test_variations_notebook_uses_runnable_capabilities_and_checks_relay_status( assert "${DOCS_MCP_URL}" not in source assert 'recomposed.add_skill_path("./skills/code-review")' in source assert 'if result.status != "succeeded":' in source + assert "detail = failure_detail(result)" in source + assert 'print(" error:", json.dumps(detail, indent=2))' in source assert "run_failures.append" in source assert "Harness variation failures" in source assert "returned no Relay telemetry reference" in source From 271e7baeaf82069ae95e04e789e4aa2de8c27b29 Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Thu, 23 Jul 2026 18:56:20 -0700 Subject: [PATCH 3/7] docs: address example review feedback Signed-off-by: Zhongxuan Wang --- examples/code_review_agent/README.md | 2 +- examples/notebooks/02_variations.ipynb | 2 +- examples/notebooks/README.md | 2 +- tests/integrations/test_harbor_runner.py | 1 + tests/python/test_code_review_example.py | 2 ++ tests/python/test_notebook_examples.py | 4 ++++ 6 files changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/code_review_agent/README.md b/examples/code_review_agent/README.md index 7de50e7bd..6fbe8d9d5 100644 --- a/examples/code_review_agent/README.md +++ b/examples/code_review_agent/README.md @@ -22,7 +22,7 @@ just build-all The default variant uses Hermes Agent with an NVIDIA-hosted model. Follow the [Hermes Agent adapter setup](../../adapters/hermes/README.md) with Python -3.11-3.13, then set `NVIDIA_API_KEY`. If Hermes Agent is installed in a +3.11–3.13, then set `NVIDIA_API_KEY`. If Hermes Agent is installed in a separate environment, select its interpreter only for the Hermes command: ```bash diff --git a/examples/notebooks/02_variations.ipynb b/examples/notebooks/02_variations.ipynb index 4f9fb84ae..1ecb63cb3 100644 --- a/examples/notebooks/02_variations.ipynb +++ b/examples/notebooks/02_variations.ipynb @@ -275,7 +275,7 @@ " run_failures.append(f\"{harness['name']}: {json.dumps(detail, sort_keys=True)}\")\n", " except Exception as error:\n", " print(f\" run failed ({type(error).__name__}: {oneline(error, 120)}).\")\n", - " print(f\" To run it, provide: {harness['needs']}\")\n", + " print(\" Attempted run failed; continuing with remaining variants.\")\n", " run_failures.append(f\"{harness['name']}: {type(error).__name__}: {error}\")\n", " print()" ] diff --git a/examples/notebooks/README.md b/examples/notebooks/README.md index 8f29026a4..921088cbc 100644 --- a/examples/notebooks/README.md +++ b/examples/notebooks/README.md @@ -25,7 +25,7 @@ variations notebook shows advanced composition against a real, maintained agent. harness's adapter and credentials must be present: - **Hermes Agent** (both notebooks): follow the [Hermes Agent adapter setup](../../adapters/hermes/README.md) with Python - 3.11-3.13 and set `NVIDIA_API_KEY`. The setup cells auto-detect + 3.11–3.13 and set `NVIDIA_API_KEY`. The setup cells auto-detect `.tmp/hermes-venv`. - **Deep Agents, Codex, Claude** (variations notebook): the matching adapter installed in the NeMo Fabric environment, plus that harness's credentials diff --git a/tests/integrations/test_harbor_runner.py b/tests/integrations/test_harbor_runner.py index c64c2800a..3e8f239eb 100644 --- a/tests/integrations/test_harbor_runner.py +++ b/tests/integrations/test_harbor_runner.py @@ -350,6 +350,7 @@ def test_swebench_bootstrap_pins_a_supported_relay_cli(): assert '"nemo-relay $relay_cli_version"' in prepare assert 'nemo-relay-cli --version "$relay_cli_version" --locked' in prepare assert '"$(uname -s)" != "Linux"' in prepare + assert '"$(uname -m)" != "x86_64"' in prepare def test_harbor_calculator_setup_and_solution_fail_fast(): diff --git a/tests/python/test_code_review_example.py b/tests/python/test_code_review_example.py index 9e0627745..b6cd4c734 100644 --- a/tests/python/test_code_review_example.py +++ b/tests/python/test_code_review_example.py @@ -53,6 +53,8 @@ def test_variant_builders_return_independent_complete_configs(): assert claude.models["default"].api_key_env == "ANTHROPIC_API_KEY" assert claude.mcp is None assert claude.skills is None + assert deepagents is not base + assert deepagents.harness is not base.harness assert deepagents.harness.adapter_id == "nvidia.fabric.langchain.deepagents" assert deepagents.mcp is None assert deepagents.skills is None diff --git a/tests/python/test_notebook_examples.py b/tests/python/test_notebook_examples.py index 0e119aa82..9f3a346db 100644 --- a/tests/python/test_notebook_examples.py +++ b/tests/python/test_notebook_examples.py @@ -77,7 +77,11 @@ def test_variations_notebook_uses_runnable_capabilities_and_checks_relay_status( assert 'if result.status != "succeeded":' in source assert "detail = failure_detail(result)" in source assert 'print(" error:", json.dumps(detail, indent=2))' in source + assert "Attempted run failed; continuing with remaining variants." in source assert "run_failures.append" in source assert "Harness variation failures" in source assert "returned no Relay telemetry reference" in source + assert "if not atof_paths:" in source + assert "produced no ATOF trace" in source assert "produced an empty ATOF trace" in source + assert "for line in atof_lines:\n json.loads(line)" in source From df04c9c982984bfe79145947abe2fc1e2790d580 Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Thu, 23 Jul 2026 19:28:10 -0700 Subject: [PATCH 4/7] fix: route example GitHub MCP natively Signed-off-by: Zhongxuan Wang --- README.md | 5 +- examples/code_review_agent/README.md | 21 +++-- examples/code_review_agent/__init__.py | 4 +- examples/code_review_agent/config.py | 6 +- examples/notebooks/02_variations.ipynb | 104 +++++++++++++---------- examples/notebooks/README.md | 10 +-- tests/python/test_code_review_example.py | 9 +- tests/python/test_notebook_examples.py | 68 ++++++++++++++- 8 files changed, 153 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 08b335230..574786125 100644 --- a/README.md +++ b/README.md @@ -126,9 +126,8 @@ Run the code-review example: ```bash export NVIDIA_API_KEY=... -export ADAPTER_PYTHON="$PWD/.tmp/hermes-venv/bin/python" - -.venv/bin/python -m examples.code_review_agent \ +ADAPTER_PYTHON="$PWD/.tmp/hermes-venv/bin/python" \ + .venv/bin/python -m examples.code_review_agent \ --input "Reply with exactly: fabric works" ``` diff --git a/examples/code_review_agent/README.md b/examples/code_review_agent/README.md index 6fbe8d9d5..2a85ed2aa 100644 --- a/examples/code_review_agent/README.md +++ b/examples/code_review_agent/README.md @@ -21,9 +21,9 @@ just build-all ``` The default variant uses Hermes Agent with an NVIDIA-hosted model. Follow the -[Hermes Agent adapter setup](../../adapters/hermes/README.md) with Python -3.11–3.13, then set `NVIDIA_API_KEY`. If Hermes Agent is installed in a -separate environment, select its interpreter only for the Hermes command: +[Hermes Agent quick start](../../README.md#quick-start-hermes-agent) through the +environment installation steps, which create `.tmp/hermes-venv`, then set +`NVIDIA_API_KEY`. Select that interpreter only for Hermes commands: ```bash ADAPTER_PYTHON="$PWD/.tmp/hermes-venv/bin/python" \ @@ -51,7 +51,8 @@ environment, and telemetry plan. Run one request through the default Hermes Agent variant: ```bash -.venv/bin/python -m examples.code_review_agent \ +ADAPTER_PYTHON="$PWD/.tmp/hermes-venv/bin/python" \ + .venv/bin/python -m examples.code_review_agent \ --input "Reply with exactly: fabric works" ``` @@ -65,7 +66,7 @@ The entrypoint exposes complete harness configs defined in | Variant | Command option | Additional setup | | --- | --- | --- | -| Hermes Agent | `--variant hermes` | Installed [Hermes Agent adapter requirements](../../adapters/hermes/README.md) and `NVIDIA_API_KEY`| +| Hermes Agent | `--variant hermes` | Created the environment from the [Hermes Agent quick start](../../README.md#quick-start-hermes-agent) and set `NVIDIA_API_KEY` | | Codex | `--variant codex` | Installed [Codex adapter](../../adapters/codex/README.md) and an existing ChatGPT or API key login | | Claude | `--variant claude` | Installed [Claude adapter requirements](../../adapters/claude/README.md) and `ANTHROPIC_API_KEY` | | Deep Agents | `--variant deepagents` | Installed [Deep Agents adapter requirements](../../adapters/deepagents/README.md) and `NVIDIA_API_KEY` | @@ -81,7 +82,8 @@ adapter environment. Refer to the for the current compatibility requirements. ```bash -.venv/bin/python -m examples.code_review_agent \ +ADAPTER_PYTHON="$PWD/.tmp/hermes-venv/bin/python" \ + .venv/bin/python -m examples.code_review_agent \ --variant hermes \ --relay \ --input "Review calculator.py" @@ -100,7 +102,7 @@ application-owned composition: from examples.code_review_agent import ( BASE_DIR, hermes_config, - with_fabric_managed_github_mcp, + with_github_mcp, with_opensandbox, with_relay, ) @@ -108,9 +110,10 @@ from examples.code_review_agent import ( config = hermes_config() relay_config = with_relay(config) sandbox_config = with_opensandbox(config) -github_config = with_fabric_managed_github_mcp(config) +github_config = with_github_mcp(config) ``` Each function returns a deep copy. The four configs can therefore be planned or run independently with `base_dir=BASE_DIR`. Set `GITHUB_MCP_URL` before running -`github_config`; the default smoke does not configure or contact that server. +`github_config`; it maps the server into the selected harness's native MCP +configuration. The default smoke does not configure or contact that server. diff --git a/examples/code_review_agent/__init__.py b/examples/code_review_agent/__init__.py index 8fe0a2c79..839cab6a5 100644 --- a/examples/code_review_agent/__init__.py +++ b/examples/code_review_agent/__init__.py @@ -10,7 +10,7 @@ codex_config, deepagents_config, hermes_config, - with_fabric_managed_github_mcp, + with_github_mcp, with_native_otel, with_opensandbox, with_relay, @@ -25,7 +25,7 @@ "codex_config", "deepagents_config", "hermes_config", - "with_fabric_managed_github_mcp", + "with_github_mcp", "with_native_otel", "with_opensandbox", "with_relay", diff --git a/examples/code_review_agent/config.py b/examples/code_review_agent/config.py index 7d584a84e..8c3107b4a 100644 --- a/examples/code_review_agent/config.py +++ b/examples/code_review_agent/config.py @@ -201,15 +201,15 @@ def with_opensandbox(base: FabricConfig) -> FabricConfig: return config -def with_fabric_managed_github_mcp(base: FabricConfig) -> FabricConfig: - """Return a copy that routes the GitHub MCP server through Fabric.""" +def with_github_mcp(base: FabricConfig) -> FabricConfig: + """Return a copy that maps GitHub MCP into the selected harness.""" config = base.model_copy(deep=True) config.add_mcp_server( "github", transport="streamable-http", url="${GITHUB_MCP_URL}", - exposure="fabric_managed", + exposure="harness_native", ) return config diff --git a/examples/notebooks/02_variations.ipynb b/examples/notebooks/02_variations.ipynb index 1ecb63cb3..484b9e9f1 100644 --- a/examples/notebooks/02_variations.ipynb +++ b/examples/notebooks/02_variations.ipynb @@ -219,10 +219,20 @@ "]\n", "\n", "\n", + "def adapter_python_available(value):\n", + " \"\"\"Return whether an adapter interpreter path or command is available.\"\"\"\n", + " path = Path(value)\n", + " if len(path.parts) == 1:\n", + " return shutil.which(str(path)) is not None\n", + " if not path.is_absolute():\n", + " path = BASE_DIR / path\n", + " return path.is_file()\n", + "\n", + "\n", "def blocker(harness):\n", " \"\"\"Return why a harness cannot run here, or None if it can.\"\"\"\n", " py = harness.get(\"python\")\n", - " if not py or not os.path.exists(py):\n", + " if not py or not adapter_python_available(py):\n", " return \"adapter interpreter not found\"\n", " # Codex authentication can use multiple credential stores, so its adapter\n", " # validates credentials when the run starts rather than in this helper.\n", @@ -322,7 +332,7 @@ "if github_mcp_url := os.environ.get(\"GITHUB_MCP_URL\"):\n", " recomposed.add_mcp_server(\n", " \"github\", transport=\"streamable-http\",\n", - " url=github_mcp_url, exposure=\"fabric_managed\",\n", + " url=github_mcp_url, exposure=\"harness_native\",\n", " )\n", "recomposed.models[\"default\"] = ModelConfig(\n", " provider=\"nvidia\", model=\"nvidia/nemotron-3-super-49b-a5b\", api_key_env=\"NVIDIA_API_KEY\"\n", @@ -366,52 +376,56 @@ " print(\"Install the Relay extra (nemo-fabric[relay]) in the adapter env to\")\n", " print(\"produce ATOF trace files under\", relay_dir)\n", "else:\n", - " if relay_dir.exists():\n", - " shutil.rmtree(relay_dir)\n", - " # Telemetry is just another field: clone the agent and enable Relay.\n", - " traced = for_harness(deep)\n", - " traced.enable_relay(\n", - " output_dir=str(relay_dir),\n", - " observability=RelayObservabilityConfig(\n", - " atof=RelayAtofConfig(\n", - " enabled=True, output_directory=str(relay_dir),\n", - " filename=\"events.atof.jsonl\", mode=\"overwrite\",\n", + " try:\n", + " if relay_dir.exists():\n", + " shutil.rmtree(relay_dir)\n", + " # Telemetry is just another field: clone the agent and enable Relay.\n", + " traced = for_harness(deep)\n", + " traced.enable_relay(\n", + " output_dir=str(relay_dir),\n", + " observability=RelayObservabilityConfig(\n", + " atof=RelayAtofConfig(\n", + " enabled=True, output_directory=str(relay_dir),\n", + " filename=\"events.atof.jsonl\", mode=\"overwrite\",\n", + " ),\n", " ),\n", - " ),\n", - " )\n", - " result = await fabric.run(traced, base_dir=BASE_DIR, input=PROMPT)\n", - " if result.status != \"succeeded\":\n", - " detail = failure_detail(result)\n", - " raise RuntimeError(f\"Deep Agents Relay run failed: {json.dumps(detail, sort_keys=True)}\")\n", - " print(\"run status :\", result.status)\n", - " telemetry_providers = [t.provider for t in result.telemetry]\n", - " if \"relay\" not in telemetry_providers:\n", - " raise RuntimeError(\"Deep Agents Relay run returned no Relay telemetry reference\")\n", - " print(\"telemetry refs :\", telemetry_providers)\n", - " traces = sorted(p for p in relay_dir.rglob(\"*\") if p.is_file())\n", - " atof_paths = sorted(relay_dir.rglob(\"events.atof.jsonl\"))\n", - " if not atof_paths:\n", - " raise RuntimeError(\"Deep Agents Relay run produced no ATOF trace\")\n", - " for atof_path in atof_paths:\n", - " atof_lines = [line for line in atof_path.read_text().splitlines() if line.strip()]\n", - " if not atof_lines:\n", - " raise RuntimeError(f\"Deep Agents Relay run produced an empty ATOF trace: {atof_path}\")\n", - " for line in atof_lines:\n", - " json.loads(line)\n", - " print(\"trace files :\", len(traces))\n", - " for path in traces:\n", - " print(\" \", path.relative_to(REPO_ROOT))\n", - "\n", - " # Print the full Relay trace: every ATOF event.\n", - " for path in traces:\n", - " print(f\"\\n===== {path.relative_to(REPO_ROOT)} =====\")\n", - " lines = [line for line in path.read_text().splitlines() if line.strip()]\n", - " for i, line in enumerate(lines, 1):\n", - " print(f\"\\n--- event {i}/{len(lines)} ---\")\n", - " print(json.dumps(json.loads(line), indent=2))\n", + " )\n", + " result = await fabric.run(traced, base_dir=BASE_DIR, input=PROMPT)\n", + " if result.status != \"succeeded\":\n", + " detail = failure_detail(result)\n", + " raise RuntimeError(f\"Deep Agents Relay run failed: {json.dumps(detail, sort_keys=True)}\")\n", + " print(\"run status :\", result.status)\n", + " telemetry_providers = [t.provider for t in result.telemetry]\n", + " if \"relay\" not in telemetry_providers:\n", + " raise RuntimeError(\"Deep Agents Relay run returned no Relay telemetry reference\")\n", + " print(\"telemetry refs :\", telemetry_providers)\n", + " traces = sorted(p for p in relay_dir.rglob(\"*\") if p.is_file())\n", + " atof_paths = sorted(relay_dir.rglob(\"events.atof.jsonl\"))\n", + " if not atof_paths:\n", + " raise RuntimeError(\"Deep Agents Relay run produced no ATOF trace\")\n", + " for atof_path in atof_paths:\n", + " atof_lines = [line for line in atof_path.read_text().splitlines() if line.strip()]\n", + " if not atof_lines:\n", + " raise RuntimeError(f\"Deep Agents Relay run produced an empty ATOF trace: {atof_path}\")\n", + " for line in atof_lines:\n", + " json.loads(line)\n", + " print(\"trace files :\", len(traces))\n", + " for path in traces:\n", + " print(\" \", path.relative_to(REPO_ROOT))\n", + "\n", + " # Print the full Relay trace: every ATOF event.\n", + " for path in traces:\n", + " print(f\"\\n===== {path.relative_to(REPO_ROOT)} =====\")\n", + " lines = [line for line in path.read_text().splitlines() if line.strip()]\n", + " for i, line in enumerate(lines, 1):\n", + " print(f\"\\n--- event {i}/{len(lines)} ---\")\n", + " print(json.dumps(json.loads(line), indent=2))\n", + " except Exception as error:\n", + " print(f\"Deep Agents Relay failed ({type(error).__name__}: {oneline(error, 120)}).\")\n", + " run_failures.append(f\"Deep Agents Relay: {type(error).__name__}: {error}\")\n", "\n", "if run_failures:\n", - " raise RuntimeError(\"Harness variation failures:\\n- \" + \"\\n- \".join(run_failures))" + " raise RuntimeError(\"Notebook execution failures:\\n- \" + \"\\n- \".join(run_failures))" ] }, { diff --git a/examples/notebooks/README.md b/examples/notebooks/README.md index 921088cbc..66eaa7dde 100644 --- a/examples/notebooks/README.md +++ b/examples/notebooks/README.md @@ -19,14 +19,14 @@ variations notebook shows advanced composition against a real, maintained agent. ## Prerequisites - Build the SDK and native extension from the repo root: `just build-all`. This - alone is enough to execute both notebooks top to bottom with unavailable live - harnesses skipped. + is enough to execute the setup, planning, and configuration-inspection cells. + Live runs require the prerequisites below. - To actually *run* a harness (rather than just inspect its config), that harness's adapter and credentials must be present: - **Hermes Agent** (both notebooks): follow the - [Hermes Agent adapter setup](../../adapters/hermes/README.md) with Python - 3.11–3.13 and set `NVIDIA_API_KEY`. The setup cells auto-detect - `.tmp/hermes-venv`. + [Hermes Agent quick start](../../README.md#quick-start-hermes-agent) through + the environment installation steps and set `NVIDIA_API_KEY`. The setup cells + auto-detect the resulting `.tmp/hermes-venv`. - **Deep Agents, Codex, Claude** (variations notebook): the matching adapter installed in the NeMo Fabric environment, plus that harness's credentials (`NVIDIA_API_KEY` for Deep Agents; an existing ChatGPT or provisioned API-key diff --git a/tests/python/test_code_review_example.py b/tests/python/test_code_review_example.py index b6cd4c734..991a13eac 100644 --- a/tests/python/test_code_review_example.py +++ b/tests/python/test_code_review_example.py @@ -16,7 +16,7 @@ from examples.code_review_agent import codex_config from examples.code_review_agent import deepagents_config from examples.code_review_agent import hermes_config -from examples.code_review_agent import with_fabric_managed_github_mcp +from examples.code_review_agent import with_github_mcp from examples.code_review_agent import with_native_otel from examples.code_review_agent import with_opensandbox from examples.code_review_agent import with_relay @@ -67,7 +67,7 @@ def test_variant_builders_return_independent_complete_configs(): def test_capability_and_telemetry_variants_do_not_mutate_their_input(): base = hermes_config() variants = ( - with_fabric_managed_github_mcp(base), + with_github_mcp(base), with_native_otel(base), with_opensandbox(base), with_relay(base), @@ -82,7 +82,7 @@ def test_capability_and_telemetry_variants_do_not_mutate_their_input(): assert base.mcp is None assert all(variant is not base for variant in variants) assert variants[0].mcp is not None - assert variants[0].mcp.servers["github"].exposure == "fabric_managed" + assert variants[0].mcp.servers["github"].exposure == "harness_native" assert variants[1].telemetry is not None assert "native" in variants[1].telemetry.providers assert variants[2].environment is not None @@ -104,6 +104,9 @@ def test_variants_plan_from_complete_configs(): assert plan.base_dir == BASE_DIR assert plan.agent_name == "code-review-agent" assert plan.adapter.adapter_id == config.harness.adapter_id + github_plan = client.plan(with_github_mcp(config), base_dir=BASE_DIR) + assert "github" in github_plan["capability_plan"]["native"]["mcp_servers"] + assert "mcp_servers" not in github_plan["capability_plan"]["unsupported"] def test_example_entrypoint_plans_without_starting_a_runtime(): diff --git a/tests/python/test_notebook_examples.py b/tests/python/test_notebook_examples.py index 9f3a346db..77e22ffef 100644 --- a/tests/python/test_notebook_examples.py +++ b/tests/python/test_notebook_examples.py @@ -5,10 +5,16 @@ from __future__ import annotations +import ast import json +import os +import shutil import sys from pathlib import Path +from unittest.mock import AsyncMock +from unittest.mock import MagicMock +import pytest from examples.code_review_agent import BASE_DIR, base_config from nemo_fabric import Fabric, HarnessConfig, ModelConfig @@ -28,6 +34,9 @@ def _variation_harness_definitions(): "base_config": base_config, "HarnessConfig": HarnessConfig, "ModelConfig": ModelConfig, + "os": os, + "Path": Path, + "shutil": shutil, "HERMES_PY": sys.executable, "FABRIC_PY": sys.executable, "INSTRUCTION": "Test instruction.", @@ -35,11 +44,11 @@ def _variation_harness_definitions(): } # Execute only the checked-in notebook source controlled by this repository. exec(compile(source, str(VARIATIONS_NOTEBOOK), "exec"), namespace) # noqa: S102 - return namespace["HARNESSES"], namespace["for_harness"] + return namespace["HARNESSES"], namespace["for_harness"], namespace["blocker"] def test_variations_notebook_harnesses_plan_with_current_adapters(): - harnesses, for_harness = _variation_harness_definitions() + harnesses, for_harness, _ = _variation_harness_definitions() client = Fabric() plans = { @@ -63,6 +72,54 @@ def test_variations_notebook_harnesses_plan_with_current_adapters(): assert plans["Codex"].config.runtime.input_schema == "text" +def test_variations_notebook_accepts_adapter_command_on_path(monkeypatch): + _, _, blocker = _variation_harness_definitions() + monkeypatch.setattr(shutil, "which", lambda _command: "/resolved/python3") + + assert blocker({"python": "python3"}) is None + + +async def test_variations_notebook_relay_failure_preserves_prior_failures(tmp_path): + notebook = json.loads(VARIATIONS_NOTEBOOK.read_text(encoding="utf-8")) + source = next( + "".join(cell["source"]) + for cell in notebook["cells"] + if "relay_dir =" in "".join(cell["source"]) + ) + traced = MagicMock() + fabric = MagicMock() + fabric.run = AsyncMock(side_effect=RuntimeError("relay boom")) + namespace = { + "BASE_DIR": BASE_DIR, + "HARNESSES": [{}, {"name": "Deep Agents"}], + "PROMPT": "test", + "RELAY_AVAILABLE": True, + "REPO_ROOT": tmp_path, + "RelayAtofConfig": MagicMock(), + "RelayObservabilityConfig": MagicMock(), + "blocker": lambda _harness: None, + "fabric": fabric, + "failure_detail": MagicMock(), + "for_harness": MagicMock(return_value=traced), + "json": json, + "oneline": lambda error, _limit: str(error), + "run_failures": ["Codex: prior authentication failure"], + "shutil": shutil, + } + code = compile( + source, + str(VARIATIONS_NOTEBOOK), + "exec", + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + ) + + with pytest.raises(RuntimeError) as caught: + await eval(code, namespace) # noqa: S307 + + assert "Codex: prior authentication failure" in str(caught.value) + assert "Deep Agents Relay: RuntimeError: relay boom" in str(caught.value) + + def test_variations_notebook_uses_runnable_capabilities_and_checks_relay_status(): notebook = json.loads(VARIATIONS_NOTEBOOK.read_text(encoding="utf-8")) source = "\n".join( @@ -74,14 +131,17 @@ def test_variations_notebook_uses_runnable_capabilities_and_checks_relay_status( assert "./skills/style-guide" not in source assert "${DOCS_MCP_URL}" not in source assert 'recomposed.add_skill_path("./skills/code-review")' in source + assert 'exposure="harness_native"' in source + assert 'exposure="fabric_managed"' not in source assert 'if result.status != "succeeded":' in source assert "detail = failure_detail(result)" in source assert 'print(" error:", json.dumps(detail, indent=2))' in source assert "Attempted run failed; continuing with remaining variants." in source assert "run_failures.append" in source - assert "Harness variation failures" in source + assert "Notebook execution failures" in source assert "returned no Relay telemetry reference" in source assert "if not atof_paths:" in source assert "produced no ATOF trace" in source assert "produced an empty ATOF trace" in source - assert "for line in atof_lines:\n json.loads(line)" in source + assert "for line in atof_lines:\n json.loads(line)" in source + assert 'run_failures.append(f"Deep Agents Relay:' in source From 313c4beafddfe20d1eaf243b7c01e79915d07e49 Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Thu, 23 Jul 2026 19:40:49 -0700 Subject: [PATCH 5/7] fix: limit notebook Relay dump to ATOF Signed-off-by: Zhongxuan Wang --- examples/notebooks/02_variations.ipynb | 2 +- tests/python/test_notebook_examples.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/examples/notebooks/02_variations.ipynb b/examples/notebooks/02_variations.ipynb index 484b9e9f1..9118ac207 100644 --- a/examples/notebooks/02_variations.ipynb +++ b/examples/notebooks/02_variations.ipynb @@ -414,7 +414,7 @@ " print(\" \", path.relative_to(REPO_ROOT))\n", "\n", " # Print the full Relay trace: every ATOF event.\n", - " for path in traces:\n", + " for path in atof_paths:\n", " print(f\"\\n===== {path.relative_to(REPO_ROOT)} =====\")\n", " lines = [line for line in path.read_text().splitlines() if line.strip()]\n", " for i, line in enumerate(lines, 1):\n", diff --git a/tests/python/test_notebook_examples.py b/tests/python/test_notebook_examples.py index 77e22ffef..97887345b 100644 --- a/tests/python/test_notebook_examples.py +++ b/tests/python/test_notebook_examples.py @@ -23,7 +23,7 @@ VARIATIONS_NOTEBOOK = ROOT / "examples" / "notebooks" / "02_variations.ipynb" -def _variation_harness_definitions(): +def _variation_harness_definitions(base_dir=BASE_DIR): notebook = json.loads(VARIATIONS_NOTEBOOK.read_text(encoding="utf-8")) source = next( "".join(cell["source"]) @@ -32,6 +32,7 @@ def _variation_harness_definitions(): ) namespace = { "base_config": base_config, + "BASE_DIR": base_dir, "HarnessConfig": HarnessConfig, "ModelConfig": ModelConfig, "os": os, @@ -72,11 +73,18 @@ def test_variations_notebook_harnesses_plan_with_current_adapters(): assert plans["Codex"].config.runtime.input_schema == "text" -def test_variations_notebook_accepts_adapter_command_on_path(monkeypatch): - _, _, blocker = _variation_harness_definitions() +def test_variations_notebook_accepts_adapter_commands_and_relative_paths( + monkeypatch, tmp_path +): + adapter_python = tmp_path / "adapter" / "python" + adapter_python.parent.mkdir() + adapter_python.touch() + _, _, blocker = _variation_harness_definitions(base_dir=tmp_path) monkeypatch.setattr(shutil, "which", lambda _command: "/resolved/python3") assert blocker({"python": "python3"}) is None + assert blocker({"python": "adapter/python"}) is None + assert blocker({"python": "adapter/missing"}) == "adapter interpreter not found" async def test_variations_notebook_relay_failure_preserves_prior_failures(tmp_path): @@ -144,4 +152,5 @@ def test_variations_notebook_uses_runnable_capabilities_and_checks_relay_status( assert "produced no ATOF trace" in source assert "produced an empty ATOF trace" in source assert "for line in atof_lines:\n json.loads(line)" in source + assert "# Print the full Relay trace: every ATOF event.\n for path in atof_paths:" in source assert 'run_failures.append(f"Deep Agents Relay:' in source From 0951e8d8e09e8da879d5efce80c7bcf2670adcfd Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Thu, 23 Jul 2026 19:51:15 -0700 Subject: [PATCH 6/7] fix: require runnable notebook interpreters Signed-off-by: Zhongxuan Wang --- examples/notebooks/02_variations.ipynb | 2 +- tests/python/test_notebook_examples.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/notebooks/02_variations.ipynb b/examples/notebooks/02_variations.ipynb index 9118ac207..4aea8a67c 100644 --- a/examples/notebooks/02_variations.ipynb +++ b/examples/notebooks/02_variations.ipynb @@ -226,7 +226,7 @@ " return shutil.which(str(path)) is not None\n", " if not path.is_absolute():\n", " path = BASE_DIR / path\n", - " return path.is_file()\n", + " return path.is_file() and os.access(path, os.X_OK)\n", "\n", "\n", "def blocker(harness):\n", diff --git a/tests/python/test_notebook_examples.py b/tests/python/test_notebook_examples.py index 97887345b..816bb906f 100644 --- a/tests/python/test_notebook_examples.py +++ b/tests/python/test_notebook_examples.py @@ -78,7 +78,8 @@ def test_variations_notebook_accepts_adapter_commands_and_relative_paths( ): adapter_python = tmp_path / "adapter" / "python" adapter_python.parent.mkdir() - adapter_python.touch() + adapter_python.write_text("#!/usr/bin/env python3\n", encoding="utf-8") + adapter_python.chmod(0o755) _, _, blocker = _variation_harness_definitions(base_dir=tmp_path) monkeypatch.setattr(shutil, "which", lambda _command: "/resolved/python3") @@ -138,6 +139,7 @@ def test_variations_notebook_uses_runnable_capabilities_and_checks_relay_status( assert "./skills/style-guide" not in source assert "${DOCS_MCP_URL}" not in source + assert "path.is_file() and os.access(path, os.X_OK)" in source assert 'recomposed.add_skill_path("./skills/code-review")' in source assert 'exposure="harness_native"' in source assert 'exposure="fabric_managed"' not in source From 8562304eb28eb2b5594b9194495ee184277c351e Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Fri, 24 Jul 2026 13:59:49 -0700 Subject: [PATCH 7/7] docs: address review terminology feedback Signed-off-by: Zhongxuan Wang --- README.md | 2 +- examples/README.md | 2 +- examples/code_review_agent/README.md | 18 +++++++++--------- examples/notebooks/02_variations.ipynb | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 574786125..ec1c630d7 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ Run the code-review example: export NVIDIA_API_KEY=... ADAPTER_PYTHON="$PWD/.tmp/hermes-venv/bin/python" \ .venv/bin/python -m examples.code_review_agent \ - --input "Reply with exactly: fabric works" + --input "Reply with exactly: NeMo Fabric works" ``` `ADAPTER_PYTHON` selects the interpreter used to launch any Python adapter. diff --git a/examples/README.md b/examples/README.md index f89f10f7b..105b091d5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -34,7 +34,7 @@ Complete the [code-review setup](code_review_agent/README.md#set-up), then run: ```bash just build-all .venv/bin/python -m examples.code_review_agent \ - --input "Reply with exactly: fabric works" + --input "Reply with exactly: NeMo Fabric works" ``` ## Harbor diff --git a/examples/code_review_agent/README.md b/examples/code_review_agent/README.md index 2a85ed2aa..b471ec978 100644 --- a/examples/code_review_agent/README.md +++ b/examples/code_review_agent/README.md @@ -23,17 +23,17 @@ just build-all The default variant uses Hermes Agent with an NVIDIA-hosted model. Follow the [Hermes Agent quick start](../../README.md#quick-start-hermes-agent) through the environment installation steps, which create `.tmp/hermes-venv`, then set -`NVIDIA_API_KEY`. Select that interpreter only for Hermes commands: +`NVIDIA_API_KEY`. Select that interpreter only for Hermes Agent commands: ```bash ADAPTER_PYTHON="$PWD/.tmp/hermes-venv/bin/python" \ .venv/bin/python -m examples.code_review_agent \ - --input "Reply with exactly: fabric works" + --input "Reply with exactly: NeMo Fabric works" ``` -Do not export a Hermes-only `ADAPTER_PYTHON` globally before running the Codex, -Claude, or Deep Agents variants. Those variants use the project interpreter -unless you select another interpreter explicitly. +Do not export a Hermes Agent-only `ADAPTER_PYTHON` globally before running the +Codex, Claude, or Deep Agents variants. Those variants use the project +interpreter unless you select another interpreter explicitly. ## Inspect the plan @@ -53,7 +53,7 @@ Run one request through the default Hermes Agent variant: ```bash ADAPTER_PYTHON="$PWD/.tmp/hermes-venv/bin/python" \ .venv/bin/python -m examples.code_review_agent \ - --input "Reply with exactly: fabric works" + --input "Reply with exactly: NeMo Fabric works" ``` The command prints a normalized `RunResult` and writes runtime artifacts under @@ -75,9 +75,9 @@ Add `--relay` to any variant to enable the Relay ATOF and ATIF configuration: Relay requirements depend on the selected adapter. The Codex and Claude adapters require an external `nemo-relay` CLI in the supported `0.6.x` range; -the Python package named `nemo-relay` does not install that command. Hermes -Agent and Deep Agents require the Relay Python package in their selected -adapter environment. Refer to the +the Python package named `nemo-relay` does not install the `nemo-relay` CLI +tool. Hermes Agent and Deep Agents require the Relay Python package in their +selected adapter environment. Refer to the [installation guide](../../docs/getting-started/install.mdx#install-nemo-relay) for the current compatibility requirements. diff --git a/examples/notebooks/02_variations.ipynb b/examples/notebooks/02_variations.ipynb index 4aea8a67c..871c3555c 100644 --- a/examples/notebooks/02_variations.ipynb +++ b/examples/notebooks/02_variations.ipynb @@ -114,8 +114,8 @@ "from examples.code_review_agent import base_config, BASE_DIR\n", "\n", "# A harness adapter runs in whatever Python environment has that adapter\n", - "# installed. Hermes can use a dedicated venv; Codex, Claude, and Deep Agents\n", - "# use this notebook's interpreter in this example.\n", + "# installed. Hermes Agent can use a dedicated venv; Codex, Claude, and\n", + "# Deep Agents use this notebook's interpreter in this example.\n", "FABRIC_PY = sys.executable\n", "_hermes_python = REPO_ROOT / \".tmp\" / \"hermes-venv\" / \"bin\" / \"python\"\n", "if not _hermes_python.is_file():\n",