diff --git a/.github/actions/changes/action.yaml b/.github/actions/changes/action.yaml index 80893adba3..3e681de52b 100644 --- a/.github/actions/changes/action.yaml +++ b/.github/actions/changes/action.yaml @@ -13,6 +13,9 @@ outputs: deps: description: "'true' if any dependency-related files changed" value: ${{ steps.filter.outputs.deps }} + fabric: + description: "'true' if the Fabric agent-eval runtime, its tests, or its dependency extra changed" + value: ${{ steps.filter.outputs.fabric }} e2e: description: "'true' if any e2e test files changed" value: ${{ steps.filter.outputs.e2e }} @@ -74,6 +77,12 @@ runs: - 'pyproject.toml' - 'uv.lock' - '.pre-commit-config.yaml' + fabric: + - 'packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/**' + - 'packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_*.py' + - 'packages/nemo_evaluator_sdk/pyproject.toml' + - '.github/workflows/ci.yaml' + - '.github/actions/changes/action.yaml' e2e: - 'e2e/**' docs: diff --git a/.github/wheel-constraints/nemo-platform-services.txt b/.github/wheel-constraints/nemo-platform-services.txt index 8285a5d277..a4ee79c0bc 100644 --- a/.github/wheel-constraints/nemo-platform-services.txt +++ b/.github/wheel-constraints/nemo-platform-services.txt @@ -42,8 +42,8 @@ langchain-openai==1.4.0 langchain==1.3.14 lark==1.3.1 nemo-anonymizer==0.3.1 -nemo-fabric==0.1.0a20260723 -nemo-relay==0.4.0 +nemo-fabric==0.1.0rc6 +nemo-relay==0.6.0 nemo-safe-synthesizer==0.1.7 nemoguardrails==0.23.0 ngcsdk==4.21.0 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a2cee01fb4..74880a2024 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -37,6 +37,7 @@ jobs: openapi: ${{ steps.changes.outputs.openapi }} test: ${{ steps.changes.outputs.test }} deps: ${{ steps.changes.outputs.deps }} + fabric: ${{ steps.changes.outputs.fabric }} e2e: ${{ steps.changes.outputs.e2e }} docs: ${{ steps.changes.outputs.docs }} web-studio: ${{ steps.changes.outputs.web-studio }} @@ -857,6 +858,96 @@ jobs: coverage.json coverage-html/ + # WHY THIS JOB EXISTS (it looks redundant, it isn't): `--extra fabric` appears nowhere else in this + # file and there is no `--all-extras`, so no other job installs nemo-fabric. Every other Fabric test + # either fakes `nemo_fabric` or `importorskip`s it, which means the hermetic suite keeps passing when + # the real API moves — that is how the enable_relay keyword, the `.cli`/`.sdk` adapter ids, and the + # entire profile mechanism each drifted underneath us while the unit tests stayed green. Deleting + # this job takes real-Fabric coverage to zero. The bare import is also the ONLY check on runtime.py's + # TYPE_CHECKING import block (RunResult/RunOutput/...), which ty cannot verify because nemo-fabric is + # absent from the type-check environment. + fabric-wheel-smoke: + name: Fabric wheel install smoke (Linux, py${{ matrix.python-version }}) + needs: [changes] + # A Fabric wheel bump lands in uv.lock (deps), and edits to the runtime's own Fabric call sites or + # its tests land under the fabric filter — either can surface API drift, so run on both. + if: > + !cancelled() && + (needs.changes.outputs.deps == 'true' || needs.changes.outputs.fabric == 'true') + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + # One interpreter on purpose. nemo-fabric-runtime ships a single cp311-abi3 wheel and the + # adapters are pure Python, so extra legs mostly re-check the same dependency closure; the + # 3.12-3.13 range is already asserted by uv.lock resolving universally. + # + # It must satisfy uv.lock's `requires-python` (currently >=3.12,<3.14) because this job runs + # `uv sync` on the WORKSPACE. That ceiling is the intersection across workspace members — + # nemo-rl, automodel, unsloth, deployments and experimentalist still cap at <3.14 — so it is + # narrower than the root pyproject's <3.15, and a 3.14 leg would fail at `uv sync` before + # installing anything. + python-version: ["3.12"] + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + # This job installs and imports third-party wheels; don't leave GITHUB_TOKEN in .git/config. + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + # Must satisfy the workspace floor in the root pyproject.toml (`requires-python`), or + # `uv sync` refuses the interpreter outright. + python-version: ${{ matrix.python-version }} + enable-cache: true + cache-dependency-glob: uv.lock + # Installs the published nemo-fabric wheels (+ codex/claude/deepagents adapters) from the lock. + # Linux is where jobs actually execute the runtime, and it validates that the manylinux + # nemo-fabric-runtime wheel installs on the runner's glibc. + - name: Install nemo-evaluator-sdk[fabric] from the lock + run: uv sync --frozen --package nemo-evaluator-sdk --extra fabric + # Import the Fabric SDK surface that agent_eval/runtimes/fabric/runtime.py depends on, so the + # published package is exercised for real instead of only via the hermetic fake-nemo_fabric tests. + - name: Import the Fabric SDK surface the runtime uses + run: | + uv run --frozen --no-sync python - <<'PY' + import nemo_fabric + from nemo_fabric import ( + EnvironmentConfig, + Fabric, + FabricConfig, + RelayAtifConfig, + RelayAtofConfig, + RelayAtofFileSinkConfig, + RelayObservabilityConfig, + RunRequest, + RunResult, + ) + print("nemo_fabric import OK:", nemo_fabric.__file__) + PY + # A bare import can't catch API drift in the runtime's *call sites* — the enable_relay keyword and + # the harness adapter ids both moved under us while the hermetic fakes kept passing. These contract + # tests exercise those call sites against the real wheels (importorskip elsewhere), so a future + # drift turns this job red instead of slipping through. + - name: Run Fabric runtime contract tests against the real wheels + # pytest lives in the root `dev` group, which the minimal `--package ... --extra fabric` sync + # above does not pull, so install it (and pytest-asyncio, for the SDK package's + # asyncio_mode=auto) into the synced venv rather than bloating the smoke env with the whole + # dev group. + # + # It must go INTO .venv — NOT via `uv run --with`, which layers an ephemeral overlay whose + # sys.prefix is a temp build dir. Fabric discovers adapter descriptors under + # `/share/nemo-fabric/adapters` (that is where the adapter wheels install their + # fabric-adapter.json data files), so under an overlay every adapter goes missing and any + # resolution fails with `unknown adapter ...; available adapters: []`. + run: | + uv pip install --python .venv/bin/python pytest pytest-asyncio + uv run --frozen --no-sync pytest \ + packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_surface.py -v + python-integration-test: name: Python integration tests needs: [policy-wasm] @@ -1820,6 +1911,7 @@ jobs: - policy-wasm - python-unit-test-tools - python-unit-test + - fabric-wheel-smoke # Enable if you want this required # - python-integration-test - require-nvskills diff --git a/packages/nemo_evaluator_sdk/examples/agentic_eval_with_fabric.ipynb b/packages/nemo_evaluator_sdk/examples/agentic_eval_with_fabric.ipynb index 3924055045..7414ba387e 100644 --- a/packages/nemo_evaluator_sdk/examples/agentic_eval_with_fabric.ipynb +++ b/packages/nemo_evaluator_sdk/examples/agentic_eval_with_fabric.ipynb @@ -22,7 +22,7 @@ "cell_type": "markdown", "id": "cell-03", "metadata": {}, - "source": "\n## 1. Install & prerequisites\n\nThis tutorial evaluates **Codex driven by NeMo Fabric**. Fabric is a native (Rust) component, so —\nunlike a pure-Python SDK — this notebook does **not** run from a plain `pip install`. You need a real\ntoolchain:\n\n- **NeMo Evaluator SDK** — `pip install \"nemo-platform-sdk[nemo-evaluator-sdk]\"` and `pip install pytest` (some metrics run the agent's tests).\n- **NeMo Fabric** (native) with the coding-agent + trajectory extras — `nemo-fabric[codex,relay]` —\n plus a **checkout of the NeMo-Fabric repo**: the Codex adapter registry (`adapters/codex-cli`) is\n resolved relative to it. The repo's `script/dev-install-fabric.sh` installs `nemo-fabric[codex,relay]`\n and builds the `nemo-relay` gateway for you.\n- **Codex CLI** on your `PATH` (`npm install -g @openai/codex`) + auth (`codex login`, or set `OPENAI_API_KEY`).\n- **`nemo-relay` gateway** binary on your `PATH` (built by the script above) — required for ATIF\n trajectory capture.\n\nSet two environment variables before launching (both read from the environment, never written here):\n\n- **`NEMO_FABRIC_REPO`** — path to your NeMo-Fabric checkout (Fabric resolves the adapter registry\n from it).\n- **`NVIDIA_BUILD_API_KEY`** — a build.nvidia.com key for the `write-docs` LLM judge. The cell below\n prompts for it if it isn't already set.\n\nEverything else — defining tasks and metrics, reading results — is plain Python you can read through." + "source": "\n## 1. Install & prerequisites\n\nThis tutorial evaluates **Codex driven by NeMo Fabric**. Fabric is a native (Rust) component, so —\nunlike a pure-Python SDK — this notebook does **not** run from a plain `pip install`. You need a real\ntoolchain:\n\n- **NeMo Evaluator SDK** — `pip install \"nemo-platform-sdk[nemo-evaluator-sdk]\"` and `pip install pytest` (some metrics run the agent's tests).\n- **NeMo Fabric harness adapters** — `pip install \"nemo-evaluator-sdk[fabric]\"`. Fabric's SDK is\n already a dependency of the evaluator SDK; this extra adds the Codex/Claude/deepagents adapters.\n No NeMo-Fabric checkout is needed: adapter descriptors install with the wheels and Fabric\n discovers them from the environment.\n- **Codex CLI** on your `PATH` (`npm install -g @openai/codex`) + auth (`codex login`, or set `OPENAI_API_KEY`).\n- **`nemo-relay` gateway** binary on your `PATH` — required for ATIF trajectory capture. It is the one\n piece not published to PyPI; the repo's `script/dev-install-fabric.sh` downloads it for you.\n\nSet one environment variable before launching (read from the environment, never written here):\n\n- **`NVIDIA_BUILD_API_KEY`** — a build.nvidia.com key for the `write-docs` LLM judge. The cell below\n prompts for it if it isn't already set.\n\nOptionally set **`CODEX_MODEL`** to pick the model; the Codex adapter requires one and has no default.\n\nEverything else — defining tasks and metrics, reading results — is plain Python you can read through." }, { "cell_type": "code", @@ -37,16 +37,7 @@ "# Prompt for the judge's API key unless it's already in the environment. getpass masks the input, so\n", "# the key never lands in the notebook or its saved output.\n", "if not os.environ.get(\"NVIDIA_BUILD_API_KEY\"):\n", - " os.environ[\"NVIDIA_BUILD_API_KEY\"] = getpass.getpass(\"build.nvidia.com API key (NVIDIA_BUILD_API_KEY): \")\n", - "\n", - "# Fabric resolves the Codex adapter registry relative to a NeMo-Fabric checkout. It's a path, not a\n", - "# secret, so prompt with plain input() (not getpass) and expand ~ if the environment doesn't set it.\n", - "if not os.environ.get(\"NEMO_FABRIC_REPO\"):\n", - " os.environ[\"NEMO_FABRIC_REPO\"] = os.path.expanduser(\n", - " input(\"Path to your NeMo-Fabric checkout (NEMO_FABRIC_REPO): \").strip()\n", - " )\n", - "if not os.environ[\"NEMO_FABRIC_REPO\"]:\n", - " raise RuntimeError(\"NEMO_FABRIC_REPO is required — set it to your NeMo-Fabric checkout.\")" + " os.environ[\"NVIDIA_BUILD_API_KEY\"] = getpass.getpass(\"build.nvidia.com API key (NVIDIA_BUILD_API_KEY): \")" ] }, { @@ -543,7 +534,7 @@ "cell_type": "markdown", "id": "cell-23", "metadata": {}, - "source": "\n## 5. Pick the target: Codex via NeMo Fabric\n\nThe **target** is what runs each task. We use **`FabricAgentRuntime`**: NeMo Fabric drives a harness\n(here the Codex CLI, selected by `harness.adapter_id`) in a fresh per-task workspace, and captures\nboth the final workspace and the agent's execution **trajectory**. Each task's `inputs[\"files\"]` are\nseeded into its workspace, the harness runs there, and the final file tree is exposed as `workspace`\nevidence (what the metrics above open). `capture_trajectory=True` additionally records the agent's\nstep-by-step actions as an ATIF `trace` (via the `nemo-relay` gateway).\n\nThe agent config is built from Fabric's own typed config objects (`FabricConfig`, `HarnessConfig`, …)\nrather than a raw dict, so the harness/runtime/environment fields are checked as you write them.\n`base_dir` points at your NeMo-Fabric checkout so Fabric can resolve the Codex adapter registry." + "source": "\n## 5. Pick the target: Codex via NeMo Fabric\n\nThe **target** is what runs each task. We use **`FabricAgentRuntime`**: NeMo Fabric drives a harness\n(here the Codex CLI, selected by `harness.adapter_id`) in a fresh per-task workspace, and captures\nboth the final workspace and the agent's execution **trajectory**. Each task's `inputs[\"files\"]` are\nseeded into its workspace, the harness runs there, and the final file tree is exposed as `workspace`\nevidence (what the metrics above open). `capture_trajectory=True` additionally records the agent's\nstep-by-step actions as an ATIF `trace` (via the `nemo-relay` gateway).\n\nThe agent config is built from Fabric's own typed config objects (`FabricConfig`, `HarnessConfig`, …)\nrather than a raw dict, so the harness/runtime/environment fields are checked as you write them. Fabric\nresolves the Codex adapter from the installed adapter wheels, so no checkout path has to be supplied." }, { "cell_type": "code", @@ -551,7 +542,39 @@ "id": "cell-24", "metadata": {}, "outputs": [], - "source": "from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime\nfrom nemo_fabric import ( # ty: ignore[unresolved-import]\n EnvironmentConfig,\n FabricConfig,\n HarnessConfig,\n MetadataConfig,\n RuntimeConfig,\n)\n\n# A typed Fabric agent config: Codex CLI harness, one-shot text in / message out, local execution.\ncodex_via_fabric = FabricConfig(\n metadata=MetadataConfig(name=\"coding-agent-eval\"),\n harness=HarnessConfig(\n adapter_id=\"nvidia.fabric.codex.cli\", # Fabric drives the Codex CLI under the hood\n resolution=\"preinstalled\",\n settings={\"sandbox\": \"workspace-write\", \"skip_git_repo_check\": True, \"timeout_seconds\": 180},\n ),\n runtime=RuntimeConfig(mode=\"oneshot\", transport=\"cli\", input_schema=\"text\", output_schema=\"message\"),\n environment=EnvironmentConfig(provider=\"local\"), # the per-task workspace is set by the runtime\n)\n\ntarget = FabricAgentRuntime(\n config=codex_via_fabric,\n model=os.environ.get(\"CODEX_MODEL\"), # None → the adapter's default model\n base_dir=Path(os.environ[\"NEMO_FABRIC_REPO\"]), # resolves adapters/codex-cli\n work_root=OUTPUT_DIR / \"fabric\",\n capture_trajectory=True, # capture the agent's ATIF trajectory as trace evidence\n)\nprint(\"Target:\", type(target).__name__)" + "source": [ + "from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime\n", + "from nemo_fabric import ( # ty: ignore[unresolved-import]\n", + " EnvironmentConfig,\n", + " FabricConfig,\n", + " HarnessConfig,\n", + " MetadataConfig,\n", + " RuntimeConfig,\n", + ")\n", + "\n", + "# A typed Fabric agent config: Codex CLI harness, one-shot text in / message out, local execution.\n", + "codex_via_fabric = FabricConfig(\n", + " metadata=MetadataConfig(name=\"coding-agent-eval\"),\n", + " harness=HarnessConfig(\n", + " adapter_id=\"nvidia.fabric.codex\", # Fabric drives the Codex CLI under the hood\n", + " resolution=\"preinstalled\",\n", + " settings={\"sandbox\": \"workspace-write\", \"skip_git_repo_check\": True, \"timeout_seconds\": 180},\n", + " ),\n", + " runtime=RuntimeConfig.from_mapping(\n", + " # mode/transport are not declared RuntimeConfig fields; Fabric keeps them in its extras bag.\n", + " {\"mode\": \"oneshot\", \"transport\": \"cli\", \"input_schema\": \"text\", \"output_schema\": \"message\"}\n", + " ),\n", + " environment=EnvironmentConfig(provider=\"local\"), # the per-task workspace is set by the runtime\n", + ")\n", + "\n", + "target = FabricAgentRuntime(\n", + " config=codex_via_fabric,\n", + " model=os.environ.get(\"CODEX_MODEL\", \"openai/gpt-5.4\"), # required: the codex adapter has no default model\n", + " work_root=OUTPUT_DIR / \"fabric\",\n", + " capture_trajectory=True, # capture the agent's ATIF trajectory as trace evidence\n", + ")\n", + "print(\"Target:\", type(target).__name__)" + ] }, { "cell_type": "markdown", @@ -668,7 +691,7 @@ "cell_type": "markdown", "id": "cell-35", "metadata": {}, - "source": "\n## 8. Where to go next\n\n- **Swap the harness.** Fabric selects the agent by `harness.adapter_id`. Point it at another adapter\n (e.g. `nvidia.fabric.hermes.cli`) to evaluate a different coding agent with the same tasks and\n metrics.\n- **Evaluate your own agent.** Implement the `AgentTaskRunner` protocol — a class with one async\n `run_tasks(tasks, config)` that runs each task and returns a trial (output + workspace evidence).\n Everything else in this notebook stays the same.\n- **Re-score without re-running the agent.** Pass precomputed `trials=` instead of `target=` to\n `run_sync` to apply new metrics to trials you already have.\n- **Grade against the trajectory.** A metric can open the `trace` evidence (ATIF) to score *how* the\n agent worked — tool calls, retries, steps — not just its final files.\n- **Add signals to a view, or weight them.** `SemanticReducer` also offers `ANY`, `MEAN`, and\n `WEIGHTED_MEAN` (with `ViewSignal(weight=...)`) — e.g. a partial-credit `correctness` from `pass_rate`.\n- **Keep ground truth held out.** Anything a metric grades on — tests, reference solutions, rubrics —\n belongs in `reference`, overlaid or checksummed at scoring time, never in the agent's workspace.\n- **Decide pass/fail in your app.** The evaluator reports scores; thresholds and gating belong to your\n CI/release process, not the evaluation itself." + "source": "\n## 8. Where to go next\n\n- **Swap the harness.** Fabric selects the agent by `harness.adapter_id`. Point it at another adapter\n (e.g. `nvidia.fabric.hermes`) to evaluate a different coding agent with the same tasks and\n metrics.\n- **Evaluate your own agent.** Implement the `AgentTaskRunner` protocol — a class with one async\n `run_tasks(tasks, config)` that runs each task and returns a trial (output + workspace evidence).\n Everything else in this notebook stays the same.\n- **Re-score without re-running the agent.** Pass precomputed `trials=` instead of `target=` to\n `run_sync` to apply new metrics to trials you already have.\n- **Grade against the trajectory.** A metric can open the `trace` evidence (ATIF) to score *how* the\n agent worked — tool calls, retries, steps — not just its final files.\n- **Add signals to a view, or weight them.** `SemanticReducer` also offers `ANY`, `MEAN`, and\n `WEIGHTED_MEAN` (with `ViewSignal(weight=...)`) — e.g. a partial-credit `correctness` from `pass_rate`.\n- **Keep ground truth held out.** Anything a metric grades on — tests, reference solutions, rubrics —\n belongs in `reference`, overlaid or checksummed at scoring time, never in the agent's workspace.\n- **Decide pass/fail in your app.** The evaluator reports scores; thresholds and gating belong to your\n CI/release process, not the evaluation itself." } ], "metadata": { @@ -683,4 +706,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/packages/nemo_evaluator_sdk/examples/fabric_container/run_e2e.py b/packages/nemo_evaluator_sdk/examples/fabric_container/run_e2e.py index 56d56c7971..6005d244c2 100644 --- a/packages/nemo_evaluator_sdk/examples/fabric_container/run_e2e.py +++ b/packages/nemo_evaluator_sdk/examples/fabric_container/run_e2e.py @@ -46,9 +46,11 @@ async def main() -> int: # chosen by harness.adapter_id, never inferred from the model. FabricConfig( metadata=MetadataConfig(name="hermes-eval"), - harness=HarnessConfig(adapter_id="nvidia.fabric.hermes.sdk", resolution="preinstalled"), + harness=HarnessConfig(adapter_id="nvidia.fabric.hermes", resolution="preinstalled"), models={"default": {"provider": "nvidia", "model": model}}, - runtime=RuntimeConfig(mode="oneshot", transport="library", input_schema="chat", output_schema="message"), + runtime=RuntimeConfig.from_mapping( + {"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"} + ), ), provider=DockerSandboxProvider(), secrets={"NVIDIA_API_KEY": SecretRef(root="NVIDIA_API_KEY")}, diff --git a/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py b/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py index ec4ff5d5bc..e2e47385d9 100644 --- a/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py +++ b/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py @@ -8,9 +8,9 @@ ``harness.adapter_id``, never inferred from a model. Across harnesses the shape differs mainly in that ``adapter_id``, ``runtime.transport``, and any harness-specific ``harness.settings``: -* **Codex CLI** (``nvidia.fabric.codex.cli``) runs the agent as a subprocess — ``transport="cli"`` — +* **Codex CLI** (``nvidia.fabric.codex``) runs the agent as a subprocess — ``transport="cli"`` — and takes codex-specific ``harness.settings`` (sandbox mode, git-repo check, ...). -* **Hermes SDK** (``nvidia.fabric.hermes.sdk``) runs in-library — ``transport="library"`` — and +* **Hermes SDK** (``nvidia.fabric.hermes``) runs in-library — ``transport="library"`` — and declares its ``input``/``output`` schemas instead. An optional ``model=`` slug (e.g. ``"openai/gpt-5.4"``) can be passed to ``FabricAgentRuntime`` to @@ -39,19 +39,21 @@ CODEX_CLI_CONFIG = FabricConfig( metadata=MetadataConfig(name="codex-eval"), harness=HarnessConfig( - adapter_id="nvidia.fabric.codex.cli", + adapter_id="nvidia.fabric.codex", settings={"sandbox": "read-only", "skip_git_repo_check": True}, ), models={"default": {"provider": "openai", "model": "gpt-5.4"}}, - runtime=RuntimeConfig(mode="oneshot", transport="cli"), + runtime=RuntimeConfig.from_mapping({"mode": "oneshot", "transport": "cli"}), ) #: Hermes SDK harness — in-library transport, explicit chat/message schemas. HERMES_SDK_CONFIG = FabricConfig( metadata=MetadataConfig(name="hermes-eval"), - harness=HarnessConfig(adapter_id="nvidia.fabric.hermes.sdk", resolution="preinstalled"), + harness=HarnessConfig(adapter_id="nvidia.fabric.hermes", resolution="preinstalled"), models={"default": {"provider": "nvidia", "model": "qwen2.5-coder-32b"}}, - runtime=RuntimeConfig(mode="oneshot", transport="library", input_schema="chat", output_schema="message"), + runtime=RuntimeConfig.from_mapping( + {"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"} + ), ) #: Named Fabric configs, one per harness, keyed by a short label. diff --git a/packages/nemo_evaluator_sdk/examples/skill_eval/README.md b/packages/nemo_evaluator_sdk/examples/skill_eval/README.md index defe07f22c..92b0247ec3 100644 --- a/packages/nemo_evaluator_sdk/examples/skill_eval/README.md +++ b/packages/nemo_evaluator_sdk/examples/skill_eval/README.md @@ -43,8 +43,9 @@ won't have the workspace on its import path): - `make bootstrap-python` — creates `.venv` and `uv sync --all-packages`, which installs the workspace packages (including `nemo_evaluator_sdk`); -- `script/dev-install-fabric.sh` — the native `nemo-fabric` + Hermes SDK adapter + - `nemo-relay` gateway (not in the lockfile, so installed separately); +- `uv sync --extra fabric` — the `nemo-fabric` SDK, its adapters, and the `nemo-relay` + Python bindings, all from the lock; +- `script/dev-install-fabric.sh` — the `nemo-relay` gateway binary, which is not on PyPI; - `NVIDIA_API_KEY` for an account **provisioned for** `MODEL` in `run_skill_eval.py`. Then, from the repo root, run with the venv interpreter: @@ -68,7 +69,7 @@ block and exits non-zero rather than showing an empty-but-tidy table. Example output (with `nvidia/nemotron-3-super-120b-a12b`): ```text -Harness: nvidia.fabric.hermes.sdk model: nvidia/nemotron-3-super-120b-a12b tasks: 2 +Harness: nvidia.fabric.hermes model: nvidia/nemotron-3-super-120b-a12b tasks: 2 runs: baseline (baseline) vs treated (treated) metric.output baseline with-skill diff --git a/packages/nemo_evaluator_sdk/examples/skill_eval/run_skill_eval.py b/packages/nemo_evaluator_sdk/examples/skill_eval/run_skill_eval.py index bcc7291ac5..0693efe21e 100644 --- a/packages/nemo_evaluator_sdk/examples/skill_eval/run_skill_eval.py +++ b/packages/nemo_evaluator_sdk/examples/skill_eval/run_skill_eval.py @@ -242,7 +242,7 @@ async def _main() -> int: fabric_config = { "metadata": {"name": "skill-eval-hermes"}, "harness": { - "adapter_id": "nvidia.fabric.hermes.sdk", + "adapter_id": "nvidia.fabric.hermes", "resolution": "preinstalled", "settings": {"max_iterations": 50}, }, diff --git a/packages/nemo_evaluator_sdk/pyproject.toml b/packages/nemo_evaluator_sdk/pyproject.toml index 4bcff065d2..b1aed04a93 100644 --- a/packages/nemo_evaluator_sdk/pyproject.toml +++ b/packages/nemo_evaluator_sdk/pyproject.toml @@ -30,7 +30,13 @@ dependencies = [ "ragas==0.4.3", "langchain-openai>=1.3.5", "langchain-nvidia-ai-endpoints>=1.4.3,<2.0.0", - "nemo-relay>=0.4.0,<0.5.0", + "nemo-relay>=0.6.0,<0.7", + # Fabric's typed config surface (FabricConfig/RuntimeConfig/ModelConfig/the relay models) — what the + # agent-eval runtimes compose against. The metapackage alone is ~2 MB / 3 packages; it requires + # nemo-fabric-runtime unconditionally as of rc4, which is why the `runtime` extra it used to expose + # is gone. The harness adapters are the weight (codex ~299 MB, claude ~231 MB) and stay in the + # `fabric` extra below, so only *resolving and running* a harness pulls them. + "nemo-fabric>=0.1.0rc6,<0.2.0", ] version = "0.0.0" @@ -54,6 +60,41 @@ harbor = [ nemo-platform = [ "nemo-platform-sdk", ] +# NeMo Fabric HARNESS ADAPTERS for the agent-eval runtimes (FabricAgentRuntime / +# FabricContainerRuntime). The Fabric SDK itself is a base dependency above; this extra adds the +# adapters needed to resolve and run a harness. +# +# WHY THE ADAPTERS ARE SPLIT OUT: they are the entire weight. Measured against the base SDK, +# `codex` adds ~299 MB and `claude` ~231 MB — both ship an agent CLI binary — while `deepagents` +# adds ~15 MB but ~50 packages of langchain. The `runtime` extra that carries the typed config +# surface is ~2 MB. Someone installing nemo-evaluator-sdk for LLM-as-judge or RAG metrics should +# not pay half a gigabyte for coding agents they never invoke. +# +# Notes: +# * Without this extra the SDK still imports, composes a Fabric config, and type-checks against +# Fabric's real models; only `Fabric.plan`/`run` fail, with `unknown adapter ...; available +# adapters: []`. Adapter descriptors install as wheel data under +# /share/nemo-fabric/adapters, which is where Fabric discovers them — so an +# ephemeral overlay env (`uv run --with ...`) hides them; install into the venv instead. +# * Fabric has not cut a final 0.1.0 yet, so the explicit prerelease in the specifier scopes uv's +# prerelease allowance to just this package (workspace prerelease = "if-necessary-or-explicit"). +# The floor stays at the oldest rc whose API this runtime targets; the lock tracks the newest. +# * claude/codex go through the metapackage extras, which pin `nemo-fabric-adapters-*==`. +# That keeps the adapters and the SDK we type against on one version by construction — they were +# pinned directly through rc5 only because rc4's extras required `nemo-fabric-adapters-*==0.1.0`, +# a version that was never published. Fixed in rc6. +# * hermes stays direct because it must omit `[harness]`, and the metapackage's `hermes-agent` +# extra applies it. `[harness]` pulls `hermes-agent`, which pins `requests==2.33.0` exactly and +# conflicts with the workspace's `requests>=2.33.1` floor; the adapter alone is what the runtime +# needs. It cannot drift from the others regardless: every adapter pins +# `nemo-fabric-adapters-common==`, so one shared common forces one shared rc. +# Python<3.14-gated upstream, which the lock's own <3.14 ceiling already satisfies. +# * deepagents is omitted to match plugins/nemo-agents (AIRCORE-952: its adapter does not support +# the Relay observability v2 config Fabric streaming generates). Nothing here exercises it. +fabric = [ + "nemo-fabric[claude,codex]>=0.1.0rc6,<0.2.0", + "nemo-fabric-adapters-hermes>=0.1.0rc6,<0.2.0; python_version < '3.14'", +] [build-system] requires = ["hatchling"] diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/_common.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/_common.py index 4ed7e12e24..909a05151c 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/_common.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/_common.py @@ -23,13 +23,16 @@ from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor -from nemo_relay.observability import AtifConfig, AtofConfig, ComponentSpec, ObservabilityConfig # Trajectory profile identity + the file-exporter output names we choose (Relay accepts these as # inputs). Shared so both runtimes select/emit the trajectory under identical names. TRAJECTORY_PROFILE_NAME = "eval_trajectory" ATIF_FILENAME_TEMPLATE = "trajectory-{session_id}.atif.json" ATOF_FILENAME = "events.atof.jsonl" +#: ATIF ``agent.version``. Both runtimes report the agent *framework* here so a consumer can group +#: host and container traces together; ``agent.name`` is what distinguishes them. Not a real version +#: yet — reporting the resolved nemo-fabric version would be the better answer. +FABRIC_AGENT_VERSION = "fabric" # Fabric telemetry-profile selectors (Relay file exporter, no OTLP endpoint). TELEMETRY_PROVIDER = "relay" TELEMETRY_MODE = "sdk" @@ -112,7 +115,19 @@ def trajectory_telemetry(*, relay_dir: str, agent_name: str, agent_version: str) Built from ``nemo_relay``'s own typed config so Relay owns its schema — no hand-maintained dict to silently drift when Relay changes it. Callers wrap this in a profile with their own name + ``runtime``/``environment`` blocks; ``relay_dir`` is where the ``trajectory-*.atif.json`` lands. + + ``nemo_relay`` is imported here rather than at module scope: it is a native extension costing + ~120ms to load, and this module is reachable from the evaluator plugin's job imports, so an + eager import would charge every consumer for trajectory capture they may never use. """ + from nemo_relay.observability import ( + AtifConfig, + AtofConfig, + AtofFileSinkConfig, + ComponentSpec, + ObservabilityConfig, + ) + observability = ComponentSpec( config=ObservabilityConfig( atif=AtifConfig( @@ -124,9 +139,13 @@ def trajectory_telemetry(*, relay_dir: str, agent_name: str, agent_version: str) ), atof=AtofConfig( enabled=True, - output_directory=relay_dir, - filename=ATOF_FILENAME, - mode="overwrite", + sinks=[ + AtofFileSinkConfig( + output_directory=relay_dir, + filename=ATOF_FILENAME, + mode="overwrite", + ) + ], ), ) ) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py index 92cb5f0274..18ba73c905 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py @@ -11,7 +11,7 @@ Per task it: -1. seeds ``/in`` with the Fabric agent config, profiles, and framed input, plus the task's workspace +1. seeds ``/in`` with the composed Fabric agent config and framed input, plus the task's workspace seed files; 2. execs Fabric's own CLI (``fabric run``), which writes a normalized ``RunResult`` to stdout and the workspace + Relay ATIF trajectory under a fixed ``/out`` layout; @@ -74,7 +74,7 @@ if TYPE_CHECKING: # nemo_fabric is an optional native dep (see FabricAgentRuntime); imported for typing only. Configs # are consumed structurally via ``to_mapping()`` at runtime, so this module stays importable without it. - from nemo_fabric import FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import] + from nemo_fabric import FabricConfig # ty: ignore[unresolved-import] # Default per-task exec budget. Timeout is really task-specific (see AALGO-323 to move it onto # AgentEvalTask); until then it is an internal default rather than a runtime-construction knob. @@ -86,7 +86,7 @@ "only inside the sandbox." ) -# Fixed in-container layout. The runtime seeds ``/in`` (agent config, profiles, input), execs Fabric's +# Fixed in-container layout. The runtime seeds ``/in`` (agent config, input), execs Fabric's # CLI, and reads the produced ``/out`` subtree back across the boundary. _IN_DIR = "/in" _OUT_DIR = "/out" @@ -98,7 +98,6 @@ _FABRIC_STDERR = f"{_LOGS_DIR}/fabric-stderr.txt" _AGENT_PATH = f"{_IN_DIR}/agent.yaml" _INPUT_PATH = f"{_IN_DIR}/input.txt" -_WORKSPACE_PROFILE_NAME = "eval_workspace" # In-sandbox root for a natively-injected skill bundle. It lives under ``/in`` (not ``/out``), so it is # never part of the downloaded ``/out`` evidence — only codex-mode skills, which must sit in the workspace # for the harness to self-discover them, need post-download cleanup. @@ -116,7 +115,6 @@ def __init__( config: FabricConfig | Mapping[str, Any], *, provider: SandboxProvider, - profiles: Sequence[FabricProfileConfig | Mapping[str, Any]] = (), secrets: Mapping[str, SecretRef] = {}, image: str | None = None, skills: Sequence[AgentSkill] | None = None, @@ -124,7 +122,6 @@ def __init__( # The Fabric agent is fully described by its ``FabricConfig`` (harness + model + runtime); it is # consumed structurally as a mapping to cross the sandbox boundary as JSON. self._config = _to_mapping(config) - self._profiles = [_to_mapping(profile) for profile in profiles] self._provider = provider # ``secrets`` maps the env-var name a Fabric harness reads its credential from (declared by the # adapter's ``requirements.env``) to a SecretRef. The runner only *declares* them; the resolver @@ -233,14 +230,14 @@ async def _run_task( # trial rather than aborting the gathered batch. skill_provenances: list[SkillProvenance] = [] try: - seed_files, profile_paths, skill_provenances = self._seed_files(task, skill_mode) + seed_files, skill_provenances = self._seed_files(task, skill_mode) spec = SandboxSpec( image=self._image, workdir=_WORKSPACE_DIR, env=dict(self._resolved_env), files=seed_files ) async with AsyncSandbox(self._provider, spec) as sandbox: await sandbox.start() await self._seed_workspace(sandbox, task) - result = await sandbox.exec(self._fabric_command(profile_paths), timeout_s=DEFAULT_FABRIC_TIMEOUT_S) + result = await sandbox.exec(self._fabric_command(), timeout_s=DEFAULT_FABRIC_TIMEOUT_S) await sandbox.download_dir(_OUT_DIR, out_dir) # Codex self-injection seeds each bundle inside the workspace so the harness discovers it during # the run; drop them from the downloaded evidence before the workspace is exposed (else the @@ -266,14 +263,12 @@ def _resolve_skill_mode(self) -> SkillMode | None: is imported lazily on the host (only when a skill is set), so the no-skill path never needs it. """ try: - from nemo_fabric import Fabric, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import] + from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] except ImportError as exc: raise RuntimeError(_MISSING_FABRIC_MSG) from exc - agent_config = FabricConfig.from_mapping(self._config) - base_profiles = [FabricProfileConfig.from_mapping(profile) for profile in self._profiles] - probe_config = agent_config.model_copy(deep=True) + probe_config = FabricConfig.from_mapping(self._config) probe_config.add_skill_path(_SKILL_PROBE_PATH) - plan = Fabric().plan(probe_config, profiles=base_profiles) + plan = Fabric().plan(probe_config) return resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness) def _adapter_id(self) -> str: @@ -282,26 +277,9 @@ def _adapter_id(self) -> str: adapter_id = harness.get("adapter_id") if isinstance(harness, Mapping) else None return str(adapter_id) if adapter_id is not None else "" - def _existing_skill_paths(self) -> list[str]: - """Skill paths the base config/profiles already declare (union, order-preserved). - - Fabric applies profile ``skills.paths`` last-wins, so the native overlay has to re-list these - alongside the evaluated skill or the treated arm would silently drop preconfigured skills (see - ``stage_skills_seed``). Read from the raw config/profile mappings the runtime was given. - """ - paths: list[str] = [] - for section in (self._config, *self._profiles): - skills = section.get("skills") if isinstance(section, Mapping) else None - declared = skills.get("paths") if isinstance(skills, Mapping) else None - for path in declared or []: - if isinstance(path, str) and path not in paths: - paths.append(path) - return paths - - def _fabric_command(self, profile_paths: Sequence[str]) -> str: + def _fabric_command(self) -> str: """The ``fabric run`` invocation: pre-create the /out dirs Fabric chdirs into, run, capture stdout.""" - profiles = " ".join(f"--profile {shlex.quote(path)}" for path in profile_paths) - run = f"fabric run {shlex.quote(_AGENT_PATH)} {profiles} --input-file {shlex.quote(_INPUT_PATH)}" + run = f"fabric run {shlex.quote(_AGENT_PATH)} --input-file {shlex.quote(_INPUT_PATH)}" return ( f"mkdir -p {_WORKSPACE_DIR} {_RELAY_DIR} {_ARTIFACTS_DIR} {_LOGS_DIR} && " f"{run} > {shlex.quote(_RESULT_PATH)} 2> {shlex.quote(_FABRIC_STDERR)}" @@ -309,22 +287,19 @@ def _fabric_command(self, profile_paths: Sequence[str]) -> str: def _seed_files( self, task: AgentEvalTask, skill_mode: SkillMode | None - ) -> tuple[dict[str, str], list[str], list[SkillProvenance]]: - """Return (files to seed into the sandbox, profile paths for --profile, skill provenances). - - Configs are written as JSON, which the Fabric CLI parses as YAML. When skills are injected each - bundle is rendered into the seed set at the harness's in-sandbox discovery path (native: - ``/in/skills/``; codex: ``/.agents/skills/``), with at most ONE merged native - overlay listing every bundle. Profiles are ordered caller-first, then the native skill overlay (if - any), then the per-task workspace + trajectory overlays — which trail so the evaluator-owned - workspace/artifacts stay authoritative (mirroring the host runtime's overlay ordering). + ) -> tuple[dict[str, str], list[SkillProvenance]]: + """Return (files to seed into the sandbox, skill provenances). + + The agent config is written as JSON, which the Fabric CLI parses as YAML. Fabric 0.1.0rc2 removed + profile overlays (``--profile`` and the ``profiles`` config key are both gone), so everything — + the runtime's in-container settings and any natively-injected skill paths — is composed into the + single agent config here. When skills are injected each bundle is also rendered into the seed set + at the harness's in-sandbox discovery path (native: ``/in/skills/``; codex: + ``/.agents/skills/``). """ - files: dict[str, str] = { - _AGENT_PATH: json.dumps(self._config), - _INPUT_PATH: task.agent_prompt(), - } - skill_profiles: list[dict[str, Any]] = [] + skill_paths: list[str] = [] provenances: list[SkillProvenance] = [] + files: dict[str, str] = {_INPUT_PATH: task.agent_prompt()} if self._skill_set.skills and skill_mode is not None: if skill_mode == SKILL_MODE_CODEX_SKILLS_DIR: _check_codex_skill_collision(self._skill_set.skills, task.inputs.get(SEED_FILES_INPUT_KEY) or {}) @@ -334,38 +309,53 @@ def _seed_files( mode=skill_mode, workspace_dir=_WORKSPACE_DIR, skills_dir=_SKILLS_DIR, - existing_skill_paths=self._existing_skill_paths(), ) files.update(seed.files) - skill_profiles = seed.profiles + skill_paths = seed.skill_paths provenances = seed.provenances - profile_paths: list[str] = [] - profiles = [*self._profiles, *skill_profiles, self._workspace_profile(), self._trajectory_profile()] - for index, profile in enumerate(profiles): - path = f"{_IN_DIR}/profile-{index}.yaml" - files[path] = json.dumps(profile) - profile_paths.append(path) - return files, profile_paths, provenances - - @staticmethod - def _workspace_profile() -> dict[str, Any]: - # Pin the harness working directory to the retrievable workspace; ``provider`` is required by the - # native planner (it does not inject the Python default into a raw overlay). - return {"name": _WORKSPACE_PROFILE_NAME, "environment": {"provider": "local", "workspace": _WORKSPACE_DIR}} - - @staticmethod - def _trajectory_profile() -> dict[str, Any]: - # Relay ATIF/ATOF file exporter (sdk mode). The telemetry block is built from nemo_relay's typed - # config via the shared helper (single source of truth with the host runtime); ``provider:local`` - # is required by the native planner in the container (it does not inject the Python default). - return { - "name": _common.TRAJECTORY_PROFILE_NAME, - "runtime": {"artifacts": _ARTIFACTS_DIR}, - "environment": {"provider": "local", "artifacts": _ARTIFACTS_DIR}, - "telemetry": _common.trajectory_telemetry( - relay_dir=_RELAY_DIR, agent_name=_RUNTIME_NAME, agent_version=_RUNTIME_NAME - ), + files[_AGENT_PATH] = json.dumps(self._composed_config(skill_paths)) + return files, provenances + + def _composed_config(self, skill_paths: Sequence[str] = ()) -> dict[str, Any]: + """The supplied agent config with the runtime's in-container settings merged on last. + + Mirrors the host runtime's ``_compose_config``: the workspace, artifact roots, trajectory + telemetry, and any natively-injected skill paths are evaluator-owned, so they are applied over + whatever the caller's config declared. Stays plain dicts rather than round-tripping through the + host's ``FabricConfig`` — the sandbox may run a different Fabric build, so the config is only + required to survive JSON transport, not to validate against the host's schema. + + Injected skill paths are APPENDED to the config's own ``skills.paths`` — mirroring + ``FabricConfig.add_skill_path`` — so skills the caller preconfigured survive injection and the + treated A/B arm differs from the baseline by exactly the injected skills. + """ + config = dict(self._config) + + # Each section is spread over the caller's, so sibling keys survive — pinning + # ``runtime.artifacts`` must not drop a configured ``runtime.transport``. + config["runtime"] = {**_section(config, "runtime"), "artifacts": _ARTIFACTS_DIR} + # ``provider: local`` is required by the native planner in the container (it does not inject the + # Python default), and the workspace pins the harness cwd to the retrievable /out subtree. + config["environment"] = { + **_section(config, "environment"), + "provider": "local", + "workspace": _WORKSPACE_DIR, + "artifacts": _ARTIFACTS_DIR, } + # Relay ATIF/ATOF file exporter (sdk mode), built from nemo_relay's typed config via the shared + # helper so it stays a single source of truth with the host runtime. Replaced wholesale. + # ``agent_name`` distinguishes this runtime from the host one; ``agent_version`` records the + # agent framework and so matches the host's value, letting an ATIF consumer group both + # runtimes' traces. (Neither is a real version yet — see _common.trajectory_telemetry.) + config["telemetry"] = _common.trajectory_telemetry( + relay_dir=_RELAY_DIR, agent_name=_RUNTIME_NAME, agent_version=_common.FABRIC_AGENT_VERSION + ) + + declared_paths = _section(config, "skills").get("paths") or [] + merged_paths = list(dict.fromkeys([*(str(path) for path in declared_paths), *skill_paths])) + if merged_paths: + config["skills"] = {**_section(config, "skills"), "paths": merged_paths} + return config async def _seed_workspace(self, sandbox: AsyncSandbox, task: AgentEvalTask) -> None: seeds = task.inputs.get(SEED_FILES_INPUT_KEY) @@ -484,8 +474,8 @@ def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunCon def _to_mapping(config: FabricConfig | Mapping[str, Any]) -> dict[str, Any]: - """Normalize a typed Fabric config/profile or a plain mapping to a plain dict for JSON transport.""" - # A typed Fabric config/profile exposes ``to_mapping()``; a plain mapping is used as-is. Both are + """Normalize a typed Fabric config or a plain mapping to a plain dict for JSON transport.""" + # A typed Fabric config exposes ``to_mapping()``; a plain mapping is used as-is. Both are # str-keyed at runtime, but the getattr + optional (unresolved) ``FabricConfig`` type defeat static # narrowing, so cast the known-good source before building the dict. to_mapping = getattr(config, "to_mapping", None) @@ -554,6 +544,12 @@ def _remove_injected_bundle(workspace_dir: Path, location: str) -> None: parent = parent.parent +def _section(config: Mapping[str, Any], name: str) -> dict[str, Any]: + """A top-level config section as a plain dict — ``{}`` when absent or not a mapping.""" + value = config.get(name) + return dict(value) if isinstance(value, Mapping) else {} + + def _find_atif(relay_dir: Path) -> Path | None: # Relay nests the trajectory under a per-run subdir (relay/runtime-/trajectory-*.atif.json), # so search recursively rather than only relay's direct children. diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/image.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/image.py index 3beca8fa49..6d218c8589 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/image.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/image.py @@ -25,7 +25,7 @@ That only works on NeMo-Fabric's ``installed-adapter-discovery`` branch (which bundles the adapters under ``python/src/nemo_fabric/adapters`` and adds ``AdapterDescriptorSource::Installed``). On today's ``main`` the wheel ships no adapter descriptors, so a wheel-only image cannot resolve e.g. -``nvidia.fabric.hermes.sdk``. Once that lands on ``main``, switch to installing the top-level +``nvidia.fabric.hermes``. Once that lands on ``main``, switch to installing the top-level ``adapters/*`` packages explicitly here instead of relying on the branch's packaging. """ diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py index eb194b93e8..34d56b7c7a 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py @@ -11,7 +11,10 @@ Per-task settings (workspace, model, trajectory capture) are composed directly onto a copy of the supplied config via the SDK's config helpers (``model_copy`` + -``enable_relay`` + ``environment``), rather than layered as profile overlays. +``enable_relay`` + ``environment``). Fabric removed profile overlays in 0.1.0rc2 — +``FabricConfig`` rejects a ``profiles`` key and ``Fabric.run`` takes no ``profiles`` +argument — so a run is described by exactly one complete typed config, and the +evaluator-owned per-task settings are authoritative simply by being applied last. Every task runs in its own fresh workspace: the runtime seeds it from ``inputs['files']`` (a no-op when there are none), runs the harness in it (via @@ -37,6 +40,7 @@ from typing import TYPE_CHECKING, Any from uuid import uuid4 +from nemo_evaluator_sdk.agent_eval.runtimes.fabric import _common from nemo_evaluator_sdk.agent_eval.runtimes.fabric.skills import ( SKILL_MODE_CODEX_SKILLS_DIR, AgentSkill, @@ -65,7 +69,7 @@ from nemo_fabric import ( # ty: ignore[unresolved-import] Fabric, FabricConfig, - FabricProfileConfig, + RelayObservabilityConfig, RunOutput, RunResult, ) @@ -85,9 +89,9 @@ # Per-task workspace: where seed files are staged and where the harness reads/writes. We # create it, point Fabric's ``environment.workspace`` at it, and expose it as ``workspace`` evidence. _WORKSPACE_SUBDIR = "workspace" -# Per-task skill staging dir (native injection): the skill's files are resolved here and a per-task -# ``skills`` profile overlay points Fabric at it. For codex self-injection the skill lands in the -# workspace instead (no overlay). +# Per-task skill staging dir (native injection): the skill's files are resolved here and the staged +# root is added to the task config's ``skills.paths``. For codex self-injection the skill lands in the +# workspace instead (no path added). _SKILL_SUBDIR = "skill" # Sentinel skill path attached only to probe Fabric's capability planner for the selected adapter's # skills routing (see ``_resolve_skill_mode``). Never staged and need not exist on disk — the planner @@ -102,11 +106,6 @@ _ATOF_FILENAME = "events.atof.jsonl" # ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence. _ATIF_ARTIFACT_KIND = "atif" -# Names for the trailing overlays that re-assert the evaluator-owned per-task settings (see -# ``_eval_lock_profiles``): Fabric applies caller profiles over the config, so these must trail them. -_WORKSPACE_PROFILE_NAME = "eval_workspace" -_MODEL_PROFILE_NAME = "eval_model" -_ARTIFACTS_PROFILE_NAME = "eval_artifacts" class FabricAgentRuntime: @@ -123,7 +122,6 @@ def __init__( self, *, config: Mapping[str, Any], - profiles: Sequence[Mapping[str, Any]] | None = None, model: str | None = None, base_dir: str | Path | None = None, work_root: str | Path | None = None, @@ -133,7 +131,6 @@ def __init__( skills: Sequence[AgentSkill] | None = None, ) -> None: self._config = config - self._profiles = list(profiles or []) self._model = model self._base_dir = Path(base_dir).expanduser() if base_dir is not None else None self._work_root = Path(work_root).expanduser() if work_root is not None else None @@ -171,7 +168,7 @@ async def run_tasks( try: # nemo_fabric ships a native (pyo3) core and is an optional dependency, so it is imported # lazily here rather than at module load. - from nemo_fabric import Fabric, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import] + from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] except ImportError as exc: raise RuntimeError(_MISSING_FABRIC_MSG) from exc @@ -189,11 +186,6 @@ async def run_tasks( import nemo_relay.observability # noqa: F401 # ty: ignore[unresolved-import] except ImportError as exc: raise RuntimeError(_MISSING_RELAY_MSG) from exc - # Caller-supplied profile overlays pass through as-is; this runtime's per-task workspace, model, - # and trajectory settings are composed directly onto a copy of the config (config-first), not - # layered as profiles. - base_profiles = [FabricProfileConfig.from_mapping(profile) for profile in self._profiles] - # ``Fabric`` (formerly ``FabricClient``) is a lightweight, reusable facade — not a lifecycle # context manager — so it is created once and reused across tasks with no cleanup. client = Fabric() @@ -205,7 +197,7 @@ async def run_tasks( # A/B comparison. Only touched when a skill is set, so the no-skill path is unaffected. skill_mode: SkillMode | None = None if self._skill_set.skills: - skill_mode = self._resolve_skill_mode(client, agent_config, base_profiles) + skill_mode = self._resolve_skill_mode(client, agent_config) if skill_mode is None: adapter_id = agent_config.harness.adapter_id raise RuntimeError( @@ -218,18 +210,11 @@ async def run_tasks( async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: async with semaphore: - return await self._run_task( - client, agent_config, base_profiles, index, task, resolved_config, skill_mode - ) + return await self._run_task(client, agent_config, index, task, resolved_config, skill_mode) return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - def _resolve_skill_mode( - self, - client: Fabric, - agent_config: FabricConfig, - base_profiles: list[FabricProfileConfig], - ) -> SkillMode | None: + def _resolve_skill_mode(self, client: Fabric, agent_config: FabricConfig) -> SkillMode | None: """Ask Fabric how a skill would reach the selected harness, or ``None`` if it can't. Probes Fabric's capability planner: plan a copy of the config with a sentinel skill path attached @@ -239,31 +224,13 @@ def _resolve_skill_mode( """ probe_config = agent_config.model_copy(deep=True) probe_config.add_skill_path(_SKILL_PROBE_PATH) - plan = client.plan(probe_config, profiles=base_profiles, base_dir=self._base_dir) + plan = client.plan(probe_config, base_dir=self._base_dir) return resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness) - def _existing_skill_paths(self) -> list[str]: - """Skill paths the base config/profiles already declare (union, order-preserved). - - Fabric applies profile ``skills.paths`` last-wins, so the native skill overlay has to re-list - these alongside the evaluated skill or the treated arm would silently drop them (see - ``install_skill``). Read from the raw config/profile mappings the runtime was given, so it covers - both config- and profile-declared skills without a Fabric round-trip. - """ - paths: list[str] = [] - for section in (self._config, *self._profiles): - skills = section.get("skills") if isinstance(section, Mapping) else None - declared = skills.get("paths") if isinstance(skills, Mapping) else None - for path in declared or []: - if isinstance(path, str) and path not in paths: - paths.append(path) - return paths - async def _run_task( self, client: Fabric, agent_config: FabricConfig, - base_profiles: list[FabricProfileConfig], index: int, task: AgentEvalTask, config: AgentEvalRunConfig, @@ -271,7 +238,7 @@ async def _run_task( ) -> AgentEvalTrial: # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules # lookup, not a re-load, so the types are used where they're constructed instead of threaded down. - from nemo_fabric import FabricProfileConfig, RunRequest # ty: ignore[unresolved-import] + from nemo_fabric import RunRequest # ty: ignore[unresolved-import] evidence_dir = self._evidence_dir(index, task, config) evidence_dir.mkdir(parents=True, exist_ok=True) @@ -290,11 +257,11 @@ async def _run_task( # instruction only, so the returned paths are unused. await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) - # Inject the skill set (if any) for this task. A native harness gets ONE ``skills`` profile - # overlay listing every staged bundle; codex self-injection stages each bundle into the - # workspace and emits no overlay. One provenance per skill is stamped on the trial for the A/B + # Inject the skill set (if any) for this task. A native harness gets each staged bundle added + # to the config's ``skills.paths``; codex self-injection stages each bundle into the + # workspace and adds no path. One provenance per skill is stamped on the trial for the A/B # diff. Blocking file I/O, off the event loop. - skill_profiles: list[FabricProfileConfig] = [] + skill_paths: list[str] = [] if self._skill_set.skills and skill_mode is not None: installation = await asyncio.to_thread( install_skills, @@ -303,25 +270,21 @@ async def _run_task( mode=skill_mode, workspace_dir=workspace_dir, skill_stage_dir=(evidence_dir / _SKILL_SUBDIR).resolve(), - existing_skill_paths=self._existing_skill_paths(), ) skill_provenances = installation.provenances - skill_profiles = [FabricProfileConfig.from_mapping(p) for p in installation.profiles] + skill_paths = installation.skill_paths + # Everything the run needs lives in one typed config: Fabric no longer layers profile + # overlays, so the per-task workspace/model/trajectory settings are composed on last and are + # authoritative by construction. ``add_skill_path`` appends, so config-declared skills survive. task_config = self._compose_config(agent_config, evidence_dir, workspace_dir) - # Caller ``base_profiles`` are applied by Fabric over the config; the evaluator-owned - # settings are re-asserted as trailing overlays so they win over any caller profile. - lock_profiles = self._eval_lock_profiles( - FabricProfileConfig, workspace_dir=workspace_dir, evidence_dir=evidence_dir - ) + for skill_path in skill_paths: + task_config.add_skill_path(skill_path) result = await asyncio.wait_for( # ``Fabric.run`` folds the per-invocation input + request id into a ``RunRequest``. client.run( task_config, - # Caller profiles, then the native skill overlay, then the evaluator lock overlays; - # the lock overlays trail so the per-task workspace/model/artifacts stay authoritative. - profiles=[*base_profiles, *skill_profiles, *lock_profiles], base_dir=self._base_dir, request=RunRequest(input=task.agent_prompt(), request_id=task.id), ), @@ -477,11 +440,10 @@ def _compose_config( ) -> FabricConfig: # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules # lookup, not a re-load, so the type is used where it's constructed instead of threaded down. - from nemo_fabric import EnvironmentConfig # ty: ignore[unresolved-import] + from nemo_fabric import EnvironmentConfig, ModelConfig # ty: ignore[unresolved-import] - # Config-first composition (the SDK's recommended in-memory pattern): copy the base config and - # apply this task's workspace, model, and trajectory settings directly onto it, rather than - # layering FabricProfileConfig overlays. + # Copy the base config and apply this task's workspace, model, and trajectory settings directly + # onto it. These land last, so they override anything the supplied config declared. cfg = agent_config.model_copy(deep=True) # Point the harness at this task's staged workspace (the codex-cli adapter resolves its cwd from @@ -495,7 +457,7 @@ def _compose_config( # Apply the model as the config's default (mirrors nemo_fabric.integrations.harbor). if self._model: provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai" - cfg.models["default"] = {"provider": provider, "model": self._model} + cfg.models["default"] = ModelConfig(provider=provider, model=self._model) if self._capture_trajectory: # Enable Relay's ATIF/ATOF file exporter under this task's durable evidence dir, and pin the @@ -505,83 +467,46 @@ def _compose_config( artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR relay_dir.mkdir(parents=True, exist_ok=True) artifacts_dir.mkdir(parents=True, exist_ok=True) - cfg.enable_relay(output_dir=str(relay_dir), config=self._relay_config(relay_dir)) + cfg.enable_relay(output_dir=str(relay_dir), observability=self._relay_config(relay_dir)) cfg.runtime.artifacts = str(artifacts_dir) cfg.environment.artifacts = str(artifacts_dir) return cfg - def _eval_lock_profiles( - self, - profile_cls: type[FabricProfileConfig], - *, - workspace_dir: Path, - evidence_dir: Path, - ) -> list[FabricProfileConfig]: - # ``_compose_config`` composes the evaluator's per-task settings onto the config, but Fabric - # applies caller-supplied profiles OVER the config (last-wins), so a caller profile could - # otherwise override them. Re-assert the evaluator-owned settings here as trailing overlays — - # applied after the caller profiles — so the per-task workspace (isolation + ``workspace`` - # evidence integrity), the model under evaluation, and the trajectory artifact location stay - # authoritative and non-overridable. - overlays = [ - profile_cls.from_mapping( - {"name": _WORKSPACE_PROFILE_NAME, "environment": {"workspace": str(workspace_dir)}} - ) - ] - if self._model: - provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai" - overlays.append( - profile_cls.from_mapping( - {"name": _MODEL_PROFILE_NAME, "models": {"default": {"provider": provider, "model": self._model}}} - ) - ) - if self._capture_trajectory: - artifacts_dir = str(evidence_dir / _ARTIFACTS_SUBDIR) - overlays.append( - profile_cls.from_mapping( - { - "name": _ARTIFACTS_PROFILE_NAME, - "runtime": {"artifacts": artifacts_dir}, - "environment": {"artifacts": artifacts_dir}, - } - ) - ) - return overlays - - def _relay_config(self, relay_dir: Path) -> dict[str, Any]: - # The observability component is built from nemo_relay's own typed config objects so Relay owns - # its schema (no hand-maintained dict that silently drifts when Relay changes it); imported - # lazily since nemo-relay, like nemo-fabric, is an optional native dependency. - try: - from nemo_relay.observability import ( # ty: ignore[unresolved-import] - AtifConfig, - AtofConfig, - ComponentSpec, - ObservabilityConfig, - ) - except ImportError as exc: - raise RuntimeError(_MISSING_RELAY_MSG) from exc + def _relay_config(self, relay_dir: Path) -> RelayObservabilityConfig: + # The ATIF/ATOF observability config is built from Fabric's own typed relay-config objects so + # Fabric owns the schema (no hand-maintained dict that silently drifts when Fabric changes it), + # mirroring nemo_fabric's own Harbor integration. It is handed straight to ``enable_relay`` via + # its ``observability=`` parameter — the SDK only configures ATIF/ATOF observability, so it needs + # neither a generic ``components`` list nor the legacy component-wrapped shape. nemo_fabric is + # already imported+validated in ``run_tasks``, so this is a cached sys.modules lookup. + from nemo_fabric import ( # ty: ignore[unresolved-import] + RelayAtifConfig, + RelayAtofConfig, + RelayAtofFileSinkConfig, + RelayObservabilityConfig, + ) relay_dir_str = str(relay_dir) - observability = ComponentSpec( - config=ObservabilityConfig( - atif=AtifConfig( - enabled=True, - output_directory=relay_dir_str, - filename_template=_ATIF_FILENAME_TEMPLATE, - agent_name=self._runtime_name, - agent_version="fabric", - ), - atof=AtofConfig( - enabled=True, - output_directory=relay_dir_str, - filename=_ATOF_FILENAME, - mode="overwrite", - ), - ) + return RelayObservabilityConfig( + atif=RelayAtifConfig( + enabled=True, + output_directory=relay_dir_str, + filename_template=_ATIF_FILENAME_TEMPLATE, + agent_name=self._runtime_name, + agent_version=_common.FABRIC_AGENT_VERSION, + ), + atof=RelayAtofConfig( + enabled=True, + sinks=[ + RelayAtofFileSinkConfig( + output_directory=relay_dir_str, + filename=_ATOF_FILENAME, + mode="overwrite", + ) + ], + ), ) - return {"version": 1, "components": [observability.to_dict()]} def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: root = self._work_root diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/skills.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/skills.py index 81d9bb6f0f..4e1a912315 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/skills.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/skills.py @@ -19,15 +19,19 @@ ``RunPlan.capability_plan``), not a hardcoded adapter list — so it tracks whatever the installed adapters declare, including end-user adapters we don't ship: -* **Native** (:data:`SKILL_MODE_NATIVE`): the adapter advertises ``accepts: ["skills", ...]`` (the - Hermes/Claude adapters do), so Fabric's planner routes skills to ``harness_native``. We stage the - bundle into an isolated ``/`` dir and hand Fabric a ``skills.paths`` profile overlay; the - adapter loads it (Hermes → harness ``skills.external_dirs``). -* **Codex skills dir** (:data:`SKILL_MODE_CODEX_SKILLS_DIR`): the Fabric ``codex`` adapter only - ``accepts: ["models"]`` (planner routes skills ``unsupported``), but the Codex CLI itself discovers - agentskills bundles from ``.agents/skills/`` in its working directory. So we place the bundle at - ``/.agents/skills//`` and let Codex discover it — same discoverable-skill semantics - as native (cross-harness A/B is apples-to-apples), no Fabric adapter change needed. +* **Native** (:data:`SKILL_MODE_NATIVE`): the adapter advertises ``accepts: ["skills", ...]``, so + Fabric's planner routes skills to ``harness_native``. We stage the bundle into an isolated + ``/`` dir and add it to the config's ``skills.paths``; the adapter loads it (Hermes → harness + ``skills.external_dirs``). As of nemo-fabric 0.1.0rc3 the hermes, claude AND **codex** adapters all + declare ``skills``, so this is the path every harness we ship currently takes. +* **Codex skills dir** (:data:`SKILL_MODE_CODEX_SKILLS_DIR`): a fallback for a codex-harness adapter + that does *not* accept the native skills config. The Codex CLI itself discovers agentskills bundles + from ``.agents/skills/`` in its working directory, so we place the bundle at + ``/.agents/skills//`` and let Codex find it — same discoverable-skill semantics as + native (cross-harness A/B stays apples-to-apples), no Fabric adapter change needed. + NOTE: the shipped codex adapter accepts ``skills`` today, so this branch is currently unreachable in + production and is exercised only by the fake-backed tests. It is kept for adapters (ours or an + end-user's) that route skills ``unsupported`` on a codex harness. If an adapter neither routes skills natively nor is a Codex harness, :func:`resolve_skill_mode` returns ``None`` and the runtime fails fast rather than silently running a skill-free trial. @@ -49,8 +53,6 @@ PRIMARY_SKILL_DOC = "SKILL.md" #: Directory Codex scans (relative to its working dir) for agentskills bundles. CODEX_SKILLS_DIR = ".agents/skills" -#: Name of the Fabric profile overlay that carries the native ``skills`` config. -SKILL_PROFILE_NAME = "eval_skill" #: How an injected skill reaches the selected harness (resolved from Fabric's capability plan). The two #: runtimes thread this value from :func:`resolve_skill_mode` down to :func:`install_skill` / @@ -134,13 +136,13 @@ class SkillProvenance(TypedDict): class SkillInstallation: """Result of installing a skill for one task. - ``profiles`` are Fabric profile-overlay mappings the runtime appends to its profile stack (the - native branch emits one ``skills`` overlay; the Codex branch emits none because placement in the - workspace is the delivery mechanism). ``provenance`` is stamped into trial metadata so the A/B - comparison is auditable. + ``skill_paths`` are staged bundle roots the runtime hands to ``FabricConfig.add_skill_path`` (the + native branch emits one; the Codex branch emits none because placement in the workspace is the + delivery mechanism). ``provenance`` is stamped into trial metadata so the A/B comparison is + auditable. """ - profiles: list[dict[str, object]] + skill_paths: list[str] provenance: SkillProvenance @@ -188,7 +190,6 @@ def install_skill( mode: SkillMode, workspace_dir: Path, skill_stage_dir: Path, - existing_skill_paths: Sequence[str] = (), ) -> SkillInstallation: """Stage ``skill`` as a ``/`` bundle and wire it into the harness per ``mode``. @@ -196,24 +197,15 @@ def install_skill( namespaced under ``/`` so it never collides with task-seeded workspace-root files; the content hash is computed over the staged bytes so provenance tracks the actual skill content. - ``existing_skill_paths`` are the skill paths the base config/profiles already declare. Fabric applies - profile ``skills.paths`` last-wins, so the native overlay must re-list them alongside the evaluated - skill — otherwise the treated arm would silently drop every preconfigured skill and the A/B would - differ by more than the injected skill. + The native branch returns the staged root for ``FabricConfig.add_skill_path``, which appends to + whatever the base config already declares. Any preconfigured skills therefore survive injection + without this function having to re-list them. """ if mode == SKILL_MODE_NATIVE: skill_root = skill_stage_dir / skill.name _stage_bundle(skill.directory, skill_root, reserved=False) - # Preserve the pre-existing skill paths (order-preserved, de-duplicated) and append the - # evaluated skill, so the last-wins overlay reproduces the baseline skill set plus this one. - paths = list(dict.fromkeys([*existing_skill_paths, str(skill_root)])) - overlay: dict[str, object] = { - "name": SKILL_PROFILE_NAME, - "description": "Make the evaluation skill available via the native Fabric skills config.", - "skills": {"paths": paths}, - } return SkillInstallation( - profiles=[overlay], + skill_paths=[str(skill_root)], provenance=_provenance(skill, _hash_directory(skill_root), mode, adapter_id, str(skill_root)), ) @@ -222,7 +214,7 @@ def install_skill( _stage_bundle(skill.directory, skill_root, reserved=True) location = (Path(CODEX_SKILLS_DIR) / skill.name).as_posix() return SkillInstallation( - profiles=[], + skill_paths=[], provenance=_provenance(skill, _hash_directory(skill_root), mode, adapter_id, location), ) @@ -233,14 +225,13 @@ def install_skill( class SkillsInstallation: """Result of installing several skills for one task (see :func:`install_skills`). - ``profiles`` is the Fabric profile-overlay stack the runtime appends: at most ONE merged native - ``skills`` overlay listing every staged bundle (Fabric applies profile ``skills.paths`` last-wins, so - all skills must ride in a single overlay or all but the last would be dropped); the Codex branch emits - none because workspace placement is the delivery mechanism. ``provenances`` is one entry per skill, in - the given order, stamped into trial metadata so a multi-skill A/B comparison is auditable. + ``skill_paths`` is every staged native bundle root, in the given order, for the runtime to feed to + ``FabricConfig.add_skill_path``; the Codex branch emits none because workspace placement is the + delivery mechanism. ``provenances`` is one entry per skill, in the given order, stamped into trial + metadata so a multi-skill A/B comparison is auditable. """ - profiles: list[dict[str, object]] + skill_paths: list[str] provenances: list[SkillProvenance] @@ -295,16 +286,14 @@ def install_skills( mode: SkillMode, workspace_dir: Path, skill_stage_dir: Path, - existing_skill_paths: Sequence[str] = (), ) -> SkillsInstallation: """Stage every skill in ``skills`` for one task and wire them all into the harness per ``mode``. - Loops :func:`install_skill` — each skill stages into its own namespaced ``/`` bundle — then, for - the native mode, merges the per-skill paths into a SINGLE ``skills`` overlay: Fabric applies profile - ``skills.paths`` last-wins, so emitting one overlay per skill would silently drop all but the last. - Pre-existing ``existing_skill_paths`` are preserved ahead of the injected skills (same reason). Skill - names must be unique (their ``/`` bundles would otherwise collide). Blocking file I/O — call via - ``asyncio.to_thread`` from the async runtime. + Loops :func:`install_skill` — each skill stages into its own namespaced ``/`` bundle — and + collects the staged roots for the native mode. ``FabricConfig.add_skill_path`` appends and + de-duplicates, so every injected skill lands alongside whatever the base config already declared, + with no re-listing. Skill names must be unique (their ``/`` bundles would otherwise collide). + Blocking file I/O — call via ``asyncio.to_thread`` from the async runtime. Installation is all-or-nothing: if any skill fails to stage, the bundles already staged in this call are rolled back before the error propagates, so a partial skill set never lingers on disk (the caller @@ -330,7 +319,6 @@ def install_skills( mode=mode, workspace_dir=workspace_dir, skill_stage_dir=skill_stage_dir, - existing_skill_paths=existing_skill_paths, ).provenance provenances.append(provenance) except Exception: @@ -338,19 +326,12 @@ def install_skills( shutil.rmtree(root, ignore_errors=True) raise - profiles: list[dict[str, object]] = [] - if mode == SKILL_MODE_NATIVE and provenances: - # One merged overlay: pre-existing skills first, then each staged bundle (a native provenance's - # ``location`` is its absolute staged skill root), order-preserved and de-duplicated. - paths = list(dict.fromkeys([*existing_skill_paths, *(prov["location"] for prov in provenances)])) - profiles = [ - { - "name": SKILL_PROFILE_NAME, - "description": "Make the evaluation skills available via the native Fabric skills config.", - "skills": {"paths": paths}, - } - ] - return SkillsInstallation(profiles=profiles, provenances=provenances) + skill_paths: list[str] = [] + if mode == SKILL_MODE_NATIVE: + # Each staged bundle root, order-preserved and de-duplicated (a native provenance's + # ``location`` is its absolute staged skill root). + skill_paths = list(dict.fromkeys(prov["location"] for prov in provenances)) + return SkillsInstallation(skill_paths=skill_paths, provenances=provenances) def _skill_stage_root(skill: AgentSkill, mode: SkillMode, workspace_dir: Path, skill_stage_dir: Path) -> Path: @@ -422,14 +403,13 @@ class SkillsSeed: The plural, containerized sibling of :class:`SkillsInstallation`: * ``files`` — the merged ``{absolute_in_sandbox_path: text}`` seed map for every staged bundle. - * ``profiles`` — at most ONE merged native ``skills`` overlay listing every bundle (Fabric applies - ``skills.paths`` last-wins, so all must ride in a single overlay or all but the last are dropped); - the codex branch emits none. + * ``skill_paths`` — every staged native bundle root, in order, for the runtime to merge into the + composed config's ``skills.paths``; the codex branch emits none. * ``provenances`` — one entry per skill, in the given order, for the multi-skill A/B trial metadata. """ files: dict[str, str] - profiles: list[dict[str, object]] + skill_paths: list[str] provenances: list[SkillProvenance] @@ -440,18 +420,16 @@ def stage_skills_seed( mode: SkillMode, workspace_dir: str, skills_dir: str, - existing_skill_paths: Sequence[str] = (), ) -> SkillsSeed: - """Render every skill in ``skills`` into one sandbox seed set + overlays for the container runtime. + """Render every skill in ``skills`` into one sandbox seed set for the container runtime. The plural, containerized sibling of :func:`install_skills`: renders each bundle (via - :func:`_render_skill_seed`) under its own ``/`` at the harness's in-sandbox discovery path, then, - for the native mode, merges the per-skill paths into a SINGLE ``skills`` overlay (Fabric applies profile - ``skills.paths`` last-wins, so one overlay per skill would silently drop all but the last). Pre-existing - ``existing_skill_paths`` are preserved ahead of the injected skills (same reason). Skill names must be - unique — their ``/`` bundles would otherwise collide. No on-disk rollback is needed (unlike - :func:`install_skills`): the seed set is an in-memory map, so a failure to render any skill just - discards the accumulated map and raises, leaving nothing staged. + :func:`_render_skill_seed`) under its own ``/`` at the harness's in-sandbox discovery path and + collects the native in-sandbox roots. The caller merges those into the composed config's + ``skills.paths`` alongside whatever it already declared, so nothing has to be re-listed here. Skill + names must be unique — their ``/`` bundles would otherwise collide. No on-disk rollback is + needed (unlike :func:`install_skills`): the seed set is an in-memory map, so a failure to render any + skill just discards the accumulated map and raises, leaving nothing staged. """ require_unique_skill_names(skills) files: dict[str, str] = {} @@ -463,19 +441,12 @@ def stage_skills_seed( files.update(rendered) provenances.append(provenance) - profiles: list[dict[str, object]] = [] - if mode == SKILL_MODE_NATIVE and provenances: - # One merged overlay: pre-existing skills first, then each staged bundle (a native provenance's - # ``location`` is its absolute in-sandbox skill root), order-preserved and de-duplicated. - paths = list(dict.fromkeys([*existing_skill_paths, *(prov["location"] for prov in provenances)])) - profiles = [ - { - "name": SKILL_PROFILE_NAME, - "description": "Make the evaluation skills available via the native Fabric skills config.", - "skills": {"paths": paths}, - } - ] - return SkillsSeed(files=files, profiles=profiles, provenances=provenances) + skill_paths: list[str] = [] + if mode == SKILL_MODE_NATIVE: + # Each staged bundle (a native provenance's ``location`` is its absolute in-sandbox skill + # root), order-preserved and de-duplicated. + skill_paths = list(dict.fromkeys(prov["location"] for prov in provenances)) + return SkillsSeed(files=files, skill_paths=skill_paths, provenances=provenances) def _stage_bundle(directory: Path, skill_root: Path, *, reserved: bool) -> None: diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_container_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_container_runtime.py index 5778c45b40..e9a6f3694b 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_container_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_container_runtime.py @@ -28,7 +28,7 @@ from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus from nemo_evaluator_sdk.values.common import SecretRef -_CONFIG = {"metadata": {"name": "eval"}, "harness": {"adapter_id": "nvidia.fabric.hermes.sdk"}} +_CONFIG = {"metadata": {"name": "eval"}, "harness": {"adapter_id": "nvidia.fabric.hermes"}} @pytest.fixture(autouse=True) @@ -192,20 +192,24 @@ async def test_failed_trial_stamps_agent_ok_false(tmp_path: Path) -> None: assert trial.metadata["agent_ok"] is False -async def test_seeds_agent_config_profiles_and_execs_cli(tmp_path: Path) -> None: +async def test_seeds_composed_agent_config_and_execs_cli(tmp_path: Path) -> None: provider = _FakeProvider() await _run(_runtime(provider), [_task()], tmp_path) assert "/in/agent.yaml" in provider.seeded and "/in/input.txt" in provider.seeded - # base profiles (none here) + the per-task workspace overlay + the trajectory profile. - profile_files = sorted(key for key in provider.seeded if key.startswith("/in/profile-")) - names = {json.loads(provider.seeded[path])["name"] for path in profile_files} - assert names == {"eval_workspace", "eval_trajectory"} + # Fabric dropped profile overlays, so everything rides in the single agent config: the caller's + # harness plus the runtime's workspace, artifact roots, and trajectory telemetry. + assert not [key for key in provider.seeded if key.startswith("/in/profile-")] + agent = json.loads(provider.seeded["/in/agent.yaml"]) + assert agent["harness"]["adapter_id"] == _CONFIG["harness"]["adapter_id"] # caller keys survive + assert agent["environment"] == {"provider": "local", "workspace": "/out/workspace", "artifacts": "/out/artifacts"} + assert agent["runtime"]["artifacts"] == "/out/artifacts" + assert agent["telemetry"]["provider"] == "relay" # Workspace seed files were staged and uploaded across the boundary. assert provider.uploaded_dirs and provider.uploaded_dirs[0][1] == "/out/workspace" # Execs Fabric's own CLI (not an in-image Python driver), redirecting the RunResult to /out. (cmd,) = provider.execs assert "fabric run /in/agent.yaml" in cmd - assert "--profile /in/profile-0.yaml" in cmd and "--input-file /in/input.txt" in cmd + assert "--profile" not in cmd and "--input-file /in/input.txt" in cmd assert "> /out/fabric_result.json" in cmd @@ -315,17 +319,22 @@ def test_empty_instruction_is_rejected() -> None: AgentEvalTask(id="x", intent="ignored", inputs={"instruction": ""}).agent_prompt() -def test_trajectory_profile_built_from_relay_types() -> None: +def test_trajectory_telemetry_built_from_relay_types() -> None: # The trajectory telemetry is built from nemo_relay's own typed config (a hard dependency), so drift # in relay's schema fails construction here rather than silently emitting a malformed profile. Runs # in CI now that nemo-relay is declared — no importorskip. Asserts the shape metrics rely on. - component = FabricContainerRuntime._trajectory_profile()["telemetry"]["config"]["components"][0] + telemetry = FabricContainerRuntime({**_CONFIG}, provider=_FakeProvider())._composed_config()["telemetry"] + component = telemetry["config"]["components"][0] assert component["kind"] == "observability" and component["enabled"] is True cfg = component["config"] - # The ATIF/ATOF file exporter is configured with the names both runtimes agree on. + # The ATIF/ATOF file exporter is configured with the names both runtimes agree on. Since + # nemo-relay 0.6 the ATOF destination lives in a typed sink list rather than flat on the config. assert cfg["atif"]["enabled"] is True assert cfg["atif"]["filename_template"] == crt._common.ATIF_FILENAME_TEMPLATE - assert cfg["atof"]["filename"] == crt._common.ATOF_FILENAME + assert cfg["atof"]["enabled"] is True + (atof_sink,) = cfg["atof"]["sinks"] + assert atof_sink["type"] == "file" + assert atof_sink["filename"] == crt._common.ATOF_FILENAME # -------------------------------------------------------------------------------------------------- @@ -335,9 +344,9 @@ def test_trajectory_profile_built_from_relay_types() -> None: # Adapters the fake planner reports as accepting the native Fabric ``skills`` config. ``acme.custom.native`` # stands in for an END-USER adapter the platform doesn't ship — the runtime learns it accepts skills purely # from the plan, with no hardcoded list. -_NATIVE_SKILL_ADAPTERS = {"nvidia.fabric.hermes.sdk", "acme.custom.native"} +_NATIVE_SKILL_ADAPTERS = {"nvidia.fabric.hermes", "acme.custom.native"} _KNOWN_HARNESSES = ("hermes", "codex", "claude") -_CODEX_CONFIG = {"metadata": {"name": "eval"}, "harness": {"adapter_id": "nvidia.fabric.codex.cli"}} +_CODEX_CONFIG = {"metadata": {"name": "eval"}, "harness": {"adapter_id": "nvidia.fabric.codex"}} def _harness_name(adapter_id: str) -> str: @@ -395,10 +404,10 @@ def __init__(self, *, capability_plan: dict[str, object], harness: str) -> None: class _FakeFabric: planned: list[dict[str, object]] = [] - def plan(self, agent: object, *, profiles: object = None, base_dir: object = None) -> _FakePlan: + def plan(self, agent: object, *, base_dir: object = None) -> _FakePlan: # Mirror Fabric's planner: a ``skills`` route appears only when a skill path is attached, and it # routes ``harness_native`` iff the selected adapter accepts native skills. - _FakeFabric.planned.append({"agent": agent, "profiles": profiles}) + _FakeFabric.planned.append({"agent": agent}) adapter_id = agent.harness.adapter_id has_skill_path = bool(getattr(agent, "skill_paths", None)) native = has_skill_path and adapter_id in _NATIVE_SKILL_ADAPTERS @@ -412,7 +421,6 @@ def _install_fake_fabric(monkeypatch: pytest.MonkeyPatch) -> type[_FakeFabric]: module = types.ModuleType("nemo_fabric") module.Fabric = _FakeFabric # type: ignore[attr-defined] module.FabricConfig = _FakeConfig # type: ignore[attr-defined] - module.FabricProfileConfig = _FakeProfile # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "nemo_fabric", module) return _FakeFabric @@ -429,17 +437,13 @@ def _skill_bundle(base: Path, *, name: str = "code-review", extra: dict[str, str return root -def _seeded_profiles(provider: _FakeProvider) -> dict[str, dict[str, object]]: - """The profile overlays the runtime seeded into /in, keyed by their ``name``.""" - profiles: dict[str, dict[str, object]] = {} - for key, value in provider.seeded.items(): - if key.startswith("/in/profile-"): - profile = json.loads(value) - profiles[profile["name"]] = profile - return profiles +def _seeded_skill_paths(provider: _FakeProvider) -> list[str]: + """``skills.paths`` on the composed agent config the runtime seeded into /in.""" + agent = json.loads(provider.seeded["/in/agent.yaml"]) + return list(agent.get("skills", {}).get("paths", [])) -async def test_native_skill_seeds_bundle_into_seed_set_with_overlay( +async def test_native_skill_seeds_bundle_into_seed_set_and_config( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: from nemo_evaluator_sdk.agent_eval.runtimes.fabric.skills import AgentSkill @@ -456,9 +460,8 @@ async def test_native_skill_seeds_bundle_into_seed_set_with_overlay( # never lands in the downloaded workspace evidence). assert provider.seeded["/in/skills/code-review/SKILL.md"].startswith("---") assert provider.seeded["/in/skills/code-review/references/r.md"] == "material" - # A native `skills` overlay points at the staged bundle dir; the eval workspace/trajectory overlays trail. - overlay = _seeded_profiles(provider)["eval_skill"] - assert overlay["skills"]["paths"][-1] == "/in/skills/code-review" + # The composed agent config's skills.paths points at the staged bundle dir. + assert _seeded_skill_paths(provider)[-1] == "/in/skills/code-review" # Provenance is stamped into trial metadata for the A/B diff. prov = trial.metadata["skill"] assert prov["name"] == "code-review" and prov["mode"] == "native" and prov["hash"] @@ -468,23 +471,18 @@ async def test_native_skill_seeds_bundle_into_seed_set_with_overlay( async def test_native_skill_preserves_preconfigured_skill_paths( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - # Fabric applies profile skills.paths last-wins, so the overlay must re-list config- and profile-declared - # skills (order-preserved) ahead of the evaluated skill, or the treated arm would drop them. + # Injected paths are APPENDED to the config's own skills.paths (order-preserved), so preconfigured + # skills survive injection and the treated arm differs by exactly the injected skill. from nemo_evaluator_sdk.agent_eval.runtimes.fabric.skills import AgentSkill _install_fake_fabric(monkeypatch) - config = {**_CONFIG, "skills": {"paths": ["/pre/existing-a"]}} + config = {**_CONFIG, "skills": {"paths": ["/pre/existing-a", "/pre/existing-b"]}} skill = AgentSkill.from_directory(_skill_bundle(tmp_path / "src")) provider = _FakeProvider() - runtime = FabricContainerRuntime( - config, # type: ignore[arg-type] - provider=provider, - profiles=[{"name": "caller", "skills": {"paths": ["/pre/existing-b"]}}], - skills=[skill], - ) + runtime = FabricContainerRuntime(config, provider=provider, skills=[skill]) # type: ignore[arg-type] await runtime.run_tasks([_task()], AgentEvalRunConfig(output_dir=tmp_path)) - paths = _seeded_profiles(provider)["eval_skill"]["skills"]["paths"] + paths = _seeded_skill_paths(provider) assert paths[:2] == ["/pre/existing-a", "/pre/existing-b"] assert paths[-1] == "/in/skills/code-review" @@ -501,7 +499,7 @@ async def test_native_skill_on_runtime_discovered_adapter(tmp_path: Path, monkey runtime = FabricContainerRuntime(custom, provider=provider, skills=[skill]) # type: ignore[arg-type] (trial,) = await runtime.run_tasks([_task()], AgentEvalRunConfig(output_dir=tmp_path)) - assert "eval_skill" in _seeded_profiles(provider) + assert "/in/skills/code-review" in _seeded_skill_paths(provider) assert trial.metadata["skill"]["mode"] == "native" @@ -527,8 +525,8 @@ async def download_dir(self, handle: SandboxHandle, source_dir: str, target_dir: # Codex discovers agentskills from .agents/skills/ in its working dir, so the bundle is seeded there in # the workspace (not /in), for the harness to self-discover during the run. assert provider.seeded["/out/workspace/.agents/skills/code-review/SKILL.md"].startswith("---") - # No native overlay: placement in the workspace is the delivery mechanism. - assert "eval_skill" not in _seeded_profiles(provider) + # No skills path added: placement in the workspace is the delivery mechanism. + assert _seeded_skill_paths(provider) == [] prov = trial.metadata["skill"] assert prov["mode"] == "codex_skills_dir" assert prov["location"] == ".agents/skills/code-review" @@ -562,11 +560,11 @@ async def test_no_skill_leaves_metadata_none_and_skips_planner(tmp_path: Path, m assert not any("/skills/" in key or "/.agents/" in key for key in provider.seeded) -async def test_multiple_native_skills_each_staged_with_one_merged_overlay( +async def test_multiple_native_skills_each_staged_and_all_in_the_config( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: # A set of skills: each bundle stages under its own /in/skills//, and all ride in ONE merged - # `eval_skill` overlay (Fabric applies skills.paths last-wins, so a per-skill overlay would drop all + # config's skills.paths (all of them must be listed, or the treated arm would drop all # but the last). Trial metadata carries one provenance per skill; the lone `skill` field is None. from nemo_evaluator_sdk.agent_eval.runtimes.fabric.skills import AgentSkill @@ -580,9 +578,8 @@ async def test_multiple_native_skills_each_staged_with_one_merged_overlay( assert provider.seeded["/in/skills/docx/SKILL.md"].startswith("---") assert provider.seeded["/in/skills/pptx/SKILL.md"].startswith("---") - # Exactly one merged overlay listing both bundle roots, in order. - overlay = _seeded_profiles(provider)["eval_skill"] - assert overlay["skills"]["paths"] == ["/in/skills/docx", "/in/skills/pptx"] + # Both bundle roots land on the composed config's skills.paths, in order. + assert _seeded_skill_paths(provider) == ["/in/skills/docx", "/in/skills/pptx"] # One provenance per skill; the historical lone `skill` field is None for a multi-skill run. names = [prov["name"] for prov in trial.metadata["skills"]] assert names == ["docx", "pptx"] @@ -610,10 +607,10 @@ async def download_dir(self, handle: SandboxHandle, source_dir: str, target_dir: runtime = FabricContainerRuntime(_CODEX_CONFIG, provider=provider, skills=skills) # type: ignore[arg-type] (trial,) = await runtime.run_tasks([_task()], AgentEvalRunConfig(output_dir=tmp_path)) - # Both bundles seeded under the codex discovery dir, no overlay, and every one scrubbed from evidence. + # Both bundles seeded under the codex discovery dir, no skills path, all scrubbed from evidence. assert provider.seeded["/out/workspace/.agents/skills/docx/SKILL.md"].startswith("---") assert provider.seeded["/out/workspace/.agents/skills/pptx/SKILL.md"].startswith("---") - assert "eval_skill" not in _seeded_profiles(provider) + assert _seeded_skill_paths(provider) == [] assert [prov["name"] for prov in trial.metadata["skills"]] == ["docx", "pptx"] workspace = Path(trial.evidence.require("workspace").ref) # type: ignore[arg-type] assert not (workspace / ".agents").exists() diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py index d58409e509..9c12c223af 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py @@ -8,8 +8,8 @@ in CI: it proves the runner -> evaluator -> metric -> evidence chain, i.e. the metric receives and reads the trajectory (ATIF) evidence for the task. - ``test_fabric_codex_live_eval_captures_atif_trajectory`` is the real fabric->codex->Relay run, gated - behind the required binaries/checkout so CI skips it; run it locally after - ``script/dev-install-fabric.sh``. + behind the required binaries so CI skips it; run it locally after ``uv sync --extra fabric`` + plus ``script/dev-install-fabric.sh`` for the relay gateway. """ from __future__ import annotations @@ -103,20 +103,39 @@ def model_copy(self, *, deep: bool = False) -> _FakeConfig: return clone def enable_relay( - self, *, project: str | None = None, output_dir: str | None = None, config: Any = None + self, + *, + project: str | None = None, + output_dir: str | None = None, + observability: Any = None, + components: Any = None, + policy: Any = None, ) -> _FakeConfig: - self.relay = {"project": project, "output_dir": output_dir, "config": config} + self.relay = { + "project": project, + "output_dir": output_dir, + "observability": observability, + "components": components, + "policy": policy, + } return self -class _FakeProfile: +class _FakeModelConfig: + """Stand-in for nemo_fabric.ModelConfig (FabricConfig.models is dict[str, ModelConfig]).""" + + def __init__(self, *, provider: str, model: str, **extra: Any) -> None: + self.provider = provider + self.model = model + self.extra = extra + + +class _FakeRelayModel: + """Stand-in for nemo_fabric's RelayObservabilityConfig/RelayAtifConfig/RelayAtofConfig (kwargs bag).""" + def __init__(self, **kwargs: Any) -> None: self.kwargs = kwargs - @classmethod - def from_mapping(cls, mapping: dict[str, Any]) -> _FakeProfile: - return cls(**mapping) - class _FakeArtifact: def __init__(self, name: str, kind: str, path: Path) -> None: @@ -139,7 +158,7 @@ def __init__(self, artifacts: list[_FakeArtifact]) -> None: self.output = {"adapter": "cli", "response": "DONE"} self.error = None self.harness = "codex" - self.adapter_id = "nvidia.fabric.codex.cli" + self.adapter_id = "nvidia.fabric.codex" self.adapter_kind = "process" self.invocation_id = "inv-1" self.artifacts = _FakeManifest(artifacts) @@ -170,15 +189,21 @@ def __init__(self, **kwargs: Any) -> None: module = types.ModuleType("nemo_fabric") module.Fabric = _FakeClient # type: ignore[attr-defined] module.FabricConfig = _FakeConfig # type: ignore[attr-defined] - module.FabricProfileConfig = _FakeProfile # type: ignore[attr-defined] module.EnvironmentConfig = _FakeEnvironment # type: ignore[attr-defined] + module.ModelConfig = _FakeModelConfig # type: ignore[attr-defined] module.RunRequest = _FakeRunRequest # type: ignore[attr-defined] + # The runtime builds the relay observability config from Fabric's own typed models (lazy import). + module.RelayObservabilityConfig = _FakeRelayModel # type: ignore[attr-defined] + module.RelayAtifConfig = _FakeRelayModel # type: ignore[attr-defined] + module.RelayAtofConfig = _FakeRelayModel # type: ignore[attr-defined] + module.RelayAtofFileSinkConfig = _FakeRelayModel # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "nemo_fabric", module) - # nemo_relay is a hard dependency (the trajectory profile is built from its real typed config), so - # it is not stubbed — only the optional native nemo_fabric SDK is faked. + # nemo_relay stays a hard (installed) dependency here so ``run_tasks``'s capture-trajectory fail-fast + # (``import nemo_relay.observability``) resolves; only the optional native nemo_fabric SDK is faked. + # The observability config itself is built from nemo_fabric's typed models (faked above), not nemo_relay's. runtime = fabric_runtime.FabricAgentRuntime( - config={"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex.cli"}}, + config={"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex"}}, work_root=tmp_path / "fabric", ) result = AgentEvaluator().run_sync( @@ -202,16 +227,30 @@ def __init__(self, **kwargs: Any) -> None: # --- gated live: real fabric -> codex -> Relay ATIF ------------------------------------------------ -_FABRIC_REPO = os.environ.get("NEMO_FABRIC_REPO", "") -_LIVE_READY = bool( - _FABRIC_REPO - and shutil.which("codex") - and shutil.which("nemo-relay") - and importlib.util.find_spec("nemo_fabric") is not None -) + +def _codex_adapter_installed() -> bool: + """Whether the codex harness adapter is installed (the ``fabric`` extra, not the base SDK). + + ``nemo_fabric`` itself is a base dependency, so importing it proves nothing about harnesses: + without the adapters Fabric resolves none and fails with ``available adapters: []``. ``find_spec`` + raises rather than returning None when the parent package is missing, hence the guard. + """ + try: + return importlib.util.find_spec("nemo_fabric_adapters.codex") is not None + except ModuleNotFoundError: + return False + + +# No NeMo-Fabric checkout in the gate: the adapter registry resolves from the installed wheels +# (/share/nemo-fabric/adapters), so `uv sync --extra fabric` is enough. +_LIVE_READY = bool(shutil.which("codex") and shutil.which("nemo-relay") and _codex_adapter_installed()) +_LIVE_MODEL = os.environ.get("NEMO_FABRIC_LIVE_MODEL", "gpt-5.6-terra") requires_live_fabric = pytest.mark.skipif( not _LIVE_READY, - reason="needs NEMO_FABRIC_REPO + codex + nemo-relay gateway + nemo_fabric (run script/dev-install-fabric.sh)", + reason=( + "needs the harness adapters (uv sync --extra fabric) + the nemo-relay gateway " + "(script/dev-install-fabric.sh) + codex on PATH" + ), ) @@ -222,18 +261,21 @@ def test_fabric_codex_live_eval_captures_atif_trajectory(tmp_path: Path) -> None "schema_version": "fabric.agent/v1alpha1", "metadata": {"name": "eval-fabric-live"}, "harness": { - "adapter_id": "nvidia.fabric.codex.cli", + "adapter_id": "nvidia.fabric.codex", "resolution": "preinstalled", "settings": {"sandbox": "workspace-write", "skip_git_repo_check": True, "timeout_seconds": 180}, }, "runtime": {"mode": "oneshot", "transport": "cli", "input_schema": "text", "output_schema": "message"}, "environment": {"provider": "local", "workspace": str(tmp_path / "ws")}, + # Fabric's codex adapter requires an explicit model provider — it does not fall back to the + # Codex CLI's own configured default, and starting without one fails the adapter lifecycle + # with `codex_invalid_configuration`. Override for an account with different model access. + "models": {"default": {"provider": "openai", "model": _LIVE_MODEL}}, "telemetry": {"enabled": False}, } (tmp_path / "ws").mkdir(parents=True, exist_ok=True) runtime = fabric_runtime.FabricAgentRuntime( config=codex_config, - base_dir=Path(_FABRIC_REPO), # so the adapter registry resolves adapters/codex-cli work_root=tmp_path / "fabric", capture_trajectory=True, ) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py index f51d674bb7..aaf599e8ac 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py @@ -50,7 +50,9 @@ def __init__(self, mapping: dict[str, Any]) -> None: self.runtime = _FakeRuntimeCfg() self.models: dict[str, Any] = dict(mapping.get("models", {})) self.relay: dict[str, Any] | None = None # records enable_relay(...) - self.skill_paths: list[str] = [] # records add_skill_path(...) (the capability-plan probe uses it) + # Mirrors FabricConfig.skills.paths: seeded from the config, then appended to by + # add_skill_path (which the capability-plan probe and native skill injection both use). + self.skill_paths: list[str] = [str(p) for p in mapping.get("skills", {}).get("paths", [])] @classmethod def from_mapping(cls, mapping: dict[str, Any]) -> _FakeConfig: @@ -65,25 +67,44 @@ def model_copy(self, *, deep: bool = False) -> _FakeConfig: clone.skill_paths = list(self.skill_paths) return clone - def add_skill_path(self, path: Any) -> None: - self.skill_paths.append(str(path)) + def add_skill_path(self, path: Any) -> _FakeConfig: + # Real SkillConfig.add_path appends only if absent, preserving order. + value = str(path) + if value not in self.skill_paths: + self.skill_paths.append(value) + return self def enable_relay( - self, *, project: str | None = None, output_dir: str | None = None, config: Any = None + self, + *, + project: str | None = None, + output_dir: str | None = None, + observability: Any = None, + components: Any = None, + policy: Any = None, ) -> _FakeConfig: - self.relay = {"project": project, "output_dir": output_dir, "config": config} + self.relay = { + "project": project, + "output_dir": output_dir, + "observability": observability, + "components": components, + "policy": policy, + } return self -class _FakeProfile: - def __init__(self, *, name: str | None = None, models: Any = None, mapping: Any = None) -> None: - self.name = name - self.models = models - self.mapping = mapping +class _FakeModelConfig: + """Stand-in for nemo_fabric.ModelConfig — FabricConfig.models is dict[str, ModelConfig], so the + runtime must build the typed model rather than assign a raw dict (which Fabric tolerates but + does not validate, and warns about on serialization).""" - @classmethod - def from_mapping(cls, mapping: dict[str, Any]) -> _FakeProfile: - return cls(name=mapping.get("name"), mapping=mapping) + def __init__(self, *, provider: str, model: str, **extra: Any) -> None: + self.provider = provider + self.model = model + self.extra = extra + + def to_dict(self) -> dict[str, Any]: + return {"provider": self.provider, "model": self.model, **self.extra} class _FakeRunRequest: @@ -94,24 +115,16 @@ def __init__(self, *, input: Any = None, request_id: str | None = None) -> None: self.request_id = request_id -class _FakeRelayConfig: - """Stand-in for nemo_relay.observability's typed config objects (AtifConfig/AtofConfig/...).""" +class _FakeRelayModel: + """Stand-in for Fabric's typed relay models (RelayObservabilityConfig/RelayAtifConfig/RelayAtofConfig). + + The runtime constructs these to hand to ``enable_relay(observability=...)``; it never introspects + them, so a plain kwargs bag suffices. + """ def __init__(self, **kwargs: Any) -> None: self.kwargs = kwargs - def to_dict(self) -> dict[str, Any]: - return {key: (value.to_dict() if hasattr(value, "to_dict") else value) for key, value in self.kwargs.items()} - - -class _FakeComponentSpec: - def __init__(self, *, config: Any, enabled: bool = True) -> None: - self.config = config - self.enabled = enabled - - def to_dict(self) -> dict[str, Any]: - return {"kind": "observability", "enabled": self.enabled, "config": self.config.to_dict()} - class _FakeArtifact: def __init__(self, name: str, kind: str, path: Path, media_type: str | None = None) -> None: @@ -152,8 +165,7 @@ def __init__(self, kind: str, message: str) -> None: # adapters/*/fabric-adapter.json. ``acme.custom.native`` stands in for an END-USER adapter the platform # doesn't ship — the runtime learns it accepts skills purely from the plan, with no hardcoded list. _NATIVE_SKILL_ADAPTERS = { - "nvidia.fabric.hermes.sdk", - "nvidia.fabric.hermes.cli", + "nvidia.fabric.hermes", "nvidia.fabric.claude", "acme.custom.native", } @@ -193,7 +205,7 @@ def __init__( self.output = output self.error = error self.harness = "codex" - self.adapter_id = "nvidia.fabric.codex.cli" + self.adapter_id = "nvidia.fabric.codex" self.adapter_kind = "process" self.invocation_id = "inv-1" self.artifacts = _FakeManifest(artifacts or []) @@ -216,10 +228,10 @@ async def run(self, agent: Any, **kwargs: Any) -> Any: _FakeClient.recorded.append({"agent": agent, **kwargs}) return handler(agent, kwargs) - def plan(self, agent: Any, *, profiles: Any = None, base_dir: Any = None) -> _FakePlan: + def plan(self, agent: Any, *, base_dir: Any = None) -> _FakePlan: # Mirror Fabric's capability planner: a ``skills`` route appears only when a skill path is # attached, and it routes ``harness_native`` iff the selected adapter accepts native skills. - _FakeClient.planned.append({"agent": agent, "profiles": profiles, "base_dir": base_dir}) + _FakeClient.planned.append({"agent": agent, "base_dir": base_dir}) adapter_id = agent.harness.adapter_id has_skill_path = bool(getattr(agent, "skill_paths", None)) native = has_skill_path and adapter_id in _NATIVE_SKILL_ADAPTERS @@ -233,19 +245,22 @@ def plan(self, agent: Any, *, profiles: Any = None, base_dir: Any = None) -> _Fa module = types.ModuleType("nemo_fabric") module.Fabric = _FakeClient # type: ignore[attr-defined] module.FabricConfig = _FakeConfig # type: ignore[attr-defined] - module.FabricProfileConfig = _FakeProfile # type: ignore[attr-defined] module.EnvironmentConfig = _FakeEnvironment # type: ignore[attr-defined] + module.ModelConfig = _FakeModelConfig # type: ignore[attr-defined] module.RunRequest = _FakeRunRequest # type: ignore[attr-defined] + # The runtime builds the relay observability config from Fabric's own typed models (lazy import). + module.RelayObservabilityConfig = _FakeRelayModel # type: ignore[attr-defined] + module.RelayAtifConfig = _FakeRelayModel # type: ignore[attr-defined] + module.RelayAtofConfig = _FakeRelayModel # type: ignore[attr-defined] + module.RelayAtofFileSinkConfig = _FakeRelayModel # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "nemo_fabric", module) - # The runtime builds the trajectory profile from nemo_relay's typed config objects (lazy import); - # stub the optional package so trajectory-capture paths resolve without the native dependency. + # ``run_tasks`` fails fast on ``import nemo_relay.observability`` when capture_trajectory is on + # (the relay gateway is a runtime requirement); stub the optional package so that guard resolves + # without the native dependency. The observability config itself is now built from nemo_fabric's + # typed models (stubbed above), not from nemo_relay, so this stand-in only needs to be importable. relay_mod = types.ModuleType("nemo_relay") observability_mod = types.ModuleType("nemo_relay.observability") - observability_mod.AtifConfig = _FakeRelayConfig # type: ignore[attr-defined] - observability_mod.AtofConfig = _FakeRelayConfig # type: ignore[attr-defined] - observability_mod.ObservabilityConfig = _FakeRelayConfig # type: ignore[attr-defined] - observability_mod.ComponentSpec = _FakeComponentSpec # type: ignore[attr-defined] relay_mod.observability = observability_mod # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "nemo_relay", relay_mod) monkeypatch.setitem(sys.modules, "nemo_relay.observability", observability_mod) @@ -253,7 +268,7 @@ def plan(self, agent: Any, *, profiles: Any = None, base_dir: Any = None) -> _Fa _TASK = AgentEvalTask(id="task/1", intent="Answer.", inputs={"instruction": "Ping?"}) -_CONFIG = {"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex.cli"}} +_CONFIG = {"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex"}} @pytest.mark.asyncio @@ -280,7 +295,7 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: assert trial.output.output_text == "PONG" # extracted from the adapter envelope's `response` assert trial.output.response == {"adapter": "cli", "response": "PONG", "returncode": 0} assert trial.metadata["harness"] == "codex" - assert trial.metadata["adapter_id"] == "nvidia.fabric.codex.cli" + assert trial.metadata["adapter_id"] == "nvidia.fabric.codex" assert trial.metadata["generated"] is True # agent_ok mirrors the Codex runtime so AgentPhaseSuccessMetric scores the phase as clean. assert trial.metadata["agent_ok"] is True @@ -294,7 +309,7 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: # Config-first: the model is set on the config's default model and relay (ATIF trajectory) is # enabled on the config, rather than layered as profile overlays. composed = client_cls.recorded[0]["agent"] - assert composed.models["default"] == {"provider": "openai", "model": "openai/gpt-5.4"} + assert (composed.models["default"].provider, composed.models["default"].model) == ("openai", "openai/gpt-5.4") assert composed.relay is not None # capture_trajectory defaults on -> enable_relay(...) called assert client_cls.recorded[0]["request"].request_id == "task/1" # Telemetry reference is preserved end-to-end (uri + trace_id), not just provider/kind. @@ -357,34 +372,16 @@ def _workspace_from_config(config: Any) -> Path: return Path(config.environment.workspace) -def _resolve_like_fabric(config: Any, profiles: list[Any], section: str, key: str) -> Any: - """Mirror Fabric's resolver: start from the config, then apply each profile as a winning overlay in - order (last wins). Used to assert what value actually reaches the harness for a config/profile key. - """ - if section == "environment": - value = getattr(config.environment, key, None) if config.environment is not None else None - elif section == "models": - value = config.models.get(key) - else: # pragma: no cover - only the two sections above are exercised - raise ValueError(section) - for profile in profiles: - overlay = getattr(profile, "mapping", None) - if isinstance(overlay, Mapping) and isinstance(overlay.get(section), Mapping): - if overlay[section].get(key) is not None: - value = overlay[section][key] - return value - - @pytest.mark.asyncio -async def test_caller_profiles_cannot_override_evaluator_owned_settings( +async def test_supplied_config_cannot_override_evaluator_owned_settings( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - # Fabric applies caller-supplied profiles over the config (last-wins), so the evaluator's per-task - # workspace (isolation + `workspace` evidence integrity) and model-under-eval must remain the final, - # authoritative layer. A caller profile that sets these must NOT win. - caller_profile = { - "name": "caller", - "environment": {"workspace": "/caller/hijacked-workspace"}, + # Since Fabric dropped profile overlays there is exactly one config, so the evaluator's per-task + # workspace (isolation + `workspace` evidence integrity) and model-under-eval stay authoritative by + # being composed on last. A caller config that pins these must NOT survive into the run. + hijacked = { + **_CONFIG, + "environment": {"provider": "local", "workspace": "/caller/hijacked-workspace"}, "models": {"default": {"provider": "openai", "model": "caller/rogue-model"}}, } @@ -392,20 +389,16 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: return _FakeResult(status="succeeded", output="ok") client_cls = _install_fake_fabric(monkeypatch, handler) - runtime = fabric_runtime.FabricAgentRuntime( - config=_CONFIG, model="openai/gpt-5.4", work_root=tmp_path / "fabric", profiles=[caller_profile] - ) + runtime = fabric_runtime.FabricAgentRuntime(config=hijacked, model="openai/gpt-5.4", work_root=tmp_path / "fabric") await runtime.run_tasks([_TASK]) config = client_cls.recorded[0]["agent"] - profiles = client_cls.recorded[0]["profiles"] - eval_workspace = config.environment.workspace # the per-task dir the evaluator composed - eval_model = config.models["default"] - - # After Fabric applies the caller profile, the evaluator's workspace + model must still win. - assert _resolve_like_fabric(config, profiles, "environment", "workspace") == eval_workspace - assert _resolve_like_fabric(config, profiles, "models", "default") == eval_model + # The config handed to Fabric carries the evaluator's per-task workspace and model, not the + # caller's — and there is no second layer that could put them back. + assert config.environment.workspace != "/caller/hijacked-workspace" + assert Path(config.environment.workspace).is_relative_to(tmp_path / "fabric") + assert (config.models["default"].provider, config.models["default"].model) == ("openai", "openai/gpt-5.4") @pytest.mark.asyncio @@ -653,11 +646,11 @@ def _skill_bundle(base: Path, *, name: str = "code-review", body: str = "Be thor return root -_HERMES_CONFIG = {"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.hermes.sdk"}} +_HERMES_CONFIG = {"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.hermes"}} @pytest.mark.asyncio -async def test_fabric_runtime_native_skill_adds_overlay_and_provenance( +async def test_fabric_runtime_native_skill_adds_skill_path_and_provenance( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: from nemo_evaluator_sdk.agent_eval.runtimes.fabric.skills import AgentSkill @@ -675,10 +668,9 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: # hardcoded adapter list. assert client_cls.planned, "expected the runtime to query Fabric.plan for skills routing" assert client_cls.planned[0]["agent"].skill_paths, "expected a probe skill path attached for planning" - # A native `skills` overlay reaches client.run pointing at the staged / skill dir. - profiles = client_cls.recorded[0]["profiles"] - skill_profile = next(p for p in profiles if p.name == "eval_skill") - assert skill_profile.mapping["skills"]["paths"][0].endswith("/code-review") + # The staged / skill dir is on the config handed to client.run. + config = client_cls.recorded[0]["agent"] + assert config.skill_paths[-1].endswith("/code-review") # Provenance is stamped into trial metadata for the A/B diff. prov = trials[0].metadata["skill"] assert prov["name"] == "code-review" @@ -690,29 +682,23 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: async def test_fabric_runtime_native_skill_preserves_preconfigured_skills( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - # Regression: Fabric applies profile skills.paths last-wins, so the native overlay must re-list any - # skills the config/profiles already declare — otherwise the treated arm would drop them and the A/B - # would differ by more than the injected skill. + # Injection appends to the config's skills.paths, so skills the config already declares survive. + # If they were dropped, the treated arm would differ from the baseline by more than the injected + # skill and the A/B would be invalid. from nemo_evaluator_sdk.agent_eval.runtimes.fabric.skills import AgentSkill def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: return _FakeResult(status="succeeded", output={"response": "ok"}) client_cls = _install_fake_fabric(monkeypatch, handler) - config = {**_HERMES_CONFIG, "skills": {"paths": ["/pre/existing-a"]}} + config = {**_HERMES_CONFIG, "skills": {"paths": ["/pre/existing-a", "/pre/existing-b"]}} skill = AgentSkill.from_directory(_skill_bundle(tmp_path)) - runtime = fabric_runtime.FabricAgentRuntime( - config=config, - work_root=tmp_path / "fabric", - skills=[skill], - profiles=[{"name": "caller", "skills": {"paths": ["/pre/existing-b"]}}], - ) + runtime = fabric_runtime.FabricAgentRuntime(config=config, work_root=tmp_path / "fabric", skills=[skill]) await runtime.run_tasks([_TASK]) - overlay = next(p for p in client_cls.recorded[0]["profiles"] if p.name == "eval_skill") - paths = overlay.mapping["skills"]["paths"] - # Config- and profile-declared skills are preserved, in order, ahead of the evaluated skill. + paths = client_cls.recorded[0]["agent"].skill_paths + # Config-declared skills are preserved, in order, ahead of the evaluated skill. assert paths[:2] == ["/pre/existing-a", "/pre/existing-b"] assert paths[-1].endswith("/code-review") @@ -736,8 +722,7 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: trials = await runtime.run_tasks([_TASK]) - profiles = client_cls.recorded[0]["profiles"] - assert any(p.name == "eval_skill" for p in profiles) + assert any(p.endswith("/code-review") for p in client_cls.recorded[0]["agent"].skill_paths) assert trials[0].metadata["skill"]["mode"] == "native" @@ -767,9 +752,8 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: # the injected files don't read as agent output to workspace-reading metrics. workspace = next((tmp_path / "fabric").glob("*/000000-task-1/workspace")) assert not (workspace / ".agents").exists() - # No skills overlay; provenance still records the codex injection. - names = [p.name for p in client_cls.recorded[0]["profiles"]] - assert "eval_skill" not in names + # No skills path added to the config; provenance still records the codex injection. + assert client_cls.recorded[0]["agent"].skill_paths == [] assert trials[0].metadata["skill"]["mode"] == "codex_skills_dir" @@ -894,12 +878,11 @@ def test_constructor_rejects_duplicate_skill_names(tmp_path: Path) -> None: @pytest.mark.asyncio -async def test_fabric_runtime_multiple_native_skills_merge_into_single_overlay( +async def test_fabric_runtime_multiple_native_skills_all_reach_the_config( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: # LAB hands the agent all of its skills on every task. A native harness must stage each skill into its - # own / bundle and list them ALL in ONE overlay — Fabric applies profile skills.paths last-wins, - # so a per-skill overlay would silently drop all but the last. + # own / bundle and every one of them must reach the config's skills.paths, in order. from nemo_evaluator_sdk.agent_eval.runtimes.fabric.skills import AgentSkill def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: @@ -911,10 +894,8 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: trials = await runtime.run_tasks([_TASK]) - # Exactly one native skills overlay reaches client.run, listing every staged / bundle in order. - overlays = [p for p in client_cls.recorded[0]["profiles"] if p.name == "eval_skill"] - assert len(overlays) == 1 - paths = overlays[0].mapping["skills"]["paths"] + # Every staged / bundle is on the config handed to client.run, in order. + paths = client_cls.recorded[0]["agent"].skill_paths assert [Path(p).name for p in paths] == ["docx", "pptx", "xlsx"] # Each bundle is staged on disk under its own / dir with its SKILL.md. for path in paths: @@ -955,8 +936,8 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: # ...then all removed (with the emptied .agents parent) before the workspace is exposed as evidence. workspace = next((tmp_path / "fabric").glob("*/000000-task-1/workspace")) assert not (workspace / ".agents").exists() - # Codex mode emits no overlay; one provenance per skill records the injection. - assert "eval_skill" not in [p.name for p in client_cls.recorded[0]["profiles"]] + # Codex mode adds no skills path; one provenance per skill records the injection. + assert client_cls.recorded[0]["agent"].skill_paths == [] provs = trials[0].metadata["skills"] assert [prov["name"] for prov in provs] == list(names) assert all(prov["mode"] == "codex_skills_dir" for prov in provs) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_skills.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_skills.py index 23e449f6d0..a9cb672f38 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_skills.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_skills.py @@ -13,7 +13,6 @@ CODEX_SKILLS_DIR, SKILL_MODE_CODEX_SKILLS_DIR, SKILL_MODE_NATIVE, - SKILL_PROFILE_NAME, AgentSkill, SkillInjectionError, install_skill, @@ -105,7 +104,7 @@ def test_install_native_stages_named_dir_and_overlay(tmp_path: Path) -> None: installation = install_skill( skill=AgentSkill.from_directory(_make_bundle(tmp_path / "src")), - adapter_id="nvidia.fabric.hermes.sdk", + adapter_id="nvidia.fabric.hermes", mode=SKILL_MODE_NATIVE, workspace_dir=workspace, skill_stage_dir=stage, @@ -117,9 +116,7 @@ def test_install_native_stages_named_dir_and_overlay(tmp_path: Path) -> None: assert not (workspace / "SKILL.md").exists() assert not (workspace / ".agents").exists() - overlay = installation.profiles[0] - assert overlay["name"] == SKILL_PROFILE_NAME - assert overlay["skills"] == {"paths": [str(skill_root)]} + assert installation.skill_paths == [str(skill_root)] prov = installation.provenance assert prov["name"] == "code-review" @@ -134,7 +131,7 @@ def test_install_native_copies_directory_tree(tmp_path: Path) -> None: install_skill( skill=AgentSkill.from_directory(src), - adapter_id="nvidia.fabric.hermes.sdk", + adapter_id="nvidia.fabric.hermes", mode=SKILL_MODE_NATIVE, workspace_dir=tmp_path / "workspace", skill_stage_dir=stage, @@ -152,7 +149,7 @@ def test_install_codex_places_under_agents_skills(tmp_path: Path) -> None: installation = install_skill( skill=AgentSkill.from_directory(_make_bundle(tmp_path / "src")), - adapter_id="nvidia.fabric.codex.cli", + adapter_id="nvidia.fabric.codex", mode=SKILL_MODE_CODEX_SKILLS_DIR, workspace_dir=workspace, skill_stage_dir=tmp_path / "stage", @@ -161,8 +158,8 @@ def test_install_codex_places_under_agents_skills(tmp_path: Path) -> None: # Codex discovers agentskills bundles from .agents/skills/ in its working directory. skill_md = workspace / ".agents" / "skills" / "code-review" / "SKILL.md" assert "Be thorough." in skill_md.read_text(encoding="utf-8") - # No profile overlay: placement in the workspace is the delivery mechanism. - assert installation.profiles == [] + # No skills path: placement in the workspace is the delivery mechanism. + assert installation.skill_paths == [] assert installation.provenance["mode"] == SKILL_MODE_CODEX_SKILLS_DIR assert installation.provenance["location"] == f"{CODEX_SKILLS_DIR}/code-review" @@ -176,7 +173,7 @@ def test_codex_bundle_does_not_collide_with_workspace_root(tmp_path: Path) -> No install_skill( skill=AgentSkill.from_directory(src), - adapter_id="nvidia.fabric.codex.cli", + adapter_id="nvidia.fabric.codex", mode=SKILL_MODE_CODEX_SKILLS_DIR, workspace_dir=workspace, skill_stage_dir=tmp_path / "stage", @@ -186,21 +183,18 @@ def test_codex_bundle_does_not_collide_with_workspace_root(tmp_path: Path) -> No assert (workspace / ".agents" / "skills" / "collide" / "data.csv").read_text(encoding="utf-8") == "skill payload" -def test_install_native_preserves_existing_skill_paths(tmp_path: Path) -> None: - # Fabric applies profile skills.paths last-wins, so the overlay must carry the pre-existing paths - # (order-preserved) alongside the evaluated skill, or the treated arm would drop them. +def test_install_native_returns_only_the_staged_path(tmp_path: Path) -> None: + # Preserving config-declared skills is FabricConfig.add_skill_path's job (it appends and + # de-duplicates), so installation reports only what it staged and never re-lists prior paths. installation = install_skill( skill=AgentSkill.from_directory(_make_bundle(tmp_path / "src")), - adapter_id="nvidia.fabric.hermes.sdk", + adapter_id="nvidia.fabric.hermes", mode=SKILL_MODE_NATIVE, workspace_dir=tmp_path / "workspace", skill_stage_dir=tmp_path / "stage", - existing_skill_paths=["/pre/a", "/pre/b", "/pre/a"], # duplicate is collapsed ) - paths = installation.profiles[0]["skills"]["paths"] - assert paths[:2] == ["/pre/a", "/pre/b"] - assert paths[-1] == str(tmp_path / "stage" / "code-review") + assert installation.skill_paths == [str(tmp_path / "stage" / "code-review")] def test_install_native_recreates_stale_stage(tmp_path: Path) -> None: @@ -209,7 +203,7 @@ def test_install_native_recreates_stale_stage(tmp_path: Path) -> None: stage = tmp_path / "stage" install_skill( skill=AgentSkill.from_directory(_make_bundle(tmp_path / "v1", extra={"old.md": "stale"})), - adapter_id="nvidia.fabric.hermes.sdk", + adapter_id="nvidia.fabric.hermes", mode=SKILL_MODE_NATIVE, workspace_dir=tmp_path / "workspace", skill_stage_dir=stage, @@ -218,7 +212,7 @@ def test_install_native_recreates_stale_stage(tmp_path: Path) -> None: install_skill( skill=AgentSkill.from_directory(_make_bundle(tmp_path / "v2")), # no old.md - adapter_id="nvidia.fabric.hermes.sdk", + adapter_id="nvidia.fabric.hermes", mode=SKILL_MODE_NATIVE, workspace_dir=tmp_path / "workspace", skill_stage_dir=stage, @@ -237,7 +231,7 @@ def test_install_codex_rejects_reserved_path_collision(tmp_path: Path) -> None: with pytest.raises(SkillInjectionError, match="reserved path"): install_skill( skill=AgentSkill.from_directory(_make_bundle(tmp_path / "src")), - adapter_id="nvidia.fabric.codex.cli", + adapter_id="nvidia.fabric.codex", mode=SKILL_MODE_CODEX_SKILLS_DIR, workspace_dir=workspace, skill_stage_dir=tmp_path / "stage", @@ -254,14 +248,14 @@ def test_hash_is_content_sensitive(tmp_path: Path) -> None: a = install_skill( skill=AgentSkill.from_directory(one), - adapter_id="nvidia.fabric.hermes.sdk", + adapter_id="nvidia.fabric.hermes", mode=SKILL_MODE_NATIVE, workspace_dir=tmp_path / "wa", skill_stage_dir=tmp_path / "sa", ) b = install_skill( skill=AgentSkill.from_directory(two), - adapter_id="nvidia.fabric.hermes.sdk", + adapter_id="nvidia.fabric.hermes", mode=SKILL_MODE_NATIVE, workspace_dir=tmp_path / "wb", skill_stage_dir=tmp_path / "sb", @@ -280,7 +274,7 @@ def test_install_skills_rolls_back_staged_bundles_on_failure(tmp_path: Path) -> with pytest.raises(SkillInjectionError): install_skills( skills=[good, bad], - adapter_id="nvidia.fabric.hermes.sdk", + adapter_id="nvidia.fabric.hermes", mode=SKILL_MODE_NATIVE, workspace_dir=tmp_path / "ws", skill_stage_dir=stage_dir, @@ -313,7 +307,7 @@ def _fail_on_late(directory: Path) -> str: with pytest.raises(OSError): install_skills( skills=[good, late], - adapter_id="nvidia.fabric.hermes.sdk", + adapter_id="nvidia.fabric.hermes", mode=SKILL_MODE_NATIVE, workspace_dir=tmp_path / "ws", skill_stage_dir=stage_dir, @@ -336,7 +330,7 @@ def test_install_skills_rollback_never_deletes_preexisting_seed_file(tmp_path: P with pytest.raises(SkillInjectionError): install_skills( skills=[good, collide], - adapter_id="nvidia.fabric.codex.cli", + adapter_id="nvidia.fabric.codex", mode=SKILL_MODE_CODEX_SKILLS_DIR, workspace_dir=workspace, skill_stage_dir=tmp_path / "stage", diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_surface.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_surface.py new file mode 100644 index 0000000000..715ad56cea --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_surface.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Contract tests binding ``FabricAgentRuntime`` to the REAL ``nemo_fabric`` API surface. + +The other fabric unit tests are hermetic: they replace ``nemo_fabric`` with a hand-written fake, so +they encode whatever call signature the fake author wrote and keep passing even when the real Fabric +API moves underneath them. That is exactly how two breakages reached us unnoticed against a newer +Fabric — the ``enable_relay`` keyword changed and the harness adapter ids dropped their ``.cli``/ +``.sdk`` suffixes — because nothing exercised the real package's call sites. + +These tests close that gap by driving the runtime's own composition against the installed Fabric: + +* **enable_relay signature** — ``_compose_config`` calls + ``FabricConfig.enable_relay(observability=RelayObservabilityConfig(...))``. The retired ``config=`` + keyword (or a shape Fabric's ``RelayObservabilityConfig`` rejects) would raise here. +* **relay observability shape** — Fabric's relay models are ``extra="allow"``, so a stale field name + is accepted *silently* and simply never takes effect (Relay's config policy warns on unknown fields + rather than failing). ``ATOF``'s destination moved onto a typed sink list, and the old flat + ``output_directory``/``filename``/``mode`` form would export nothing at all. Asserting the composed + values — not just that the block exists — is what makes that visible. +* **adapter-id resolution** — the bare harness id the runtime forwards must resolve against Fabric's + adapter registry via the planner. A suffixed ``nvidia.fabric.codex.cli`` would raise instead. + +``importorskip('nemo_fabric')`` makes the whole module inert wherever the native Fabric wheels are not +installed (the hermetic-only 3.11 lanes), so it never competes with the fake-backed unit tests. It is +meant to run where the ``fabric`` extra is present — e.g. the Linux ``fabric-wheel-smoke`` CI job, or a +local ``uv sync --extra fabric`` (Fabric publishes a macOS arm64 wheel as of 0.1.0rc2). +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +pytest.importorskip("nemo_fabric") + +from nemo_evaluator_sdk.agent_eval.runtimes.fabric import runtime as fabric_runtime +from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime + +# The ``fabric`` extra installs claude/codex/deepagents/hermes. Codex and hermes are both covered +# below because they exercise *different* skill routing: codex self-discovers bundles from its +# workspace, hermes accepts them natively through Fabric's ``skills`` config. + + +def _adapter_installed(name: str) -> bool: + """Whether a harness adapter is installed (the ``fabric`` extra, not the base SDK). + + ``nemo_fabric`` itself is a base dependency, so importing it proves nothing about harnesses: + without the adapters Fabric resolves none and fails with ``available adapters: []``. ``find_spec`` + raises rather than returning None when the parent package is missing, hence the guard. + """ + try: + return importlib.util.find_spec(f"nemo_fabric_adapters.{name}") is not None + except ModuleNotFoundError: + return False + + +def _codex_adapter_installed() -> bool: + return _adapter_installed("codex") + + +requires_harness_adapters = pytest.mark.skipif( + not _codex_adapter_installed(), + reason="needs the harness adapters: uv sync --extra fabric", +) + +requires_hermes_adapter = pytest.mark.skipif( + not _adapter_installed("hermes"), + reason="needs the hermes harness adapter: uv sync --extra fabric", +) + +_CODEX_ADAPTER_ID = "nvidia.fabric.codex" +_HERMES_ADAPTER_ID = "nvidia.fabric.hermes" +_HERMES_CONFIG = { + "metadata": {"name": "fabric-surface-hermes"}, + "harness": {"adapter_id": _HERMES_ADAPTER_ID, "resolution": "preinstalled"}, + "runtime": {"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"}, + "environment": {"provider": "local"}, +} +_CODEX_CONFIG = { + "metadata": {"name": "fabric-surface"}, + "harness": {"adapter_id": _CODEX_ADAPTER_ID, "resolution": "preinstalled"}, + "runtime": {"mode": "oneshot", "transport": "cli", "input_schema": "text", "output_schema": "message"}, + "environment": {"provider": "local"}, +} + + +def test_compose_config_enables_relay_via_current_signature(tmp_path: Path) -> None: + """The runtime's real ``enable_relay(observability=...)`` call is accepted by installed Fabric. + + Exercises ``_compose_config`` (which calls ``enable_relay`` with the observability config built by + ``_relay_config``) rather than executing a harness, so it needs only the Fabric wheels — no codex + CLI, relay gateway, or model. The dropped ``config=`` keyword would raise a ``TypeError`` here. + """ + from nemo_fabric import ( # ty: ignore[unresolved-import] + FabricConfig, + RelayConfig, + RelayObservabilityConfig, + ) + + runtime = FabricAgentRuntime(config=_CODEX_CONFIG, capture_trajectory=True) + agent_config = FabricConfig.from_mapping(_CODEX_CONFIG) + evidence_dir = tmp_path / "evidence" + workspace_dir = evidence_dir / "workspace" + evidence_dir.mkdir() + workspace_dir.mkdir() + + composed = runtime._compose_config(agent_config, evidence_dir, workspace_dir) + + # enable_relay accepted the runtime's call and stored Fabric's own typed relay models: a populated + # RelayConfig whose observability carries the ATIF/ATOF shape ``_relay_config`` built. The dropped + # ``config=`` keyword — or an observability shape Fabric rejects — would have raised before this. + assert isinstance(composed.relay, RelayConfig) + observability = composed.relay.observability + assert isinstance(observability, RelayObservabilityConfig) + + # The exporters must land on the *declared fields*, not in the extras bag. Fabric's models allow + # extras, so asserting `is not None` alone would still pass with a stale field name that exports + # nothing — these assertions are what actually pin the current schema. + relay_dir = str(evidence_dir / "relay") + atif = observability.atif + assert atif is not None and atif.enabled is True + assert str(atif.output_directory) == relay_dir + assert atif.filename_template == fabric_runtime._ATIF_FILENAME_TEMPLATE + + atof = observability.atof + assert atof is not None and atof.enabled is True + (atof_sink,) = atof.sinks or [] + assert str(atof_sink.output_directory) == relay_dir + assert atof_sink.filename == fabric_runtime._ATOF_FILENAME + + +@requires_harness_adapters +def test_compose_config_is_a_complete_config_fabric_accepts(tmp_path: Path) -> None: + """The composed per-task config is self-contained — no profile layer is needed or possible. + + Fabric 0.1.0rc2 deleted profile overlays: ``FabricProfileConfig`` is gone, ``Fabric.run``/``plan`` + take no ``profiles``, and ``FabricConfig.from_mapping`` raises on a ``profiles`` key. This asserts + the runtime's composed config carries the evaluator-owned settings itself and round-trips through + Fabric's own validator, which is the whole contract now. + """ + from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] + + runtime = FabricAgentRuntime(config=_CODEX_CONFIG, model="openai/gpt-5.4", capture_trajectory=False) + evidence_dir = tmp_path / "evidence" + workspace_dir = evidence_dir / "workspace" + workspace_dir.mkdir(parents=True) + + composed = runtime._compose_config(FabricConfig.from_mapping(_CODEX_CONFIG), evidence_dir, workspace_dir) + composed.add_skill_path(str(tmp_path / "staged-skill")) + + # Evaluator-owned per-task settings live on the config itself, not in a trailing overlay. + assert composed.environment is not None and str(composed.environment.workspace) == str(workspace_dir) + assert composed.models["default"].model == "openai/gpt-5.4" + assert str(tmp_path / "staged-skill") in [str(p) for p in composed.skills.paths] + + # It survives Fabric's own round-trip and planner, so it is a complete config by Fabric's rules. + assert Fabric().plan(FabricConfig.from_mapping(composed.to_mapping())).adapter.adapter_id == _CODEX_ADAPTER_ID + + +@requires_harness_adapters +def test_runtime_adapter_id_resolves_against_registry() -> None: + """The bare harness id the runtime forwards resolves through Fabric's real planner. + + Planning resolves the selected adapter from the registry without starting the runtime, so it needs + only the installed adapter package. A retired ``nvidia.fabric.codex.cli`` id would raise a + ``FabricConfigError`` instead of resolving. + """ + from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] + + plan = Fabric().plan(FabricConfig.from_mapping(_CODEX_CONFIG)) + + assert plan.adapter.adapter_id == _CODEX_ADAPTER_ID + + +@requires_hermes_adapter +def test_hermes_routes_skills_natively_per_the_real_planner() -> None: + """Fabric really does route skills to the hermes harness natively. + + This is the one capability the hermetic tests cannot check: they fake the planner and hardcode + ``_NATIVE_SKILL_ADAPTERS``, so the entire native-injection branch — ``SKILL_MODE_NATIVE``, + ``add_skill_path``, and the native-vs-workspace split in ``install_skills`` — is otherwise + validated only against our own assumption. If Fabric stopped routing hermes' skills + ``harness_native``, every fake-backed test would keep passing while real runs silently fell back. + + Drives our own ``resolve_skill_mode`` over a real ``RunPlan`` so the assertion covers the code + the runtime actually executes, not just Fabric's output shape. + """ + from nemo_evaluator_sdk.agent_eval.runtimes.fabric.skills import SKILL_MODE_NATIVE, resolve_skill_mode + from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] + + # A skill path must be attached for the planner to emit a skills route at all — the sentinel need + # not exist on disk, mirroring how FabricAgentRuntime probes capabilities. + probe = FabricConfig.from_mapping(_HERMES_CONFIG) + probe.add_skill_path(fabric_runtime._SKILL_PROBE_PATH) + plan = Fabric().plan(probe) + + assert plan.adapter.adapter_id == _HERMES_ADAPTER_ID + assert resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness) == SKILL_MODE_NATIVE + + +@requires_harness_adapters +def test_codex_also_routes_skills_natively_so_the_workspace_branch_is_a_fallback() -> None: + """The shipped codex adapter accepts the native skills config, so codex plans ``native`` too. + + This is not what our fake-backed tests encode: they hardcode codex as non-native and exercise + ``SKILL_MODE_CODEX_SKILLS_DIR`` (staging into ``/.agents/skills/``). That branch is a + genuine fallback for a codex-harness adapter which routes skills ``unsupported``, but the adapter + we ship no longer takes it. Pinning the real answer here keeps the two from silently diverging — + and would catch a revert, which would change where bundles are staged and whether the workspace + needs post-run cleanup. + """ + from nemo_evaluator_sdk.agent_eval.runtimes.fabric.skills import SKILL_MODE_NATIVE, resolve_skill_mode + from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] + + probe = FabricConfig.from_mapping(_CODEX_CONFIG) + probe.add_skill_path(fabric_runtime._SKILL_PROBE_PATH) + plan = Fabric().plan(probe) + + assert "skills" in plan.capability_plan.get("routes", [])[0].get("kind", "") + assert resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness) == SKILL_MODE_NATIVE diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_skill_used_metric.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_skill_used_metric.py index 1d22582172..870efe1de6 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_skill_used_metric.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_skill_used_metric.py @@ -17,7 +17,7 @@ "name": "code-review", "hash": "deadbeef", "mode": "codex_skills_dir", - "adapter_id": "nvidia.fabric.codex.cli", + "adapter_id": "nvidia.fabric.codex", "location": _LOCATION, } @@ -26,7 +26,7 @@ "name": "summarize", "hash": "cafebabe", "mode": "codex_skills_dir", - "adapter_id": "nvidia.fabric.codex.cli", + "adapter_id": "nvidia.fabric.codex", "location": _LOCATION_B, } diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml index cc6b317210..7ad21652b8 100644 --- a/packages/nemo_platform/pyproject.toml +++ b/packages/nemo_platform/pyproject.toml @@ -294,7 +294,8 @@ nemo-evaluator-sdk = [ "ragas==0.4.3", "langchain-openai>=1.3.5", "langchain-nvidia-ai-endpoints>=1.4.3,<2.0.0", - "nemo-relay>=0.4.0,<0.5.0", + "nemo-relay>=0.6.0,<0.7", + "nemo-fabric>=0.1.0rc6,<0.2.0", ] # Generated from [tool.bundle-package]; do not edit by hand. diff --git a/plugins/nemo-agents/pyproject.toml b/plugins/nemo-agents/pyproject.toml index 044d057383..a1bc1db833 100644 --- a/plugins/nemo-agents/pyproject.toml +++ b/plugins/nemo-agents/pyproject.toml @@ -63,14 +63,16 @@ fabric = [ # TODO(AIRCORE-932): Move Fabric into the default plugin dependencies once evaluator has migrated # to the 0.1.0a20260724+ config-first SDK API and Fabric packaging is stable across Platform environments. # TODO(AIRCORE-897): Move this to a stable Fabric version before release once available. and add [relay] - "nemo-fabric>=0.1.0rc4,<0.2.0", - "nemo-fabric-adapters-claude[harness]>=0.1.0rc4,<0.2.0", - "nemo-fabric-adapters-codex[harness]>=0.1.0rc4,<0.2.0", + # claude/codex ride the metapackage extras, which pin the adapters to the metapackage's own + # version. They were pinned directly through rc5 only because rc4's extras required + # `nemo-fabric-adapters-*==0.1.0`, never published; fixed in rc6. + "nemo-fabric[claude,codex]>=0.1.0rc6,<0.2.0", # TODO(AIRCORE-952): Re-enable once the DeepAgents adapter supports Relay observability v2 configs # generated by Fabric streaming. - # "nemo-fabric-adapters-deepagents[harness]>=0.1.0rc4,<0.2.0", - # TODO(AIRCORE-952): Switch to [harness] once hermes-agent relaxes vulnerable exact dependency pins. - "nemo-fabric-adapters-hermes>=0.1.0rc4,<0.2.0; python_version < '3.14'", + # "nemo-fabric[deepagents]>=0.1.0rc6,<0.2.0", + # TODO(AIRCORE-952): Switch to the metapackage's `hermes-agent` extra once hermes-agent relaxes its + # vulnerable exact dependency pins — that extra applies [harness], which pins requests==2.33.0. + "nemo-fabric-adapters-hermes>=0.1.0rc6,<0.2.0; python_version < '3.14'", ] container = [ "jinja2>=3.1", diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml index ae6e66cd4d..68ffee6c31 100644 --- a/plugins/nemo-evaluator/openapi/openapi.yaml +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -2982,20 +2982,12 @@ components: type: object title: Config description: Inline NeMo Fabric agent config (an ``agent.yaml`` as a JSON-shaped - mapping). Its ``harness.adapter_id`` selects the harness, e.g. ``nvidia.fabric.codex.cli`` + mapping). Its ``harness.adapter_id`` selects the harness, e.g. ``nvidia.fabric.codex`` for Codex. - profiles: - items: - additionalProperties: true - type: object - type: array - title: Profiles - description: Ordered Fabric profile overlays applied after the base config, - before ``model``. model: title: Model - description: Optional ``provider/model`` slug applied as a final profile - overlay; the harness default is used when omitted. + description: Optional ``provider/model`` slug applied as the config's default + model; the harness default is used when omitted. type: string timeout_s: type: integer @@ -3014,17 +3006,13 @@ components: required: - config title: FabricRunnerTarget - description: 'Generate trials by driving an agent harness through the NeMo Fabric - runtime. - - - Fabric is harness-agnostic: the harness (Codex, Hermes, ...) is selected by - the supplied - - config''s ``harness.adapter_id`` and is never inferred from ``model``. ``model`` - is applied as a - - final profile overlay when given.' + description: "Generate trials by driving an agent harness through the NeMo Fabric\ + \ runtime.\n\nFabric is harness-agnostic: the harness (Codex, Hermes, ...)\ + \ is selected by the supplied\nconfig's ``harness.adapter_id`` and is never\ + \ inferred from ``model``. ``model`` is applied as the\nconfig's default model\ + \ when given.\n\nA run is described by exactly one complete ``config``. Fabric\ + \ 0.1.0rc2 removed profile overlays,\nso the former ``profiles`` field is\ + \ gone \u2014 fold any overlay you were passing into ``config``." FieldMapping: properties: input: diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py index 8e997dee61..008e1338a7 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py @@ -267,7 +267,6 @@ def _resolve_target( if isinstance(target, FabricRunnerTarget): fabric_runtime = FabricAgentRuntime( config=target.config, - profiles=target.profiles, model=target.model, timeout_s=target.timeout_s, capture_trajectory=target.capture_trajectory, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py index f71a42b6fa..afd688cf08 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py @@ -80,8 +80,11 @@ class FabricRunnerTarget(BaseModel): """Generate trials by driving an agent harness through the NeMo Fabric runtime. Fabric is harness-agnostic: the harness (Codex, Hermes, ...) is selected by the supplied - config's ``harness.adapter_id`` and is never inferred from ``model``. ``model`` is applied as a - final profile overlay when given. + config's ``harness.adapter_id`` and is never inferred from ``model``. ``model`` is applied as the + config's default model when given. + + A run is described by exactly one complete ``config``. Fabric 0.1.0rc2 removed profile overlays, + so the former ``profiles`` field is gone — fold any overlay you were passing into ``config``. """ model_config = ConfigDict(extra="forbid") @@ -89,15 +92,11 @@ class FabricRunnerTarget(BaseModel): kind: Literal["fabric"] = "fabric" config: dict[str, Any] = Field( description="Inline NeMo Fabric agent config (an ``agent.yaml`` as a JSON-shaped mapping). Its " - "``harness.adapter_id`` selects the harness, e.g. ``nvidia.fabric.codex.cli`` for Codex.", - ) - profiles: list[dict[str, Any]] = Field( - default_factory=list, - description="Ordered Fabric profile overlays applied after the base config, before ``model``.", + "``harness.adapter_id`` selects the harness, e.g. ``nvidia.fabric.codex`` for Codex.", ) model: str | None = Field( default=None, - description="Optional ``provider/model`` slug applied as a final profile overlay; the harness " + description="Optional ``provider/model`` slug applied as the config's default model; the harness " "default is used when omitted.", ) timeout_s: int = Field(default=600, ge=1, description="Per-task timeout for the Fabric run, in seconds.") diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py index fdbb3ed23d..8c03b81a34 100644 --- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py +++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py @@ -222,7 +222,7 @@ def test_resolve_target_builds_codex_runtime_from_runner_target(tmp_path: Path) def test_resolve_target_builds_fabric_runtime_from_runner_target(tmp_path: Path) -> None: ctx = _job_context(tmp_path) fabric_target = FabricRunnerTarget( - config={"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex.cli"}}, + config={"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex"}}, model="openai/gpt-5.4", ) target, prompt_template, params = AgentEvalJob._resolve_target(fabric_target, ctx) @@ -441,9 +441,7 @@ def _assert_agent_eval_step_entrypoint(job_spec: PlatformJobSpec) -> None: [ (CodexRunnerTarget(model="gpt-5.5"), "codex", None), ( - FabricRunnerTarget( - config={"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex.cli"}} - ), + FabricRunnerTarget(config={"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex"}}), "fabric", None, ), diff --git a/script/dev-install-fabric.sh b/script/dev-install-fabric.sh index 6c966fc324..d560455bcd 100755 --- a/script/dev-install-fabric.sh +++ b/script/dev-install-fabric.sh @@ -2,22 +2,28 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Dev-only: set up the local dependencies the Fabric eval runner needs so the type checker and a -# live FabricAgentRuntime run work end-to-end: -# 1. the native `nemo-fabric` SDK (with the codex + relay extras) into the project venv, and -# 2. the `nemo-relay` gateway binary (required for ATIF trajectory capture on the codex harness). -# This is an imperative install — it does NOT touch uv.lock, and CI intentionally runs without it -# (the `# ty: ignore[unresolved-import]` in agent_eval/runtimes/fabric/runtime.py covers the CI case). +# Dev-only: install the `nemo-relay` GATEWAY BINARY, the one Fabric eval dependency that cannot come +# from a wheel. It is required for live ATIF trajectory capture on out-of-process harnesses (codex). # -# nemo-fabric and the nemo-relay gateway are private/native builds with no published wheel/binary in -# our index, so they can't be locked dependencies yet (see -# plugins/nemo-evaluator/docs/design/fabric-runner-integration.md, Tier 3). A live codex run also -# needs the `codex` CLI + `codex login` auth. +# Everything else is in the lock — `uv sync --extra fabric` installs the nemo-fabric SDK, the +# codex/claude/deepagents adapters, and the nemo-relay Python bindings. The pip `nemo-relay` package +# is bindings-only (its wheel declares no console script and contains no executable), so the daemon is +# published solely as a GitHub release asset. +# +# The version defaults to the `nemo-relay` bindings installed in the venv, so the daemon and the +# bindings cannot drift apart when the lock moves. +# +# To run against an unreleased Fabric instead of the locked wheels, install the checkout directly: +# uv pip install --python .venv/bin/python "/path/to/NeMo-Fabric[codex,relay,runtime]" +# and `uv sync --extra fabric` to get back to the locked state. (That needs cargo — Fabric builds a +# Rust/pyo3 extension from source.) +# +# A live codex run additionally needs the `codex` CLI + `codex login` auth. +# See plugins/nemo-evaluator/docs/design/fabric-runner-integration.md. # # Usage: -# script/dev-install-fabric.sh # NeMo-Fabric+NeMo-Relay under $HOME/workspace -# NEMO_FABRIC_REPO=... NEMO_RELAY_REPO=... script/dev-install-fabric.sh -# script/dev-install-fabric.sh --uninstall # restore the CI-equivalent (no nemo-fabric) state +# script/dev-install-fabric.sh # version matching the installed bindings +# NEMO_RELAY_VERSION=0.6.0-rc.4 script/dev-install-fabric.sh # pin a specific gateway release set -euo pipefail VENV_PY=".venv/bin/python" @@ -26,58 +32,71 @@ if [ ! -x "$VENV_PY" ]; then exit 1 fi -if [ "${1:-}" = "--uninstall" ]; then - uv pip uninstall --python "$VENV_PY" nemo-fabric - echo "Removed nemo-fabric; venv is back to the lock-consistent / CI-equivalent state." - echo "(The nemo-relay gateway binary, if installed, is left in place — remove it from ~/.cargo/bin manually if desired.)" - exit 0 +# NeMo-Relay tags releases with the version PyPI publishes, but PyPI normalizes prereleases +# (0.6.0rc4) while the git tag is semver (0.6.0-rc.4), so convert. +if [ -z "${NEMO_RELAY_VERSION:-}" ]; then + bindings_version="$("$VENV_PY" -c 'import importlib.metadata as m; print(m.version("nemo-relay"))' 2>/dev/null || true)" + if [ -z "$bindings_version" ]; then + echo "nemo-relay is not installed in $VENV_PY, so the gateway version cannot be derived." >&2 + echo "Run 'uv sync --extra fabric' first, or pass NEMO_RELAY_VERSION= explicitly." >&2 + exit 1 + fi + NEMO_RELAY_VERSION="$(printf '%s' "$bindings_version" | sed -E 's/([0-9])(a|b|rc)\.?([0-9]+)$/\1-\2.\3/')" + echo "Using nemo-relay gateway ${NEMO_RELAY_VERSION} to match the installed bindings (${bindings_version})." fi -# nemo-fabric builds a Rust/pyo3 extension via maturin and the relay gateway is a Rust CLI, so cargo -# must be on PATH. -if ! command -v cargo >/dev/null 2>&1 && [ -f "$HOME/.cargo/env" ]; then - # shellcheck disable=SC1091 - . "$HOME/.cargo/env" -fi -if ! command -v cargo >/dev/null 2>&1; then - echo "cargo (Rust toolchain) not found; install it to build the native components: https://rustup.rs" >&2 - exit 1 +# Skip only when an existing nemo-relay already matches, so an explicit version request is honored +# rather than silently short-circuited by any PATH match. +if command -v nemo-relay >/dev/null 2>&1; then + # `|| true`: a broken nemo-relay on PATH exits non-zero, and under `set -e -o pipefail` that would + # abort here — in the very branch that exists to replace it. An empty version just means "reinstall". + installed_relay_ver="$(nemo-relay --version 2>/dev/null | awk '{print $NF}' || true)" + if [ "$installed_relay_ver" = "$NEMO_RELAY_VERSION" ]; then + echo "nemo-relay gateway already on PATH at ${installed_relay_ver}: $(command -v nemo-relay)" + exit 0 + fi + echo "nemo-relay ${installed_relay_ver:-?} on PATH differs from requested ${NEMO_RELAY_VERSION}; (re)installing ..." fi -# 1. nemo-fabric SDK (+ codex and relay extras) into the project venv. -FABRIC_REPO="${NEMO_FABRIC_REPO:-$HOME/workspace/NeMo-Fabric}" -if [ ! -d "$FABRIC_REPO" ]; then - echo "NeMo-Fabric checkout not found at: $FABRIC_REPO" >&2 - echo "Clone it (gh repo clone NVIDIA/NeMo-Fabric) or set NEMO_FABRIC_REPO=/path/to/NeMo-Fabric." >&2 - exit 1 -fi -echo "Building + installing nemo-fabric[codex,relay] from $FABRIC_REPO into $VENV_PY ..." -uv pip install --python "$VENV_PY" "${FABRIC_REPO}[codex,relay]" -"$VENV_PY" -c "import nemo_fabric; from nemo_fabric import Fabric, RunResult; print('nemo_fabric OK:', nemo_fabric.__file__)" +# Host platform -> NeMo-Relay release target triple. Every platform in the workspace's +# [tool.uv] environments has a published asset. +case "$(uname -s):$(uname -m)" in + Darwin:arm64) relay_target="aarch64-apple-darwin" ;; + Linux:x86_64) relay_target="x86_64-unknown-linux-musl" ;; + Linux:aarch64 | Linux:arm64) relay_target="aarch64-unknown-linux-musl" ;; + *) + echo "No published nemo-relay gateway for $(uname -s):$(uname -m)." >&2 + echo "See https://github.com/NVIDIA/NeMo-Relay/releases for available targets." >&2 + exit 1 + ;; +esac -# 2. nemo-relay gateway binary (codex -> OTLP -> gateway -> trajectory-*.atif.json). Required for -# trajectory capture; the pip `nemo-relay` package does NOT ship this executable. -if command -v nemo-relay >/dev/null 2>&1; then - echo "nemo-relay gateway already on PATH: $(command -v nemo-relay) ($(nemo-relay --version 2>/dev/null || echo '?'))" +RELAY_BIN_DIR="${CARGO_HOME:-$HOME/.cargo}/bin" +asset="nemo-relay-cli-${relay_target}-${NEMO_RELAY_VERSION}" +base="https://github.com/NVIDIA/NeMo-Relay/releases/download/${NEMO_RELAY_VERSION}" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +echo "Downloading nemo-relay gateway ${NEMO_RELAY_VERSION} (${relay_target}) from GitHub releases ..." +curl -fsSL -o "${tmp}/nemo-relay" "${base}/${asset}" +curl -fsSL -o "${tmp}/nemo-relay.sha256" "${base}/${asset}.sha256" +want="$(awk '{print $1}' "${tmp}/nemo-relay.sha256")" +if command -v sha256sum >/dev/null 2>&1; then + got="$(sha256sum "${tmp}/nemo-relay" | awk '{print $1}')" else - RELAY_REPO="${NEMO_RELAY_REPO:-$HOME/workspace/NeMo-Relay}" - if [ ! -d "$RELAY_REPO" ]; then - echo "nemo-relay gateway not on PATH and NeMo-Relay checkout not found at: $RELAY_REPO" >&2 - echo "Clone it (gh repo clone NVIDIA/NeMo-Relay) or set NEMO_RELAY_REPO=/path/to/NeMo-Relay." >&2 - echo "Trajectory (ATIF) capture will fail without the nemo-relay gateway." >&2 - exit 1 - fi - echo "Building + installing the nemo-relay gateway from $RELAY_REPO (cargo install) ..." - cargo install --path "$RELAY_REPO/crates/cli" --locked - echo "nemo-relay gateway installed: $(command -v nemo-relay) ($(nemo-relay --version 2>/dev/null || echo '?'))" + got="$(shasum -a 256 "${tmp}/nemo-relay" | awk '{print $1}')" +fi +if [ "$want" != "$got" ]; then + echo "Checksum mismatch for ${asset}: expected ${want}, got ${got}" >&2 + exit 1 fi -cat <<'EOF' - -Done. Fabric's real types now resolve (ty enforces them; you'll see 2 harmless "unused ty: ignore" -warnings while nemo-fabric is installed), and live FabricAgentRuntime runs can capture ATIF -trajectories (needs the `codex` CLI + `codex login` for a codex-harness run). +mkdir -p "$RELAY_BIN_DIR" +install -m 0755 "${tmp}/nemo-relay" "${RELAY_BIN_DIR}/nemo-relay" +echo "nemo-relay gateway installed: ${RELAY_BIN_DIR}/nemo-relay ($("${RELAY_BIN_DIR}/nemo-relay" --version 2>/dev/null || echo '?'))" -Restore the CI-equivalent state with: - script/dev-install-fabric.sh --uninstall -EOF +# Warn if the install dir isn't on PATH — live tests resolve the gateway via shutil.which(). +case ":${PATH}:" in + *":${RELAY_BIN_DIR}:"*) : ;; + *) echo "NOTE: ${RELAY_BIN_DIR} is not on PATH — add it so 'nemo-relay' is found: export PATH=\"${RELAY_BIN_DIR}:\$PATH\"" >&2 ;; +esac diff --git a/sdk/python/nemo-platform/pyproject.toml b/sdk/python/nemo-platform/pyproject.toml index 2b9e51278e..c2174eed92 100644 --- a/sdk/python/nemo-platform/pyproject.toml +++ b/sdk/python/nemo-platform/pyproject.toml @@ -60,7 +60,8 @@ nemo-evaluator-sdk = [ "ragas==0.4.3", "langchain-openai>=1.3.5", "langchain-nvidia-ai-endpoints>=1.4.3,<2.0.0", - "nemo-relay>=0.4.0,<0.5.0", + "nemo-relay>=0.6.0,<0.7", + "nemo-fabric>=0.1.0rc6,<0.2.0", ] [project.entry-points."nemo.skills"] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/_common.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/_common.py index 47fd59e63b..0d455fc9cf 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/_common.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/_common.py @@ -23,13 +23,16 @@ from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor -from nemo_relay.observability import AtifConfig, AtofConfig, ComponentSpec, ObservabilityConfig # Trajectory profile identity + the file-exporter output names we choose (Relay accepts these as # inputs). Shared so both runtimes select/emit the trajectory under identical names. TRAJECTORY_PROFILE_NAME = "eval_trajectory" ATIF_FILENAME_TEMPLATE = "trajectory-{session_id}.atif.json" ATOF_FILENAME = "events.atof.jsonl" +#: ATIF ``agent.version``. Both runtimes report the agent *framework* here so a consumer can group +#: host and container traces together; ``agent.name`` is what distinguishes them. Not a real version +#: yet — reporting the resolved nemo-fabric version would be the better answer. +FABRIC_AGENT_VERSION = "fabric" # Fabric telemetry-profile selectors (Relay file exporter, no OTLP endpoint). TELEMETRY_PROVIDER = "relay" TELEMETRY_MODE = "sdk" @@ -112,7 +115,19 @@ def trajectory_telemetry(*, relay_dir: str, agent_name: str, agent_version: str) Built from ``nemo_relay``'s own typed config so Relay owns its schema — no hand-maintained dict to silently drift when Relay changes it. Callers wrap this in a profile with their own name + ``runtime``/``environment`` blocks; ``relay_dir`` is where the ``trajectory-*.atif.json`` lands. + + ``nemo_relay`` is imported here rather than at module scope: it is a native extension costing + ~120ms to load, and this module is reachable from the evaluator plugin's job imports, so an + eager import would charge every consumer for trajectory capture they may never use. """ + from nemo_relay.observability import ( + AtifConfig, + AtofConfig, + AtofFileSinkConfig, + ComponentSpec, + ObservabilityConfig, + ) + observability = ComponentSpec( config=ObservabilityConfig( atif=AtifConfig( @@ -124,9 +139,13 @@ def trajectory_telemetry(*, relay_dir: str, agent_name: str, agent_version: str) ), atof=AtofConfig( enabled=True, - output_directory=relay_dir, - filename=ATOF_FILENAME, - mode="overwrite", + sinks=[ + AtofFileSinkConfig( + output_directory=relay_dir, + filename=ATOF_FILENAME, + mode="overwrite", + ) + ], ), ) ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py index f93c043f51..59fda190e6 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py @@ -11,7 +11,7 @@ Per task it: -1. seeds ``/in`` with the Fabric agent config, profiles, and framed input, plus the task's workspace +1. seeds ``/in`` with the composed Fabric agent config and framed input, plus the task's workspace seed files; 2. execs Fabric's own CLI (``fabric run``), which writes a normalized ``RunResult`` to stdout and the workspace + Relay ATIF trajectory under a fixed ``/out`` layout; @@ -74,7 +74,7 @@ if TYPE_CHECKING: # nemo_fabric is an optional native dep (see FabricAgentRuntime); imported for typing only. Configs # are consumed structurally via ``to_mapping()`` at runtime, so this module stays importable without it. - from nemo_fabric import FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import] + from nemo_fabric import FabricConfig # ty: ignore[unresolved-import] # Default per-task exec budget. Timeout is really task-specific (see AALGO-323 to move it onto # AgentEvalTask); until then it is an internal default rather than a runtime-construction knob. @@ -86,7 +86,7 @@ "only inside the sandbox." ) -# Fixed in-container layout. The runtime seeds ``/in`` (agent config, profiles, input), execs Fabric's +# Fixed in-container layout. The runtime seeds ``/in`` (agent config, input), execs Fabric's # CLI, and reads the produced ``/out`` subtree back across the boundary. _IN_DIR = "/in" _OUT_DIR = "/out" @@ -98,7 +98,6 @@ _FABRIC_STDERR = f"{_LOGS_DIR}/fabric-stderr.txt" _AGENT_PATH = f"{_IN_DIR}/agent.yaml" _INPUT_PATH = f"{_IN_DIR}/input.txt" -_WORKSPACE_PROFILE_NAME = "eval_workspace" # In-sandbox root for a natively-injected skill bundle. It lives under ``/in`` (not ``/out``), so it is # never part of the downloaded ``/out`` evidence — only codex-mode skills, which must sit in the workspace # for the harness to self-discover them, need post-download cleanup. @@ -116,7 +115,6 @@ def __init__( config: FabricConfig | Mapping[str, Any], *, provider: SandboxProvider, - profiles: Sequence[FabricProfileConfig | Mapping[str, Any]] = (), secrets: Mapping[str, SecretRef] = {}, image: str | None = None, skills: Sequence[AgentSkill] | None = None, @@ -124,7 +122,6 @@ def __init__( # The Fabric agent is fully described by its ``FabricConfig`` (harness + model + runtime); it is # consumed structurally as a mapping to cross the sandbox boundary as JSON. self._config = _to_mapping(config) - self._profiles = [_to_mapping(profile) for profile in profiles] self._provider = provider # ``secrets`` maps the env-var name a Fabric harness reads its credential from (declared by the # adapter's ``requirements.env``) to a SecretRef. The runner only *declares* them; the resolver @@ -233,14 +230,14 @@ async def _run_task( # trial rather than aborting the gathered batch. skill_provenances: list[SkillProvenance] = [] try: - seed_files, profile_paths, skill_provenances = self._seed_files(task, skill_mode) + seed_files, skill_provenances = self._seed_files(task, skill_mode) spec = SandboxSpec( image=self._image, workdir=_WORKSPACE_DIR, env=dict(self._resolved_env), files=seed_files ) async with AsyncSandbox(self._provider, spec) as sandbox: await sandbox.start() await self._seed_workspace(sandbox, task) - result = await sandbox.exec(self._fabric_command(profile_paths), timeout_s=DEFAULT_FABRIC_TIMEOUT_S) + result = await sandbox.exec(self._fabric_command(), timeout_s=DEFAULT_FABRIC_TIMEOUT_S) await sandbox.download_dir(_OUT_DIR, out_dir) # Codex self-injection seeds each bundle inside the workspace so the harness discovers it during # the run; drop them from the downloaded evidence before the workspace is exposed (else the @@ -266,14 +263,12 @@ def _resolve_skill_mode(self) -> SkillMode | None: is imported lazily on the host (only when a skill is set), so the no-skill path never needs it. """ try: - from nemo_fabric import Fabric, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import] + from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] except ImportError as exc: raise RuntimeError(_MISSING_FABRIC_MSG) from exc - agent_config = FabricConfig.from_mapping(self._config) - base_profiles = [FabricProfileConfig.from_mapping(profile) for profile in self._profiles] - probe_config = agent_config.model_copy(deep=True) + probe_config = FabricConfig.from_mapping(self._config) probe_config.add_skill_path(_SKILL_PROBE_PATH) - plan = Fabric().plan(probe_config, profiles=base_profiles) + plan = Fabric().plan(probe_config) return resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness) def _adapter_id(self) -> str: @@ -282,26 +277,9 @@ def _adapter_id(self) -> str: adapter_id = harness.get("adapter_id") if isinstance(harness, Mapping) else None return str(adapter_id) if adapter_id is not None else "" - def _existing_skill_paths(self) -> list[str]: - """Skill paths the base config/profiles already declare (union, order-preserved). - - Fabric applies profile ``skills.paths`` last-wins, so the native overlay has to re-list these - alongside the evaluated skill or the treated arm would silently drop preconfigured skills (see - ``stage_skills_seed``). Read from the raw config/profile mappings the runtime was given. - """ - paths: list[str] = [] - for section in (self._config, *self._profiles): - skills = section.get("skills") if isinstance(section, Mapping) else None - declared = skills.get("paths") if isinstance(skills, Mapping) else None - for path in declared or []: - if isinstance(path, str) and path not in paths: - paths.append(path) - return paths - - def _fabric_command(self, profile_paths: Sequence[str]) -> str: + def _fabric_command(self) -> str: """The ``fabric run`` invocation: pre-create the /out dirs Fabric chdirs into, run, capture stdout.""" - profiles = " ".join(f"--profile {shlex.quote(path)}" for path in profile_paths) - run = f"fabric run {shlex.quote(_AGENT_PATH)} {profiles} --input-file {shlex.quote(_INPUT_PATH)}" + run = f"fabric run {shlex.quote(_AGENT_PATH)} --input-file {shlex.quote(_INPUT_PATH)}" return ( f"mkdir -p {_WORKSPACE_DIR} {_RELAY_DIR} {_ARTIFACTS_DIR} {_LOGS_DIR} && " f"{run} > {shlex.quote(_RESULT_PATH)} 2> {shlex.quote(_FABRIC_STDERR)}" @@ -309,22 +287,19 @@ def _fabric_command(self, profile_paths: Sequence[str]) -> str: def _seed_files( self, task: AgentEvalTask, skill_mode: SkillMode | None - ) -> tuple[dict[str, str], list[str], list[SkillProvenance]]: - """Return (files to seed into the sandbox, profile paths for --profile, skill provenances). - - Configs are written as JSON, which the Fabric CLI parses as YAML. When skills are injected each - bundle is rendered into the seed set at the harness's in-sandbox discovery path (native: - ``/in/skills/``; codex: ``/.agents/skills/``), with at most ONE merged native - overlay listing every bundle. Profiles are ordered caller-first, then the native skill overlay (if - any), then the per-task workspace + trajectory overlays — which trail so the evaluator-owned - workspace/artifacts stay authoritative (mirroring the host runtime's overlay ordering). + ) -> tuple[dict[str, str], list[SkillProvenance]]: + """Return (files to seed into the sandbox, skill provenances). + + The agent config is written as JSON, which the Fabric CLI parses as YAML. Fabric 0.1.0rc2 removed + profile overlays (``--profile`` and the ``profiles`` config key are both gone), so everything — + the runtime's in-container settings and any natively-injected skill paths — is composed into the + single agent config here. When skills are injected each bundle is also rendered into the seed set + at the harness's in-sandbox discovery path (native: ``/in/skills/``; codex: + ``/.agents/skills/``). """ - files: dict[str, str] = { - _AGENT_PATH: json.dumps(self._config), - _INPUT_PATH: task.agent_prompt(), - } - skill_profiles: list[dict[str, Any]] = [] + skill_paths: list[str] = [] provenances: list[SkillProvenance] = [] + files: dict[str, str] = {_INPUT_PATH: task.agent_prompt()} if self._skill_set.skills and skill_mode is not None: if skill_mode == SKILL_MODE_CODEX_SKILLS_DIR: _check_codex_skill_collision(self._skill_set.skills, task.inputs.get(SEED_FILES_INPUT_KEY) or {}) @@ -334,38 +309,53 @@ def _seed_files( mode=skill_mode, workspace_dir=_WORKSPACE_DIR, skills_dir=_SKILLS_DIR, - existing_skill_paths=self._existing_skill_paths(), ) files.update(seed.files) - skill_profiles = seed.profiles + skill_paths = seed.skill_paths provenances = seed.provenances - profile_paths: list[str] = [] - profiles = [*self._profiles, *skill_profiles, self._workspace_profile(), self._trajectory_profile()] - for index, profile in enumerate(profiles): - path = f"{_IN_DIR}/profile-{index}.yaml" - files[path] = json.dumps(profile) - profile_paths.append(path) - return files, profile_paths, provenances - - @staticmethod - def _workspace_profile() -> dict[str, Any]: - # Pin the harness working directory to the retrievable workspace; ``provider`` is required by the - # native planner (it does not inject the Python default into a raw overlay). - return {"name": _WORKSPACE_PROFILE_NAME, "environment": {"provider": "local", "workspace": _WORKSPACE_DIR}} - - @staticmethod - def _trajectory_profile() -> dict[str, Any]: - # Relay ATIF/ATOF file exporter (sdk mode). The telemetry block is built from nemo_relay's typed - # config via the shared helper (single source of truth with the host runtime); ``provider:local`` - # is required by the native planner in the container (it does not inject the Python default). - return { - "name": _common.TRAJECTORY_PROFILE_NAME, - "runtime": {"artifacts": _ARTIFACTS_DIR}, - "environment": {"provider": "local", "artifacts": _ARTIFACTS_DIR}, - "telemetry": _common.trajectory_telemetry( - relay_dir=_RELAY_DIR, agent_name=_RUNTIME_NAME, agent_version=_RUNTIME_NAME - ), + files[_AGENT_PATH] = json.dumps(self._composed_config(skill_paths)) + return files, provenances + + def _composed_config(self, skill_paths: Sequence[str] = ()) -> dict[str, Any]: + """The supplied agent config with the runtime's in-container settings merged on last. + + Mirrors the host runtime's ``_compose_config``: the workspace, artifact roots, trajectory + telemetry, and any natively-injected skill paths are evaluator-owned, so they are applied over + whatever the caller's config declared. Stays plain dicts rather than round-tripping through the + host's ``FabricConfig`` — the sandbox may run a different Fabric build, so the config is only + required to survive JSON transport, not to validate against the host's schema. + + Injected skill paths are APPENDED to the config's own ``skills.paths`` — mirroring + ``FabricConfig.add_skill_path`` — so skills the caller preconfigured survive injection and the + treated A/B arm differs from the baseline by exactly the injected skills. + """ + config = dict(self._config) + + # Each section is spread over the caller's, so sibling keys survive — pinning + # ``runtime.artifacts`` must not drop a configured ``runtime.transport``. + config["runtime"] = {**_section(config, "runtime"), "artifacts": _ARTIFACTS_DIR} + # ``provider: local`` is required by the native planner in the container (it does not inject the + # Python default), and the workspace pins the harness cwd to the retrievable /out subtree. + config["environment"] = { + **_section(config, "environment"), + "provider": "local", + "workspace": _WORKSPACE_DIR, + "artifacts": _ARTIFACTS_DIR, } + # Relay ATIF/ATOF file exporter (sdk mode), built from nemo_relay's typed config via the shared + # helper so it stays a single source of truth with the host runtime. Replaced wholesale. + # ``agent_name`` distinguishes this runtime from the host one; ``agent_version`` records the + # agent framework and so matches the host's value, letting an ATIF consumer group both + # runtimes' traces. (Neither is a real version yet — see _common.trajectory_telemetry.) + config["telemetry"] = _common.trajectory_telemetry( + relay_dir=_RELAY_DIR, agent_name=_RUNTIME_NAME, agent_version=_common.FABRIC_AGENT_VERSION + ) + + declared_paths = _section(config, "skills").get("paths") or [] + merged_paths = list(dict.fromkeys([*(str(path) for path in declared_paths), *skill_paths])) + if merged_paths: + config["skills"] = {**_section(config, "skills"), "paths": merged_paths} + return config async def _seed_workspace(self, sandbox: AsyncSandbox, task: AgentEvalTask) -> None: seeds = task.inputs.get(SEED_FILES_INPUT_KEY) @@ -484,8 +474,8 @@ def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunCon def _to_mapping(config: FabricConfig | Mapping[str, Any]) -> dict[str, Any]: - """Normalize a typed Fabric config/profile or a plain mapping to a plain dict for JSON transport.""" - # A typed Fabric config/profile exposes ``to_mapping()``; a plain mapping is used as-is. Both are + """Normalize a typed Fabric config or a plain mapping to a plain dict for JSON transport.""" + # A typed Fabric config exposes ``to_mapping()``; a plain mapping is used as-is. Both are # str-keyed at runtime, but the getattr + optional (unresolved) ``FabricConfig`` type defeat static # narrowing, so cast the known-good source before building the dict. to_mapping = getattr(config, "to_mapping", None) @@ -554,6 +544,12 @@ def _remove_injected_bundle(workspace_dir: Path, location: str) -> None: parent = parent.parent +def _section(config: Mapping[str, Any], name: str) -> dict[str, Any]: + """A top-level config section as a plain dict — ``{}`` when absent or not a mapping.""" + value = config.get(name) + return dict(value) if isinstance(value, Mapping) else {} + + def _find_atif(relay_dir: Path) -> Path | None: # Relay nests the trajectory under a per-run subdir (relay/runtime-/trajectory-*.atif.json), # so search recursively rather than only relay's direct children. diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/image.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/image.py index 3beca8fa49..6d218c8589 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/image.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/image.py @@ -25,7 +25,7 @@ That only works on NeMo-Fabric's ``installed-adapter-discovery`` branch (which bundles the adapters under ``python/src/nemo_fabric/adapters`` and adds ``AdapterDescriptorSource::Installed``). On today's ``main`` the wheel ships no adapter descriptors, so a wheel-only image cannot resolve e.g. -``nvidia.fabric.hermes.sdk``. Once that lands on ``main``, switch to installing the top-level +``nvidia.fabric.hermes``. Once that lands on ``main``, switch to installing the top-level ``adapters/*`` packages explicitly here instead of relying on the branch's packaging. """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py index c345aadc46..77423a4547 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py @@ -11,7 +11,10 @@ Per-task settings (workspace, model, trajectory capture) are composed directly onto a copy of the supplied config via the SDK's config helpers (``model_copy`` + -``enable_relay`` + ``environment``), rather than layered as profile overlays. +``enable_relay`` + ``environment``). Fabric removed profile overlays in 0.1.0rc2 — +``FabricConfig`` rejects a ``profiles`` key and ``Fabric.run`` takes no ``profiles`` +argument — so a run is described by exactly one complete typed config, and the +evaluator-owned per-task settings are authoritative simply by being applied last. Every task runs in its own fresh workspace: the runtime seeds it from ``inputs['files']`` (a no-op when there are none), runs the harness in it (via @@ -37,6 +40,7 @@ from typing import TYPE_CHECKING, Any from uuid import uuid4 +from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric import _common from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.skills import ( SKILL_MODE_CODEX_SKILLS_DIR, AgentSkill, @@ -65,7 +69,7 @@ from nemo_fabric import ( # ty: ignore[unresolved-import] Fabric, FabricConfig, - FabricProfileConfig, + RelayObservabilityConfig, RunOutput, RunResult, ) @@ -85,9 +89,9 @@ # Per-task workspace: where seed files are staged and where the harness reads/writes. We # create it, point Fabric's ``environment.workspace`` at it, and expose it as ``workspace`` evidence. _WORKSPACE_SUBDIR = "workspace" -# Per-task skill staging dir (native injection): the skill's files are resolved here and a per-task -# ``skills`` profile overlay points Fabric at it. For codex self-injection the skill lands in the -# workspace instead (no overlay). +# Per-task skill staging dir (native injection): the skill's files are resolved here and the staged +# root is added to the task config's ``skills.paths``. For codex self-injection the skill lands in the +# workspace instead (no path added). _SKILL_SUBDIR = "skill" # Sentinel skill path attached only to probe Fabric's capability planner for the selected adapter's # skills routing (see ``_resolve_skill_mode``). Never staged and need not exist on disk — the planner @@ -102,11 +106,6 @@ _ATOF_FILENAME = "events.atof.jsonl" # ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence. _ATIF_ARTIFACT_KIND = "atif" -# Names for the trailing overlays that re-assert the evaluator-owned per-task settings (see -# ``_eval_lock_profiles``): Fabric applies caller profiles over the config, so these must trail them. -_WORKSPACE_PROFILE_NAME = "eval_workspace" -_MODEL_PROFILE_NAME = "eval_model" -_ARTIFACTS_PROFILE_NAME = "eval_artifacts" class FabricAgentRuntime: @@ -123,7 +122,6 @@ def __init__( self, *, config: Mapping[str, Any], - profiles: Sequence[Mapping[str, Any]] | None = None, model: str | None = None, base_dir: str | Path | None = None, work_root: str | Path | None = None, @@ -133,7 +131,6 @@ def __init__( skills: Sequence[AgentSkill] | None = None, ) -> None: self._config = config - self._profiles = list(profiles or []) self._model = model self._base_dir = Path(base_dir).expanduser() if base_dir is not None else None self._work_root = Path(work_root).expanduser() if work_root is not None else None @@ -171,7 +168,7 @@ async def run_tasks( try: # nemo_fabric ships a native (pyo3) core and is an optional dependency, so it is imported # lazily here rather than at module load. - from nemo_fabric import Fabric, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import] + from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] except ImportError as exc: raise RuntimeError(_MISSING_FABRIC_MSG) from exc @@ -189,11 +186,6 @@ async def run_tasks( import nemo_relay.observability # noqa: F401 # ty: ignore[unresolved-import] except ImportError as exc: raise RuntimeError(_MISSING_RELAY_MSG) from exc - # Caller-supplied profile overlays pass through as-is; this runtime's per-task workspace, model, - # and trajectory settings are composed directly onto a copy of the config (config-first), not - # layered as profiles. - base_profiles = [FabricProfileConfig.from_mapping(profile) for profile in self._profiles] - # ``Fabric`` (formerly ``FabricClient``) is a lightweight, reusable facade — not a lifecycle # context manager — so it is created once and reused across tasks with no cleanup. client = Fabric() @@ -205,7 +197,7 @@ async def run_tasks( # A/B comparison. Only touched when a skill is set, so the no-skill path is unaffected. skill_mode: SkillMode | None = None if self._skill_set.skills: - skill_mode = self._resolve_skill_mode(client, agent_config, base_profiles) + skill_mode = self._resolve_skill_mode(client, agent_config) if skill_mode is None: adapter_id = agent_config.harness.adapter_id raise RuntimeError( @@ -218,18 +210,11 @@ async def run_tasks( async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: async with semaphore: - return await self._run_task( - client, agent_config, base_profiles, index, task, resolved_config, skill_mode - ) + return await self._run_task(client, agent_config, index, task, resolved_config, skill_mode) return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - def _resolve_skill_mode( - self, - client: Fabric, - agent_config: FabricConfig, - base_profiles: list[FabricProfileConfig], - ) -> SkillMode | None: + def _resolve_skill_mode(self, client: Fabric, agent_config: FabricConfig) -> SkillMode | None: """Ask Fabric how a skill would reach the selected harness, or ``None`` if it can't. Probes Fabric's capability planner: plan a copy of the config with a sentinel skill path attached @@ -239,31 +224,13 @@ def _resolve_skill_mode( """ probe_config = agent_config.model_copy(deep=True) probe_config.add_skill_path(_SKILL_PROBE_PATH) - plan = client.plan(probe_config, profiles=base_profiles, base_dir=self._base_dir) + plan = client.plan(probe_config, base_dir=self._base_dir) return resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness) - def _existing_skill_paths(self) -> list[str]: - """Skill paths the base config/profiles already declare (union, order-preserved). - - Fabric applies profile ``skills.paths`` last-wins, so the native skill overlay has to re-list - these alongside the evaluated skill or the treated arm would silently drop them (see - ``install_skill``). Read from the raw config/profile mappings the runtime was given, so it covers - both config- and profile-declared skills without a Fabric round-trip. - """ - paths: list[str] = [] - for section in (self._config, *self._profiles): - skills = section.get("skills") if isinstance(section, Mapping) else None - declared = skills.get("paths") if isinstance(skills, Mapping) else None - for path in declared or []: - if isinstance(path, str) and path not in paths: - paths.append(path) - return paths - async def _run_task( self, client: Fabric, agent_config: FabricConfig, - base_profiles: list[FabricProfileConfig], index: int, task: AgentEvalTask, config: AgentEvalRunConfig, @@ -271,7 +238,7 @@ async def _run_task( ) -> AgentEvalTrial: # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules # lookup, not a re-load, so the types are used where they're constructed instead of threaded down. - from nemo_fabric import FabricProfileConfig, RunRequest # ty: ignore[unresolved-import] + from nemo_fabric import RunRequest # ty: ignore[unresolved-import] evidence_dir = self._evidence_dir(index, task, config) evidence_dir.mkdir(parents=True, exist_ok=True) @@ -290,11 +257,11 @@ async def _run_task( # instruction only, so the returned paths are unused. await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) - # Inject the skill set (if any) for this task. A native harness gets ONE ``skills`` profile - # overlay listing every staged bundle; codex self-injection stages each bundle into the - # workspace and emits no overlay. One provenance per skill is stamped on the trial for the A/B + # Inject the skill set (if any) for this task. A native harness gets each staged bundle added + # to the config's ``skills.paths``; codex self-injection stages each bundle into the + # workspace and adds no path. One provenance per skill is stamped on the trial for the A/B # diff. Blocking file I/O, off the event loop. - skill_profiles: list[FabricProfileConfig] = [] + skill_paths: list[str] = [] if self._skill_set.skills and skill_mode is not None: installation = await asyncio.to_thread( install_skills, @@ -303,25 +270,21 @@ async def _run_task( mode=skill_mode, workspace_dir=workspace_dir, skill_stage_dir=(evidence_dir / _SKILL_SUBDIR).resolve(), - existing_skill_paths=self._existing_skill_paths(), ) skill_provenances = installation.provenances - skill_profiles = [FabricProfileConfig.from_mapping(p) for p in installation.profiles] + skill_paths = installation.skill_paths + # Everything the run needs lives in one typed config: Fabric no longer layers profile + # overlays, so the per-task workspace/model/trajectory settings are composed on last and are + # authoritative by construction. ``add_skill_path`` appends, so config-declared skills survive. task_config = self._compose_config(agent_config, evidence_dir, workspace_dir) - # Caller ``base_profiles`` are applied by Fabric over the config; the evaluator-owned - # settings are re-asserted as trailing overlays so they win over any caller profile. - lock_profiles = self._eval_lock_profiles( - FabricProfileConfig, workspace_dir=workspace_dir, evidence_dir=evidence_dir - ) + for skill_path in skill_paths: + task_config.add_skill_path(skill_path) result = await asyncio.wait_for( # ``Fabric.run`` folds the per-invocation input + request id into a ``RunRequest``. client.run( task_config, - # Caller profiles, then the native skill overlay, then the evaluator lock overlays; - # the lock overlays trail so the per-task workspace/model/artifacts stay authoritative. - profiles=[*base_profiles, *skill_profiles, *lock_profiles], base_dir=self._base_dir, request=RunRequest(input=task.agent_prompt(), request_id=task.id), ), @@ -477,11 +440,10 @@ def _compose_config( ) -> FabricConfig: # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules # lookup, not a re-load, so the type is used where it's constructed instead of threaded down. - from nemo_fabric import EnvironmentConfig # ty: ignore[unresolved-import] + from nemo_fabric import EnvironmentConfig, ModelConfig # ty: ignore[unresolved-import] - # Config-first composition (the SDK's recommended in-memory pattern): copy the base config and - # apply this task's workspace, model, and trajectory settings directly onto it, rather than - # layering FabricProfileConfig overlays. + # Copy the base config and apply this task's workspace, model, and trajectory settings directly + # onto it. These land last, so they override anything the supplied config declared. cfg = agent_config.model_copy(deep=True) # Point the harness at this task's staged workspace (the codex-cli adapter resolves its cwd from @@ -495,7 +457,7 @@ def _compose_config( # Apply the model as the config's default (mirrors nemo_fabric.integrations.harbor). if self._model: provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai" - cfg.models["default"] = {"provider": provider, "model": self._model} + cfg.models["default"] = ModelConfig(provider=provider, model=self._model) if self._capture_trajectory: # Enable Relay's ATIF/ATOF file exporter under this task's durable evidence dir, and pin the @@ -505,83 +467,46 @@ def _compose_config( artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR relay_dir.mkdir(parents=True, exist_ok=True) artifacts_dir.mkdir(parents=True, exist_ok=True) - cfg.enable_relay(output_dir=str(relay_dir), config=self._relay_config(relay_dir)) + cfg.enable_relay(output_dir=str(relay_dir), observability=self._relay_config(relay_dir)) cfg.runtime.artifacts = str(artifacts_dir) cfg.environment.artifacts = str(artifacts_dir) return cfg - def _eval_lock_profiles( - self, - profile_cls: type[FabricProfileConfig], - *, - workspace_dir: Path, - evidence_dir: Path, - ) -> list[FabricProfileConfig]: - # ``_compose_config`` composes the evaluator's per-task settings onto the config, but Fabric - # applies caller-supplied profiles OVER the config (last-wins), so a caller profile could - # otherwise override them. Re-assert the evaluator-owned settings here as trailing overlays — - # applied after the caller profiles — so the per-task workspace (isolation + ``workspace`` - # evidence integrity), the model under evaluation, and the trajectory artifact location stay - # authoritative and non-overridable. - overlays = [ - profile_cls.from_mapping( - {"name": _WORKSPACE_PROFILE_NAME, "environment": {"workspace": str(workspace_dir)}} - ) - ] - if self._model: - provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai" - overlays.append( - profile_cls.from_mapping( - {"name": _MODEL_PROFILE_NAME, "models": {"default": {"provider": provider, "model": self._model}}} - ) - ) - if self._capture_trajectory: - artifacts_dir = str(evidence_dir / _ARTIFACTS_SUBDIR) - overlays.append( - profile_cls.from_mapping( - { - "name": _ARTIFACTS_PROFILE_NAME, - "runtime": {"artifacts": artifacts_dir}, - "environment": {"artifacts": artifacts_dir}, - } - ) - ) - return overlays - - def _relay_config(self, relay_dir: Path) -> dict[str, Any]: - # The observability component is built from nemo_relay's own typed config objects so Relay owns - # its schema (no hand-maintained dict that silently drifts when Relay changes it); imported - # lazily since nemo-relay, like nemo-fabric, is an optional native dependency. - try: - from nemo_relay.observability import ( # ty: ignore[unresolved-import] - AtifConfig, - AtofConfig, - ComponentSpec, - ObservabilityConfig, - ) - except ImportError as exc: - raise RuntimeError(_MISSING_RELAY_MSG) from exc + def _relay_config(self, relay_dir: Path) -> RelayObservabilityConfig: + # The ATIF/ATOF observability config is built from Fabric's own typed relay-config objects so + # Fabric owns the schema (no hand-maintained dict that silently drifts when Fabric changes it), + # mirroring nemo_fabric's own Harbor integration. It is handed straight to ``enable_relay`` via + # its ``observability=`` parameter — the SDK only configures ATIF/ATOF observability, so it needs + # neither a generic ``components`` list nor the legacy component-wrapped shape. nemo_fabric is + # already imported+validated in ``run_tasks``, so this is a cached sys.modules lookup. + from nemo_fabric import ( # ty: ignore[unresolved-import] + RelayAtifConfig, + RelayAtofConfig, + RelayAtofFileSinkConfig, + RelayObservabilityConfig, + ) relay_dir_str = str(relay_dir) - observability = ComponentSpec( - config=ObservabilityConfig( - atif=AtifConfig( - enabled=True, - output_directory=relay_dir_str, - filename_template=_ATIF_FILENAME_TEMPLATE, - agent_name=self._runtime_name, - agent_version="fabric", - ), - atof=AtofConfig( - enabled=True, - output_directory=relay_dir_str, - filename=_ATOF_FILENAME, - mode="overwrite", - ), - ) + return RelayObservabilityConfig( + atif=RelayAtifConfig( + enabled=True, + output_directory=relay_dir_str, + filename_template=_ATIF_FILENAME_TEMPLATE, + agent_name=self._runtime_name, + agent_version=_common.FABRIC_AGENT_VERSION, + ), + atof=RelayAtofConfig( + enabled=True, + sinks=[ + RelayAtofFileSinkConfig( + output_directory=relay_dir_str, + filename=_ATOF_FILENAME, + mode="overwrite", + ) + ], + ), ) - return {"version": 1, "components": [observability.to_dict()]} def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: root = self._work_root diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/skills.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/skills.py index 81d9bb6f0f..4e1a912315 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/skills.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/skills.py @@ -19,15 +19,19 @@ ``RunPlan.capability_plan``), not a hardcoded adapter list — so it tracks whatever the installed adapters declare, including end-user adapters we don't ship: -* **Native** (:data:`SKILL_MODE_NATIVE`): the adapter advertises ``accepts: ["skills", ...]`` (the - Hermes/Claude adapters do), so Fabric's planner routes skills to ``harness_native``. We stage the - bundle into an isolated ``/`` dir and hand Fabric a ``skills.paths`` profile overlay; the - adapter loads it (Hermes → harness ``skills.external_dirs``). -* **Codex skills dir** (:data:`SKILL_MODE_CODEX_SKILLS_DIR`): the Fabric ``codex`` adapter only - ``accepts: ["models"]`` (planner routes skills ``unsupported``), but the Codex CLI itself discovers - agentskills bundles from ``.agents/skills/`` in its working directory. So we place the bundle at - ``/.agents/skills//`` and let Codex discover it — same discoverable-skill semantics - as native (cross-harness A/B is apples-to-apples), no Fabric adapter change needed. +* **Native** (:data:`SKILL_MODE_NATIVE`): the adapter advertises ``accepts: ["skills", ...]``, so + Fabric's planner routes skills to ``harness_native``. We stage the bundle into an isolated + ``/`` dir and add it to the config's ``skills.paths``; the adapter loads it (Hermes → harness + ``skills.external_dirs``). As of nemo-fabric 0.1.0rc3 the hermes, claude AND **codex** adapters all + declare ``skills``, so this is the path every harness we ship currently takes. +* **Codex skills dir** (:data:`SKILL_MODE_CODEX_SKILLS_DIR`): a fallback for a codex-harness adapter + that does *not* accept the native skills config. The Codex CLI itself discovers agentskills bundles + from ``.agents/skills/`` in its working directory, so we place the bundle at + ``/.agents/skills//`` and let Codex find it — same discoverable-skill semantics as + native (cross-harness A/B stays apples-to-apples), no Fabric adapter change needed. + NOTE: the shipped codex adapter accepts ``skills`` today, so this branch is currently unreachable in + production and is exercised only by the fake-backed tests. It is kept for adapters (ours or an + end-user's) that route skills ``unsupported`` on a codex harness. If an adapter neither routes skills natively nor is a Codex harness, :func:`resolve_skill_mode` returns ``None`` and the runtime fails fast rather than silently running a skill-free trial. @@ -49,8 +53,6 @@ PRIMARY_SKILL_DOC = "SKILL.md" #: Directory Codex scans (relative to its working dir) for agentskills bundles. CODEX_SKILLS_DIR = ".agents/skills" -#: Name of the Fabric profile overlay that carries the native ``skills`` config. -SKILL_PROFILE_NAME = "eval_skill" #: How an injected skill reaches the selected harness (resolved from Fabric's capability plan). The two #: runtimes thread this value from :func:`resolve_skill_mode` down to :func:`install_skill` / @@ -134,13 +136,13 @@ class SkillProvenance(TypedDict): class SkillInstallation: """Result of installing a skill for one task. - ``profiles`` are Fabric profile-overlay mappings the runtime appends to its profile stack (the - native branch emits one ``skills`` overlay; the Codex branch emits none because placement in the - workspace is the delivery mechanism). ``provenance`` is stamped into trial metadata so the A/B - comparison is auditable. + ``skill_paths`` are staged bundle roots the runtime hands to ``FabricConfig.add_skill_path`` (the + native branch emits one; the Codex branch emits none because placement in the workspace is the + delivery mechanism). ``provenance`` is stamped into trial metadata so the A/B comparison is + auditable. """ - profiles: list[dict[str, object]] + skill_paths: list[str] provenance: SkillProvenance @@ -188,7 +190,6 @@ def install_skill( mode: SkillMode, workspace_dir: Path, skill_stage_dir: Path, - existing_skill_paths: Sequence[str] = (), ) -> SkillInstallation: """Stage ``skill`` as a ``/`` bundle and wire it into the harness per ``mode``. @@ -196,24 +197,15 @@ def install_skill( namespaced under ``/`` so it never collides with task-seeded workspace-root files; the content hash is computed over the staged bytes so provenance tracks the actual skill content. - ``existing_skill_paths`` are the skill paths the base config/profiles already declare. Fabric applies - profile ``skills.paths`` last-wins, so the native overlay must re-list them alongside the evaluated - skill — otherwise the treated arm would silently drop every preconfigured skill and the A/B would - differ by more than the injected skill. + The native branch returns the staged root for ``FabricConfig.add_skill_path``, which appends to + whatever the base config already declares. Any preconfigured skills therefore survive injection + without this function having to re-list them. """ if mode == SKILL_MODE_NATIVE: skill_root = skill_stage_dir / skill.name _stage_bundle(skill.directory, skill_root, reserved=False) - # Preserve the pre-existing skill paths (order-preserved, de-duplicated) and append the - # evaluated skill, so the last-wins overlay reproduces the baseline skill set plus this one. - paths = list(dict.fromkeys([*existing_skill_paths, str(skill_root)])) - overlay: dict[str, object] = { - "name": SKILL_PROFILE_NAME, - "description": "Make the evaluation skill available via the native Fabric skills config.", - "skills": {"paths": paths}, - } return SkillInstallation( - profiles=[overlay], + skill_paths=[str(skill_root)], provenance=_provenance(skill, _hash_directory(skill_root), mode, adapter_id, str(skill_root)), ) @@ -222,7 +214,7 @@ def install_skill( _stage_bundle(skill.directory, skill_root, reserved=True) location = (Path(CODEX_SKILLS_DIR) / skill.name).as_posix() return SkillInstallation( - profiles=[], + skill_paths=[], provenance=_provenance(skill, _hash_directory(skill_root), mode, adapter_id, location), ) @@ -233,14 +225,13 @@ def install_skill( class SkillsInstallation: """Result of installing several skills for one task (see :func:`install_skills`). - ``profiles`` is the Fabric profile-overlay stack the runtime appends: at most ONE merged native - ``skills`` overlay listing every staged bundle (Fabric applies profile ``skills.paths`` last-wins, so - all skills must ride in a single overlay or all but the last would be dropped); the Codex branch emits - none because workspace placement is the delivery mechanism. ``provenances`` is one entry per skill, in - the given order, stamped into trial metadata so a multi-skill A/B comparison is auditable. + ``skill_paths`` is every staged native bundle root, in the given order, for the runtime to feed to + ``FabricConfig.add_skill_path``; the Codex branch emits none because workspace placement is the + delivery mechanism. ``provenances`` is one entry per skill, in the given order, stamped into trial + metadata so a multi-skill A/B comparison is auditable. """ - profiles: list[dict[str, object]] + skill_paths: list[str] provenances: list[SkillProvenance] @@ -295,16 +286,14 @@ def install_skills( mode: SkillMode, workspace_dir: Path, skill_stage_dir: Path, - existing_skill_paths: Sequence[str] = (), ) -> SkillsInstallation: """Stage every skill in ``skills`` for one task and wire them all into the harness per ``mode``. - Loops :func:`install_skill` — each skill stages into its own namespaced ``/`` bundle — then, for - the native mode, merges the per-skill paths into a SINGLE ``skills`` overlay: Fabric applies profile - ``skills.paths`` last-wins, so emitting one overlay per skill would silently drop all but the last. - Pre-existing ``existing_skill_paths`` are preserved ahead of the injected skills (same reason). Skill - names must be unique (their ``/`` bundles would otherwise collide). Blocking file I/O — call via - ``asyncio.to_thread`` from the async runtime. + Loops :func:`install_skill` — each skill stages into its own namespaced ``/`` bundle — and + collects the staged roots for the native mode. ``FabricConfig.add_skill_path`` appends and + de-duplicates, so every injected skill lands alongside whatever the base config already declared, + with no re-listing. Skill names must be unique (their ``/`` bundles would otherwise collide). + Blocking file I/O — call via ``asyncio.to_thread`` from the async runtime. Installation is all-or-nothing: if any skill fails to stage, the bundles already staged in this call are rolled back before the error propagates, so a partial skill set never lingers on disk (the caller @@ -330,7 +319,6 @@ def install_skills( mode=mode, workspace_dir=workspace_dir, skill_stage_dir=skill_stage_dir, - existing_skill_paths=existing_skill_paths, ).provenance provenances.append(provenance) except Exception: @@ -338,19 +326,12 @@ def install_skills( shutil.rmtree(root, ignore_errors=True) raise - profiles: list[dict[str, object]] = [] - if mode == SKILL_MODE_NATIVE and provenances: - # One merged overlay: pre-existing skills first, then each staged bundle (a native provenance's - # ``location`` is its absolute staged skill root), order-preserved and de-duplicated. - paths = list(dict.fromkeys([*existing_skill_paths, *(prov["location"] for prov in provenances)])) - profiles = [ - { - "name": SKILL_PROFILE_NAME, - "description": "Make the evaluation skills available via the native Fabric skills config.", - "skills": {"paths": paths}, - } - ] - return SkillsInstallation(profiles=profiles, provenances=provenances) + skill_paths: list[str] = [] + if mode == SKILL_MODE_NATIVE: + # Each staged bundle root, order-preserved and de-duplicated (a native provenance's + # ``location`` is its absolute staged skill root). + skill_paths = list(dict.fromkeys(prov["location"] for prov in provenances)) + return SkillsInstallation(skill_paths=skill_paths, provenances=provenances) def _skill_stage_root(skill: AgentSkill, mode: SkillMode, workspace_dir: Path, skill_stage_dir: Path) -> Path: @@ -422,14 +403,13 @@ class SkillsSeed: The plural, containerized sibling of :class:`SkillsInstallation`: * ``files`` — the merged ``{absolute_in_sandbox_path: text}`` seed map for every staged bundle. - * ``profiles`` — at most ONE merged native ``skills`` overlay listing every bundle (Fabric applies - ``skills.paths`` last-wins, so all must ride in a single overlay or all but the last are dropped); - the codex branch emits none. + * ``skill_paths`` — every staged native bundle root, in order, for the runtime to merge into the + composed config's ``skills.paths``; the codex branch emits none. * ``provenances`` — one entry per skill, in the given order, for the multi-skill A/B trial metadata. """ files: dict[str, str] - profiles: list[dict[str, object]] + skill_paths: list[str] provenances: list[SkillProvenance] @@ -440,18 +420,16 @@ def stage_skills_seed( mode: SkillMode, workspace_dir: str, skills_dir: str, - existing_skill_paths: Sequence[str] = (), ) -> SkillsSeed: - """Render every skill in ``skills`` into one sandbox seed set + overlays for the container runtime. + """Render every skill in ``skills`` into one sandbox seed set for the container runtime. The plural, containerized sibling of :func:`install_skills`: renders each bundle (via - :func:`_render_skill_seed`) under its own ``/`` at the harness's in-sandbox discovery path, then, - for the native mode, merges the per-skill paths into a SINGLE ``skills`` overlay (Fabric applies profile - ``skills.paths`` last-wins, so one overlay per skill would silently drop all but the last). Pre-existing - ``existing_skill_paths`` are preserved ahead of the injected skills (same reason). Skill names must be - unique — their ``/`` bundles would otherwise collide. No on-disk rollback is needed (unlike - :func:`install_skills`): the seed set is an in-memory map, so a failure to render any skill just - discards the accumulated map and raises, leaving nothing staged. + :func:`_render_skill_seed`) under its own ``/`` at the harness's in-sandbox discovery path and + collects the native in-sandbox roots. The caller merges those into the composed config's + ``skills.paths`` alongside whatever it already declared, so nothing has to be re-listed here. Skill + names must be unique — their ``/`` bundles would otherwise collide. No on-disk rollback is + needed (unlike :func:`install_skills`): the seed set is an in-memory map, so a failure to render any + skill just discards the accumulated map and raises, leaving nothing staged. """ require_unique_skill_names(skills) files: dict[str, str] = {} @@ -463,19 +441,12 @@ def stage_skills_seed( files.update(rendered) provenances.append(provenance) - profiles: list[dict[str, object]] = [] - if mode == SKILL_MODE_NATIVE and provenances: - # One merged overlay: pre-existing skills first, then each staged bundle (a native provenance's - # ``location`` is its absolute in-sandbox skill root), order-preserved and de-duplicated. - paths = list(dict.fromkeys([*existing_skill_paths, *(prov["location"] for prov in provenances)])) - profiles = [ - { - "name": SKILL_PROFILE_NAME, - "description": "Make the evaluation skills available via the native Fabric skills config.", - "skills": {"paths": paths}, - } - ] - return SkillsSeed(files=files, profiles=profiles, provenances=provenances) + skill_paths: list[str] = [] + if mode == SKILL_MODE_NATIVE: + # Each staged bundle (a native provenance's ``location`` is its absolute in-sandbox skill + # root), order-preserved and de-duplicated. + skill_paths = list(dict.fromkeys(prov["location"] for prov in provenances)) + return SkillsSeed(files=files, skill_paths=skill_paths, provenances=provenances) def _stage_bundle(directory: Path, skill_root: Path, *, reserved: bool) -> None: diff --git a/third_party/licenses.jsonl b/third_party/licenses.jsonl index 60f209eb03..1e6d19fbf9 100644 --- a/third_party/licenses.jsonl +++ b/third_party/licenses.jsonl @@ -163,6 +163,8 @@ {"name": "multiprocess", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "mypy-extensions", "license": "MIT", "compatible": true} {"name": "nemo-anonymizer", "license": "APACHE-2.0", "compatible": true} +{"name": "nemo-fabric", "license": "APACHE-2.0", "compatible": true} +{"name": "nemo-fabric-runtime", "license": "APACHE-2.0", "compatible": true} {"name": "nemo-relay", "license": "APACHE-2.0", "compatible": true} {"name": "nemo-safe-synthesizer", "license": "APACHE-2.0", "compatible": true} {"name": "nemoguardrails", "license": "APACHE-2.0", "compatible": true} diff --git a/third_party/osv-licenses.json b/third_party/osv-licenses.json index 4810ac5ab7..7c7f98b1ec 100644 --- a/third_party/osv-licenses.json +++ b/third_party/osv-licenses.json @@ -3407,10 +3407,30 @@ "Apache-2.0" ] }, + { + "package": { + "name": "nemo-fabric", + "version": "0.1.0rc6", + "ecosystem": "PyPI" + }, + "licenses": [ + "UNKNOWN" + ] + }, + { + "package": { + "name": "nemo-fabric-runtime", + "version": "0.1.0rc6", + "ecosystem": "PyPI" + }, + "licenses": [ + "UNKNOWN" + ] + }, { "package": { "name": "nemo-relay", - "version": "0.4.0", + "version": "0.6.0", "ecosystem": "PyPI" }, "licenses": [ @@ -8325,7 +8345,7 @@ }, { "name": "UNKNOWN", - "count": 4 + "count": 6 } ] } \ No newline at end of file diff --git a/third_party/requirements-main.txt b/third_party/requirements-main.txt index df436f8649..d4a537dd59 100644 --- a/third_party/requirements-main.txt +++ b/third_party/requirements-main.txt @@ -1470,10 +1470,16 @@ mypy-extensions==1.0.0 ; (platform_machine == 'arm64' and sys_platform == 'darwi nemo-anonymizer==0.3.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:342cba7427df553afc214e85bd2ee947328ae60f60adaa75dad2e09096c7ba4d # via nemo-anonymizer-plugin -nemo-relay==0.4.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ - --hash=sha256:0f92883b81540076e4b6c5e754eb7726225336634204ebd22ff730ccbcfb2723 \ - --hash=sha256:6a5a5f5dec1428085f6c41c3645f1773f9cfb154cfa47306c93d6514091a8006 \ - --hash=sha256:d5f14fe5c8e5fbcc26827cc88e5b867f4ad37d6f983100cbf31705234c39f9d3 +nemo-fabric==0.1.0rc6 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ + --hash=sha256:624950b21151824b975232500d246e717ae0cde3225f0a54aac96b251a36c8b7 + # via nemo-evaluator-sdk +nemo-fabric-runtime==0.1.0rc6 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ + --hash=sha256:908c586b46419bc75b3b39371718932e4c1d6553c8f6fd01d1aac284a6ee8ed3 + # via nemo-fabric +nemo-relay==0.6.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ + --hash=sha256:849daa9e45158ac581e54506e0fcc7a24f557d1ed06dbdc074f5de7a00393cbc \ + --hash=sha256:ad5dae6febf6532d7b113abc2a404679c8feffc499df3034b93d9a078185d2bb \ + --hash=sha256:c0cd9570f64c6956fe3bfb82af1cdb3ee70cb50b51098cdb0de831c3f9b4e904 # via nemo-evaluator-sdk nemo-safe-synthesizer==0.1.7 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:07ad037e6ded8f7020039fa88efdab4aaaf805ea52fb8238380086e0811eaaf0 @@ -2147,6 +2153,7 @@ pydantic==2.12.5 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or # nemo-evaluator-plugin # nemo-evaluator-sdk # nemo-experimentalist-plugin + # nemo-fabric-runtime # nemo-insights-plugin # nemo-platform-ext # nemo-platform-plugin @@ -2931,6 +2938,7 @@ typing-extensions==4.15.0 ; (platform_machine == 'arm64' and sys_platform == 'da # langsmith # mcp # mlflow-skinny + # nemo-fabric-runtime # nemo-platform-sdk # nemo-safe-synthesizer # openai diff --git a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml index 20ba55d1cc..0abdb406fd 100644 --- a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml +++ b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml @@ -83,6 +83,11 @@ overrides: nvidia-cudnn-frontend: NVIDIA Proprietary Software nvidia-nvshmem-cu12: NVIDIA Proprietary Software triton: MIT + # NeMo Fabric — Apache-2.0 per PyPI (`license_expression`) and deps.dev, but osv-scanner reports + # UNKNOWN for the 0.1.x rc series, the same way it does for safetensors 0.8.0rc1 above. Revisit + # once Fabric cuts a non-prerelease 0.1.0. + nemo-fabric: Apache-2.0 # https://pypi.org/project/nemo-fabric/ + nemo-fabric-runtime: Apache-2.0 # https://pypi.org/project/nemo-fabric-runtime/ # ONNX Runtime onnxruntime: MIT # https://github.com/microsoft/onnxruntime diff --git a/uv.lock b/uv.lock index 06bf00f42d..7c45c420f9 100644 --- a/uv.lock +++ b/uv.lock @@ -3565,9 +3565,7 @@ container = [ { name = "python-on-whales", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] fabric = [ - { name = "nemo-fabric", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nemo-fabric-adapters-claude", extra = ["harness"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nemo-fabric-adapters-codex", extra = ["harness"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-fabric", extra = ["claude", "codex"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-fabric-adapters-hermes", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] test = [ @@ -3592,10 +3590,8 @@ requires-dist = [ { name = "nemo-agents-example-calculator", editable = "plugins/nemo-agents/examples/calculator-agent" }, { name = "nemo-agents-example-email-phishing", editable = "plugins/nemo-agents/examples/email-phishing-analyzer" }, { name = "nemo-deployments-plugin", editable = "plugins/nemo-deployments" }, - { name = "nemo-fabric", marker = "extra == 'fabric'", specifier = ">=0.1.0rc4,<0.2.0" }, - { name = "nemo-fabric-adapters-claude", extras = ["harness"], marker = "extra == 'fabric'", specifier = ">=0.1.0rc4,<0.2.0" }, - { name = "nemo-fabric-adapters-codex", extras = ["harness"], marker = "extra == 'fabric'", specifier = ">=0.1.0rc4,<0.2.0" }, - { name = "nemo-fabric-adapters-hermes", marker = "python_full_version < '3.14' and extra == 'fabric'", specifier = ">=0.1.0rc4,<0.2.0" }, + { name = "nemo-fabric", extras = ["claude", "codex"], marker = "extra == 'fabric'", specifier = ">=0.1.0rc6,<0.2.0" }, + { name = "nemo-fabric-adapters-hermes", marker = "python_full_version < '3.14' and extra == 'fabric'", specifier = ">=0.1.0rc6,<0.2.0" }, { name = "nemo-platform", editable = "packages/nemo_platform" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, { name = "nvidia-nat-config-optimizer", specifier = ">=1.8.0,<1.9" }, @@ -3988,6 +3984,7 @@ dependencies = [ { name = "jsonschema", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "langchain-nvidia-ai-endpoints", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "langchain-openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-fabric", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-relay", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pandas", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -4002,6 +3999,10 @@ dependencies = [ agent-runtimes = [ { name = "openai-agents", extra = ["docker"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] +fabric = [ + { name = "nemo-fabric", extra = ["claude", "codex"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-fabric-adapters-hermes", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] harbor = [ { name = "harbor", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] @@ -4023,8 +4024,11 @@ requires-dist = [ { name = "jsonschema", specifier = ">=4.23.0" }, { name = "langchain-nvidia-ai-endpoints", specifier = ">=1.4.3,<2.0.0" }, { name = "langchain-openai", specifier = ">=1.3.5" }, + { name = "nemo-fabric", specifier = ">=0.1.0rc6,<0.2.0" }, + { name = "nemo-fabric", extras = ["claude", "codex"], marker = "extra == 'fabric'", specifier = ">=0.1.0rc6,<0.2.0" }, + { name = "nemo-fabric-adapters-hermes", marker = "python_full_version < '3.14' and extra == 'fabric'", specifier = ">=0.1.0rc6,<0.2.0" }, { name = "nemo-platform-sdk", marker = "extra == 'nemo-platform'", editable = "sdk/python/nemo-platform" }, - { name = "nemo-relay", specifier = ">=0.4.0,<0.5.0" }, + { name = "nemo-relay", specifier = ">=0.6.0,<0.7" }, { name = "openai", specifier = ">=1.61.0" }, { name = "openai-agents", extras = ["docker"], marker = "extra == 'agent-runtimes'", specifier = ">=0.17.3,<0.18" }, { name = "pandas", specifier = ">=1.5.3" }, @@ -4034,7 +4038,7 @@ requires-dist = [ { name = "rouge-score", specifier = "==0.1.2" }, { name = "sacrebleu", specifier = ">=2.5.1" }, ] -provides-extras = ["agent-runtimes", "engine", "harbor", "nemo-platform"] +provides-extras = ["agent-runtimes", "engine", "harbor", "nemo-platform", "fabric"] [package.metadata.requires-dev] dev = [ @@ -4079,22 +4083,30 @@ requires-dist = [ [[package]] name = "nemo-fabric" -version = "0.1.0rc4" +version = "0.1.0rc6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nemo-fabric-runtime", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/10/82/17097fda6af7ff5c3597383bc7ff1a8c6d3427d27ea040196bc80d09b8e8/nemo_fabric-0.1.0rc4.tar.gz", hash = "sha256:dbbd1097886cdde50a4b8a65071c5e0b16362de948d48d843d4ade1ed8fd8445", size = 6783, upload-time = "2026-07-29T00:49:32.835Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/dd/a670260d802a268c3a3707a15cce80fd636e732fe80671bd19bb71389145/nemo_fabric-0.1.0rc6.tar.gz", hash = "sha256:624950b21151824b975232500d246e717ae0cde3225f0a54aac96b251a36c8b7", size = 6577, upload-time = "2026-07-30T15:08:21.112Z" } + +[package.optional-dependencies] +claude = [ + { name = "nemo-fabric-adapters-claude", extra = ["harness"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +codex = [ + { name = "nemo-fabric-adapters-codex", extra = ["harness"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] [[package]] name = "nemo-fabric-adapters-claude" -version = "0.1.0rc4" +version = "0.1.0rc6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nemo-fabric-adapters-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "tomli-w", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/63/2b/442f862ba8ac0433cde7f34e955260818168e6cae8efcb12c3f222af5492/nemo_fabric_adapters_claude-0.1.0rc4.tar.gz", hash = "sha256:b43e295ebffd3d58b903a2b13fe1cfd7361cf27dcde05e857beb694f760fbb67", size = 8487, upload-time = "2026-07-29T00:49:30.747Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/6b/0bfc2c63b5aa265c9975828f28b46ea98791742c3d4d4de32011f783734c/nemo_fabric_adapters_claude-0.1.0rc6.tar.gz", hash = "sha256:104c65263c4e0e7650718450608c1abea23519379a47592c863510b28fd35348", size = 8475, upload-time = "2026-07-30T15:08:27.014Z" } [package.optional-dependencies] harness = [ @@ -4103,13 +4115,13 @@ harness = [ [[package]] name = "nemo-fabric-adapters-codex" -version = "0.1.0rc4" +version = "0.1.0rc6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nemo-fabric-adapters-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "tomli-w", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/28/97/cfcf99aa528d5b583b37b85337a0486c1db81c1b3f44d3c7c42dfd231857/nemo_fabric_adapters_codex-0.1.0rc4.tar.gz", hash = "sha256:1749f61857bb8493229b988e55e2497be8a2c93b3f36e9d402dd0f4747bc8afc", size = 7818, upload-time = "2026-07-29T00:49:35.732Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/9b/428ce1e1ef93a5844e1f732084c00d029e234846d48950d58ad97119f9fe/nemo_fabric_adapters_codex-0.1.0rc6.tar.gz", hash = "sha256:82fa9469d522d9c4ca14d98f07a4c19a05ad45464579947602cd363d2684b3a7", size = 7860, upload-time = "2026-07-30T15:08:27.443Z" } [package.optional-dependencies] harness = [ @@ -4118,28 +4130,28 @@ harness = [ [[package]] name = "nemo-fabric-adapters-common" -version = "0.1.0rc4" +version = "0.1.0rc6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7f/ca/58c0381807a80154e76f33f8d47821207ae4613705783a4f7c51114cd844/nemo_fabric_adapters_common-0.1.0rc4.tar.gz", hash = "sha256:b82bb79e94533538a4778a553545bde735b8839712fc92e6c7f61750fce4edec", size = 5787, upload-time = "2026-07-29T00:49:34.164Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/15/8b176c6fb4c5ebea30a5d2de5146b774169959c3a9a7f99c331f767c4195/nemo_fabric_adapters_common-0.1.0rc6.tar.gz", hash = "sha256:c3238f5a75e19d4809f3e57566ca281c66a02c04e16263d5869e77970b47202b", size = 5508, upload-time = "2026-07-30T15:08:30.843Z" } [[package]] name = "nemo-fabric-adapters-hermes" -version = "0.1.0rc4" +version = "0.1.0rc6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nemo-fabric-adapters-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3d/8c/fa0c4003c3b28e32414566095808248114b63ab20cf2a37aa86c47fb4d1d/nemo_fabric_adapters_hermes-0.1.0rc4.tar.gz", hash = "sha256:2d0439208e2c09dea3c8bb22f7ba388916d9a7beec546b9169bbbb27ad478a7d", size = 6714, upload-time = "2026-07-29T00:49:37.704Z" } +sdist = { url = "https://files.pythonhosted.org/packages/39/de/3418eee6dab4bc3b50a26af2a7d548bd7834f85224b9712420b32b6d3705/nemo_fabric_adapters_hermes-0.1.0rc6.tar.gz", hash = "sha256:484d88159beeebc9e41facc1a033bf820c42ec6fe530fc62550011b715844c85", size = 6996, upload-time = "2026-07-30T15:08:24.946Z" } [[package]] name = "nemo-fabric-runtime" -version = "0.1.0rc4" +version = "0.1.0rc6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic", extra = ["email"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/20/9d/d92bb0f18c6a1e15708323347ebcbd292ff46ca3e5db7bac5156211d513c/nemo_fabric_runtime-0.1.0rc4.tar.gz", hash = "sha256:87279b38ddee05fab1ef82f50451f569a98490c4a540580cf95860e944349725", size = 5268, upload-time = "2026-07-29T00:49:28.985Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/12/f480441bef8d9d4d308a0786526b179468e37cf74aa4443e15fd44a21136/nemo_fabric_runtime-0.1.0rc6.tar.gz", hash = "sha256:908c586b46419bc75b3b39371718932e4c1d6553c8f6fd01d1aac284a6ee8ed3", size = 5013, upload-time = "2026-07-30T15:08:24.35Z" } [[package]] name = "nemo-guardrails-plugin" @@ -4562,6 +4574,7 @@ nemo-evaluator-sdk = [ { name = "jsonschema", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "langchain-nvidia-ai-endpoints", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "langchain-openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-fabric", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-relay", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pandas", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -5035,6 +5048,7 @@ requires-dist = [ { name = "nemo-evaluator-sdk", marker = "extra == 'nemo-evaluator-plugin'", editable = "packages/nemo_evaluator_sdk" }, { name = "nemo-evaluator-sdk", marker = "extra == 'plugins'", editable = "packages/nemo_evaluator_sdk" }, { name = "nemo-evaluator-sdk", marker = "extra == 'services'", editable = "packages/nemo_evaluator_sdk" }, + { name = "nemo-fabric", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=0.1.0rc6,<0.2.0" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, { name = "nemo-platform-plugin", marker = "extra == 'all'", editable = "packages/nemo_platform_plugin" }, { name = "nemo-platform-plugin", marker = "extra == 'core-service'", editable = "packages/nemo_platform_plugin" }, @@ -5061,7 +5075,7 @@ requires-dist = [ { name = "nemo-platform-sdk", marker = "extra == 'nmp-common'", editable = "sdk/python/nemo-platform" }, { name = "nemo-platform-sdk", marker = "extra == 'plugins'", editable = "sdk/python/nemo-platform" }, { name = "nemo-platform-sdk", marker = "extra == 'services'", editable = "sdk/python/nemo-platform" }, - { name = "nemo-relay", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=0.4.0,<0.5.0" }, + { name = "nemo-relay", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=0.6.0,<0.7" }, { name = "nemo-safe-synthesizer", marker = "extra == 'all'", specifier = "==0.1.7" }, { name = "nemo-safe-synthesizer", marker = "extra == 'nemo-safe-synthesizer-plugin'", specifier = "==0.1.7" }, { name = "nemo-safe-synthesizer", marker = "extra == 'plugins'", specifier = "==0.1.7" }, @@ -5518,6 +5532,7 @@ nemo-evaluator-sdk = [ { name = "jsonschema", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "langchain-nvidia-ai-endpoints", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "langchain-openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-fabric", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-relay", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pandas", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -5560,8 +5575,9 @@ requires-dist = [ { name = "jsonschema", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=4.23.0" }, { name = "langchain-nvidia-ai-endpoints", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=1.4.3,<2.0.0" }, { name = "langchain-openai", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=1.3.5" }, + { name = "nemo-fabric", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=0.1.0rc6,<0.2.0" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, - { name = "nemo-relay", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=0.4.0,<0.5.0" }, + { name = "nemo-relay", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=0.6.0,<0.7" }, { name = "ngcsdk", specifier = ">=4.8.2" }, { name = "nvidia-ml-py", specifier = ">=13.0.0" }, { name = "openai" }, @@ -5660,12 +5676,12 @@ test = [ [[package]] name = "nemo-relay" -version = "0.4.0" +version = "0.6.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/f0/1626672dd86005570afbcd4e368f5f271bcd2104ceaee883e839b11bfc81/nemo_relay-0.4.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:d5f14fe5c8e5fbcc26827cc88e5b867f4ad37d6f983100cbf31705234c39f9d3", size = 6674344, upload-time = "2026-06-12T23:24:02.707Z" }, - { url = "https://files.pythonhosted.org/packages/c1/35/956142c558334ec8594d97da6a1a717e70ba43685659dbdf18965c87c3f5/nemo_relay-0.4.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a5a5f5dec1428085f6c41c3645f1773f9cfb154cfa47306c93d6514091a8006", size = 5989959, upload-time = "2026-06-12T23:24:04.496Z" }, - { url = "https://files.pythonhosted.org/packages/6c/56/e5172ec8486deb23dabf5c948be81b8172840efdfcda42b1da1a8d9ef969/nemo_relay-0.4.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f92883b81540076e4b6c5e754eb7726225336634204ebd22ff730ccbcfb2723", size = 6332992, upload-time = "2026-06-12T23:24:06.288Z" }, + { url = "https://files.pythonhosted.org/packages/25/65/d320016505457cc30971f575e8dadffb923b7cfc780ab8bb25a4ce9d305c/nemo_relay-0.6.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:ad5dae6febf6532d7b113abc2a404679c8feffc499df3034b93d9a078185d2bb", size = 9917779, upload-time = "2026-07-22T20:07:48.961Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c0/f33250e71c4206da1b339072893f9a1e39295fe1aceb9a2fef4b8620a0f2/nemo_relay-0.6.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0cd9570f64c6956fe3bfb82af1cdb3ee70cb50b51098cdb0de831c3f9b4e904", size = 8888375, upload-time = "2026-07-22T20:07:51.049Z" }, + { url = "https://files.pythonhosted.org/packages/a3/f4/d1dfaed022da0f6f14765a122867f976a69cc520fe1faaf99757f5719d1f/nemo_relay-0.6.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:849daa9e45158ac581e54506e0fcc7a24f557d1ed06dbdc074f5de7a00393cbc", size = 9336372, upload-time = "2026-07-22T20:07:53.224Z" }, ] [[package]]