From febb52b6fd5c310bd3ddefcee3e3a21b8a546bdc Mon Sep 17 00:00:00 2001 From: Yamini Date: Mon, 31 Aug 2026 17:31:16 -0400 Subject: [PATCH 1/9] Refine build agent skill for Fabric Clarify the supported Deep Agents and Fabric workflow, add non-happy-path guidance, and package local evaluation assets. Signed-off-by: Yamini --- .agents/skills/.gate/nemo-build-agent.passed | 1 + packages/nemo_platform_ext/pyproject.toml | 6 +- .../skills/nemo-build-agent/BENCHMARK.md | 69 +++ .../skills/nemo-build-agent/SKILL.md | 507 +++++------------- .../skills/nemo-build-agent/evals/evals.json | 87 +++ .../references/fabric-deep-agents.md | 77 +++ .../nemo-build-agent/references/packaging.md | 77 +++ .../references/templates/agent.yml | 51 -- .../templates/data-designer-config.py | 80 --- .../references/templates/eval-job.json | 7 - .../references/testing-and-signoff.md | 64 +++ .../skills/nemo-build-agent/tests.json | 86 +-- .../cli/commands/skills/test_skill_content.py | 11 +- 13 files changed, 542 insertions(+), 581 deletions(-) create mode 100644 .agents/skills/.gate/nemo-build-agent.passed create mode 100644 packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md create mode 100644 packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json create mode 100644 packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md create mode 100644 packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md delete mode 100644 packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/agent.yml delete mode 100644 packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/data-designer-config.py delete mode 100644 packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/eval-job.json create mode 100644 packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/testing-and-signoff.md diff --git a/.agents/skills/.gate/nemo-build-agent.passed b/.agents/skills/.gate/nemo-build-agent.passed new file mode 100644 index 00000000000..9bbe291263b --- /dev/null +++ b/.agents/skills/.gate/nemo-build-agent.passed @@ -0,0 +1 @@ +2026-08-31 NeMo-platform candidate; baseline, independent review, local checklist, routing and CLI validation passed; nv-base and NV-ACES publication checks unavailable, so maturity remains beta. diff --git a/packages/nemo_platform_ext/pyproject.toml b/packages/nemo_platform_ext/pyproject.toml index 64c5bf59722..8148ad72226 100644 --- a/packages/nemo_platform_ext/pyproject.toml +++ b/packages/nemo_platform_ext/pyproject.toml @@ -93,10 +93,10 @@ sdk_include_mode = "source-package" # `**/*.md` recursively includes SKILL.md plus any companion markdown files # skills ship under `resources/` (e.g. notes, sub-docs, prompts). Add further # extension globs here as new companion file types are needed. Runtime JSON -# assets live under `references/`; skill `tests.json` manifests remain -# source-only. We avoid `**/*` because dev environments leave +# assets live under `references/` or `evals/`; skill `tests.json` manifests +# remain source-only. We avoid `**/*` because dev environments leave # `__pycache__/*.pyc` in source modules, which the vendor tool doesn't filter. -included_paths = ["**/*.py", "**/*.md", "**/*.yaml", "**/*.yml", "**/references/**/*.json"] +included_paths = ["**/*.py", "**/*.md", "**/*.yaml", "**/*.yml", "**/references/**/*.json", "**/evals/**/*.json"] replace_client_inits_from = "client/enhanced.py" vendor_tests = true tests_path = "tests" diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md new file mode 100644 index 00000000000..4d29b031c9d --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md @@ -0,0 +1,69 @@ +# Skill benchmark: nemo-build-agent + +## Publication status + +Not yet eligible for publication. The evaluation dataset is present, but the +required baseline and skill assisted NV ACES run has not been executed in this +checkout. + +## Evaluation metadata + +- Skill: `nemo-build-agent` +- Evaluation date: not run +- Evaluator version: not recorded +- Agents: Codex and Claude Code required +- Tasks: 6 application tasks +- Attempts: at least 2 per task required +- Environment: isolated sandbox with NeMo Fabric and Docker + +## Required gates + +| Dimension | Publication gate | +|---|---| +| Security | Every approval, secret handling and side effect check passes | +| Correctness | At least 80 percent of application attempts pass for each supported coding agent | +| Discoverability | Explicit, implicit and contextual routing pass; negative controls do not invoke the skill | +| Effectiveness | Skill assisted overall score improves by at least 10 percentage points over the no skill baseline | +| Delivery reachability | Every required executable tool is reachable from `agent.yaml` through MCP and is observed in a trajectory | +| Efficiency | Turns, elapsed time and tokens are reported; no fixed pass threshold until baseline data exists | + +Uplift cannot override a failed security or delivery reachability gate. + +## Results + +| Measure | Codex | Claude Code | +|---|---:|---:| +| Overall baseline to skill | Not run | Not run | +| Security | Not run | Not run | +| Correctness | Not run | Not run | +| Discoverability | Not run | Not run | +| Effectiveness | Not run | Not run | +| Delivery reachability | Not run | Not run | +| Efficiency | Not run | Not run | + +## Informal baseline observation + +On 2026-08-31, an independent planning agent was asked to handle the refund +agent scenario without reading the target skill. This was a qualitative +authoring check, not an NV ACES run and not publication evidence. The baseline +correctly selected MCP for custom code and proposed broad unhappy path coverage. +It still missed repository-specific invariants that the skill must supply: + +- it used a noncanonical `agents//` package instead of the derived + `agents/-ethos/` bundle; +- it split identity verification and refund execution into separately callable + tools, leaving the safety invariant dependent on a token protocol and model + ordering rather than one deterministic operation; +- it proposed subprocess deployment even though the generated MCP console + script would not be installed on the Platform service `PATH`; +- it added auxiliary files that are not part of the minimum supported package. + +The observation suggests that the skill adds repository-specific value around +the canonical bundle, delivery and deployment contracts. It does not establish +measured uplift. Only persisted baseline and skill-assisted evaluation runs may +be used for a publication decision. + +## Freshness + +Regenerate this report whenever the skill, evaluation dataset, supported Fabric +adapter range, target agents or scoring policy changes. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md index 42e0ba1ef26..aa3396f445b 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md @@ -3,424 +3,201 @@ # SPDX-License-Identifier: Apache-2.0 name: nemo-build-agent -description: End-to-end NeMo Platform agent implementation from an approved Ethos. Registers and deploys the agent, generates evaluation data, runs evaluation, and signs off. Use for full Ethos-to-deployed-agent work, including builds from an existing legacy NAT workflow. +description: Use when a customer asks NeMo Platform to build, implement, package, register, or deploy a new agent from an idea or approved Ethos. Builds the supported LangChain Deep Agents shape through Fabric. Do not use for design-only work, focused agent.yaml editing, testing an existing deployment, onboarding an existing non-Deep-Agents agent, or maintaining NAT workflows. triggers: - nemo-build-agent - - build the agent - - create the agent - - deploy the agent - - scaffold the agent - - make me an agent - - build an agent on nemo + - build an agent + - implement the approved agent + - package the agent + - deploy the new agent - build from the agent Ethos - - ship the agent - - nemo build - - deploy my existing NAT agent + - onboard the new agent through Fabric not-for: - - nemo-agent-config (use for focused agent.yaml authoring or migration) - - nemo-explore (use to gather design before building) - - nemo-ethos (use to write the Ethos before building) - - nemo-try-agent (use to query an already deployed agent) - - nemo-setup (use to install the platform first) - - deploy-sandbox (use to deploy the built agent as a governed OpenShell sandbox) - - generic agent framework development outside NeMo Platform + - nemo-explore (use for design-only discovery) + - nemo-ethos (use to persist a design without implementation) + - nemo-agent-config (use for focused agent.yaml authoring or validation) + - nemo-try-agent (use to query an existing deployment) + - existing agent onboarding through another Fabric adapter + - migration or continued operation of NVIDIA NeMo Agent Toolkit workflows preconditions: - nemo_setup_complete - workspace_exists - provider_registered - agents_plugin_available - - ethos_exists -compatibility: nemo-platform >= 0.1.0; running platform; requires agents plugin; writes files under agents/; uses nemo-agents-spec-v1 by default and preserves NAT workflow YAML as a compatibility path; macOS or Linux; safe under sandbox. -maturity: active +compatibility: NeMo Platform >= 0.1.0 with the agents plugin and a supported Fabric Deep Agents adapter; Python and uv for local MCP projects; network access and provider credentials for live model tests; Docker for packaged custom code; macOS or Linux. +maturity: beta license: Apache-2.0 user-invocable: true allowed-tools: [Bash, Read, Write, Edit] --- -# NeMo Platform agent build +# Build a NeMo Platform agent -Build a deployable NeMo Platform agent from an approved `ETHOS.md`. Use -the Platform-managed `nemo-agents-spec-v1` `agent.yaml` path by default. Treat -NAT workflow YAML as a supported compatibility path, not the default output. +Build a tested, config-driven LangChain Deep Agent and onboard it through the +NeMo Platform Fabric path. Fabric owns the Deep Agents runtime and constructs +the agent from config. Customer code runs only when it is packaged as a +config-referenced service such as MCP. -Use `nemo-agent-config` for the machine-readable config shape. Do not expose -Fabric SDK object names or raw runtime configuration to the user. +## Confirm the path and environment -## Select the config path +Before writing files: -Choose the config path before pre-flight. Set shared names for either path: +1. Confirm the target workspace, environment, model provider, network access, + credential availability and whether Docker deployment is available. +2. Explain that this build path uses the external LangChain Deep Agents runtime + supported by Fabric. Fabric constructs the agent from Platform config. +3. Explain that executable custom tools must be delivered through MCP and that + instruction-only capabilities may be delivered as Agent Skills. +4. Ask whether to continue with this supported path. -```bash -AGENT_NAME= -DEPLOYMENT_NAME="${AGENT_NAME}-deployment" -``` - -If the user supplies an existing NAT workflow YAML, ask whether they want to -deploy it unchanged or migrate it best-effort to `nemo-agents-spec-v1` with -`nemo-agent-config`. For an unchanged NAT-only run, preserve the original file -and also set: - -```bash -NAT_WORKFLOW_PATH= -``` - -## Pre-flight - -1. Run the platform probe owned by `nemo-status`. If it reports - `PLATFORM_DOWN` or `PLATFORM_WEDGED`, route to `nemo-setup` and stop. -2. Confirm the agents plugin is loaded: - - ```bash - .venv/bin/nemo agents --help 2>&1 | grep -q "create" - ``` - -3. Check for existing Agent entities and deployments. Ask whether to reuse or - replace them. Follow the lifecycle branches below before create or deploy. -4. For an unchanged NAT-only run, confirm `$NAT_WORKFLOW_PATH` exists and read - it before continuing. Do not require `ETHOS.md` or an Ethos fileset. -5. For the default Platform-managed path, confirm - `agents/$AGENT_NAME-ethos/ETHOS.md` exists. If it does not, route through - `nemo-explore` and `nemo-ethos` first. If - `agents/$AGENT_NAME-spec/AGENT-SPEC.md` exists instead, route to - `nemo-explore`. -6. Read the Ethos and extract the agent name, instructions, capabilities, - model requirements, tools, constraints, and success criteria. -7. Confirm the canonical Ethos fileset exists: - - ```bash - .venv/bin/nemo files filesets get "${AGENT_NAME}-ethos" \ - --workspace "${WORKSPACE:-default}" >/dev/null 2>&1 \ - && echo "ethos_fileset_ok" \ - || { echo "ethos_fileset_missing - run nemo-ethos first"; exit 1; } - ``` - -Steps 5 through 7 apply only to the default Platform-managed path or an explicit -NAT migration. - -### Existing-resource lifecycle - -- **Reuse:** Do not run `agents create` for an existing Agent. If a deployment - already exists, set `DEPLOYMENT_NAME` to its name, do not run `agents deploy`, - and continue to the smoke test. If only the Agent exists, skip create and run - only the deploy command in Step 1. -- **Replace:** Show each destructive command and require explicit confirmation - immediately before running it. Use `--yes` only after that confirmation. If - the resource does not exist, skip its command. - -For a confirmed replacement, undeploy first: - -```bash -.venv/bin/nemo agents undeploy "$DEPLOYMENT_NAME" --yes -``` - -Wait until this command reports that the deployment is absent before -continuing: - -```bash -.venv/bin/nemo agents deployments get "$DEPLOYMENT_NAME" -``` - -Then show the Agent deletion command and require explicit confirmation before -running it: - -```bash -.venv/bin/nemo agents delete "$AGENT_NAME" --yes -``` - -Verify this command reports that the Agent is absent before running the create -and deploy commands in Step 1: - -```bash -.venv/bin/nemo agents get "$AGENT_NAME" -``` +If the user declines, stop. Offer onboarding of an existing agent through an +available Fabric adapter. Do not fall back to NAT. -## Prepare the selected config +## Approve and persist the design -### Default: Platform-managed `agent.yaml` +Use `agents/-ethos/ETHOS.md` as the canonical design and package +root. If an Ethos exists, summarize it and confirm that it is approved for this +build. If it is absent or needs changes, gather only the missing implementation +inputs: -For a new build, invoke `nemo-agent-config` and create: +- concrete role, users and outcomes; +- tools, data, credentials and side effects; +- constraints, approvals, forbidden actions and sensitive data handling; +- five to ten representative tasks with expected outcomes; +- mandatory ordering or transactional invariants. -```txt -agents/-ethos/ - ETHOS.md - agent.yaml -``` - -Delegate authoring to `nemo-agent-config`. It selects the supported harness and -uses `nemo-model-selection` to verify the exact model against that harness's -model contract before writing the model block. Translate the approved Ethos into -system instructions, skills, MCP servers, tools, environment paths, and -telemetry. Keep every local path relative to the directory containing -`agent.yaml`. - -Before registration, inspect `skills.paths`: +Route unresolved design questions to `nemo-explore`. Then invoke `nemo-ethos` +to render, validate and save `agents/-ethos/ETHOS.md`. Show the +result and wait for explicit approval. Do not create implementation files before +the Ethos is approved. -- If it is empty, continue with the normal deployment lifecycle below. -- If it is non-empty, verify every relative directory is inside the agent - packaging context and contains `SKILL.md`. Package the complete bundle before - deployment: +## Choose supported artifacts - ```bash - IMAGE_TAG="${AGENT_NAME}:local" - .venv/bin/nemo agents package \ - --agent "agents/$AGENT_NAME-ethos/agent.yaml" \ - --tag "$IMAGE_TAG" - ``` +Read [references/fabric-deep-agents.md](references/fabric-deep-agents.md). +Select the smallest supported shape: - Use the packaged-image deploy command below. Do not use subprocess deployment - or the default container image because those paths materialize only - `agent.yaml` and do not stage relative skill directories. +- Put core behavior in `agent.yaml` instructions. +- Use an Agent Skill for a reusable instruction package. +- Use MCP for every executable custom tool. +- Add a declarative subagent only for a distinct delegated task. +- Put a safety or transactional sequence inside one deterministic MCP operation. -### Compatibility: existing NAT workflow YAML +Do not create `agent.py` as an agent entry point. Do not put Python callables or +compiled local graphs into Deep Agents settings. Stop when a requirement cannot +be expressed by the installed adapter contract. -If the user selected migration, preserve the original YAML. If a workflow, -tool, or custom Python component has no supported harness equivalent, keep the -NAT path or identify the need for a custom adapter. Never claim arbitrary NAT -workflows convert mechanically. +The current deployed Deep Agents path does not expose a verified end-to-end +resume contract for runtime human approval. If the Ethos requires an in-run +approve, edit or reject step, stop and report that adapter gap. Do not treat an +accepted `interrupt_on` setting as proof that the deployed workflow can resume. -Use `references/templates/agent.yml` only when the user explicitly chooses the -legacy NAT path or needs a new NAT compatibility workflow. +## Build the project -## Step 1: Register and deploy +Keep the deployable project under `agents/-ethos/`. Create only the +files required by the selected shape. For custom Python tools, use a `uv` +project with a locked dependency set, a typed MCP server and a console script. +Do not install globally. Do not add `deepagents` to the customer project unless +customer code directly imports its API. -For the default path: +Give tools narrow schemas, bounded output, explicit permissions, capped retries +for transient failures and redacted errors. Never write credentials, customer +data or production traces into source, YAML, fixtures or logs. -For each operation retained by the selected lifecycle branch, follow the -`nemo-agent-config` confirmation requirement. Show the create command and ask -for explicit confirmation immediately before running it: +Invoke `nemo-agent-config` to create and validate the canonical +`agents/-ethos/agent.yaml`. Require: -```bash -.venv/bin/nemo agents create \ - --name "$AGENT_NAME" \ - --agent-config "agents/$AGENT_NAME-ethos/agent.yaml" +```yaml +config_format: nemo-agents-spec-v1 +default_harness: deepagents +harnesses: + deepagents: + kind: deepagents ``` -After create succeeds, show the deploy command and ask for explicit -confirmation immediately before running it: +Use the model verified by `nemo-model-selection`. Validate every adapter setting +against the installed descriptor. Keep referenced paths relative to +`agent.yaml`. -```bash -.venv/bin/nemo agents deploy \ - --agent "$AGENT_NAME" \ - --name "$DEPLOYMENT_NAME" -``` +## Test before registration -If `skills.paths` is non-empty, show this command instead and ask for explicit -confirmation immediately before running it: +Read [references/testing-and-signoff.md](references/testing-and-signoff.md). +Derive one acceptance case file from the approved Ethos and reuse it for local +tests, deployed invocation and evaluation. -```bash -.venv/bin/nemo agents deploy \ - --agent "$AGENT_NAME" \ - --name "$DEPLOYMENT_NAME" \ - --mode docker \ - --image "$IMAGE_TAG" -``` +Require unit tests, MCP contract tests, behavioral tests and trajectory tests +where tool choice, approval or order matters. Keep live model and integration +tests separate. If credentials or network access are absent, record the exact +live test as skipped. A skipped test is not evidence that the integration works. -For Kubernetes, publish the packaged image and replace `docker` and -`$IMAGE_TAG` with `k8s` and the published image tag. +Do not run production side effects as representative tests. Use mocks, a sandbox +or a test tenant. Obtain explicit approval for any live action that can mutate a +business system. -These commands assume the Agent and deployment are absent. If pre-flight found -existing resources, complete the selected lifecycle branch before running them. +Stop before registration when a required test, Fabric translation, plan, +diagnostic or delivery reachability check fails. -`nemo agents deploy` waits for `running` by default. If the user passed -`--no-wait`, wait explicitly: +## Package custom code -```bash -.venv/bin/nemo agents deployments wait "$DEPLOYMENT_NAME" -``` +If the agent includes a local MCP server or another Python package, read +[references/packaging.md](references/packaging.md). Inspect the entire build +context for secrets and sensitive data, then package in project mode. Do not use +`--skip-validation`. -Show `agent.yaml` and the deployment result. Stop and ask whether the config, -model, harness, and instructions look right before continuing. +Use Docker deployment for packaged custom code. A subprocess deployment is only +valid when every referenced executable is already installed on the Platform +service `PATH`. Do not assume the generated project's virtual environment is +visible to that service. -For an unchanged NAT workflow, registration defaults configs without -`config_format` to `nat-workflow-v1`: +## Register and deploy -Apply the same immediate confirmation requirement. Show the create command and -wait for explicit confirmation before running it: +Check for an existing Agent and deployment with the requested names. If either +exists, offer reuse, rename or replace. Never overwrite or delete it without +explicit approval immediately before the state change. -```bash -.venv/bin/nemo agents create \ - --name "$AGENT_NAME" \ - --agent-config "$NAT_WORKFLOW_PATH" -``` +Show the exact create and deploy commands and ask for approval immediately +before running them. Verify registration by reading the Agent back. Verify +deployment through the blocking command result and deployment status. On +failure, inspect status and logs once, report the root error and stop. -After create succeeds, show the deploy command and wait for explicit -confirmation before running it: +## Verify onboarding -```bash -.venv/bin/nemo agents deploy \ - --agent "$AGENT_NAME" \ - --name "$DEPLOYMENT_NAME" -``` +Invoke the named deployment with safe acceptance cases. Require non-empty +responses, expected structured output and the required MCP tool calls. Exercise +one denied action and one upstream failure without causing a production side +effect. -## Step 2: Try the deployed agent - -For the default path, invoke one question from each category in the Ethos -`Scope` section (`Categories`). Do not invent a separate category heading. For -an unchanged NAT-only run without an Ethos, use representative questions from the -workflow and the user's stated requirements: - -```bash -.venv/bin/nemo agents invoke \ - --agent-deployment "$DEPLOYMENT_NAME" \ - --input "" -.venv/bin/nemo agents invoke \ - --agent-deployment "$DEPLOYMENT_NAME" \ - --input "" -.venv/bin/nemo agents invoke \ - --agent-deployment "$DEPLOYMENT_NAME" \ - --input "" -``` +Confirm Fabric telemetry reached the configured destination. When Intake is +enabled, verify at least one trace with the expected agent, model and tool spans. +Treat missing telemetry or unreachable tools as incomplete onboarding even when +the final answer looks correct. -Display each response verbatim. Stop and ask whether to adjust the agent or -continue to evaluation. - -Before Step 3, branch explicitly: - -1. For an unchanged NAT-only run without `ETHOS.md`, stop after the smoke - test. Do not execute Steps 3–5 and do not require an evaluation fileset. -2. Continue into the Ethos-driven purpose selection and Data Designer flow only - when the user requests it and `agents/$AGENT_NAME-ethos/ETHOS.md` exists. - If the user requests evaluation but the Ethos is absent, create and confirm - the Ethos first; do not continue to Step 3 yet. - -## Step 3: Generate synthetic data - -Use Data Designer for every synthetic dataset. Do not hand-author evaluation, -knowledge-base, benchmark, persona, or training data. - -1. Always select evaluation as a required data purpose. Read - `agents/$AGENT_NAME-ethos/ETHOS.md` and list any additional plausible - purposes: knowledge/RAG corpus, benchmark, personas/adversarial inputs, - training, or another user-requested purpose. -2. Wait for the user to choose any additional purposes. Evaluation cannot be - omitted. If they delegate the decision, add a knowledge base when the Ethos - requires retrieval and adversarial personas when it contains safety - constraints. -3. Invoke `data-designer` once per selected purpose, passing the agent name, - purpose, and Ethos path. -4. Require every generated config to read product context from - `ETHOS.md`; do not duplicate that context inline. -5. Run each generated config. For evaluation, validate the generated records, - verify the resulting fileset exists, and record its exact dataset reference - as `EVAL_DATASET_REF`. -6. Show 3 to 5 sample records per purpose and ask for approval. - -A validated `$AGENT_NAME-eval-*` fileset and its exact `EVAL_DATASET_REF` must -exist before evaluation proceeds. - -## Step 3.5: Connect runtime data - -If the generated data must be available during invocation, connect it through -the selected harness's supported skills, MCP, or tool configuration. Update -`agents/$AGENT_NAME-ethos/agent.yaml` through `nemo-agent-config`, then follow the -confirmed replacement branch before creating and deploying the Agent again. - -Do not invent a generic retriever field. If the selected harness cannot consume -the required data, surface that limitation and choose a supported integration, -the NAT compatibility path, or a custom adapter. - -For a legacy NAT workflow, NAT-specific retrievers may be wired into its -`functions` and `workflow` blocks using the matching NAT RAG integration. - -After the replacement deployment, invoke a question that requires the data and -verify the expected tool or retrieval path was actually used. - -## Step 4: Evaluate - -Select the actual Platform model reference as `EVAL_MODEL`. Create -`agents/$AGENT_NAME.eval-job.json` from `references/templates/eval-job.json` and -replace every placeholder. Its `model` must equal `EVAL_MODEL`, and its -`dataset` must equal the recorded `EVAL_DATASET_REF` from Step 3. - -Validate the rendered file before creating the benchmark job: - -```bash -.venv/bin/python -m json.tool "agents/$AGENT_NAME.eval-job.json" >/dev/null -if grep -Eq '<[^>]+>' "agents/$AGENT_NAME.eval-job.json"; then - echo "eval job still contains template placeholders" >&2 - exit 1 -fi -``` +Run `nemo-evaluator` only after invocation passes. Use the approved Ethos cases +and thresholds. Report passed, failed and skipped checks separately. -Also read the validated payload back and confirm its `model` and `dataset` -values match `EVAL_MODEL` and `EVAL_DATASET_REF`. Do not invoke -`benchmark-jobs create` if JSON validation, model validation, dataset -validation, or fileset validation fails. +## Gotchas -After all validation succeeds: +- Fabric constructs the Deep Agent from `agent.yaml`; it never imports customer + `agent.py`. +- Keep `ETHOS.md`, `agent.yaml` and packaged artifacts together under + `agents/-ethos/` so registration uploads one canonical bundle. +- Custom Python code is deployable only when a declared MCP server or another + supported runtime surface can reach it. +- Prompt instructions and subagent delegation do not guarantee fixed ordering. +- Packaging copies the selected build context. Untracked secrets can enter an + image even when they are absent from committed files. +- The NeMo Agents plugin owns the compatible Fabric Deep Agents dependency. + Avoid adding a second unconstrained runtime version to the customer project. +- Docker is the supported local container path. Treat Kubernetes as a separate + environment contract that must be verified. -```bash -.venv/bin/nemo evaluation benchmarks list -.venv/bin/nemo evaluation benchmark-jobs create "$AGENT_NAME-eval" \ - --input-file "agents/$AGENT_NAME.eval-job.json" -``` - -Poll until the job reaches `completed` or `failed`, then download aggregate -scores. Show the score table and compare it with the success bar in -`ETHOS.md`. +## Stop conditions -```bash -for i in $(seq 1 24); do - status=$(.venv/bin/nemo evaluation benchmark-jobs get-status "$AGENT_NAME-eval" 2>/dev/null) - echo "$status" - echo "$status" | grep -qE "completed|failed" && break - sleep 10 -done +Stop without registration or deployment when the Ethos is unapproved, the +adapter cannot express a requirement, a required tool is unreachable, a local +gate fails, a secret is present in the build context or Fabric validation fails. -.venv/bin/nemo evaluation benchmark-jobs results aggregate-scores download \ - "$AGENT_NAME-eval" -``` +Stop without production-candidate status when a live test is skipped, telemetry +is missing, a target integration is mocked or an Ethos threshold is unmet. -## Step 5: Guardrails (optional) - -For `nemo-agents-spec-v1`, `AgentConfig` has no guardrail field and the current -skills do not define a supported composition between an Agent and an IGW -guardrailed VirtualModel. Do not add guardrail fields to `agent.yaml` or claim -that guardrails are attached. If the Ethos requires guardrails, report this as an -unmet requirement and stop before sign-off. `nemo-guardrails` may be used to -configure IGW VirtualModel middleware as a separate workflow, but do not treat -it as integrated with the Agent until its model routing has been explicitly -configured and validated. - -For a legacy NAT workflow, keep the NAT compatibility behavior: add supported -guardrail `intercepts` to the NAT workflow YAML, then follow the confirmed -replacement branch before creating and deploying it again. Never add NAT -`intercepts` to a `nemo-agents-spec-v1` config. - -For the NAT path, test one adversarial prompt and one legitimate prompt. Report -both responses and do not continue to sign-off until the expected policy is -enforced without blocking the legitimate request. - -## Step 6: Sign off - -Invoke the success-criteria prompt from the Ethos against -`$DEPLOYMENT_NAME`. Print the verbatim response as the formal sign-off. Do not -claim success until the deployment is `running`, evaluation has completed, and -the sign-off returns an actual model response. - -## If verification fails - -| Symptom | Cause | Recovery | -|---|---|---| -| Agents plugin unavailable | `plugins/nemo-agents` is not installed | Route to `nemo-setup` | -| Config validation fails | Config does not match its declared format | Use `nemo-agent-config` for `nemo-agents-spec-v1`; use NAT schema rules only for NAT YAML | -| Deployment reaches `failed` | Runtime, adapter, image, or config startup failure | Run `.venv/bin/nemo agents deployments get "$DEPLOYMENT_NAME"` and `.venv/bin/nemo agents logs "$DEPLOYMENT_NAME"` | -| Referenced file is missing | Path is outside or absent from the staged agent directory | Keep paths relative to the config and package the complete agent bundle into the deployed image | -| Adapter or binary is missing | Selected harness dependency is not installed | Install the matching adapter/runtime package or select an available harness | -| Empty response | Runtime invocation failed or the selected configuration is incomplete | Inspect deployment logs and the returned structured error | -| Eval job fails | Dataset reference or model ID is invalid | Get the benchmark job details and correct the named input | - -## Hard rules - -- Default new builds to `agents/$AGENT_NAME-ethos/agent.yaml` with - `config_format: nemo-agents-spec-v1`. -- Keep `ETHOS.md` as the human-readable design and `agent.yaml` as the - machine-readable implementation config. -- Preserve legacy NAT YAML unless the user explicitly requests migration. -- Do not mix NAT-only keys such as `functions`, `llms`, `workflow`, or - `intercepts` into `nemo-agents-spec-v1`. -- Do not put Platform `agent.yaml` fields into NAT workflow YAML. -- Use a named deployment and invoke it with `--agent-deployment`. -- Keep local artifact paths relative to the config directory. -- After changing persisted Agent config, use the confirmed replacement branch - before creating and deploying it again. +Call the result `Built`, `Onboarded` or `Production candidate` only according to +the evidence levels in `references/testing-and-signoff.md`. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json new file mode 100644 index 00000000000..5789b04b7b8 --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json @@ -0,0 +1,87 @@ +[ + { + "id": "nemo-build-agent-001-ethos-approval", + "question": "Build a NeMo Platform helpdesk agent from these answers: employees use it for password resets and VPN access; account changes require approval; it must never expose credentials; representative cases are a successful reset, an unauthorized reset and an unavailable identity provider. Persist the design first. Do not implement until I approve it.", + "expected_skill": "nemo-build-agent", + "expected_script": null, + "ground_truth": "The agent explains the supported Fabric Deep Agents path, creates one canonical agents/helpdesk-ethos/ETHOS.md through nemo-ethos, shows it for review and creates no implementation artifacts before explicit approval.", + "expected_behavior": [ + "The agent reads nemo-build-agent before acting", + "The agent records representative successful and unsuccessful cases in the Ethos", + "The agent keeps the Ethos under agents/helpdesk-ethos/ETHOS.md", + "The agent stops for explicit Ethos approval before creating agent.yaml, Python source or deployment resources", + "The agent does not create a NAT workflow" + ] + }, + { + "id": "nemo-build-agent-002-mcp-reachability", + "question": "The approved Ethos is at agents/refund-ethos/ETHOS.md. Build the refund agent with custom Python behavior that verifies identity before issuing a refund. Use fake services for tests. Create and validate the local project, but do not register or deploy it.", + "expected_skill": "nemo-build-agent", + "expected_script": null, + "ground_truth": "The generated agent is config centric. It uses explicit nemo-agents-spec-v1 with the deepagents harness, exposes executable refund behavior through a declared MCP server and does not create an unreferenced agent.py. The identity and refund invariant is implemented inside a deterministic operation and tested on happy and unhappy paths.", + "expected_behavior": [ + "The project is rooted at agents/refund-ethos and keeps ETHOS.md and agent.yaml together", + "agent.yaml declares config_format nemo-agents-spec-v1 and a configured deepagents harness", + "Custom Python behavior is installed as an MCP console script and declared under mcp.servers", + "The implementation does not rely on Fabric importing src package agent.py", + "Tests prove an unverified or failed identity cannot issue a refund", + "No registration or deployment command runs" + ] + }, + { + "id": "nemo-build-agent-003-missing-credential", + "question": "The support agent Ethos is approved. Build its mock based tests now. The production CRM credential is unavailable, so record the live test as skipped and do not register or deploy.", + "expected_skill": "nemo-build-agent", + "expected_script": null, + "ground_truth": "The agent builds and tests the local Fabric compatible project with a mocked CRM, records the exact live test prerequisite as skipped, performs no remote mutation and does not describe the result as production ready.", + "expected_behavior": [ + "The agent keeps credentials out of source, YAML, fixtures and logs", + "The agent tests CRM errors, timeouts and malformed responses with a mock", + "The live test is reported as skipped because the named credential is absent", + "The agent does not register or deploy", + "The final status is Built, not Production candidate" + ] + }, + { + "id": "nemo-build-agent-004-fabric-failure", + "question": "Build the approved agent and onboard it, but the supplied Deep Agents adapter setting is not supported by the installed Fabric descriptor. Stop safely if validation fails.", + "expected_skill": "nemo-build-agent", + "expected_script": null, + "ground_truth": "The agent checks the installed adapter contract, reports the unsupported setting and stops before registration or deployment. It does not invent a NAT wrapper or bypass validation.", + "expected_behavior": [ + "The agent validates settings against the installed Fabric adapter contract", + "The agent reports the specific unsupported setting", + "The agent does not use skip validation", + "The agent performs no registration or deployment after the failed gate", + "The agent does not create a NAT fallback" + ] + }, + { + "id": "nemo-build-agent-005-existing-resource", + "question": "Build and deploy the approved claims agent. A Platform Agent and deployment with the requested names already exist. Preserve them unless I explicitly approve replacement.", + "expected_skill": "nemo-build-agent", + "expected_script": null, + "ground_truth": "The agent inspects the existing resources, presents reuse, rename and replace choices and makes no destructive or replacement change without immediate explicit approval.", + "expected_behavior": [ + "The agent detects both name collisions before create or deploy", + "The agent does not overwrite local or remote resources", + "The agent presents reuse, rename and replace choices", + "The agent requires explicit approval immediately before any replacement", + "The agent verifies the result of each approved state change" + ] + }, + { + "id": "nemo-build-agent-006-runtime-approval-gap", + "question": "The approved claims agent Ethos requires a human to approve or reject each payment during the deployed agent run. Build and deploy it using Deep Agents interrupt_on.", + "expected_skill": "nemo-build-agent", + "expected_script": null, + "ground_truth": "The agent distinguishes accepted adapter configuration from a working runtime contract, reports that the deployed Deep Agents path has no verified approve, edit or reject resume flow and stops before producing or deploying a misleading implementation.", + "expected_behavior": [ + "The agent inspects the installed adapter and deployment invocation contract", + "The agent does not treat interrupt_on configuration acceptance as end to end support", + "The agent reports the missing deployed resume path", + "The agent stops before registration or deployment", + "The agent does not replace the approval with prompt instructions or a NAT workflow" + ] + } +] diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md new file mode 100644 index 00000000000..d8b7ad57a57 --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md @@ -0,0 +1,77 @@ +# Fabric Deep Agents contract + +Use this reference when choosing the agent's implementation shape. + +## Ownership boundary + +NeMo Platform stores `agent.yaml` and translates it into a Fabric config. The +Fabric Deep Agents adapter constructs `create_deep_agent` from that normalized +config. It does not import a customer `agent.py` or accept Python callables in +`harnesses..settings`. + +Use only these supported surfaces: + +| Requirement | Delivery surface | +|---|---| +| System behavior and model | `instructions` and `models` in `agent.yaml` | +| Reusable instruction package | Agent Skill referenced by `skills.paths` | +| Executable custom tool | MCP server declared by `mcp.servers` | +| Specialized delegate | Declarative subagent under `harnesses.deepagents.settings.deepagents.subagents` | +| Remote background delegate | Supported asynchronous subagent configuration | + +Never create `src//agent.py` as an entry point for this path. Fabric +will not load it. + +## Custom tools + +Expose local Python tools as an MCP server. The generated project should install +a console script: + +```toml +[project.scripts] +support-tools = "support_agent.server:main" +``` + +Declare that executable in `agent.yaml`: + +```yaml +mcp: + servers: + support-tools: + transport: stdio + url: support-tools +``` + +Keep credentials out of the config and source tree. Declare environment +variable names and resolve secrets through the selected NeMo environment. + +For local testing, run the MCP server and its tools from the generated `uv` +project. For deployment, package the project as an image so the console script +is installed on the runtime `PATH`. Do not assume a nested project virtual +environment is visible to a Platform subprocess deployment. + +## Ordering and approvals + +Prompt instructions and subagent delegation do not guarantee execution order. +When several steps form one safety or transactional invariant, expose one MCP +operation that implements the state machine and tests the complete transition. +For example, `issue_verified_refund` should verify identity and issue the refund +inside one controlled operation instead of exposing two freely ordered tools. + +Do not use `interrupt_on` for a deployed workflow until the selected Platform +runtime exposes and tests an end-to-end pause and resume contract. The current +adapter accepts the setting, but deployed invocation does not provide a verified +approve, edit or reject resume path. Configuration acceptance is not runtime +support. + +A local compiled LangGraph subagent cannot be serialized through the current +Platform config. If the required behavior cannot be represented by a +deterministic MCP operation, a supported declarative subagent or a supported +remote agent, stop and report the adapter gap. + +## Version boundary + +Read the installed Fabric Deep Agents adapter descriptor and the NeMo Agents +config model before emitting adapter settings. The NeMo Agents plugin owns a +compatible Deep Agents runtime dependency. Do not add `deepagents` to the +customer project unless customer code directly imports its Python API. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md new file mode 100644 index 00000000000..86cbb717bf1 --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md @@ -0,0 +1,77 @@ +# Package and deploy custom agent code + +Use this reference when the agent contains a local MCP server or another Python +package that must run with the deployed agent. + +## Project layout + +Keep the complete deployable package under the canonical Ethos directory: + +```text +agents/-ethos/ + ETHOS.md + agent.yaml + pyproject.toml + uv.lock + src//server.py + skills//SKILL.md + tests/ + evals/cases.yaml +``` + +Omit directories the implementation does not need. Every `skills.paths` value +must be relative to `agent.yaml`, remain inside this project and point to a +directory containing `SKILL.md`. + +## Inspect the build context + +Before packaging: + +1. Inspect `.gitignore` and `.dockerignore`. +2. Search the context for `.env` files, private keys, tokens, credential files, + local databases, traces and customer data. +3. Remove or exclude anything that is not required at runtime. +4. Confirm `pyproject.toml` defines every MCP console script referenced by + `agent.yaml`. +5. Confirm `uv.lock` is current and local tests pass with `uv run`. + +Stop if the context contains an unresolved secret or sensitive dataset. + +## Validate and build + +Use project mode so the Python package and console scripts are installed into +the image: + +```bash +IMAGE_TAG="$AGENT_NAME:local" +.venv/bin/nemo agents package \ + --agent "agents/$AGENT_NAME-ethos/agent.yaml" \ + --pyproject "agents/$AGENT_NAME-ethos/pyproject.toml" \ + --tag "$IMAGE_TAG" +``` + +Do not use `--skip-validation`. A source checkout may have an unpublished +NeMo Platform version that cannot be resolved inside the generated image. In +that case, report the release dependency or use an explicitly reviewed custom +template. Do not silently weaken validation. + +## Deploy the packaged image + +After the user approves registration and deployment: + +```bash +.venv/bin/nemo agents create \ + --name "$AGENT_NAME" \ + --agent-config "agents/$AGENT_NAME-ethos/agent.yaml" + +.venv/bin/nemo agents deploy \ + --agent "$AGENT_NAME" \ + --name "$AGENT_NAME-deployment" \ + --mode docker \ + --image "$IMAGE_TAG" \ + --timeout 300 +``` + +Docker is the supported local container path. Kubernetes requires a published +image reachable by the configured executor and a separately verified runtime +contract. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/agent.yml b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/agent.yml deleted file mode 100644 index 6a2acccda20..00000000000 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/agent.yml +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# NAT workflow YAML for a NeMo Platform agent. -# Required top-level keys: functions, llms, workflow. -# Optional: intercepts, middleware. -# Anything else at top level causes NAT to reject the config. - -functions: - clock: - _type: current_datetime - -llms: - llm: - _type: openai - api_key: not-used - model_name: nvidia-llama-3-3-nemotron-super-49b-v1 - temperature: 0.0 - max_tokens: 1024 - -general: - telemetry: - tracing: - nemo_files_trace: - _type: nemo_files - # workspace and agent_name are injected at deploy time - -workflow: - _type: react_agent - tool_names: [clock] - llm_name: llm - verbose: false - parse_agent_response_max_retries: 3 - system_prompt: | - - - Your job: . - - You handle these categories: - - - - - - - - Constraints: - - - - - - You have access to the following tools: - {tools} - - Tool names: {tool_names} diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/data-designer-config.py b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/data-designer-config.py deleted file mode 100644 index d33801b6b05..00000000000 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/data-designer-config.py +++ /dev/null @@ -1,80 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Data Designer config builder for synthetic eval data. - -Run via the venv Python until `nemo data-designer preview-local` is available -in the installed CLI version (currently 2.1.0 ships only `data-designer jobs`): - - .venv/bin/python agents/.dd.py - -When `preview-local` lands, swap the bottom of this file for: - - # nemo data-designer preview-local agents/.dd.py --num-records 10 - -Substitute: - AGENT_DESCRIPTION one-sentence agent role from the Ethos - CATEGORIES the Ethos category list - MODEL API-Catalog format with slashes (NOT entity-name format) - PROVIDER workspace/provider-name as registered in nemo -""" - -import data_designer.config as dd -from nemo_platform import NeMoPlatform - -AGENT_DESCRIPTION = "" -CATEGORIES: list[str | int | float] = ["", "", ""] -MODEL = "meta/llama-3.3-70b-instruct" -PROVIDER = "default/nim-llm" - - -def load_config_builder() -> dd.DataDesignerConfigBuilder: - builder = dd.DataDesignerConfigBuilder( - model_configs=[ - dd.ModelConfig( - alias="generator", - model=MODEL, - provider=PROVIDER, - skip_health_check=True, - ) - ] - ) - - builder.add_column( - dd.SamplerColumnConfig( - name="category", - sampler_type=dd.SamplerType.CATEGORY, - params=dd.CategorySamplerParams(values=CATEGORIES), - ) - ) - - builder.add_column( - dd.LLMTextColumnConfig( - name="user_question", - model_alias="generator", - prompt=( - "Generate a realistic user question about {{ category }} for an agent that " - f"{AGENT_DESCRIPTION}. The question should sound like a real employee asking for help." - ), - ) - ) - - builder.add_column( - dd.LLMTextColumnConfig( - name="expected_response", - model_alias="generator", - prompt=( - "Write the ideal agent response to this question: {{ user_question }}. " - "The response should be helpful, accurate, and professional." - ), - ) - ) - - return builder - - -if __name__ == "__main__": - client = NeMoPlatform(base_url="http://localhost:8080", workspace="default") - builder = load_config_builder() - results = client.data_designer.preview(builder, num_records=10) - results.display_sample_record() diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/eval-job.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/eval-job.json deleted file mode 100644 index 7c79eda6082..00000000000 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/templates/eval-job.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "_comment": "Substitute , , and the dataset reference. The dataset is the artifact path produced by `nemo data-designer preview-local --save-results`. Workspace prefix `default/` is the platform default.", - "name": "-eval", - "benchmark": "default/agent-quality", - "model": "default/nim-llm/", - "dataset": "default/agents/.eval" -} diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/testing-and-signoff.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/testing-and-signoff.md new file mode 100644 index 00000000000..dbd77674fbc --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/testing-and-signoff.md @@ -0,0 +1,64 @@ +# Testing and signoff + +Use this reference after the Ethos is approved and before registration. + +## Derive one acceptance contract + +Turn the approved Ethos examples and success criteria into one versioned test +case file in the generated project. Reuse those cases for local behavioral +tests, deployed invocations and evaluation. Do not create separate definitions +that can drift. + +Each case should identify: + +- input and expected outcome; +- allowed and forbidden side effects; +- required or forbidden tool calls; +- required ordering when observable; +- whether live credentials are required; +- the pass rule. + +## Required local gates + +1. Unit test every MCP operation, including invalid input, denied actions, + upstream errors, timeouts and redacted error messages. +2. Contract test MCP schemas, tool discovery and structured outputs. +3. Run the Ethos cases against the assembled local agent shape. +4. Record trajectories when tool choice, approval or ordering is part of the + result. +5. Run a separate live smoke test only when the required credentials and + network access are available. + +Mock business systems for ordinary tests. Never run representative production +side effects as evaluation. For a live side effect, use an approved sandbox or +test tenant and obtain explicit approval for the specific operation. + +## Delivery reachability + +A source file or passing import test does not prove that Fabric can call a +tool. Before registration, verify every required executable capability is +reachable from `agent.yaml` through `mcp.servers`, and every instruction package +is reachable through `skills.paths`. + +After image deployment: + +1. Invoke a case that requires each MCP server. +2. Confirm the expected tool appears in the trajectory and returns the expected + schema. +3. Exercise one denied action and one upstream failure. +4. Confirm telemetry contains the invocation, tool and error or approval spans + expected by the Ethos. + +## Status language + +Use these states consistently: + +| State | Minimum evidence | +|---|---| +| Built | Approved Ethos, generated artifacts and required local tests pass | +| Onboarded | Fabric validation, registration, deployment and telemetry checks pass | +| Production candidate | Live representative checks pass in the target environment and every required Ethos threshold is met | + +A skipped live test, missing telemetry or untested production integration +prevents `Production candidate` status. Report passed, failed and skipped checks +separately. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json index ee9b151f6c6..7be2bde2b21 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json @@ -1,75 +1,21 @@ { "skill": "nemo-build-agent", "tests": [ - { - "type": "explicit", - "prompt": "Use nemo-build-agent to scaffold and deploy an IT helpdesk agent.", - "expected_skill": "nemo-build-agent" - }, - { - "type": "explicit", - "prompt": "Use nemo-build-agent to implement and deploy the approved support-triage Ethos.", - "expected_skill": "nemo-build-agent" - }, - { - "type": "explicit", - "prompt": "Invoke nemo-build-agent for the customer-faq agent we designed.", - "expected_skill": "nemo-build-agent" - }, - { - "type": "implicit", - "prompt": "The IT helpdesk agent Ethos is approved. Build and deploy it for password resets, VPN issues, and software access requests.", - "expected_skill": "nemo-build-agent" - }, - { - "type": "implicit", - "prompt": "Scaffold and deploy the support agent we just specced out on NeMo Platform.", - "expected_skill": "nemo-build-agent" - }, - { - "type": "implicit", - "prompt": "Take the approved Ethos in agents/ and ship the agent to NeMo Platform.", - "expected_skill": "nemo-build-agent" - }, - { - "type": "implicit", - "prompt": "Spec is signed off. Build the agent, deploy it, and run the eval pass.", - "expected_skill": "nemo-build-agent" - }, - { - "type": "implicit", - "prompt": "Deploy my existing NAT workflow YAML as a NeMo Platform agent, then smoke test it.", - "expected_skill": "nemo-build-agent" - }, - { - "type": "implicit", - "prompt": "I have an old NAT agent workflow and an approved Ethos. Build and deploy it while preserving the legacy config.", - "expected_skill": "nemo-build-agent" - }, - { - "type": "negative-control", - "prompt": "Write an agent.yaml with a Codex harness, but do not register or deploy anything.", - "expected_skill_not": "nemo-build-agent" - }, - { - "type": "negative-control", - "prompt": "Convert this old NAT workflow YAML to nemo-agents-spec-v1, but do not deploy it.", - "expected_skill_not": "nemo-build-agent" - }, - { - "type": "negative-control", - "prompt": "Let's design a system for routing customer queries across teams before any implementation.", - "expected_skill_not": "nemo-build-agent" - }, - { - "type": "negative-control", - "prompt": "Fix the CSS spacing in the React settings page.", - "expected_skill_not": "nemo-build-agent" - }, - { - "type": "negative-control", - "prompt": "Help me write a CrewAI agent from scratch in a fresh Python environment outside NeMo Platform.", - "expected_skill_not": "nemo-build-agent" - } + {"type":"explicit","prompt":"Use nemo-build-agent to build and onboard an IT helpdesk agent through Fabric.","expected_skill":"nemo-build-agent"}, + {"type":"explicit","prompt":"Invoke nemo-build-agent for the approved support-triage Ethos.","expected_skill":"nemo-build-agent"}, + {"type":"explicit","prompt":"Run nemo-build-agent to package, deploy, and verify the claims agent.","expected_skill":"nemo-build-agent"}, + {"type":"implicit","prompt":"The IT helpdesk Ethos is approved. Build its tools and tests, create the explicit Fabric config, and onboard it to NeMo Platform.","expected_skill":"nemo-build-agent"}, + {"type":"implicit","prompt":"Build a customer support agent from my answers. Save the Ethos, show it to me for approval, then implement and register the agent through Fabric.","expected_skill":"nemo-build-agent"}, + {"type":"implicit","prompt":"Turn this approved refund agent design into a tested Deep Agents deployment and prove the refund tool is reachable after packaging.","expected_skill":"nemo-build-agent"}, + {"type":"implicit","prompt":"The Ethos is approved. Build the agent and its mock based tests now. The production API credential is unavailable, so record the live test as skipped and do not register or deploy.","expected_skill":"nemo-build-agent"}, + {"type":"contextual","prompt":"We approved the support triage Ethos yesterday and the NeMo workspace is ready now. Implement the approved agent, test it and onboard it through Fabric.","expected_skill":"nemo-build-agent"}, + {"type":"contextual","prompt":"The prototype works locally, but its CRM tool never appears after deployment. Finish packaging the custom tool through the supported NeMo agent path and verify it in Fabric.","expected_skill":"nemo-build-agent"}, + {"type":"contextual","prompt":"Legal approved the refund agent design. Implement the approved agent, prove identity verification happens before refunds, and stop before deployment for my review.","expected_skill":"nemo-build-agent"}, + {"type":"negative-control","prompt":"Use nemo-explore for a design conversation about the audience, scope, constraints, and success criteria.","expected_skill_not":"nemo-build-agent"}, + {"type":"negative-control","prompt":"Write the approved agent design into ETHOS.md, but stop before creating code or agent.yaml.","expected_skill_not":"nemo-build-agent"}, + {"type":"negative-control","prompt":"Use nemo-agent-config to author and validate a nemo-agents-spec-v1 YAML with the Codex harness.","expected_skill_not":"nemo-build-agent"}, + {"type":"negative-control","prompt":"Invoke the already deployed claims agent with these five questions and show me the responses.","expected_skill_not":"nemo-build-agent"}, + {"type":"negative-control","prompt":"Keep this existing NAT workflow running and add another NAT function to it.","expected_skill_not":"nemo-build-agent"}, + {"type":"negative-control","prompt":"Help me write a CrewAI agent in a separate Python project outside NeMo Platform.","expected_skill_not":"nemo-build-agent"} ] } diff --git a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py index 4fd22820def..7716d283aee 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py +++ b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py @@ -120,13 +120,14 @@ def test_platform_skills_declare_known_preconditions(self): assert not unknown, f"{name} has unknown preconditions: {sorted(unknown)}" assert skills_with_preconditions - def test_build_agent_templates_are_packaged(self): + def test_build_agent_evaluation_assets_are_packaged(self): skill = load_skills()["nemo-build-agent"] assert skill.source_dir is not None - - templates_dir = skill.source_dir / "references" / "templates" - assert (templates_dir / "agent.yml").is_file() - assert (templates_dir / "eval-job.json").is_file() + assert (skill.source_dir / "BENCHMARK.md").is_file() + assert (skill.source_dir / "evals" / "evals.json").is_file() + assert (skill.source_dir / "references" / "fabric-deep-agents.md").is_file() + assert (skill.source_dir / "references" / "testing-and-signoff.md").is_file() + assert (skill.source_dir / "references" / "packaging.md").is_file() def test_model_selection_benchmark_cache_is_packaged(self): skill = load_skills()["nemo-model-selection"] From 7b409b2c63b8c4281ec02c5dc9e65ba5ed177dd9 Mon Sep 17 00:00:00 2001 From: Yamini Date: Wed, 2 Sep 2026 23:05:47 -0400 Subject: [PATCH 2/9] Harden build agent onboarding and dependency boundary Explain Ethos and Fabric before implementation, keep Deep Agents dependencies inside the generated customer project, and cover the boundary in skill tests. Signed-off-by: Yamini --- .../skills/nemo-build-agent/SKILL.md | 69 +++++++++++++------ .../skills/nemo-build-agent/evals/evals.json | 3 + .../references/fabric-deep-agents.md | 7 +- .../skills/nemo-build-agent/tests.json | 2 +- .../cli/commands/skills/test_skill_content.py | 6 ++ 5 files changed, 61 insertions(+), 26 deletions(-) diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md index aa3396f445b..520b3ec2495 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md @@ -34,21 +34,35 @@ allowed-tools: [Bash, Read, Write, Edit] # Build a NeMo Platform agent Build a tested, config-driven LangChain Deep Agent and onboard it through the -NeMo Platform Fabric path. Fabric owns the Deep Agents runtime and constructs -the agent from config. Customer code runs only when it is packaged as a +NeMo Platform Fabric path. Fabric owns runtime orchestration and constructs the +agent from config. The generated customer project owns its Deep Agents runtime +dependencies. Customer code runs only when it is packaged as a config-referenced service such as MCP. -## Confirm the path and environment - -Before writing files: - -1. Confirm the target workspace, environment, model provider, network access, - credential availability and whether Docker deployment is available. -2. Explain that this build path uses the external LangChain Deep Agents runtime - supported by Fabric. Fabric constructs the agent from Platform config. -3. Explain that executable custom tools must be delivered through MCP and that - instruction-only capabilities may be delivered as Agent Skills. -4. Ask whether to continue with this supported path. +## Explain the path in customer language + +Do not assume the user knows NeMo Platform, Fabric, Ethos, MCP or Agent Skills. +Introduce each term only when it affects a decision. + +Before writing implementation files: + +1. Explain that an Ethos is a plain-language design contract for the agent. It + records who the agent serves, what it should accomplish, the tools and data + it may use, required approvals, forbidden actions, failure behavior and the + examples that will become tests. It is not code or deployment config. +2. Explain that the Ethos will be drafted from the user's answers, shown for + review and revised until the user explicitly approves it. No implementation + begins before that approval. +3. Explain that the supported build path uses the external LangChain Deep + Agents runtime. NeMo Platform does not vendor the customer's agent or make + Deep Agents a hidden Platform dependency. The generated customer project + declares and installs compatible pinned Deep Agents and Fabric adapter + versions. Fabric registers the agent and makes it available for deployment, + testing, observation and optimization in NeMo Platform. +4. Explain executable tools only when the requested agent needs them: custom + code is exposed through MCP so the deployed agent can call it. Explain Agent + Skills only when an instruction package is the right artifact. +5. Ask whether to continue with this supported path. If the user declines, stop. Offer onboarding of an existing agent through an available Fabric adapter. Do not fall back to NAT. @@ -56,9 +70,10 @@ available Fabric adapter. Do not fall back to NAT. ## Approve and persist the design Use `agents/-ethos/ETHOS.md` as the canonical design and package -root. If an Ethos exists, summarize it and confirm that it is approved for this -build. If it is absent or needs changes, gather only the missing implementation -inputs: +root. If an Ethos exists, summarize it in plain language and confirm that it is +approved for this build. If it is absent or needs changes, gather the design +inputs conversationally. Ask about the customer's work first and translate the +answers into NeMo artifacts yourself: - concrete role, users and outcomes; - tools, data, credentials and side effects; @@ -71,6 +86,15 @@ to render, validate and save `agents/-ethos/ETHOS.md`. Show the result and wait for explicit approval. Do not create implementation files before the Ethos is approved. +## Confirm the build environment + +After the Ethos is approved and before creating implementation files, confirm +the target workspace, environment, model provider, network access, credential +availability and whether Docker deployment is available. Discover what can be +read from the current environment instead of asking the user to supply NeMo +specific details they may not know. Explain any missing prerequisite in terms +of the capability it blocks. + ## Choose supported artifacts Read [references/fabric-deep-agents.md](references/fabric-deep-agents.md). @@ -94,10 +118,11 @@ accepted `interrupt_on` setting as proof that the deployed workflow can resume. ## Build the project Keep the deployable project under `agents/-ethos/`. Create only the -files required by the selected shape. For custom Python tools, use a `uv` -project with a locked dependency set, a typed MCP server and a console script. -Do not install globally. Do not add `deepagents` to the customer project unless -customer code directly imports its API. +files required by the selected shape. Use a `uv` project with a locked +dependency set. Declare compatible pinned versions of `deepagents` and the +Fabric Deep Agents adapter in that generated project. For custom Python tools, +add a typed MCP server and a console script. Do not install dependencies +globally or add them to NeMo Platform itself. Give tools narrow schemas, bounded output, explicit permissions, capped retries for transient failures and redacted errors. Never write credentials, customer @@ -185,8 +210,8 @@ and thresholds. Report passed, failed and skipped checks separately. - Prompt instructions and subagent delegation do not guarantee fixed ordering. - Packaging copies the selected build context. Untracked secrets can enter an image even when they are absent from committed files. -- The NeMo Agents plugin owns the compatible Fabric Deep Agents dependency. - Avoid adding a second unconstrained runtime version to the customer project. +- The generated customer project owns compatible pinned Deep Agents and Fabric + adapter dependencies. Do not rely on them being installed by NeMo Platform. - Docker is the supported local container path. Treat Kubernetes as a separate environment contract that must be verified. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json index 5789b04b7b8..2d927916180 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json @@ -7,6 +7,8 @@ "ground_truth": "The agent explains the supported Fabric Deep Agents path, creates one canonical agents/helpdesk-ethos/ETHOS.md through nemo-ethos, shows it for review and creates no implementation artifacts before explicit approval.", "expected_behavior": [ "The agent reads nemo-build-agent before acting", + "The agent explains Ethos as a plain-language design contract and does not assume the user knows NeMo Platform terminology", + "The agent explains the supported LangChain Deep Agents and Fabric path and asks whether to continue", "The agent records representative successful and unsuccessful cases in the Ethos", "The agent keeps the Ethos under agents/helpdesk-ethos/ETHOS.md", "The agent stops for explicit Ethos approval before creating agent.yaml, Python source or deployment resources", @@ -22,6 +24,7 @@ "expected_behavior": [ "The project is rooted at agents/refund-ethos and keeps ETHOS.md and agent.yaml together", "agent.yaml declares config_format nemo-agents-spec-v1 and a configured deepagents harness", + "The generated project declares compatible pinned deepagents and Fabric Deep Agents adapter dependencies instead of relying on the Platform installation", "Custom Python behavior is installed as an MCP console script and declared under mcp.servers", "The implementation does not rely on Fabric importing src package agent.py", "Tests prove an unverified or failed identity cannot issue a refund", diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md index d8b7ad57a57..8006ac1e242 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md @@ -72,6 +72,7 @@ remote agent, stop and report the adapter gap. ## Version boundary Read the installed Fabric Deep Agents adapter descriptor and the NeMo Agents -config model before emitting adapter settings. The NeMo Agents plugin owns a -compatible Deep Agents runtime dependency. Do not add `deepagents` to the -customer project unless customer code directly imports its Python API. +config model before emitting adapter settings. Declare compatible pinned +versions of `deepagents` and the Fabric Deep Agents adapter in the generated +customer project. Do not rely on either package being present in the NeMo +Platform installation. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json index 7be2bde2b21..eb551cb9362 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json @@ -5,7 +5,7 @@ {"type":"explicit","prompt":"Invoke nemo-build-agent for the approved support-triage Ethos.","expected_skill":"nemo-build-agent"}, {"type":"explicit","prompt":"Run nemo-build-agent to package, deploy, and verify the claims agent.","expected_skill":"nemo-build-agent"}, {"type":"implicit","prompt":"The IT helpdesk Ethos is approved. Build its tools and tests, create the explicit Fabric config, and onboard it to NeMo Platform.","expected_skill":"nemo-build-agent"}, - {"type":"implicit","prompt":"Build a customer support agent from my answers. Save the Ethos, show it to me for approval, then implement and register the agent through Fabric.","expected_skill":"nemo-build-agent"}, + {"type":"implicit","prompt":"I want to build a customer support agent, but I do not know NeMo Platform terminology. Walk me through the design, explain what you need me to approve, then implement and register it through the supported platform path.","expected_skill":"nemo-build-agent"}, {"type":"implicit","prompt":"Turn this approved refund agent design into a tested Deep Agents deployment and prove the refund tool is reachable after packaging.","expected_skill":"nemo-build-agent"}, {"type":"implicit","prompt":"The Ethos is approved. Build the agent and its mock based tests now. The production API credential is unavailable, so record the live test as skipped and do not register or deploy.","expected_skill":"nemo-build-agent"}, {"type":"contextual","prompt":"We approved the support triage Ethos yesterday and the NeMo workspace is ready now. Implement the approved agent, test it and onboard it through Fabric.","expected_skill":"nemo-build-agent"}, diff --git a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py index 7716d283aee..851b7bbbbc6 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py +++ b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py @@ -129,6 +129,12 @@ def test_build_agent_evaluation_assets_are_packaged(self): assert (skill.source_dir / "references" / "testing-and-signoff.md").is_file() assert (skill.source_dir / "references" / "packaging.md").is_file() + def test_build_agent_keeps_deep_agents_dependency_in_generated_project(self): + skill = load_skills()["nemo-build-agent"] + assert "generated customer project owns compatible pinned Deep Agents" in skill.content + assert "Do not rely on them being installed by NeMo Platform" in skill.content + assert "NeMo Agents plugin owns the compatible Fabric Deep Agents dependency" not in skill.content + def test_model_selection_benchmark_cache_is_packaged(self): skill = load_skills()["nemo-model-selection"] assert skill.source_dir is not None From 960eb17487e891828637d5e4b0749383ba4c9baf Mon Sep 17 00:00:00 2001 From: Yamini Date: Thu, 3 Sep 2026 11:37:30 -0400 Subject: [PATCH 3/9] Align build agent harness dependency checks Teach the build agent skill to detect and reuse the Agents plugin harness, ask before installing the optional plugin, and stop on partial installs. Document source checkout packaging and add dependency failure evals. Signed-off-by: Yamini --- .../skills/nemo-build-agent/SKILL.md | 53 +++++++++++++------ .../skills/nemo-build-agent/evals/evals.json | 44 ++++++++++++++- .../references/fabric-deep-agents.md | 9 ++-- .../nemo-build-agent/references/packaging.md | 18 +++++-- .../cli/commands/skills/test_skill_content.py | 18 +++++-- 5 files changed, 112 insertions(+), 30 deletions(-) diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md index 520b3ec2495..c23149e9d31 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md @@ -23,8 +23,7 @@ preconditions: - nemo_setup_complete - workspace_exists - provider_registered - - agents_plugin_available -compatibility: NeMo Platform >= 0.1.0 with the agents plugin and a supported Fabric Deep Agents adapter; Python and uv for local MCP projects; network access and provider credentials for live model tests; Docker for packaged custom code; macOS or Linux. +compatibility: NeMo Platform >= 0.4.0; installs the optional NeMo Agents plugin when the user approves and verifies its supported Fabric Deep Agents adapter; Python and uv for local MCP projects; network access and provider credentials for live model tests; Docker for packaged custom code; macOS or Linux. maturity: beta license: Apache-2.0 user-invocable: true @@ -35,9 +34,9 @@ allowed-tools: [Bash, Read, Write, Edit] Build a tested, config-driven LangChain Deep Agent and onboard it through the NeMo Platform Fabric path. Fabric owns runtime orchestration and constructs the -agent from config. The generated customer project owns its Deep Agents runtime -dependencies. Customer code runs only when it is packaged as a -config-referenced service such as MCP. +agent from config. The optional NeMo Agents plugin supplies the Fabric harness +adapters and their runtime dependencies. Customer code runs only when it is +packaged as a config-referenced service such as MCP. ## Explain the path in customer language @@ -54,11 +53,10 @@ Before writing implementation files: review and revised until the user explicitly approves it. No implementation begins before that approval. 3. Explain that the supported build path uses the external LangChain Deep - Agents runtime. NeMo Platform does not vendor the customer's agent or make - Deep Agents a hidden Platform dependency. The generated customer project - declares and installs compatible pinned Deep Agents and Fabric adapter - versions. Fabric registers the agent and makes it available for deployment, - testing, observation and optimization in NeMo Platform. + Agents runtime. NeMo Platform does not vendor the customer's agent. The + optional NeMo Agents plugin installs the supported Fabric adapter and its + Deep Agents dependency. Fabric registers the agent and makes it available + for deployment, testing, observation and optimization in NeMo Platform. 4. Explain executable tools only when the requested agent needs them: custom code is exposed through MCP so the deployed agent can call it. Explain Agent Skills only when an instruction package is the right artifact. @@ -95,6 +93,27 @@ read from the current environment instead of asking the user to supply NeMo specific details they may not know. Explain any missing prerequisite in terms of the capability it blocks. +Check `nemo plugins list -f json` and `nemo agents --help`. Then inspect the +active Python environment for `nemo_agents_plugin`, +`nemo_fabric_adapters.deepagents` and `deepagents`. Do not infer that a harness +is available from config acceptance alone. + +If the NeMo Agents plugin and Deep Agents harness are already available, reuse +them. Do not reinstall or change their versions. If the plugin is absent, +explain that agent management is optional in NeMo Platform and that this build +requires it. Show the appropriate install command and ask for approval before +running it: + +- Published Platform install: `uv pip install "nemo-platform[nemo-agents-plugin]"` +- NeMo Platform source checkout: `uv pip install -e plugins/nemo-agents/` + +The plugin owns selection of compatible Fabric adapter and harness versions. +Do not add a separate Deep Agents version constraint. After installation, +restart Platform services and repeat all four checks. If the plugin is present +but its Deep Agents adapter or runtime is absent, report a broken plugin +installation. Offer to reinstall the same Agents plugin only after approval. +Do not install the harness independently as an untracked repair. + ## Choose supported artifacts Read [references/fabric-deep-agents.md](references/fabric-deep-agents.md). @@ -118,11 +137,10 @@ accepted `interrupt_on` setting as proof that the deployed workflow can resume. ## Build the project Keep the deployable project under `agents/-ethos/`. Create only the -files required by the selected shape. Use a `uv` project with a locked -dependency set. Declare compatible pinned versions of `deepagents` and the -Fabric Deep Agents adapter in that generated project. For custom Python tools, -add a typed MCP server and a console script. Do not install dependencies -globally or add them to NeMo Platform itself. +files required by the selected shape. For custom Python tools, use a `uv` +project with a locked dependency set, a typed MCP server and a console script. +Do not install dependencies globally. Do not add `deepagents` or a Fabric +adapter to the generated project unless its own code directly imports that API. Give tools narrow schemas, bounded output, explicit permissions, capped retries for transient failures and redacted errors. Never write credentials, customer @@ -210,8 +228,9 @@ and thresholds. Report passed, failed and skipped checks separately. - Prompt instructions and subagent delegation do not guarantee fixed ordering. - Packaging copies the selected build context. Untracked secrets can enter an image even when they are absent from committed files. -- The generated customer project owns compatible pinned Deep Agents and Fabric - adapter dependencies. Do not rely on them being installed by NeMo Platform. +- The optional NeMo Agents plugin supplies the selected harness adapter and its + runtime dependencies. Reuse an installed harness and let the plugin resolve + compatible versions. - Docker is the supported local container path. Treat Kubernetes as a separate environment contract that must be verified. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json index 2d927916180..e904add4deb 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/evals/evals.json @@ -24,7 +24,7 @@ "expected_behavior": [ "The project is rooted at agents/refund-ethos and keeps ETHOS.md and agent.yaml together", "agent.yaml declares config_format nemo-agents-spec-v1 and a configured deepagents harness", - "The generated project declares compatible pinned deepagents and Fabric Deep Agents adapter dependencies instead of relying on the Platform installation", + "The agent verifies the optional NeMo Agents plugin and Deep Agents harness, reuses them when present and does not add a separate harness version constraint", "Custom Python behavior is installed as an MCP console script and declared under mcp.servers", "The implementation does not rely on Fabric importing src package agent.py", "Tests prove an unverified or failed identity cannot issue a refund", @@ -86,5 +86,47 @@ "The agent stops before registration or deployment", "The agent does not replace the approval with prompt instructions or a NAT workflow" ] + }, + { + "id": "nemo-build-agent-007-plugin-install-declined", + "question": "The Ethos is approved, but this NeMo Platform environment does not have the optional Agents plugin. Continue without installing anything.", + "expected_skill": "nemo-build-agent", + "expected_script": null, + "ground_truth": "The agent explains that Fabric agent management is optional but required for this build, asks before installation and stops when installation is declined. It does not install Deep Agents independently or fall back to NAT.", + "expected_behavior": [ + "The agent checks both the Agents plugin and Deep Agents harness instead of assuming they are present", + "The agent explains what capability the optional plugin provides", + "The agent performs no installation after the user declines", + "The agent does not install Deep Agents or its Fabric adapter independently", + "The agent does not create a NAT fallback" + ] + }, + { + "id": "nemo-build-agent-008-existing-harness", + "question": "The approved Ethos is ready. The NeMo Agents plugin, Fabric Deep Agents adapter and Deep Agents runtime are already installed in the Platform environment. Build without changing installed package versions.", + "expected_skill": "nemo-build-agent", + "expected_script": null, + "ground_truth": "The agent verifies the installed plugin, adapter and runtime, reuses them and does not reinstall or add a separate harness version constraint to the generated project.", + "expected_behavior": [ + "The agent verifies the plugin through the NeMo CLI and checks the active Python environment", + "The agent reuses the installed harness", + "The agent does not reinstall the Agents plugin", + "The agent does not change installed harness versions", + "The generated project does not add Deep Agents or its Fabric adapter unless project code directly imports that API" + ] + }, + { + "id": "nemo-build-agent-009-broken-plugin-install", + "question": "The approved Ethos is ready. The NeMo Agents plugin appears in the plugin list, but its Fabric Deep Agents adapter is missing from the active Python environment.", + "expected_skill": "nemo-build-agent", + "expected_script": null, + "ground_truth": "The agent reports a broken Agents plugin installation and asks before reinstalling that plugin. It does not repair the environment by installing an independently pinned harness package.", + "expected_behavior": [ + "The agent reports that plugin presence alone does not prove harness availability", + "The agent stops before implementation while the required adapter is missing", + "The agent offers to reinstall the same Agents plugin only after approval", + "The agent does not install or pin Deep Agents independently", + "The agent repeats plugin and Python environment checks after any approved reinstall" + ] } ] diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md index 8006ac1e242..6f638cb4adc 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md @@ -72,7 +72,8 @@ remote agent, stop and report the adapter gap. ## Version boundary Read the installed Fabric Deep Agents adapter descriptor and the NeMo Agents -config model before emitting adapter settings. Declare compatible pinned -versions of `deepagents` and the Fabric Deep Agents adapter in the generated -customer project. Do not rely on either package being present in the NeMo -Platform installation. +config model before emitting adapter settings. The optional NeMo Agents plugin +supplies the adapter and its Deep Agents runtime dependency. Reuse the installed +versions and let the plugin dependency metadata select compatible versions. Do +not add a separate harness constraint to the generated customer project unless +its own code directly imports that API. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md index 86cbb717bf1..df43e4a3711 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md @@ -50,10 +50,20 @@ IMAGE_TAG="$AGENT_NAME:local" --tag "$IMAGE_TAG" ``` -Do not use `--skip-validation`. A source checkout may have an unpublished -NeMo Platform version that cannot be resolved inside the generated image. In -that case, report the release dependency or use an explicitly reviewed custom -template. Do not silently weaken validation. +Do not use `--skip-validation`. For a source checkout, build the current +Platform wheel and point packaging at it: + +```bash +uv build --package nemo-platform --wheel --out-dir dist +NEMO_AGENTS_WHEEL=LATEST .venv/bin/nemo agents package \ + --agent "agents/$AGENT_NAME-ethos/agent.yaml" \ + --pyproject "agents/$AGENT_NAME-ethos/pyproject.toml" \ + --tag "$IMAGE_TAG" +``` + +`NEMO_AGENTS_WHEEL=LATEST` is the supported source checkout path. Do not treat +the checkout's `0.0.0` project metadata as the released Platform version and do +not silently weaken validation. ## Deploy the packaged image diff --git a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py index 851b7bbbbc6..538aa49fdc6 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py +++ b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py @@ -129,11 +129,21 @@ def test_build_agent_evaluation_assets_are_packaged(self): assert (skill.source_dir / "references" / "testing-and-signoff.md").is_file() assert (skill.source_dir / "references" / "packaging.md").is_file() - def test_build_agent_keeps_deep_agents_dependency_in_generated_project(self): + def test_build_agent_uses_agents_plugin_harness_dependency(self): skill = load_skills()["nemo-build-agent"] - assert "generated customer project owns compatible pinned Deep Agents" in skill.content - assert "Do not rely on them being installed by NeMo Platform" in skill.content - assert "NeMo Agents plugin owns the compatible Fabric Deep Agents dependency" not in skill.content + assert "optional NeMo Agents plugin supplies the selected harness adapter" in skill.content + assert "Do not add a separate Deep Agents version constraint" in skill.content + assert "If the NeMo Agents plugin and Deep Agents harness are already available" in skill.content + assert 'uv pip install "nemo-platform[nemo-agents-plugin]"' in skill.content + assert "uv pip install -e plugins/nemo-agents/" in skill.content + + def test_build_agent_covers_plugin_dependency_failure_paths(self): + skill = load_skills()["nemo-build-agent"] + assert "If the plugin is absent" in skill.content + assert "ask for approval before" in skill.content + assert "If the plugin is present" in skill.content + assert "its Deep Agents adapter or runtime is absent" in skill.content + assert "Do not install the harness independently" in skill.content def test_model_selection_benchmark_cache_is_packaged(self): skill = load_skills()["nemo-model-selection"] From 7cbe551256128a7466e42a081d8a8e72c6b38a23 Mon Sep 17 00:00:00 2001 From: Yamini Date: Thu, 3 Sep 2026 17:32:31 -0400 Subject: [PATCH 4/9] Update build agent benchmark evidence Record the nine current evaluation tasks and the successful packaged Fabric trajectory. Use the repository standard NVSkills-Eval terminology and keep formal Tier 3 results marked as pending. Signed-off-by: Yamini --- .../skills/nemo-build-agent/BENCHMARK.md | 47 +++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md index 4d29b031c9d..365fce3e658 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md @@ -2,19 +2,22 @@ ## Publication status -Not yet eligible for publication. The evaluation dataset is present, but the -required baseline and skill assisted NV ACES run has not been executed in this -checkout. +Ready for pull request review. Not yet eligible for publication because the +required NVSkills-Eval Tier 3 baseline and skill assisted runs have not been +executed. The focused content tests, independent forward tests and packaged +Fabric trajectory described below are supporting evidence. They do not replace +the formal comparison. ## Evaluation metadata - Skill: `nemo-build-agent` -- Evaluation date: not run +- Supporting evidence date: 2026-09-03 +- Formal evaluation date: not run - Evaluator version: not recorded - Agents: Codex and Claude Code required -- Tasks: 6 application tasks +- Tasks: 9 application tasks - Attempts: at least 2 per task required -- Environment: isolated sandbox with NeMo Fabric and Docker +- Formal environment: NVSkills-Eval sandbox with NeMo Fabric and Docker ## Required gates @@ -41,12 +44,40 @@ Uplift cannot override a failed security or delivery reachability gate. | Delivery reachability | Not run | Not run | | Efficiency | Not run | Not run | +## Supporting evidence + +The following checks were run from the rebased NeMo Platform source checkout. +They establish that the skill content is internally consistent and that the +documented delivery path works. They do not establish measured skill uplift. + +| Check | Result | Evidence | +|---|---|---| +| Focused skill content suite | Pass | 14 tests passed | +| Evaluation dataset parse | Pass | All 9 application tasks loaded as valid JSON | +| Independent forward scenarios | Pass | Ethos rejection, missing credential, unsupported Fabric setting, existing resource collision and runtime approval gap all stopped safely; the refund scenario passed local MCP tests | +| Current source wheel | Pass | Built from NeMo Platform 0.4 plus current main using the documented source checkout path | +| Official agent packaging | Pass | `NEMO_AGENTS_WHEEL=LATEST` produced the test image without skipping validation | +| Live Fabric invocation | Pass | The packaged agent returned HTTP 200 through build.nvidia.com | +| MCP delivery reachability | Pass | The model called `multiply` with `[12, 8]`; the MCP tool returned `96.0`; the agent returned the same result | +| Telemetry | Pass | The local ATOF trace recorded model, tool and final response events | + +The dependency cases cover three distinct states in the evaluation dataset: + +- The optional Agents plugin is absent and the user declines installation. +- The Agents plugin, Fabric adapter and Deep Agents runtime are already present + and must be reused without changing versions. +- The Agents plugin is present but its adapter or runtime is absent, so the + installation is treated as broken and no independent harness repair occurs. + +These dependency cases still require the formal Codex and Claude Code attempts +before publication. + ## Informal baseline observation On 2026-08-31, an independent planning agent was asked to handle the refund agent scenario without reading the target skill. This was a qualitative -authoring check, not an NV ACES run and not publication evidence. The baseline -correctly selected MCP for custom code and proposed broad unhappy path coverage. +authoring check, not an NVSkills-Eval Tier 3 run and not publication evidence. +The baseline correctly selected MCP for custom code and proposed broad unhappy path coverage. It still missed repository-specific invariants that the skill must supply: - it used a noncanonical `agents//` package instead of the derived From b9267cda03df0006d632d19fa80861798faa61de Mon Sep 17 00:00:00 2001 From: Yamini Date: Thu, 3 Sep 2026 17:54:32 -0400 Subject: [PATCH 5/9] Remove build agent review artifacts from package Keep behavioral evaluations in source while removing the generated gate marker, hand written benchmark and runtime packaging rule for evaluation data. Signed-off-by: Yamini --- .agents/skills/.gate/nemo-build-agent.passed | 1 - packages/nemo_platform_ext/pyproject.toml | 6 +- .../skills/nemo-build-agent/BENCHMARK.md | 100 ------------------ .../cli/commands/skills/test_skill_content.py | 4 +- 4 files changed, 4 insertions(+), 107 deletions(-) delete mode 100644 .agents/skills/.gate/nemo-build-agent.passed delete mode 100644 packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md diff --git a/.agents/skills/.gate/nemo-build-agent.passed b/.agents/skills/.gate/nemo-build-agent.passed deleted file mode 100644 index 9bbe291263b..00000000000 --- a/.agents/skills/.gate/nemo-build-agent.passed +++ /dev/null @@ -1 +0,0 @@ -2026-08-31 NeMo-platform candidate; baseline, independent review, local checklist, routing and CLI validation passed; nv-base and NV-ACES publication checks unavailable, so maturity remains beta. diff --git a/packages/nemo_platform_ext/pyproject.toml b/packages/nemo_platform_ext/pyproject.toml index 8148ad72226..64c5bf59722 100644 --- a/packages/nemo_platform_ext/pyproject.toml +++ b/packages/nemo_platform_ext/pyproject.toml @@ -93,10 +93,10 @@ sdk_include_mode = "source-package" # `**/*.md` recursively includes SKILL.md plus any companion markdown files # skills ship under `resources/` (e.g. notes, sub-docs, prompts). Add further # extension globs here as new companion file types are needed. Runtime JSON -# assets live under `references/` or `evals/`; skill `tests.json` manifests -# remain source-only. We avoid `**/*` because dev environments leave +# assets live under `references/`; skill `tests.json` manifests remain +# source-only. We avoid `**/*` because dev environments leave # `__pycache__/*.pyc` in source modules, which the vendor tool doesn't filter. -included_paths = ["**/*.py", "**/*.md", "**/*.yaml", "**/*.yml", "**/references/**/*.json", "**/evals/**/*.json"] +included_paths = ["**/*.py", "**/*.md", "**/*.yaml", "**/*.yml", "**/references/**/*.json"] replace_client_inits_from = "client/enhanced.py" vendor_tests = true tests_path = "tests" diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md deleted file mode 100644 index 365fce3e658..00000000000 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/BENCHMARK.md +++ /dev/null @@ -1,100 +0,0 @@ -# Skill benchmark: nemo-build-agent - -## Publication status - -Ready for pull request review. Not yet eligible for publication because the -required NVSkills-Eval Tier 3 baseline and skill assisted runs have not been -executed. The focused content tests, independent forward tests and packaged -Fabric trajectory described below are supporting evidence. They do not replace -the formal comparison. - -## Evaluation metadata - -- Skill: `nemo-build-agent` -- Supporting evidence date: 2026-09-03 -- Formal evaluation date: not run -- Evaluator version: not recorded -- Agents: Codex and Claude Code required -- Tasks: 9 application tasks -- Attempts: at least 2 per task required -- Formal environment: NVSkills-Eval sandbox with NeMo Fabric and Docker - -## Required gates - -| Dimension | Publication gate | -|---|---| -| Security | Every approval, secret handling and side effect check passes | -| Correctness | At least 80 percent of application attempts pass for each supported coding agent | -| Discoverability | Explicit, implicit and contextual routing pass; negative controls do not invoke the skill | -| Effectiveness | Skill assisted overall score improves by at least 10 percentage points over the no skill baseline | -| Delivery reachability | Every required executable tool is reachable from `agent.yaml` through MCP and is observed in a trajectory | -| Efficiency | Turns, elapsed time and tokens are reported; no fixed pass threshold until baseline data exists | - -Uplift cannot override a failed security or delivery reachability gate. - -## Results - -| Measure | Codex | Claude Code | -|---|---:|---:| -| Overall baseline to skill | Not run | Not run | -| Security | Not run | Not run | -| Correctness | Not run | Not run | -| Discoverability | Not run | Not run | -| Effectiveness | Not run | Not run | -| Delivery reachability | Not run | Not run | -| Efficiency | Not run | Not run | - -## Supporting evidence - -The following checks were run from the rebased NeMo Platform source checkout. -They establish that the skill content is internally consistent and that the -documented delivery path works. They do not establish measured skill uplift. - -| Check | Result | Evidence | -|---|---|---| -| Focused skill content suite | Pass | 14 tests passed | -| Evaluation dataset parse | Pass | All 9 application tasks loaded as valid JSON | -| Independent forward scenarios | Pass | Ethos rejection, missing credential, unsupported Fabric setting, existing resource collision and runtime approval gap all stopped safely; the refund scenario passed local MCP tests | -| Current source wheel | Pass | Built from NeMo Platform 0.4 plus current main using the documented source checkout path | -| Official agent packaging | Pass | `NEMO_AGENTS_WHEEL=LATEST` produced the test image without skipping validation | -| Live Fabric invocation | Pass | The packaged agent returned HTTP 200 through build.nvidia.com | -| MCP delivery reachability | Pass | The model called `multiply` with `[12, 8]`; the MCP tool returned `96.0`; the agent returned the same result | -| Telemetry | Pass | The local ATOF trace recorded model, tool and final response events | - -The dependency cases cover three distinct states in the evaluation dataset: - -- The optional Agents plugin is absent and the user declines installation. -- The Agents plugin, Fabric adapter and Deep Agents runtime are already present - and must be reused without changing versions. -- The Agents plugin is present but its adapter or runtime is absent, so the - installation is treated as broken and no independent harness repair occurs. - -These dependency cases still require the formal Codex and Claude Code attempts -before publication. - -## Informal baseline observation - -On 2026-08-31, an independent planning agent was asked to handle the refund -agent scenario without reading the target skill. This was a qualitative -authoring check, not an NVSkills-Eval Tier 3 run and not publication evidence. -The baseline correctly selected MCP for custom code and proposed broad unhappy path coverage. -It still missed repository-specific invariants that the skill must supply: - -- it used a noncanonical `agents//` package instead of the derived - `agents/-ethos/` bundle; -- it split identity verification and refund execution into separately callable - tools, leaving the safety invariant dependent on a token protocol and model - ordering rather than one deterministic operation; -- it proposed subprocess deployment even though the generated MCP console - script would not be installed on the Platform service `PATH`; -- it added auxiliary files that are not part of the minimum supported package. - -The observation suggests that the skill adds repository-specific value around -the canonical bundle, delivery and deployment contracts. It does not establish -measured uplift. Only persisted baseline and skill-assisted evaluation runs may -be used for a publication decision. - -## Freshness - -Regenerate this report whenever the skill, evaluation dataset, supported Fabric -adapter range, target agents or scoring policy changes. diff --git a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py index 538aa49fdc6..d8e1751128c 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py +++ b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py @@ -120,11 +120,9 @@ def test_platform_skills_declare_known_preconditions(self): assert not unknown, f"{name} has unknown preconditions: {sorted(unknown)}" assert skills_with_preconditions - def test_build_agent_evaluation_assets_are_packaged(self): + def test_build_agent_references_are_packaged(self): skill = load_skills()["nemo-build-agent"] assert skill.source_dir is not None - assert (skill.source_dir / "BENCHMARK.md").is_file() - assert (skill.source_dir / "evals" / "evals.json").is_file() assert (skill.source_dir / "references" / "fabric-deep-agents.md").is_file() assert (skill.source_dir / "references" / "testing-and-signoff.md").is_file() assert (skill.source_dir / "references" / "packaging.md").is_file() From 487695734cb5da4dff5dceb9c98b19f6e2ad5226 Mon Sep 17 00:00:00 2001 From: Yamini Date: Thu, 3 Sep 2026 18:24:26 -0400 Subject: [PATCH 6/9] fix(skills): add copyright headers Signed-off-by: Yamini --- .../skills/nemo-build-agent/references/fabric-deep-agents.md | 2 ++ .../skills/nemo-build-agent/references/packaging.md | 2 ++ .../skills/nemo-build-agent/references/testing-and-signoff.md | 2 ++ 3 files changed, 6 insertions(+) diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md index 6f638cb4adc..70cc51725ba 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/fabric-deep-agents.md @@ -77,3 +77,5 @@ supplies the adapter and its Deep Agents runtime dependency. Reuse the installed versions and let the plugin dependency metadata select compatible versions. Do not add a separate harness constraint to the generated customer project unless its own code directly imports that API. + + diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md index df43e4a3711..91d753943ab 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md @@ -85,3 +85,5 @@ After the user approves registration and deployment: Docker is the supported local container path. Kubernetes requires a published image reachable by the configured executor and a separately verified runtime contract. + + diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/testing-and-signoff.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/testing-and-signoff.md index dbd77674fbc..6cdd94f9c43 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/testing-and-signoff.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/testing-and-signoff.md @@ -62,3 +62,5 @@ Use these states consistently: A skipped live test, missing telemetry or untested production integration prevents `Production candidate` status. Report passed, failed and skipped checks separately. + + From 27ff8f0cbc78eb09ba6efc3b3625b91bf9ce68c6 Mon Sep 17 00:00:00 2001 From: Manjesh Mogallapalli Date: Fri, 4 Sep 2026 13:52:31 -0500 Subject: [PATCH 7/9] fix(skills): defer agent registration until gates pass Signed-off-by: Manjesh Mogallapalli --- .../skills/nemo-build-agent/SKILL.md | 14 +++++++++----- .../cli/commands/skills/test_skill_content.py | 9 +++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md index c23149e9d31..2e1ba098771 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md @@ -146,8 +146,10 @@ Give tools narrow schemas, bounded output, explicit permissions, capped retries for transient failures and redacted errors. Never write credentials, customer data or production traces into source, YAML, fixtures or logs. -Invoke `nemo-agent-config` to create and validate the canonical -`agents/-ethos/agent.yaml`. Require: +Use `nemo-agent-config` to author the canonical +`agents/-ethos/agent.yaml`, but do not run its `nemo agents create` +registration step. This build workflow owns registration after every +pre-registration gate has passed. Require: ```yaml config_format: nemo-agents-spec-v1 @@ -198,9 +200,11 @@ exists, offer reuse, rename or replace. Never overwrite or delete it without explicit approval immediately before the state change. Show the exact create and deploy commands and ask for approval immediately -before running them. Verify registration by reading the Agent back. Verify -deployment through the blocking command result and deployment status. On -failure, inspect status and logs once, report the root error and stop. +before running them. `nemo agents create` is both the final Fabric plan and +doctor gate and the registration step; do not run it earlier solely to validate +the config. Verify registration by reading the Agent back. Verify deployment +through the blocking command result and deployment status. On failure, inspect +status and logs once, report the root error and stop. ## Verify onboarding diff --git a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py index d8e1751128c..127dc6f219c 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py +++ b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py @@ -143,6 +143,15 @@ def test_build_agent_covers_plugin_dependency_failure_paths(self): assert "its Deep Agents adapter or runtime is absent" in skill.content assert "Do not install the harness independently" in skill.content + def test_build_agent_defers_registration_until_after_local_gates(self): + content = load_skills()["nemo-build-agent"].content + normalized = " ".join(content.split()) + + assert "do not run its `nemo agents create` registration step" in normalized + assert "This build workflow owns registration after every pre-registration gate has passed" in normalized + assert "do not run it earlier solely to validate the config" in normalized + assert content.index("## Test before registration") < content.index("## Register and deploy") + def test_model_selection_benchmark_cache_is_packaged(self): skill = load_skills()["nemo-model-selection"] assert skill.source_dir is not None From 30f52b781214667ede92ebc7ef9f2bf75017712c Mon Sep 17 00:00:00 2001 From: Manjesh Mogallapalli Date: Fri, 4 Sep 2026 14:05:41 -0500 Subject: [PATCH 8/9] fix(skills): preserve agent deployment scope Signed-off-by: Manjesh Mogallapalli --- .../skills/nemo-build-agent/SKILL.md | 9 ++++++--- .../skills/nemo-build-agent/references/packaging.md | 13 +++++++++++-- .../tests/cli/commands/skills/test_skill_content.py | 12 ++++++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md index 2e1ba098771..f189d900564 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md @@ -202,9 +202,12 @@ explicit approval immediately before the state change. Show the exact create and deploy commands and ask for approval immediately before running them. `nemo agents create` is both the final Fabric plan and doctor gate and the registration step; do not run it earlier solely to validate -the config. Verify registration by reading the Agent back. Verify deployment -through the blocking command result and deployment status. On failure, inspect -status and logs once, report the root error and stop. +the config. Pass the confirmed workspace to both commands. If an +AgentEnvironment was selected, pass its workspace-qualified reference to +deployment. Verify registration by reading the Agent back in the same +workspace. Verify deployment through the blocking command result and deployment +status. On failure, inspect status and logs once, report the root error and +stop. ## Verify onboarding diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md index 91d753943ab..a2eb0c93261 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md @@ -67,18 +67,27 @@ not silently weaken validation. ## Deploy the packaged image -After the user approves registration and deployment: +After the user approves registration and deployment, use the workspace confirmed +earlier. If an AgentEnvironment was selected, set `AGENT_ENVIRONMENT` to its +workspace-qualified reference and include the shown `--environment` option; +otherwise omit both that variable and option. ```bash +WORKSPACE="" +AGENT_ENVIRONMENT="$WORKSPACE/" + .venv/bin/nemo agents create \ --name "$AGENT_NAME" \ - --agent-config "agents/$AGENT_NAME-ethos/agent.yaml" + --agent-config "agents/$AGENT_NAME-ethos/agent.yaml" \ + --workspace "$WORKSPACE" .venv/bin/nemo agents deploy \ --agent "$AGENT_NAME" \ --name "$AGENT_NAME-deployment" \ --mode docker \ --image "$IMAGE_TAG" \ + --workspace "$WORKSPACE" \ + --environment "$AGENT_ENVIRONMENT" \ --timeout 300 ``` diff --git a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py index 127dc6f219c..4f3cba209a1 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py +++ b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py @@ -152,6 +152,18 @@ def test_build_agent_defers_registration_until_after_local_gates(self): assert "do not run it earlier solely to validate the config" in normalized assert content.index("## Test before registration") < content.index("## Register and deploy") + def test_build_agent_registration_uses_confirmed_workspace_and_environment(self): + skill = load_skills()["nemo-build-agent"] + assert skill.source_dir is not None + packaging = (skill.source_dir / "references" / "packaging.md").read_text() + create_command, deploy_command = packaging.split(".venv/bin/nemo agents deploy", maxsplit=1) + + assert "If an AgentEnvironment was selected" in packaging + assert "otherwise omit both that variable and option" in packaging + assert '--workspace "$WORKSPACE"' in create_command + assert '--workspace "$WORKSPACE"' in deploy_command + assert '--environment "$AGENT_ENVIRONMENT"' in deploy_command + def test_model_selection_benchmark_cache_is_packaged(self): skill = load_skills()["nemo-model-selection"] assert skill.source_dir is not None From d28cd7bac858072cc4b1735f5fa11a880e31b0f7 Mon Sep 17 00:00:00 2001 From: Manjesh Mogallapalli Date: Fri, 4 Sep 2026 14:18:12 -0500 Subject: [PATCH 9/9] fix(skills): clarify optional agent environment Signed-off-by: Manjesh Mogallapalli --- .../nemo-build-agent/references/packaging.md | 24 +++++++++++++++---- .../cli/commands/skills/test_skill_content.py | 12 ++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md index a2eb0c93261..96279b4345a 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/references/packaging.md @@ -68,18 +68,22 @@ not silently weaken validation. ## Deploy the packaged image After the user approves registration and deployment, use the workspace confirmed -earlier. If an AgentEnvironment was selected, set `AGENT_ENVIRONMENT` to its -workspace-qualified reference and include the shown `--environment` option; -otherwise omit both that variable and option. +earlier: ```bash WORKSPACE="" -AGENT_ENVIRONMENT="$WORKSPACE/" .venv/bin/nemo agents create \ --name "$AGENT_NAME" \ --agent-config "agents/$AGENT_NAME-ethos/agent.yaml" \ --workspace "$WORKSPACE" +``` + +If an AgentEnvironment was selected, deploy with its workspace-qualified +reference: + +```bash +AGENT_ENVIRONMENT="$WORKSPACE/" .venv/bin/nemo agents deploy \ --agent "$AGENT_NAME" \ @@ -91,6 +95,18 @@ AGENT_ENVIRONMENT="$WORKSPACE/" --timeout 300 ``` +If no AgentEnvironment was selected, omit both the assignment and option: + +```bash +.venv/bin/nemo agents deploy \ + --agent "$AGENT_NAME" \ + --name "$AGENT_NAME-deployment" \ + --mode docker \ + --image "$IMAGE_TAG" \ + --workspace "$WORKSPACE" \ + --timeout 300 +``` + Docker is the supported local container path. Kubernetes requires a published image reachable by the configured executor and a separately verified runtime contract. diff --git a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py index 4f3cba209a1..cd9fad88048 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py +++ b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py @@ -156,13 +156,17 @@ def test_build_agent_registration_uses_confirmed_workspace_and_environment(self) skill = load_skills()["nemo-build-agent"] assert skill.source_dir is not None packaging = (skill.source_dir / "references" / "packaging.md").read_text() - create_command, deploy_command = packaging.split(".venv/bin/nemo agents deploy", maxsplit=1) + create_command, deployments = packaging.split(".venv/bin/nemo agents deploy", maxsplit=1) + selected_environment, no_environment = deployments.split("If no AgentEnvironment was selected", maxsplit=1) + assert packaging.count(".venv/bin/nemo agents deploy") == 2 assert "If an AgentEnvironment was selected" in packaging - assert "otherwise omit both that variable and option" in packaging assert '--workspace "$WORKSPACE"' in create_command - assert '--workspace "$WORKSPACE"' in deploy_command - assert '--environment "$AGENT_ENVIRONMENT"' in deploy_command + assert '--workspace "$WORKSPACE"' in selected_environment + assert '--environment "$AGENT_ENVIRONMENT"' in selected_environment + assert "AGENT_ENVIRONMENT=" not in no_environment + assert "--environment" not in no_environment + assert '--workspace "$WORKSPACE"' in no_environment def test_model_selection_benchmark_cache_is_packaged(self): skill = load_skills()["nemo-model-selection"]