From f361fb751297f08e7c1d5c76ce608ced21309577 Mon Sep 17 00:00:00 2001 From: Vinay Bagade Date: Tue, 21 Apr 2026 20:00:24 +0000 Subject: [PATCH 01/10] feat(router): add model router integration with complexity-based routing Add support for the NVIDIA LLM Router v3 (prefill-based) as an inference provider in NemoClaw. The router automatically selects the most efficient model for each query based on a learned complexity signal. Architecture: - Router runs on the host (started by onboard, not inside the sandbox) - Sandbox reaches it through the OpenShell gateway's L7 proxy - Credentials flow through the OpenShell provider system Changes: - Add llm-router v3 as a git submodule at nemoclaw-blueprint/router/llm-router/ - Add pool-config.yaml defining the model pool and routing parameters - Add "Model Router" as a provider option in the onboard wizard - Add routed inference profile to blueprint.yaml (model: nvidia-routed) - Remove router lifecycle code from blueprint runner (onboard owns it) - Remove model-router-toolkit install from Dockerfile (host-only) - Add model-router-toolkit install to scripts/install.sh - Update README with Model Router documentation Made-with: Cursor --- .../SKILL.md | 144 +++++++++++++ .claude/skills | 1 - .gitmodules | 4 + CLAUDE.md | 195 +++++++++++++++++- Dockerfile | 4 + README.md | 55 +++++ nemoclaw-blueprint/blueprint.yaml | 14 ++ nemoclaw-blueprint/router/llm-router | 1 + nemoclaw-blueprint/router/pool-config.yaml | 36 ++++ nemoclaw/src/blueprint/runner.test.ts | 69 +++++++ nemoclaw/src/blueprint/runner.ts | 33 ++- schemas/blueprint.schema.json | 21 ++ scripts/install.sh | 8 + src/lib/onboard.ts | 173 +++++++++++++++- 14 files changed, 752 insertions(+), 6 deletions(-) create mode 100644 .agents/skills/nemoclaw-user-triage-instructions/SKILL.md delete mode 120000 .claude/skills create mode 100644 .gitmodules mode change 120000 => 100644 CLAUDE.md create mode 160000 nemoclaw-blueprint/router/llm-router create mode 100644 nemoclaw-blueprint/router/pool-config.yaml diff --git a/.agents/skills/nemoclaw-user-triage-instructions/SKILL.md b/.agents/skills/nemoclaw-user-triage-instructions/SKILL.md new file mode 100644 index 00000000000..52d7ecd0c6b --- /dev/null +++ b/.agents/skills/nemoclaw-user-triage-instructions/SKILL.md @@ -0,0 +1,144 @@ +--- +name: "nemoclaw-user-triage-instructions" +description: "AI-assisteds label triage instructions for NVIDIA/NemoClaw issues and PRs. Single source of truth for the nemoclaw-maintainer-triage CLI skill and the nvoss-velocity dashboard." +--- + + + + +# NemoClaw User Triage Instructions + +AI-assisted label triage instructions for NVIDIA/NemoClaw issues and PRs. Single source of truth for the nemoclaw-maintainer-triage CLI skill and the nvoss-velocity dashboard. + +This document is the single source of truth for AI-assisted label triage on NVIDIA/NemoClaw issues and PRs. +It is read at runtime by the `nemoclaw-maintainer-triage` CLI skill and fetched at generation time by the nvoss-velocity dashboard. + +--- + +## Step 1: Role + +You are a GitHub issue and PR labeler for NemoClaw, NVIDIA's open-source agentic AI assistant framework. + +For each item: + +1. Assign 1–5 labels from the provided list that best match the content. Be thorough — if a bug also involves a specific platform and is a good first issue, assign all applicable labels. Only skip a label if it genuinely does not apply. +2. Write a short triage comment appropriate to the item's tier (see Comment Tiers below). + +--- + +## Step 2: Output Format + +Return ONLY valid JSON — no markdown fences, no explanation: + +```json +{"results": [{"number": 123, "labels": ["bug", "good first issue"], "reason": "One sentence explaining label choices.", "comment": "Comment text."}]} +``` + +Fields: + +- `number` — the issue or PR number +- `labels` — array of label names, exactly as provided in the label list +- `reason` — one concise sentence explaining why these labels apply +- `comment` — triage comment text (see Comment Tiers) + +--- + +## Step 3: Label Assignment Rules + +- Use only label names exactly as provided in the label list +- Assign 1–5 labels per item — apply every label that genuinely fits +- If a specific `enhancement: *` sub-label is assigned, do NOT also assign the bare `enhancement` label — the sub-label is sufficient +- If genuinely unclear, assign `question` + +--- + +## Step 4: Skip Labels + +Never assign these — they require human judgment: + +- `duplicate` +- `invalid` +- `wontfix` +- `priority: medium` +- `priority: low` +- `status: triage` +- `NV QA` + +`priority: high` is allowed ONLY when the issue clearly blocks critical functionality, causes data loss, or describes a production outage — not based on the author's frustration or urgency language alone. + +--- + +## Step 5: Label Guide + +Use these descriptions to match labels to issue/PR content: + +- `bug`: User reports something broken — unexpected error, crash, exception, traceback, "not working", "fails", "broken", unexpected behavior +- `enhancement`: Generic enhancement — use only if none of the specific `enhancement: *` sub-types clearly apply +- `enhancement: feature`: Request for a new capability — "would be great if", "feature request", "add support for", "please add" +- `enhancement: inference`: Inference routing, model support, provider configuration +- `enhancement: security`: Security controls, policies, audit logging +- `enhancement: policy`: Network policy, egress rules, sandbox policy +- `enhancement: ui`: CLI UX, output formatting, terminal display +- `enhancement: platform`: Cross-platform support (pair with a `Platform: *` label) +- `enhancement: provider`: Cloud or inference provider support (pair with a `Provider: *` label) +- `enhancement: performance`: Speed, resource usage, memory, latency +- `enhancement: reliability`: Stability, error handling, recovery, retries +- `enhancement: testing`: Test coverage, CI/CD quality, test infrastructure +- `enhancement: MCP`: MCP protocol support, tool integration +- `enhancement: CI/CD`: Pipeline, build system, automation +- `enhancement: documentation`: Docs improvements, examples, guides +- `question`: Asking how to do something — "how do I", "is it possible", "does X support" +- `documentation`: Missing or incorrect docs, README errors, API doc gaps +- `good first issue`: Small well-scoped fix, doc typo, clear simple change — easy entry point for new contributors +- `help wanted`: Clear fix or improvement that needs a community contribution +- `security`: Auth issues, API key exposure, CVE, vulnerability, unauthorized access +- `status: needs-info`: Issue or PR has no description, no reproduction steps, or so little detail the team cannot act on it +- `priority: high`: Issue blocks critical functionality, causes data loss, or describes a production outage — apply only when the report clearly describes severe, reproducible impact +- `Platform: MacOS`: Issue specific to macOS, Mac OS X, or Apple Silicon (M1/M2/M3/M4). Apply when the user mentions macOS, Darwin, Homebrew, or Mac-specific behavior +- `Platform: Windows`: Issue specific to Windows OS. Apply when the user mentions Windows, Win32, PowerShell, WSL, or Windows-specific errors +- `Platform: Linux`: Issue specific to Linux. Apply when the user mentions a Linux distro (Ubuntu, CentOS, RHEL, Debian, etc.) or Linux-specific behavior +- `Platform: DGX Spark`: Issue specific to DGX Spark hardware or software environment +- `Platform: Brev`: Issue specific to the Brev.dev cloud environment +- `Platform: ARM64`: Issue specific to ARM64 / aarch64 architecture +- `Integration: Slack`: Issue or feature involving the Slack integration or Slack bridge +- `Integration: Discord`: Issue or feature involving the Discord integration +- `Integration: Telegram`: Issue or feature involving the Telegram integration +- `Integration: GitHub`: Issue or feature involving GitHub-specific behavior (not the repo itself) +- `Provider: NVIDIA`: Issue or feature specific to NVIDIA inference endpoints or NIM +- `Provider: OpenAI`: Issue or feature specific to OpenAI API or models +- `Provider: Anthropic`: Issue or feature specific to Anthropic / Claude models +- `Provider: Azure`: Issue or feature specific to Azure OpenAI or Azure cloud +- `Provider: AWS`: Issue or feature specific to AWS Bedrock or AWS cloud +- `Provider: GCP`: Issue or feature specific to Google Cloud / Vertex AI + +--- + +## Step 6: Comment Tiers + +Items are classified as `quality_tier` or `standard_tier` before generation. This is passed in the item metadata. + +- **quality_tier** (influencer author, company-affiliated author, or body > 800 chars): Write 2–3 sentences. Start with "Thanks," then naturally reference specific details from the body. Avoid "I've taken a look at", "I've reviewed", "it appears to", "I can see that" — these sound bot-generated. Write like a human maintainer giving a warm, specific response. +- **standard_tier**: Write 1 sentence acknowledging the report and mentioning the labels applied. + +--- + +## Step 7: Tone Rules (strictly enforced) + +- Use "could" not "should"; use "may" not "will" — this is a first response, not a commitment +- Never say "Thanks for fixing" — say "Thanks for the proposed fix" or "Thanks for submitting this" +- Never say "Thanks for adding" — say "Thanks for the suggested addition" +- Never claim the submission accomplishes something before review +- Do not say "I'll" or "we'll" +- For issues (bugs, questions, enhancements): use "this identifies a..." or "this reports a..." +- For PRs: use "this proposes a way to..." +- For security-related items: never confirm a vulnerability is real; use neutral language +- Do NOT open with praise about detail or thoroughness. Only reference the quality of the report if the body is genuinely exceptional — multiple reproduction steps, version info, logs, and clear expected vs actual behavior. For most reports, skip the praise entirely and go straight to the triage acknowledgment. +- Do not add generic closing filler phrases +- If a "Spam signal:" line is present in the item metadata, assign only `status: needs-info` and ask for more detail politely +- If a "Note: Author also opened..." line is present, briefly acknowledge if the relationship is plausible + +--- + +## Related Skills + +- `nemoclaw-user-skills-coding` — Agent Skills — all available maintainer and user skills diff --git a/.claude/skills b/.claude/skills deleted file mode 120000 index 2b7a412b8fa..00000000000 --- a/.claude/skills +++ /dev/null @@ -1 +0,0 @@ -../.agents/skills \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..9a144020624 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "nemoclaw-blueprint/router/llm-router"] + path = nemoclaw-blueprint/router/llm-router + url = https://github.com/NVIDIA-AI-Blueprints/llm-router.git + branch = v3 diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 120000 index 47dc3e3d863..00000000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000000..82099073f85 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,194 @@ +# Agent Instructions + +## Project Overview + +NVIDIA NemoClaw is an open-source reference stack for running [OpenClaw](https://openclaw.ai) always-on assistants inside [NVIDIA OpenShell](https://github.com/NVIDIA/OpenShell) sandboxes more safely. It provides CLI tooling, a blueprint for sandbox orchestration, and security hardening. + +**Status:** Alpha (March 2026+). Interfaces may change without notice. + +## Agent Skills + +This repo ships agent skills under `.agents/skills/`, organized into three audience buckets: `nemoclaw-user-*` (end users), `nemoclaw-maintainer-*` (project maintainers), and `nemoclaw-contributor-*` (codebase contributors). Load the `nemoclaw-skills-guide` skill for a full catalog and quick decision guide mapping tasks to skills. + +## Architecture + +| Path | Language | Purpose | +|------|----------|---------| +| `bin/` | JavaScript (CJS) | CLI launcher (`nemoclaw.js`) and small compatibility helpers | +| `src/lib/` | TypeScript | Core CLI logic: onboard, credentials, inference, policies, preflight, runner | +| `nemoclaw/` | TypeScript | Plugin project (Commander CLI extension for OpenClaw) | +| `nemoclaw/src/blueprint/` | TypeScript | Runner, snapshot, SSRF validation, state management | +| `nemoclaw/src/commands/` | TypeScript | Slash commands, migration state | +| `nemoclaw/src/onboard/` | TypeScript | Onboarding config | +| `nemoclaw-blueprint/` | YAML | Blueprint definition and network policies | +| `scripts/` | Bash/JS/TS | Install helpers, setup, automation, E2E tooling | +| `test/` | JavaScript (ESM) | Root-level integration tests (Vitest) | +| `test/e2e/` | Bash/JS | End-to-end tests (Brev cloud instances) | +| `docs/` | Markdown (MyST) | User-facing docs (Sphinx) | +| `k8s/` | YAML | Kubernetes deployment manifests | + +## Quick Reference + +| Task | Command | +|------|---------| +| Install all deps | `npm install && cd nemoclaw && npm install && npm run build && cd .. && cd nemoclaw-blueprint && uv sync && cd ..` | +| Build plugin | `cd nemoclaw && npm run build` | +| Watch mode | `cd nemoclaw && npm run dev` | +| Run all tests | `npm test` | +| Run plugin tests | `cd nemoclaw && npm test` | +| Run all linters | `make check` | +| Run all hooks manually | `npx prek run --all-files` | +| Type-check CLI | `npm run typecheck:cli` | +| Auto-format | `make format` | +| Build docs | `make docs` | +| Serve docs locally | `make docs-live` | + +## Key Architecture Decisions + +### Dual-Language Stack + +- **CLI and plugin**: TypeScript (`src/`, `nemoclaw/src/`) with a small CommonJS launcher in `bin/`; ESM in `test/` +- **Blueprint**: YAML configuration (`nemoclaw-blueprint/`) +- **Docs**: Sphinx/MyST Markdown +- **Tooling scripts**: Bash and Python + +The `bin/` directory uses CommonJS intentionally for the launcher and a few compatibility helpers so the CLI still has a stable executable entry point. The main CLI implementation lives in `src/` and compiles to `dist/`. The `nemoclaw/` plugin uses TypeScript and requires compilation. + +### Testing Strategy + +Tests are organized into three Vitest projects defined in `vitest.config.ts`: + +1. **`cli`** — `test/**/*.test.{js,ts}` — integration tests for CLI behavior +2. **`plugin`** — `nemoclaw/src/**/*.test.ts` — unit tests co-located with source +3. **`e2e-brev`** — `test/e2e/brev-e2e.test.js` — cloud E2E (requires `BREV_API_TOKEN`) + +When writing tests: + +- Root-level tests (`test/`) use ESM imports +- Plugin tests use TypeScript and are co-located with their source files +- Mock external dependencies; don't call real NVIDIA APIs in unit tests +- E2E tests run on ephemeral Brev cloud instances + +### Security Model + +NemoClaw isolates agents inside OpenShell sandboxes with: + +- Network policies (`nemoclaw-blueprint/policies/`) controlling egress +- Credential sanitization to prevent leaks +- SSRF validation (`nemoclaw/src/blueprint/ssrf.ts`) +- Docker capability drops and process limits + +Security-sensitive code paths require extra test coverage. + +## Code Style and Conventions + +### Commit Messages + +Conventional Commits required. Enforced by commitlint via prek `commit-msg` hook. + +```text +(): +``` + +Types: `feat`, `fix`, `docs`, `chore`, `refactor`, `test`, `ci`, `perf`, `merge` + +### SPDX Headers + +Every source file must include an SPDX license header. The pre-commit hook auto-inserts them: + +```javascript +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +``` + +For shell scripts use `#` comments. For Markdown use HTML comments. + +### JavaScript + +- `bin/` launcher and remaining `scripts/*.js`: **CommonJS** (`require`/`module.exports`), Node.js 22.16+ +- `test/`: **ESM** (`import`/`export`) +- ESLint config in `eslint.config.mjs` +- Cyclomatic complexity limit: 20 (ratcheting down to 15) +- Unused vars pattern: prefix with `_` + +### TypeScript + +- Plugin code in `nemoclaw/src/` with its own ESLint config +- CLI type-checking via `tsconfig.cli.json` +- Plugin type-checking via `nemoclaw/tsconfig.json` + +### Shell Scripts + +- ShellCheck enforced (`.shellcheckrc` at root) +- `shfmt` for formatting +- All scripts must have shebangs and be executable + +### No External Project Links + +Do not add links to third-party code repositories, community collections, or unofficial resources. Links to official tool documentation (Node.js, Python, uv) are acceptable. + +## Git Hooks (prek) + +All hooks managed by [prek](https://prek.j178.dev/) (installed via `npm install`): + +| Hook | What runs | +|------|-----------| +| **pre-commit** | File fixers, formatters, linters, Vitest (plugin) | +| **commit-msg** | commitlint (Conventional Commits) | +| **pre-push** | TypeScript type check (tsc --noEmit for plugin, JS, CLI) | + +## Working with This Repo + +### Before Making Changes + +1. Read `CONTRIBUTING.md` for the full contributor guide +2. Run `make check` to verify your environment is set up correctly +3. Check that `npm test` passes before starting + +### Common Patterns + +**Adding a CLI command:** + +- Entry point: `bin/nemoclaw.js` (launches the compiled CLI in `dist/`) +- Main CLI implementation lives in `src/lib/` and compiles to `dist/lib/` +- Add tests in `test/` + +**Adding a plugin feature:** + +- Source: `nemoclaw/src/` +- Co-locate tests as `*.test.ts` +- Build with `cd nemoclaw && npm run build` + +**Adding a network policy preset:** + +- Add YAML to `nemoclaw-blueprint/policies/presets/` +- Follow existing preset structure (see `slack.yaml`, `discord.yaml`) + +### Gotchas + +- `npm install` at root triggers `prek install` which sets up git hooks. If hooks fail, check that `core.hooksPath` is unset: `git config --unset core.hooksPath` +- The `nemoclaw/` subdirectory has its own `package.json`, `node_modules/`, and ESLint config — it's a separate npm project +- SPDX headers are auto-inserted by pre-commit hooks; don't worry about adding them manually +- Coverage thresholds are ratcheted in `ci/coverage-threshold-*.json` — new code should not decrease CLI or plugin coverage +- The `.claude/skills` symlink points to `.agents/skills` — both paths resolve to the same content + +## Documentation + +- Source of truth: `docs/` directory +- `.agents/skills/nemoclaw-user-*/*.md` is **autogenerated** — never edit directly +- After changing docs, regenerate skills: + + ```bash + python3 scripts/docs-to-skills.py docs/ .agents/skills/ --prefix nemoclaw-user + ``` + +- Follow style guide in `docs/CONTRIBUTING.md` + +## PR Requirements + +- Create feature branch from `main` +- Run `make check` and `npm test` before submitting +- Follow PR template (`.github/PULL_REQUEST_TEMPLATE.md`) +- Update docs for any user-facing behavior changes +- No secrets, API keys, or credentials committed +- Limit open PRs to fewer than 10 diff --git a/Dockerfile b/Dockerfile index 70815cc740e..83bf001bded 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,10 @@ COPY nemoclaw-blueprint/ /opt/nemoclaw-blueprint/ WORKDIR /opt/nemoclaw RUN npm ci --omit=dev +# NOTE: model-router-toolkit is NOT installed in the sandbox image. +# The router runs on the host (started by onboard) and the sandbox +# reaches it through the OpenShell gateway's L7 proxy. + # Set up blueprint for local resolution. # Blueprints are immutable at runtime; DAC protection (root ownership) is applied # later since /sandbox/.nemoclaw is Landlock read_write for plugin state (#804). diff --git a/README.md b/README.md index 303c6f9abee..ab6bee22afc 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,58 @@ Alternatively, send a single message and print the response: openclaw agent --agent main --local -m "hello" --session-id test ``` +### Model Router (Complexity-Based Routing) + +NemoClaw includes an optional model router that automatically picks the most efficient model for each query. Instead of sending every request to a single large model, the router uses a lightweight encoder to predict which model in a pool can handle each query correctly, then routes to the cheapest one that meets an accuracy threshold. + +The router uses the [NVIDIA LLM Router v3](https://github.com/NVIDIA-AI-Blueprints/llm-router/tree/v3) prefill routing engine and runs on the host as a LiteLLM proxy. The sandbox reaches it through the OpenShell gateway. + +#### Enable during onboard + +Select **Model Router (complexity-based routing)** during the onboard wizard, or set `NEMOCLAW_PROVIDER=routed` for non-interactive mode: + +```bash +NEMOCLAW_PROVIDER=routed nemoclaw onboard --non-interactive +``` + +The onboard wizard starts the router, configures the OpenShell provider, and creates the sandbox. The router process runs on the host on port 4000. + +#### Configure the model pool + +Edit `nemoclaw-blueprint/router/pool-config.yaml` to define which models the router can choose from: + +```yaml +routing: + method: prefill + checkpoint: llm-router/checkpoints/prefill_router_qwen08b.pt + tolerance: 0.20 + encoder: Qwen/Qwen3.5-0.8B + +models: + - name: nano + litellm_model: "openai/nvidia/nvidia/Nemotron-3-Nano-30B-A3B" + cost_per_m_input_tokens: 0.05 + api_base: "https://inference-api.nvidia.com" + + - name: super + litellm_model: "openai/nvidia/nvidia/nemotron-3-super-v3" + cost_per_m_input_tokens: 0.10 + api_base: "https://inference-api.nvidia.com" +``` + +The `tolerance` parameter controls the accuracy-cost tradeoff: 0.0 always picks the most accurate model, 1.0 always picks the cheapest, and 0.20 (default) allows up to 20 percentage points below the best for a cheaper model. + +#### Architecture + +The router runs on the host, not inside the sandbox: + +```text +Sandbox (OpenClaw) ──> OpenShell Gateway (L7 proxy) ──> Model Router (:4000) ──> NVIDIA API + └── PrefillRouter selects model +``` + +Credentials flow through the OpenShell provider system. The sandbox never sees raw API keys. + ### Uninstall To remove NemoClaw and all resources created during setup, run the uninstall script: @@ -177,6 +229,9 @@ NemoClaw/ │ ├── commands/ # Slash commands, migration state │ └── onboard/ # Onboarding config ├── nemoclaw-blueprint/ # Blueprint YAML and network policies +│ └── router/ +│ ├── pool-config.yaml # Model pool and routing config +│ └── llm-router/ # LLM Router v3 submodule (prefill routing engine) ├── scripts/ # Install helpers, setup, automation ├── test/ # Integration and E2E tests └── docs/ # User-facing docs (Sphinx/MyST) diff --git a/nemoclaw-blueprint/blueprint.yaml b/nemoclaw-blueprint/blueprint.yaml index d61a4e72e02..b678877a9ed 100644 --- a/nemoclaw-blueprint/blueprint.yaml +++ b/nemoclaw-blueprint/blueprint.yaml @@ -18,6 +18,7 @@ profiles: - ncp - nim-local - vllm + - routed description: | NemoClaw blueprint: orchestrates OpenClaw sandbox creation, migration, @@ -68,6 +69,19 @@ components: credential_default: "dummy" timeout_secs: 180 + routed: + provider_type: "openai" + provider_name: "nvidia-router" + endpoint: "http://localhost:4000/v1" + model: "nvidia-routed" + credential_env: "NVIDIA_API_KEY" + timeout_secs: 180 + + router: + enabled: true + port: 4000 + pool_config_path: "router/pool-config.yaml" + policy: base: "sandboxes/openclaw/policy.yaml" additions: diff --git a/nemoclaw-blueprint/router/llm-router b/nemoclaw-blueprint/router/llm-router new file mode 160000 index 00000000000..2bd8dfaa751 --- /dev/null +++ b/nemoclaw-blueprint/router/llm-router @@ -0,0 +1 @@ +Subproject commit 2bd8dfaa751efb60aa4e7e49b270490dfbc0a68a diff --git a/nemoclaw-blueprint/router/pool-config.yaml b/nemoclaw-blueprint/router/pool-config.yaml new file mode 100644 index 00000000000..39bdf67472b --- /dev/null +++ b/nemoclaw-blueprint/router/pool-config.yaml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Model Router Toolkit pool config for complexity-based routing. +# Uses the NVIDIA LLM Router v3 (https://github.com/NVIDIA-AI-Blueprints/llm-router) +# to route requests to the most efficient model that meets an accuracy threshold. +# +# The router runs an encoder (Qwen3.5-0.8B) on each query to predict P(correct) +# per model, then selects the cheapest model above the tolerance threshold. +# +# tolerance controls the accuracy-cost tradeoff: +# 0.0 = always pick the highest-confidence model (most expensive) +# 0.20 = allow up to 20pp below the best for a cheaper model (default) +# 1.0 = always pick the cheapest model + +routing: + method: prefill + checkpoint: llm-router/checkpoints/prefill_router_qwen08b.pt + tolerance: 0.20 + encoder: Qwen/Qwen3.5-0.8B + encoder_backend: transformers + +models: + - name: gpt-5.2 + display_name: "GPT-5.2" + litellm_model: "openai/nvidia/nvidia/Nemotron-3-Nano-30B-A3B" + cost_per_m_input_tokens: 0.05 + cost_per_m_output_tokens: 0.20 + api_base: "https://inference-api.nvidia.com" + + - name: nemotron-3-super + display_name: "Nemotron 3 Super 120B" + litellm_model: "openai/nvidia/nvidia/nemotron-3-super-v3" + cost_per_m_input_tokens: 0.10 + cost_per_m_output_tokens: 0.40 + api_base: "https://inference-api.nvidia.com" diff --git a/nemoclaw/src/blueprint/runner.test.ts b/nemoclaw/src/blueprint/runner.test.ts index 53fd4e6d672..e3dc7bfe632 100644 --- a/nemoclaw/src/blueprint/runner.test.ts +++ b/nemoclaw/src/blueprint/runner.test.ts @@ -123,6 +123,38 @@ function minimalBlueprint(overrides?: Record): Record { + return { + version: "1.0", + components: { + inference: { + profiles: { + routed: { + provider_type: "openai", + provider_name: "nvidia-router", + endpoint: "http://localhost:4000/v1", + model: "routed", + credential_env: "NVIDIA_API_KEY", + credential_default: "router-local", + timeout_secs: 180, + }, + }, + }, + sandbox: { + image: "openclaw", + name: "test-sandbox", + forward_ports: [18789], + }, + router: { + enabled: true, + port: 4000, + pool_config_path: "router/pool-config.yaml", + }, + policy: { additions: {} }, + }, + }; +} + function seedBlueprintFile(bp?: Record): void { addFile("blueprint.yaml", YAML.stringify(bp ?? minimalBlueprint())); } @@ -258,6 +290,25 @@ describe("runner", () => { expect(out).toContain("PROGRESS:10:Validating blueprint"); expect(out).toContain("PROGRESS:100:Plan complete"); }); + + it("includes router info when router is enabled", async () => { + captureStdout(); + mockExeca.mockResolvedValue({ exitCode: 0 }); + + const plan = await actionPlan("routed", routedBlueprint()); + expect(plan.router.enabled).toBe(true); + expect(plan.router.port).toBe(4000); + expect(plan.router.pool_config_path).toBe("router/pool-config.yaml"); + }); + + it("defaults router to disabled when not in blueprint", async () => { + captureStdout(); + mockExeca.mockResolvedValue({ exitCode: 0 }); + + const plan = await actionPlan("default", minimalBlueprint()); + expect(plan.router.enabled).toBe(false); + expect(plan.router.port).toBe(4000); + }); }); describe("actionApply", () => { @@ -558,6 +609,24 @@ describe("runner", () => { if (!inferenceCall) throw new Error("inference set call not found"); expect(inferenceCall[1]).not.toContain("--timeout"); }); + + it("passes endpoint as-is from blueprint (no rewriting)", async () => { + process.env.NVIDIA_API_KEY = "test-key"; + try { + await actionApply("routed", routedBlueprint()); + + const providerCall = mockExeca.mock.calls.find( + (c) => Array.isArray(c[1]) && c[1].includes("provider"), + ); + if (!providerCall) throw new Error("provider create call not found"); + const configArg = (providerCall[1] as string[]).find((a: string) => + a.startsWith("OPENAI_BASE_URL="), + ); + expect(configArg).toBe("OPENAI_BASE_URL=http://localhost:4000/v1"); + } finally { + delete process.env.NVIDIA_API_KEY; + } + }); }); describe("actionStatus", () => { diff --git a/nemoclaw/src/blueprint/runner.ts b/nemoclaw/src/blueprint/runner.ts index a4b21fa7491..c54f5ce3a88 100644 --- a/nemoclaw/src/blueprint/runner.ts +++ b/nemoclaw/src/blueprint/runner.ts @@ -57,6 +57,7 @@ interface Blueprint { profiles?: Record; }; sandbox?: SandboxConfig; + router?: RouterConfig; policy?: { additions?: Record; }; @@ -79,6 +80,14 @@ interface SandboxConfig { forward_ports?: number[]; } +interface RouterConfig { + enabled?: boolean; + port?: number; + pool_config_path?: string; +} + +const DEFAULT_ROUTER_PORT = 4000; + export function loadBlueprint(): Blueprint { const blueprintPath = process.env.NEMOCLAW_BLUEPRINT_PATH ?? "."; const bpFile = join(blueprintPath, "blueprint.yaml"); @@ -124,6 +133,7 @@ async function resolveRunConfig( inferenceProfiles: Record; inferenceCfg: InferenceProfile; sandboxCfg: SandboxConfig; + routerCfg: RouterConfig; }> { const inferenceProfiles = blueprint.components?.inference?.profiles ?? {}; if (!(profile in inferenceProfiles)) { @@ -143,7 +153,8 @@ async function resolveRunConfig( } const sandboxCfg = blueprint.components?.sandbox ?? {}; - return { inferenceProfiles, inferenceCfg, sandboxCfg }; + const routerCfg = blueprint.components?.router ?? {}; + return { inferenceProfiles, inferenceCfg, sandboxCfg, routerCfg }; } // ── Actions ───────────────────────────────────────────────────── @@ -163,6 +174,11 @@ export interface RunPlan { model: string | undefined; credential_env: string | undefined; }; + router: { + enabled: boolean; + port: number; + pool_config_path: string | undefined; + }; policy_additions: Record; dry_run: boolean; } @@ -175,7 +191,7 @@ export async function actionPlan( const rid = emitRunId(); progress(10, "Validating blueprint"); - const { inferenceCfg, sandboxCfg } = await resolveRunConfig( + const { inferenceCfg, sandboxCfg, routerCfg } = await resolveRunConfig( profile, blueprint, options?.endpointUrl, @@ -188,6 +204,9 @@ export async function actionPlan( ); } + const routerEnabled = routerCfg.enabled === true; + const routerPort = routerCfg.port ?? DEFAULT_ROUTER_PORT; + const plan: RunPlan = { run_id: rid, profile, @@ -203,6 +222,11 @@ export async function actionPlan( model: inferenceCfg.model, credential_env: inferenceCfg.credential_env, }, + router: { + enabled: routerEnabled, + port: routerPort, + pool_config_path: routerCfg.pool_config_path, + }, policy_additions: blueprint.components?.policy?.additions ?? {}, dry_run: options?.dryRun ?? false, }; @@ -403,7 +427,10 @@ export async function actionRollback(rid: string): Promise { const planFile = join(stateDir, "plan.json"); try { const planData = readFileSync(planFile, "utf-8"); - const plan = JSON.parse(planData) as { sandbox_name?: string }; + const plan = JSON.parse(planData) as { + sandbox_name?: string; + }; + const sandboxName = plan.sandbox_name ?? "openclaw"; progress(30, `Stopping sandbox ${sandboxName}`); diff --git a/schemas/blueprint.schema.json b/schemas/blueprint.schema.json index ffaa3e96122..abd8de24f21 100644 --- a/schemas/blueprint.schema.json +++ b/schemas/blueprint.schema.json @@ -79,6 +79,27 @@ } } }, + "router": { + "type": "object", + "description": "Model router configuration for complexity-based routing.", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether the model router is active." + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535, + "description": "Port the router proxy listens on." + }, + "pool_config_path": { + "type": "string", + "description": "Path to the router pool config YAML relative to the blueprint directory." + } + } + }, "policy": { "type": "object", "required": ["base"], diff --git a/scripts/install.sh b/scripts/install.sh index d91c52d656c..af4992daeb1 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -920,6 +920,9 @@ install_nemoclaw() { spin "Installing NemoClaw dependencies" bash -c "cd \"$NEMOCLAW_SOURCE_ROOT\" && npm install --ignore-scripts" spin "Building NemoClaw CLI modules" bash -c "cd \"$NEMOCLAW_SOURCE_ROOT\" && npm run --if-present build:cli" spin "Building NemoClaw plugin" bash -c "cd \"$NEMOCLAW_SOURCE_ROOT\"/nemoclaw && npm install --ignore-scripts && npm run build" + if command_exists pip3 && [ -d "$NEMOCLAW_SOURCE_ROOT/nemoclaw-blueprint/router/llm-router" ]; then + spin "Installing model router" pip3 install --quiet --user "$NEMOCLAW_SOURCE_ROOT/nemoclaw-blueprint/router/llm-router[prefill,proxy]" + fi spin "Linking NemoClaw CLI" bash -c "cd \"$NEMOCLAW_SOURCE_ROOT\" && npm link" else if [[ -f "$package_json" ]]; then @@ -938,6 +941,8 @@ install_nemoclaw() { mkdir -p "$(dirname "$nemoclaw_src")" NEMOCLAW_SOURCE_ROOT="$nemoclaw_src" spin "Cloning NemoClaw source" git clone --depth 1 --branch "$release_ref" https://github.com/NVIDIA/NemoClaw.git "$nemoclaw_src" + # Initialize submodules (e.g., llm-router for model routing support) + git -C "$nemoclaw_src" submodule update --init --depth 1 2>/dev/null || true # Fetch version tags into the shallow clone so `git describe --tags # --match "v*"` works at runtime (the shallow clone only has the # single ref we asked for). @@ -953,6 +958,9 @@ install_nemoclaw() { spin "Installing NemoClaw dependencies" bash -c "cd \"$nemoclaw_src\" && npm install --ignore-scripts" spin "Building NemoClaw CLI modules" bash -c "cd \"$nemoclaw_src\" && npm run --if-present build:cli" spin "Building NemoClaw plugin" bash -c "cd \"$nemoclaw_src\"/nemoclaw && npm install --ignore-scripts && npm run build" + if command_exists pip3 && [ -d "$nemoclaw_src/nemoclaw-blueprint/router/llm-router" ]; then + spin "Installing model router" pip3 install --quiet --user "$nemoclaw_src/nemoclaw-blueprint/router/llm-router[prefill,proxy]" + fi spin "Linking NemoClaw CLI" bash -c "cd \"$nemoclaw_src\" && npm link" fi diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index c2613c6c377..7a8219c31b4 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -454,6 +454,107 @@ function getBlueprintMaxOpenshellVersion(rootDir = ROOT) { return getBlueprintVersionField("max_openshell_version", rootDir); } +/** + * Load a named inference profile and router config from blueprint.yaml. + * Returns null if the blueprint or profile is missing. + */ +function loadBlueprintProfile(profileName, rootDir = ROOT) { + try { + const YAML = require("yaml"); + const blueprintPath = path.join(rootDir, "nemoclaw-blueprint", "blueprint.yaml"); + if (!fs.existsSync(blueprintPath)) return null; + const raw = fs.readFileSync(blueprintPath, "utf8"); + const parsed = YAML.parse(raw); + const profile = parsed?.components?.inference?.profiles?.[profileName]; + if (!profile) return null; + const router = parsed?.components?.router || {}; + return { ...profile, router }; + } catch { + return null; + } +} + +const ROUTER_HEALTH_RETRIES = 15; +const ROUTER_HEALTH_INTERVAL_MS = 2000; + +/** + * Start the model-router proxy and wait for it to become healthy. + * Follows the same pattern as Ollama startup (spawn detached, poll health). + * Returns the PID of the child process. + */ +async function startModelRouter(routerCfg) { + const port = routerCfg.port || 4000; + const blueprintDir = path.join(ROOT, "nemoclaw-blueprint"); + const poolConfigPath = path.join(blueprintDir, routerCfg.pool_config_path || "router/pool-config.yaml"); + const stateDir = path.join(os.homedir(), ".nemoclaw", "state"); + const litellmConfigPath = path.join(stateDir, "litellm-proxy.yaml"); + + fs.mkdirSync(stateDir, { recursive: true }); + + const proxyConfigResult = spawnSync( + "model-router", + ["proxy-config", "--config", poolConfigPath, "--output", litellmConfigPath], + { encoding: "utf8", timeout: 30_000, cwd: blueprintDir }, + ); + if (proxyConfigResult.status !== 0) { + throw new Error( + `model-router proxy-config failed: ${proxyConfigResult.stderr || proxyConfigResult.error || "unknown error"}`, + ); + } + + const { buildSubprocessEnv } = require("./subprocess-env"); + const credEnvVars = {}; + const credName = routerCfg.credential_env || "NVIDIA_API_KEY"; + if (process.env[credName]) credEnvVars[credName] = process.env[credName]; + if (process.env.OPENAI_API_KEY) credEnvVars.OPENAI_API_KEY = process.env.OPENAI_API_KEY; + const _providerKey = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); + if (_providerKey) { + if (!credEnvVars[credName]) credEnvVars[credName] = _providerKey; + if (!credEnvVars.OPENAI_API_KEY) credEnvVars.OPENAI_API_KEY = _providerKey; + } + + const child = spawn( + "model-router", + [ + "proxy", + "--litellm-config", litellmConfigPath, + "--router-config", poolConfigPath, + "--port", String(port), + ], + { + detached: true, + stdio: "ignore", + cwd: blueprintDir, + env: buildSubprocessEnv(credEnvVars), + }, + ); + child.unref(); + + const pid = child.pid; + if (!pid) { + throw new Error("Failed to start model-router proxy: no PID returned"); + } + + const http = require("http"); + for (let attempt = 0; attempt < ROUTER_HEALTH_RETRIES; attempt++) { + await new Promise((resolve) => setTimeout(resolve, ROUTER_HEALTH_INTERVAL_MS)); + const healthy = await new Promise((resolve) => { + http + .get(`http://127.0.0.1:${port}/health`, (res) => resolve(res.statusCode >= 200 && res.statusCode < 300)) + .on("error", () => resolve(false)); + }); + if (healthy) return pid; + } + try { + process.kill(pid, "SIGTERM"); + } catch { + // already dead + } + throw new Error( + `Model router failed to become healthy on port ${port} after ${ROUTER_HEALTH_RETRIES} attempts`, + ); +} + function getStableGatewayImageRef(versionOutput = null) { const version = getInstalledOpenshellVersion(versionOutput); if (!version) return null; @@ -1801,11 +1902,12 @@ function getNonInteractiveProvider() { "custom", "nim-local", "vllm", + "routed", ]); if (!validProviders.has(normalized)) { console.error(` Unsupported NEMOCLAW_PROVIDER: ${providerKey}`); console.error( - " Valid values: build, openai, anthropic, anthropicCompatible, gemini, ollama, custom, nim-local, vllm", + " Valid values: build, openai, anthropic, anthropicCompatible, gemini, ollama, custom, nim-local, vllm, routed", ); process.exit(1); } @@ -2934,6 +3036,12 @@ async function setupNim(gpu) { label: "Local vLLM [experimental] — running", }); } + // Model Router: complexity-based routing via blueprint config + const blueprintRouterCfg = loadBlueprintProfile("routed"); + if (blueprintRouterCfg && blueprintRouterCfg.router?.enabled === true) { + options.push({ key: "routed", label: "Model Router (complexity-based routing)" }); + } + // On macOS without Ollama, offer to install it if (!hasOllama && process.platform === "darwin") { options.push({ key: "install-ollama", label: "Install Ollama (macOS)" }); @@ -3509,6 +3617,57 @@ async function setupNim(gpu) { } preferredInferenceApi = "openai-completions"; break; + } else if (selected.key === "routed") { + const bp = loadBlueprintProfile("routed"); + if (!bp || bp.router?.enabled !== true) { + console.error(" Router is not enabled in nemoclaw-blueprint/blueprint.yaml."); + if (isNonInteractive()) process.exit(1); + continue selectionLoop; + } + credentialEnv = bp.credential_env || "OPENAI_API_KEY"; + const _providerKeyHint = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); + if (_providerKeyHint && !process.env[credentialEnv]) { + process.env[credentialEnv] = _providerKeyHint; + } + if (isNonInteractive()) { + if (!process.env[credentialEnv] && !bp.credential_default) { + console.error( + ` ${credentialEnv} (or NEMOCLAW_PROVIDER_KEY) is required for Model Router in non-interactive mode.`, + ); + process.exit(1); + } + } else { + if (!process.env[credentialEnv] && !bp.credential_default) { + await ensureNamedCredential(credentialEnv, "Model Router API key", null); + } + } + console.log(" Starting model router..."); + try { + const routerPid = await startModelRouter(bp.router); + const routerPort = bp.router.port || 4000; + console.log(` ✓ Model router started (PID ${routerPid}) on port ${routerPort}`); + onboardSession.updateSession((current) => { + current.routerPid = routerPid; + return current; + }); + } catch (err) { + console.error(` ✗ Failed to start model router: ${err.message}`); + if (isNonInteractive()) process.exit(1); + console.log(" Returning to provider selection."); + console.log(""); + continue selectionLoop; + } + provider = bp.provider_name || "nvidia-router"; + model = bp.model; + const { HOST_GATEWAY_URL } = require("./local-inference"); + endpointUrl = bp.endpoint || ""; + if (endpointUrl.match(/localhost|127\.0\.0\.1/)) { + const u = new URL(endpointUrl); + endpointUrl = `${HOST_GATEWAY_URL}:${u.port}${u.pathname}`; + } + preferredInferenceApi = "openai-completions"; + console.log(` ✓ Using Model Router: ${provider} / ${model}`); + break; } } } @@ -3664,6 +3823,18 @@ async function setupInference( console.error(` ${probe.message}`); process.exit(1); } + } else { + // Blueprint profile provider (e.g., nvidia-router for the routed profile). + // Same pattern as vllm-local: upsert the provider and set the inference route. + const credentialValue = hydrateCredentialEnv(credentialEnv); + const env = credentialEnv && credentialValue ? { [credentialEnv]: credentialValue } : {}; + const providerResult = upsertProvider(provider, "openai", credentialEnv, endpointUrl, env); + if (!providerResult.ok) { + console.error(` ${providerResult.message}`); + process.exit(providerResult.status || 1); + } + const inferenceArgs = ["inference", "set", "--no-verify", "--provider", provider, "--model", model]; + runOpenshell(inferenceArgs); } verifyInferenceRoute(provider, model); From 4b88dcab59d00a4a209d9cf6d7b5048d1a611309 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 6 May 2026 10:32:52 -0700 Subject: [PATCH 02/10] fix(installer): initialize router submodule before install --- scripts/install.sh | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index a47171d0838..798bdc90079 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1320,6 +1320,26 @@ is_source_checkout() { return 1 } +init_nemoclaw_submodules() { + local root="$1" + [[ -f "$root/.gitmodules" ]] || return 0 + git -C "$root" rev-parse --git-dir >/dev/null 2>&1 || return 0 + git -C "$root" submodule update --init --depth 1 2>/dev/null +} + +install_model_router_if_present() { + local root="$1" + local router_dir="$root/nemoclaw-blueprint/router/llm-router" + + command_exists pip3 || return 0 + [[ -d "$router_dir" ]] || return 0 + if [[ ! -f "$router_dir/pyproject.toml" && ! -f "$router_dir/setup.py" ]]; then + warn "Skipping model router install — llm-router submodule is not initialized." + return 0 + fi + spin "Installing model router" pip3 install --quiet --user "${router_dir}[prefill,proxy]" +} + install_nemoclaw() { command_exists git || error "git was not found on PATH." local repo_root package_json @@ -1335,12 +1355,12 @@ install_nemoclaw() { spin "Preparing OpenClaw package" bash -c "$(declare -f info warn resolve_openclaw_version pre_extract_openclaw); pre_extract_openclaw \"\$1\"" _ "$NEMOCLAW_SOURCE_ROOT" \ || warn "Pre-extraction failed — npm install may fail if openclaw tarball is broken" fi + spin "Initializing ${_CLI_DISPLAY} submodules" init_nemoclaw_submodules "$NEMOCLAW_SOURCE_ROOT" \ + || warn "Submodule initialization failed — model router support may be unavailable" spin "Installing ${_CLI_DISPLAY} dependencies" bash -c "cd \"$NEMOCLAW_SOURCE_ROOT\" && npm install --ignore-scripts" spin "Building ${_CLI_DISPLAY} CLI modules" bash -c "cd \"$NEMOCLAW_SOURCE_ROOT\" && npm run --if-present build:cli" spin "Building ${_CLI_DISPLAY} plugin" bash -c "cd \"$NEMOCLAW_SOURCE_ROOT\"/nemoclaw && npm install --ignore-scripts && npm run build" - if command_exists pip3 && [ -d "$NEMOCLAW_SOURCE_ROOT/nemoclaw-blueprint/router/llm-router" ]; then - spin "Installing model router" pip3 install --quiet --user "$NEMOCLAW_SOURCE_ROOT/nemoclaw-blueprint/router/llm-router[prefill,proxy]" - fi + install_model_router_if_present "$NEMOCLAW_SOURCE_ROOT" spin "Linking ${_CLI_DISPLAY} CLI" bash -c "cd \"$NEMOCLAW_SOURCE_ROOT\" && npm link" else if [[ -f "$package_json" ]]; then @@ -1359,8 +1379,8 @@ install_nemoclaw() { mkdir -p "$(dirname "$nemoclaw_src")" NEMOCLAW_SOURCE_ROOT="$nemoclaw_src" spin "Cloning ${_CLI_DISPLAY} source" clone_nemoclaw_ref "$release_ref" "$nemoclaw_src" - # Initialize submodules (e.g., llm-router for model routing support). - git -C "$nemoclaw_src" submodule update --init --depth 1 2>/dev/null || true + spin "Initializing ${_CLI_DISPLAY} submodules" init_nemoclaw_submodules "$nemoclaw_src" \ + || warn "Submodule initialization failed — model router support may be unavailable" # Fetch version tags into the shallow clone so `git describe --tags # --match "v*"` works at runtime (the shallow clone only has the # single ref we asked for). @@ -1376,9 +1396,7 @@ install_nemoclaw() { spin "Installing ${_CLI_DISPLAY} dependencies" bash -c "cd \"$nemoclaw_src\" && npm install --ignore-scripts" spin "Building ${_CLI_DISPLAY} CLI modules" bash -c "cd \"$nemoclaw_src\" && npm run --if-present build:cli" spin "Building ${_CLI_DISPLAY} plugin" bash -c "cd \"$nemoclaw_src\"/nemoclaw && npm install --ignore-scripts && npm run build" - if command_exists pip3 && [ -d "$nemoclaw_src/nemoclaw-blueprint/router/llm-router" ]; then - spin "Installing model router" pip3 install --quiet --user "$nemoclaw_src/nemoclaw-blueprint/router/llm-router[prefill,proxy]" - fi + install_model_router_if_present "$nemoclaw_src" spin "Linking ${_CLI_DISPLAY} CLI" bash -c "cd \"$nemoclaw_src\" && npm link" # Install/upgrade the OpenShell CLI on the GitHub-clone path (curl|bash). From 1935c91295ed6652ff066e2c41a4ba642c22d4a9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 6 May 2026 11:01:08 -0700 Subject: [PATCH 03/10] fix(router): clarify gateway route for model router Signed-off-by: Aaron Erickson --- README.md | 2 +- src/lib/onboard-providers.ts | 4 ++ src/lib/onboard.ts | 6 +- test/onboard.test.ts | 119 +++++++++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 13372b169a6..a2a094e771a 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ openclaw agent --agent main --local -m "hello" --session-id test NemoClaw includes an optional model router that automatically picks the most efficient model for each query. Instead of sending every request to a single large model, the router uses a lightweight encoder to predict which model in a pool can handle each query correctly, then routes to the cheapest one that meets an accuracy threshold. -The router uses the [NVIDIA LLM Router v3](https://github.com/NVIDIA-AI-Blueprints/llm-router/tree/v3) prefill routing engine and runs on the host as a LiteLLM proxy. The sandbox reaches it through the OpenShell gateway. +The router uses the [NVIDIA LLM Router v3](https://github.com/NVIDIA-AI-Blueprints/llm-router/tree/v3) prefill routing engine and runs on the host as a LiteLLM proxy. The sandbox reaches it through the OpenShell gateway and continues to call `https://inference.local/v1`; do not probe `localhost:4000` or `host.openshell.internal` directly from inside the sandbox. #### Enable during onboard diff --git a/src/lib/onboard-providers.ts b/src/lib/onboard-providers.ts index 7e3c9144000..a76f287fa41 100644 --- a/src/lib/onboard-providers.ts +++ b/src/lib/onboard-providers.ts @@ -339,6 +339,10 @@ function getSandboxInferenceConfig(model, provider = null, preferredInferenceApi supportsStore: false, }; break; + case "nvidia-router": + providerKey = "inference"; + primaryModelRef = `inference/${model}`; + break; case "nvidia-prod": case "nvidia-nim": default: diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index b5f8d731ebd..9247dd86ea9 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -802,7 +802,10 @@ function loadBlueprintProfile( const parsed = YAML.parse(raw); const profile = parsed?.components?.inference?.profiles?.[profileName]; if (!profile) return null; - const router = parsed?.components?.router || {}; + const router = { + ...(parsed?.components?.router || {}), + credential_env: profile.credential_env, + }; return { ...profile, router } as BlueprintInferenceProfile; } catch { return null; @@ -857,6 +860,7 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise { ); }); + it("maps Model Router sandboxes through managed inference.local", () => { + assert.deepEqual(getSandboxInferenceConfig("nvidia-routed", "nvidia-router"), { + providerKey: "inference", + primaryModelRef: "inference/nvidia-routed", + inferenceBaseUrl: "https://inference.local/v1", + inferenceApi: "openai-completions", + inferenceCompat: null, + }); + }); + it("adds Kimi K2.6 compat for NVIDIA Endpoints on the routed inference provider", () => { assert.deepEqual( getSandboxInferenceConfig("moonshotai/kimi-k2.6", "nvidia-prod", "openai-completions"), @@ -2410,6 +2420,115 @@ const { setupInference } = require(${onboardPath}); assert.equal(payload.nvidiaApiKey, "nvapi-secret-value"); }); + it("configures Model Router as a host provider while sandboxes keep inference.local", () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-router-inference-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "setup-router-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); + const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); + const registryPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "registry.js")); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); + +const commands = []; +runner.run = (command, opts = {}) => { + const cmd = _n(command); + commands.push({ command: cmd, env: opts.env || null }); + if (cmd.includes("provider get")) return { status: 1, stdout: "", stderr: "" }; + return { status: 0, stdout: "", stderr: "" }; +}; +runner.runCapture = (command) => { + const cmd = _n(command); + if (cmd.includes("inference") && cmd.includes("get")) { + return [ + "Gateway inference:", + "", + " Route: inference.local", + " Provider: nvidia-router", + " Model: nvidia-routed", + " Version: 1", + ].join("\\n"); + } + return ""; +}; +registry.updateSandbox = () => true; + +process.env.NVIDIA_API_KEY = "nvapi-router-secret"; + +const { setupInference, getSandboxInferenceConfig } = require(${onboardPath}); + +(async () => { + await setupInference( + "router-box", + "nvidia-routed", + "nvidia-router", + "http://host.openshell.internal:4000/v1", + "NVIDIA_API_KEY", + ); + console.log(JSON.stringify({ + commands, + sandboxConfig: getSandboxInferenceConfig("nvidia-routed", "nvidia-router", "openai-completions"), + })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson<{ + commands: CommandEntry[]; + sandboxConfig: SandboxInferenceConfig; + }>(result.stdout); + const providerCommand = payload.commands.find((entry) => + /provider create/.test(entry.command), + ); + assert.ok(providerCommand, JSON.stringify(payload.commands)); + assert.match(providerCommand.command, /--name nvidia-router/); + assert.match(providerCommand.command, /--credential NVIDIA_API_KEY/); + assert.match( + providerCommand.command, + /OPENAI_BASE_URL=http:\/\/host\.openshell\.internal:4000\/v1/, + ); + assert.doesNotMatch(providerCommand.command, /nvapi-router-secret/); + assert.equal(providerCommand.env?.NVIDIA_API_KEY, "nvapi-router-secret"); + + const inferenceCommand = payload.commands.find((entry) => + /inference set/.test(entry.command), + ); + assert.ok(inferenceCommand, JSON.stringify(payload.commands)); + assert.match(inferenceCommand.command, /--provider nvidia-router/); + assert.match(inferenceCommand.command, /--model nvidia-routed/); + + assert.deepEqual(payload.sandboxConfig, { + providerKey: "inference", + primaryModelRef: "inference/nvidia-routed", + inferenceBaseUrl: "https://inference.local/v1", + inferenceApi: "openai-completions", + inferenceCompat: null, + }); + }); + it("does not delete saved OpenAI credentials when configuring local vLLM", () => { const repoRoot = path.join(import.meta.dirname, ".."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-local-vllm-")); From 02a4ac4b0f7332d8ec032d53e6d031e26fca1f60 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 6 May 2026 11:59:55 -0700 Subject: [PATCH 04/10] fix(router): harden routed setup checks Signed-off-by: Aaron Erickson --- scripts/install.sh | 29 +++++++++++++++++----- src/lib/onboard.ts | 59 +++++++++++++++++++++++++++++++++++--------- test/onboard.test.ts | 6 +++++ 3 files changed, 76 insertions(+), 18 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 798bdc90079..e44b4d7b146 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1327,13 +1327,26 @@ init_nemoclaw_submodules() { git -C "$root" submodule update --init --depth 1 2>/dev/null } +is_routed_provider_requested() { + local provider="${NEMOCLAW_PROVIDER:-}" + provider="${provider,,}" + [[ "$provider" == "routed" ]] +} + install_model_router_if_present() { local root="$1" local router_dir="$root/nemoclaw-blueprint/router/llm-router" - command_exists pip3 || return 0 - [[ -d "$router_dir" ]] || return 0 + if ! command_exists pip3; then + is_routed_provider_requested && error "pip3 is required for routed inference." + return 0 + fi + if [[ ! -d "$router_dir" ]]; then + is_routed_provider_requested && error "llm-router is required for routed inference but is missing." + return 0 + fi if [[ ! -f "$router_dir/pyproject.toml" && ! -f "$router_dir/setup.py" ]]; then + is_routed_provider_requested && error "llm-router is required for routed inference but is not initialized." warn "Skipping model router install — llm-router submodule is not initialized." return 0 fi @@ -1355,8 +1368,10 @@ install_nemoclaw() { spin "Preparing OpenClaw package" bash -c "$(declare -f info warn resolve_openclaw_version pre_extract_openclaw); pre_extract_openclaw \"\$1\"" _ "$NEMOCLAW_SOURCE_ROOT" \ || warn "Pre-extraction failed — npm install may fail if openclaw tarball is broken" fi - spin "Initializing ${_CLI_DISPLAY} submodules" init_nemoclaw_submodules "$NEMOCLAW_SOURCE_ROOT" \ - || warn "Submodule initialization failed — model router support may be unavailable" + if ! spin "Initializing ${_CLI_DISPLAY} submodules" init_nemoclaw_submodules "$NEMOCLAW_SOURCE_ROOT"; then + is_routed_provider_requested && error "Failed to initialize the llm-router submodule required for routed inference." + warn "Submodule initialization failed — model router support may be unavailable" + fi spin "Installing ${_CLI_DISPLAY} dependencies" bash -c "cd \"$NEMOCLAW_SOURCE_ROOT\" && npm install --ignore-scripts" spin "Building ${_CLI_DISPLAY} CLI modules" bash -c "cd \"$NEMOCLAW_SOURCE_ROOT\" && npm run --if-present build:cli" spin "Building ${_CLI_DISPLAY} plugin" bash -c "cd \"$NEMOCLAW_SOURCE_ROOT\"/nemoclaw && npm install --ignore-scripts && npm run build" @@ -1379,8 +1394,10 @@ install_nemoclaw() { mkdir -p "$(dirname "$nemoclaw_src")" NEMOCLAW_SOURCE_ROOT="$nemoclaw_src" spin "Cloning ${_CLI_DISPLAY} source" clone_nemoclaw_ref "$release_ref" "$nemoclaw_src" - spin "Initializing ${_CLI_DISPLAY} submodules" init_nemoclaw_submodules "$nemoclaw_src" \ - || warn "Submodule initialization failed — model router support may be unavailable" + if ! spin "Initializing ${_CLI_DISPLAY} submodules" init_nemoclaw_submodules "$nemoclaw_src"; then + is_routed_provider_requested && error "Failed to initialize the llm-router submodule required for routed inference." + warn "Submodule initialization failed — model router support may be unavailable" + fi # Fetch version tags into the shallow clone so `git describe --tags # --match "v*"` works at runtime (the shallow clone only has the # single ref we asked for). diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 9247dd86ea9..b275876694b 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -832,6 +832,16 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise + new Promise((resolve) => { + http + .get(`http://127.0.0.1:${port}/health`, (res: import("node:http").IncomingMessage) => + resolve((res.statusCode || 0) >= 200 && (res.statusCode || 0) < 300), + ) + .on("error", () => resolve(false)); + }); + const proxyConfigResult = spawnSync( "model-router", ["proxy-config", "--config", poolConfigPath, "--output", litellmConfigPath], @@ -854,6 +864,12 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise { + childExited = true; + childExitDetail = `child exited with code ${code ?? "null"}${signal ? ` signal ${signal}` : ""}`; + }); - const http = require("http"); for (let attempt = 0; attempt < ROUTER_HEALTH_RETRIES; attempt++) { await new Promise((resolve) => setTimeout(resolve, ROUTER_HEALTH_INTERVAL_MS)); - const healthy = await new Promise((resolve) => { - http - .get(`http://127.0.0.1:${port}/health`, (res: import("node:http").IncomingMessage) => - resolve((res.statusCode || 0) >= 200 && (res.statusCode || 0) < 300), - ) - .on("error", () => resolve(false)); - }); - if (healthy) return pid; + if (childExited) break; + const healthy = await isRouterHealthy(); + let processAlive = true; + try { + process.kill(pid, 0); + } catch { + processAlive = false; + } + if (healthy && processAlive) return pid; + if (!processAlive) { + childExited = true; + if (!childExitDetail) childExitDetail = "child process is no longer running"; + break; + } } try { process.kill(pid, "SIGTERM"); @@ -895,7 +922,8 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise string; + providerNameToOptionKey: (name?: string | null) => string | null; parsePolicyPresetEnv: (value: string | null) => string[]; patchStagedDockerfile: ShimFn; pullAndResolveBaseImageDigest: () => { digest: string; ref: string } | null; @@ -199,6 +200,7 @@ const { configureWebSearch, isLoopbackHostname, normalizeProviderBaseUrl, + providerNameToOptionKey, parsePolicyPresetEnv, patchStagedDockerfile, pullAndResolveBaseImageDigest, @@ -839,6 +841,10 @@ describe("onboard helpers", () => { }); }); + it("maps persisted Model Router provider back to the routed provider option", () => { + assert.equal(providerNameToOptionKey("nvidia-router"), "routed"); + }); + it("leaves Kimi K2.6 compat to the model-specific setup registry", () => { assert.deepEqual( getSandboxInferenceConfig("moonshotai/kimi-k2.6", "nvidia-prod", "openai-completions"), From 56e68b6e99e172d8e46e5500a2e5492ee7913767 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 6 May 2026 12:41:19 -0700 Subject: [PATCH 05/10] fix(router): reconcile router during inference setup --- scripts/install.sh | 9 +++- src/lib/onboard-session.ts | 8 +++ src/lib/onboard.ts | 107 ++++++++++++++++++++++++++----------- test/onboard.test.ts | 32 +++++++++++ 4 files changed, 124 insertions(+), 32 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index e44b4d7b146..b49d95cbbfe 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1329,7 +1329,7 @@ init_nemoclaw_submodules() { is_routed_provider_requested() { local provider="${NEMOCLAW_PROVIDER:-}" - provider="${provider,,}" + provider="$(printf '%s' "$provider" | tr '[:upper:]' '[:lower:]')" [[ "$provider" == "routed" ]] } @@ -1350,7 +1350,12 @@ install_model_router_if_present() { warn "Skipping model router install — llm-router submodule is not initialized." return 0 fi - spin "Installing model router" pip3 install --quiet --user "${router_dir}[prefill,proxy]" + if ! spin "Installing model router" pip3 install --quiet --user "${router_dir}[prefill,proxy]"; then + if is_routed_provider_requested; then + error "pip3 install of llm-router failed" + fi + warn "Skipping model router install — pip3 install failed" + fi } install_nemoclaw() { diff --git a/src/lib/onboard-session.ts b/src/lib/onboard-session.ts index e1e9cecb232..afd9b6e2c81 100644 --- a/src/lib/onboard-session.ts +++ b/src/lib/onboard-session.ts @@ -74,6 +74,7 @@ export interface Session { credentialEnv: string | null; preferredInferenceApi: string | null; nimContainer: string | null; + routerPid: number | null; webSearchConfig: WebSearchConfig | null; policyPresets: string[] | null; messagingChannels: string[] | null; @@ -122,6 +123,7 @@ export interface SessionUpdates { credentialEnv?: string; preferredInferenceApi?: string; nimContainer?: string; + routerPid?: number; webSearchConfig?: WebSearchConfig | null; policyPresets?: string[]; messagingChannels?: string[]; @@ -189,6 +191,10 @@ function readString(value: SessionJsonValue | undefined): string | null { return typeof value === "string" ? value : null; } +function readPositiveInteger(value: SessionJsonValue | undefined): number | null { + return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null; +} + function readStringArray(value: SessionJsonValue | undefined): string[] | null { if (!Array.isArray(value)) return null; return value.filter((entry): entry is string => typeof entry === "string"); @@ -297,6 +303,7 @@ export function createSession(overrides: Partial = {}): Session { credentialEnv: overrides.credentialEnv ?? null, preferredInferenceApi: overrides.preferredInferenceApi ?? null, nimContainer: overrides.nimContainer ?? null, + routerPid: readPositiveInteger(overrides.routerPid), webSearchConfig: overrides.webSearchConfig?.fetchEnabled === true ? { fetchEnabled: true } : null, policyPresets: readStringArray(overrides.policyPresets), @@ -333,6 +340,7 @@ export function normalizeSession(data: Session | SessionJsonValue | undefined): credentialEnv: readString(data.credentialEnv), preferredInferenceApi: readString(data.preferredInferenceApi), nimContainer: readString(data.nimContainer), + routerPid: readPositiveInteger(data.routerPid), webSearchConfig: parseWebSearchConfig(data.webSearchConfig), policyPresets: readStringArray(data.policyPresets), messagingChannels: readStringArray(data.messagingChannels), diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index b275876694b..7dfac774645 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -814,6 +814,29 @@ function loadBlueprintProfile( const ROUTER_HEALTH_RETRIES = 15; const ROUTER_HEALTH_INTERVAL_MS = 2000; +const ROUTER_HEALTH_TIMEOUT_MS = 3000; + +async function isRouterHealthy(port: number, timeoutMs = ROUTER_HEALTH_TIMEOUT_MS): Promise { + const http = require("http"); + return new Promise((resolve) => { + let settled = false; + const settle = (healthy: boolean) => { + if (settled) return; + settled = true; + resolve(healthy); + }; + const request = http + .get(`http://127.0.0.1:${port}/health`, (res: import("node:http").IncomingMessage) => { + res.resume(); + settle((res.statusCode || 0) >= 200 && (res.statusCode || 0) < 300); + }) + .on("error", () => settle(false)); + request.setTimeout(timeoutMs, () => { + request.destroy(); + settle(false); + }); + }); +} /** * Start the model-router proxy and wait for it to become healthy. @@ -832,16 +855,6 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise - new Promise((resolve) => { - http - .get(`http://127.0.0.1:${port}/health`, (res: import("node:http").IncomingMessage) => - resolve((res.statusCode || 0) >= 200 && (res.statusCode || 0) < 300), - ) - .on("error", () => resolve(false)); - }); - const proxyConfigResult = spawnSync( "model-router", ["proxy-config", "--config", poolConfigPath, "--output", litellmConfigPath], @@ -864,7 +877,7 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise setTimeout(resolve, ROUTER_HEALTH_INTERVAL_MS)); if (childExited) break; - const healthy = await isRouterHealthy(); + const healthy = await isRouterHealthy(port); let processAlive = true; try { process.kill(pid, 0); @@ -927,6 +940,41 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise { + const bp = getRoutedProfile(); + const routerPort = bp.router.port || 4000; + const routerCredentialEnv = bp.router.credential_env || bp.credential_env || "NVIDIA_API_KEY"; + hydrateCredentialEnv(routerCredentialEnv); + + if (await isRouterHealthy(routerPort)) { + console.log(` ✓ Model router is already healthy on port ${routerPort}`); + return; + } + + console.log(" Starting model router..."); + const routerPid = await startModelRouter(bp.router); + console.log(` ✓ Model router started (PID ${routerPid}) on port ${routerPort}`); + onboardSession.updateSession((current: Session) => { + current.routerPid = routerPid; + return current; + }); +} + // ── Base image digest resolution ──────────────────────────────── // Pulls the sandbox-base image from GHCR and inspects it to get the // actual repo digest. This avoids the registry mismatch that broke @@ -6681,23 +6729,6 @@ async function setupNim( await ensureNamedCredential(routerCredentialEnv, "Model Router API key", null); } } - console.log(" Starting model router..."); - try { - const routerPid = await startModelRouter(bp.router); - const routerPort = bp.router.port || 4000; - console.log(` ✓ Model router started (PID ${routerPid}) on port ${routerPort}`); - onboardSession.updateSession((current) => { - (current as Session & { routerPid?: number }).routerPid = routerPid; - return current; - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - console.error(` ✗ Failed to start model router: ${message}`); - if (isNonInteractive()) process.exit(1); - console.log(" Returning to provider selection."); - console.log(""); - continue selectionLoop; - } provider = bp.provider_name || "nvidia-router"; model = bp.model; const { HOST_GATEWAY_URL } = require("./local-inference"); @@ -6958,9 +6989,15 @@ async function setupInference( // Do not mutate ~/.nemoclaw/credentials.json here: local Ollama now uses // OLLAMA_PROXY_CREDENTIAL_ENV, so any saved OPENAI_API_KEY remains available // to unrelated OpenAI-backed sandboxes. - } else if (provider === "nvidia-router") { + } else if (isRoutedInferenceProvider(provider)) { // Blueprint profile provider (e.g., nvidia-router for the routed profile). // Same pattern as vllm-local: upsert the provider and set the inference route. + try { + await reconcileModelRouter(); + } catch (err) { + console.error(` ✗ Failed to start model router: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } const resolvedCredentialEnv = credentialEnv || "NVIDIA_API_KEY"; const credentialValue = hydrateCredentialEnv(resolvedCredentialEnv); const env = credentialValue ? { [resolvedCredentialEnv]: credentialValue } : {}; @@ -9407,6 +9444,16 @@ async function onboard(opts: OnboardOptions = {}): Promise { const resumeInference = !forceProviderSelection && resume && isInferenceRouteReady(provider, model); if (resumeInference) { + if (isRoutedInferenceProvider(provider)) { + try { + await reconcileModelRouter(); + } catch (err) { + console.error( + ` ✗ Failed to reconcile model router: ${err instanceof Error ? err.message : String(err)}`, + ); + process.exit(1); + } + } skippedStepMessage("inference", `${provider} / ${model}`); if (nimContainer && sandboxName) { registry.updateSandbox(sandboxName, { nimContainer }); diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 32d7efc0b6f..19b1e8fbd07 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -149,6 +149,8 @@ function isOnboardTestInternals( typeof value.agentSupportsWebSearch === "function" && typeof value.configureWebSearch === "function" && typeof value.formatSandboxBuildEstimateNote === "function" && + Object.prototype.hasOwnProperty.call(value, "providerNameToOptionKey") && + typeof value.providerNameToOptionKey === "function" && typeof value.shouldRunCompatibleEndpointSandboxSmoke === "function" && typeof value.writeSandboxConfigSyncFile === "function" ); @@ -2435,6 +2437,36 @@ const { setupInference } = require(${onboardPath}); fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755, }); + fs.writeFileSync( + path.join(fakeBin, "model-router"), + [ + "#!/usr/bin/env node", + 'const fs = require("fs");', + 'const http = require("http");', + 'const path = require("path");', + "const args = process.argv.slice(2);", + 'if (args[0] === "proxy-config") {', + ' const output = args[args.indexOf("--output") + 1];', + " fs.mkdirSync(path.dirname(output), { recursive: true });", + ' fs.writeFileSync(output, "model_list: []\\n");', + " process.exit(0);", + "}", + 'if (args[0] === "proxy") {', + ' const port = Number(args[args.indexOf("--port") + 1] || "4000");', + " const server = http.createServer((req, res) => {", + ' if (req.url === "/health") { res.statusCode = 200; res.end("ok"); return; }', + " res.statusCode = 404;", + " res.end();", + " });", + ' server.listen(port, "127.0.0.1");', + " setTimeout(() => process.exit(0), 10000);", + "} else {", + " process.exit(1);", + "}", + "", + ].join("\n"), + { mode: 0o755 }, + ); const script = String.raw` const runner = require(${runnerPath}); From 0c26b52c797210f8b6e8a146c1d6710ea75ea6ea Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 6 May 2026 12:55:02 -0700 Subject: [PATCH 06/10] fix(router): validate credentials before reconcile --- src/lib/onboard-session.ts | 3 +++ src/lib/onboard.ts | 16 +++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/lib/onboard-session.ts b/src/lib/onboard-session.ts index afd9b6e2c81..1b5e93368a5 100644 --- a/src/lib/onboard-session.ts +++ b/src/lib/onboard-session.ts @@ -700,6 +700,9 @@ export function filterSafeUpdates(updates: SessionUpdates): Partial { if (typeof updates.preferredInferenceApi === "string") safe.preferredInferenceApi = updates.preferredInferenceApi; if (typeof updates.nimContainer === "string") safe.nimContainer = updates.nimContainer; + if (typeof updates.routerPid === "number" && Number.isInteger(updates.routerPid) && updates.routerPid > 0) { + safe.routerPid = updates.routerPid; + } if (isObject(updates.webSearchConfig) && updates.webSearchConfig.fetchEnabled === true) { safe.webSearchConfig = { fetchEnabled: true }; } else if (updates.webSearchConfig === null) { diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 7dfac774645..d0a83f91192 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -802,10 +802,10 @@ function loadBlueprintProfile( const parsed = YAML.parse(raw); const profile = parsed?.components?.inference?.profiles?.[profileName]; if (!profile) return null; - const router = { - ...(parsed?.components?.router || {}), - credential_env: profile.credential_env, - }; + const router = { ...(parsed?.components?.router || {}) }; + if (typeof profile.credential_env === "string" && profile.credential_env.trim().length > 0) { + router.credential_env = profile.credential_env; + } return { ...profile, router } as BlueprintInferenceProfile; } catch { return null; @@ -959,7 +959,13 @@ async function reconcileModelRouter(): Promise { const bp = getRoutedProfile(); const routerPort = bp.router.port || 4000; const routerCredentialEnv = bp.router.credential_env || bp.credential_env || "NVIDIA_API_KEY"; - hydrateCredentialEnv(routerCredentialEnv); + const routerCredential = + hydrateCredentialEnv(routerCredentialEnv) || + normalizeCredentialValue(process.env[routerCredentialEnv] || ""); + if (!routerCredential) { + throw new Error(`${routerCredentialEnv} is required to start Model Router.`); + } + process.env[routerCredentialEnv] = routerCredential; if (await isRouterHealthy(routerPort)) { console.log(` ✓ Model router is already healthy on port ${routerPort}`); From 89aebdadf225d893cde0c11117007742325c7da0 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 6 May 2026 13:10:03 -0700 Subject: [PATCH 07/10] fix(router): restart stale router credentials --- src/lib/onboard-session.ts | 7 ++++ src/lib/onboard.ts | 79 +++++++++++++++++++++++++++++++++----- 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/src/lib/onboard-session.ts b/src/lib/onboard-session.ts index 1b5e93368a5..f3fb36df9da 100644 --- a/src/lib/onboard-session.ts +++ b/src/lib/onboard-session.ts @@ -75,6 +75,7 @@ export interface Session { preferredInferenceApi: string | null; nimContainer: string | null; routerPid: number | null; + routerCredentialHash: string | null; webSearchConfig: WebSearchConfig | null; policyPresets: string[] | null; messagingChannels: string[] | null; @@ -124,6 +125,7 @@ export interface SessionUpdates { preferredInferenceApi?: string; nimContainer?: string; routerPid?: number; + routerCredentialHash?: string; webSearchConfig?: WebSearchConfig | null; policyPresets?: string[]; messagingChannels?: string[]; @@ -304,6 +306,7 @@ export function createSession(overrides: Partial = {}): Session { preferredInferenceApi: overrides.preferredInferenceApi ?? null, nimContainer: overrides.nimContainer ?? null, routerPid: readPositiveInteger(overrides.routerPid), + routerCredentialHash: overrides.routerCredentialHash ?? null, webSearchConfig: overrides.webSearchConfig?.fetchEnabled === true ? { fetchEnabled: true } : null, policyPresets: readStringArray(overrides.policyPresets), @@ -341,6 +344,7 @@ export function normalizeSession(data: Session | SessionJsonValue | undefined): preferredInferenceApi: readString(data.preferredInferenceApi), nimContainer: readString(data.nimContainer), routerPid: readPositiveInteger(data.routerPid), + routerCredentialHash: readString(data.routerCredentialHash), webSearchConfig: parseWebSearchConfig(data.webSearchConfig), policyPresets: readStringArray(data.policyPresets), messagingChannels: readStringArray(data.messagingChannels), @@ -703,6 +707,9 @@ export function filterSafeUpdates(updates: SessionUpdates): Partial { if (typeof updates.routerPid === "number" && Number.isInteger(updates.routerPid) && updates.routerPid > 0) { safe.routerPid = updates.routerPid; } + if (typeof updates.routerCredentialHash === "string") { + safe.routerCredentialHash = updates.routerCredentialHash; + } if (isObject(updates.webSearchConfig) && updates.webSearchConfig.fetchEnabled === true) { safe.webSearchConfig = { fetchEnabled: true }; } else if (updates.webSearchConfig === null) { diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index d0a83f91192..24b484d1d72 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -838,6 +838,37 @@ async function isRouterHealthy(port: number, timeoutMs = ROUTER_HEALTH_TIMEOUT_M }); } +function isProcessRunning(pid: number | null | undefined): boolean { + if (!Number.isInteger(pid) || Number(pid) <= 0) return false; + try { + process.kill(Number(pid), 0); + return true; + } catch { + return false; + } +} + +async function stopModelRouterProcess(pid: number, port: number): Promise { + try { + process.kill(pid, "SIGTERM"); + } catch { + return; + } + for (let attempt = 0; attempt < 10; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 500)); + if (!isProcessRunning(pid) && !(await isRouterHealthy(port, 1000))) return; + } + try { + process.kill(pid, "SIGKILL"); + } catch { + // already stopped + } + for (let attempt = 0; attempt < 5; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 500)); + if (!isProcessRunning(pid) && !(await isRouterHealthy(port, 1000))) return; + } +} + /** * Start the model-router proxy and wait for it to become healthy. * Follows the same pattern as Ollama startup (spawn detached, poll health). @@ -899,18 +930,27 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise { + childExited = true; + childExitDetail = `child failed to start: ${err.message}`; + }); child.once("exit", (code: number | null, signal: string | null) => { childExited = true; - childExitDetail = `child exited with code ${code ?? "null"}${signal ? ` signal ${signal}` : ""}`; + if (!childExitDetail) { + childExitDetail = `child exited with code ${code ?? "null"}${signal ? ` signal ${signal}` : ""}`; + } }); + child.unref(); + + const pid = child.pid; + if (!pid) { + throw new Error( + "Failed to start model-router proxy: no PID returned" + + (childExitDetail ? ` (${childExitDetail})` : ""), + ); + } for (let attempt = 0; attempt < ROUTER_HEALTH_RETRIES; attempt++) { await new Promise((resolve) => setTimeout(resolve, ROUTER_HEALTH_INTERVAL_MS)); @@ -966,10 +1006,28 @@ async function reconcileModelRouter(): Promise { throw new Error(`${routerCredentialEnv} is required to start Model Router.`); } process.env[routerCredentialEnv] = routerCredential; + const routerCredentialHash = hashCredential(routerCredential); + const session = onboardSession.loadSession(); + const recordedPid = session?.routerPid ?? null; + const recordedCredentialHash = session?.routerCredentialHash ?? null; if (await isRouterHealthy(routerPort)) { - console.log(` ✓ Model router is already healthy on port ${routerPort}`); - return; + if ( + routerCredentialHash && + recordedCredentialHash === routerCredentialHash && + isProcessRunning(recordedPid) + ) { + console.log(` ✓ Model router is already healthy on port ${routerPort}`); + return; + } + if (isProcessRunning(recordedPid)) { + console.log(" Restarting model router with updated credentials..."); + await stopModelRouterProcess(requireValue(recordedPid, "Expected recorded router PID"), routerPort); + } else { + throw new Error( + `Port ${routerPort} already has a healthy router endpoint, but its credential state is unknown. Stop the existing model-router process and rerun onboarding.`, + ); + } } console.log(" Starting model router..."); @@ -977,6 +1035,7 @@ async function reconcileModelRouter(): Promise { console.log(` ✓ Model router started (PID ${routerPid}) on port ${routerPort}`); onboardSession.updateSession((current: Session) => { current.routerPid = routerPid; + current.routerCredentialHash = routerCredentialHash; return current; }); } @@ -6711,7 +6770,7 @@ async function setupNim( if (isNonInteractive()) process.exit(1); continue selectionLoop; } - const routerCredentialEnv = bp.credential_env || "OPENAI_API_KEY"; + const routerCredentialEnv = bp.router?.credential_env || bp.credential_env || "OPENAI_API_KEY"; credentialEnv = routerCredentialEnv; const routedCredential = hydrateCredentialEnv(routerCredentialEnv) || From d776a064904eed251af807d7da6db6be9937676c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 6 May 2026 13:42:21 -0700 Subject: [PATCH 08/10] fix(router): align pool model identity --- nemoclaw-blueprint/router/pool-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nemoclaw-blueprint/router/pool-config.yaml b/nemoclaw-blueprint/router/pool-config.yaml index 39bdf67472b..231a85493a9 100644 --- a/nemoclaw-blueprint/router/pool-config.yaml +++ b/nemoclaw-blueprint/router/pool-config.yaml @@ -21,8 +21,8 @@ routing: encoder_backend: transformers models: - - name: gpt-5.2 - display_name: "GPT-5.2" + - name: nemotron-3-nano-reasoning + display_name: "Nemotron 3 Nano (Reasoning)" litellm_model: "openai/nvidia/nvidia/Nemotron-3-Nano-30B-A3B" cost_per_m_input_tokens: 0.05 cost_per_m_output_tokens: 0.20 From 8f735c6132c14512eeb88f13eca656f27f62244e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 6 May 2026 13:53:40 -0700 Subject: [PATCH 09/10] fix(router): use credential resolver for router keys --- src/lib/onboard.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 24b484d1d72..7c68d113ed7 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -900,8 +900,13 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise = {}; const credName = routerCfg.credential_env || "NVIDIA_API_KEY"; - if (process.env[credName]) credEnvVars[credName] = process.env[credName]; - if (process.env.OPENAI_API_KEY) credEnvVars.OPENAI_API_KEY = process.env.OPENAI_API_KEY; + const routedCredential = resolveProviderCredential(credName); + const openAiCredential = resolveProviderCredential("OPENAI_API_KEY"); + if (routedCredential) { + credEnvVars[credName] = routedCredential; + if (!openAiCredential) credEnvVars.OPENAI_API_KEY = routedCredential; + } + if (openAiCredential) credEnvVars.OPENAI_API_KEY = openAiCredential; const _providerKey = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); if (_providerKey) { if (!credEnvVars[credName]) credEnvVars[credName] = _providerKey; @@ -1001,11 +1006,11 @@ async function reconcileModelRouter(): Promise { const routerCredentialEnv = bp.router.credential_env || bp.credential_env || "NVIDIA_API_KEY"; const routerCredential = hydrateCredentialEnv(routerCredentialEnv) || - normalizeCredentialValue(process.env[routerCredentialEnv] || ""); + normalizeCredentialValue(bp.credential_default || ""); if (!routerCredential) { throw new Error(`${routerCredentialEnv} is required to start Model Router.`); } - process.env[routerCredentialEnv] = routerCredential; + saveCredential(routerCredentialEnv, routerCredential); const routerCredentialHash = hashCredential(routerCredential); const session = onboardSession.loadSession(); const recordedPid = session?.routerPid ?? null; @@ -6776,21 +6781,21 @@ async function setupNim( hydrateCredentialEnv(routerCredentialEnv) || normalizeCredentialValue(bp.credential_default || ""); if (routedCredential) { - process.env[routerCredentialEnv] = routedCredential; + saveCredential(routerCredentialEnv, routedCredential); } const _providerKeyHint = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); - if (_providerKeyHint && !process.env[routerCredentialEnv]) { - process.env[routerCredentialEnv] = _providerKeyHint; + if (_providerKeyHint && !resolveProviderCredential(routerCredentialEnv)) { + saveCredential(routerCredentialEnv, _providerKeyHint); } if (isNonInteractive()) { - if (!process.env[routerCredentialEnv]) { + if (!resolveProviderCredential(routerCredentialEnv)) { console.error( ` ${routerCredentialEnv} (or NEMOCLAW_PROVIDER_KEY) is required for Model Router in non-interactive mode.`, ); process.exit(1); } } else { - if (!process.env[routerCredentialEnv]) { + if (!resolveProviderCredential(routerCredentialEnv)) { await ensureNamedCredential(routerCredentialEnv, "Model Router API key", null); } } From 6dd22dc2e5ad25c60ebb34d8f675bcd0cacc57a6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 6 May 2026 14:04:35 -0700 Subject: [PATCH 10/10] test(router): use moved registry module path --- test/onboard.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/onboard.test.ts b/test/onboard.test.ts index fe96790d2c7..8e813a2a857 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -2431,7 +2431,7 @@ const { setupInference } = require(${onboardPath}); const scriptPath = path.join(tmpDir, "setup-router-check.js"); const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); - const registryPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "registry.js")); + const registryPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "state", "registry.js")); fs.mkdirSync(fakeBin, { recursive: true }); fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", {