diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 2b72110bd7c..837d4944296 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -196,6 +196,23 @@ reviews: gh workflow run nightly-e2e.yaml --ref -f jobs=cloud-e2e,sandbox-operations-e2e,rebuild-openclaw-e2e,messaging-compatible-endpoint-e2e,hermes-discord-e2e,hermes-slack-e2e,openshell-gateway-upgrade-e2e ``` + - path: "src/lib/actions/inference-set.ts" + instructions: | + This file switches the OpenShell inference route and patches the + selected running agent config. Changes here affect OpenClaw and + Hermes model switching without a sandbox rebuild. + + **E2E test recommendation:** + - `openclaw-inference-switch-e2e` — OpenClaw route + config patch + + live OpenClaw request after switch + - `hermes-inference-switch-e2e` — Hermes route + config.yaml patch + + live Hermes request after switch + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=openclaw-inference-switch-e2e,hermes-inference-switch-e2e + ``` + - path: "src/nemoclaw.ts" instructions: | This file contains CLI dispatch, status, recovery, and connect @@ -274,6 +291,8 @@ reviews: **E2E test recommendation:** - `hermes-e2e` — Hermes onboard + health probe + live inference + - `hermes-inference-switch-e2e` — Hermes route + config.yaml patch + + live Hermes request after switch - `hermes-discord-e2e` — Hermes Discord config schema + placeholder isolation - `hermes-slack-e2e` — Hermes Slack policy + Python placeholder egress @@ -282,7 +301,7 @@ reviews: To run selectively: ``` - gh workflow run nightly-e2e.yaml --ref -f jobs=hermes-e2e,hermes-discord-e2e,hermes-slack-e2e,rebuild-hermes-e2e,rebuild-hermes-stale-base-e2e + gh workflow run nightly-e2e.yaml --ref -f jobs=hermes-e2e,hermes-inference-switch-e2e,hermes-discord-e2e,hermes-slack-e2e,rebuild-hermes-e2e,rebuild-hermes-stale-base-e2e ``` - path: "test/e2e/test-hermes-slack-e2e.sh" @@ -357,6 +376,38 @@ reviews: gh workflow run nightly-e2e.yaml --ref -f jobs=cloud-inference-e2e ``` + - path: "test/e2e/test-openclaw-inference-switch.sh" + instructions: | + This script validates OpenClaw model/provider switching with + `nemoclaw inference set`, including OpenShell route state, + /sandbox/.openclaw/openclaw.json, config hash recomputation, and + live requests after the switch. + + **E2E test recommendation:** + - `openclaw-inference-switch-e2e` — OpenClaw route + config patch + + live OpenClaw request after switch + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=openclaw-inference-switch-e2e + ``` + + - path: "test/e2e/test-hermes-inference-switch.sh" + instructions: | + This script validates Hermes model/provider switching with + `nemohermes inference set`, including OpenShell route state, + /sandbox/.hermes/config.yaml, config hash recomputation, .env + preservation, and live requests after the switch. + + **E2E test recommendation:** + - `hermes-inference-switch-e2e` — Hermes route + config.yaml patch + + live Hermes request after switch + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=hermes-inference-switch-e2e + ``` + - path: "test/e2e/test-skill-agent-e2e.sh" instructions: &e2e-skill-agent | This script tests skill injection into the sandbox and verifies diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index f5516028c75..3bae7710ee3 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -24,10 +24,16 @@ # processes are restarted after an OpenShell upgrade. # hermes-e2e Hermes Agent E2E — install → onboard --agent hermes → health # probe → live inference. Validates the multi-agent architecture. +# hermes-inference-switch-e2e +# Switches a running Hermes sandbox with `nemohermes inference set` +# and verifies route, config.yaml, hashes, and live requests. # hermes-discord-e2e Hermes Discord onboarding — validates the top-level Hermes # Discord schema plus OpenShell placeholder/token isolation. # hermes-slack-e2e Hermes Slack onboarding — validates the Hermes Slack policy, # Slack providers, and Python placeholder egress path. +# openclaw-inference-switch-e2e +# Switches a running OpenClaw sandbox with `nemoclaw inference set` +# and verifies route, openclaw.json, hashes, and live requests. # credential-migration-e2e Validates legacy ~/.nemoclaw/credentials.json migration to the # OpenShell gateway, secure zero-fill on unlink, allowlist filter # on non-credential env keys, and symlink-safe deletion. @@ -62,8 +68,10 @@ on: kimi-inference-compat-e2e, token-rotation-e2e, sandbox-survival-e2e, openshell-gateway-upgrade-e2e, - issue-2478-crash-loop-recovery-e2e, hermes-e2e, hermes-discord-e2e, + issue-2478-crash-loop-recovery-e2e, hermes-e2e, + hermes-inference-switch-e2e, hermes-discord-e2e, hermes-slack-e2e, sandbox-operations-e2e, inference-routing-e2e, + openclaw-inference-switch-e2e, network-policy-e2e, deployment-services-e2e, diagnostics-e2e, credential-migration-e2e, snapshot-commands-e2e, shields-config-e2e, rebuild-openclaw-e2e, @@ -511,6 +519,41 @@ jobs: path: /tmp/nemoclaw-e2e-hermes-install.log if-no-files-found: ignore + # ── Hermes inference switch E2E ───────────────────────────────── + # Validates `nemohermes inference set` against a running Hermes sandbox: + # OpenShell route, config.yaml patch, config hashes, no automatic restart, + # and live requests after the switch. + hermes-inference-switch-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-inference-switch-e2e,')) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Run Hermes inference switch E2E test + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-hermes-inference-switch" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_AGENT: "hermes" + GITHUB_TOKEN: ${{ github.token }} + run: bash test/e2e/test-hermes-inference-switch.sh + + - name: Upload install log on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: hermes-inference-switch-install-log + path: /tmp/nemoclaw-e2e-hermes-inference-switch-install.log + if-no-files-found: ignore + # ── Hermes Discord E2E ─────────────────────────────────────── # Validates Hermes onboarding with Discord enabled. Proves the Hermes # sandbox gets top-level discord: config, never platforms.discord, and only @@ -867,6 +910,40 @@ jobs: path: test-inference-routing-*.log if-no-files-found: ignore + # ── OpenClaw inference switch E2E ─────────────────────────────── + # Validates `nemoclaw inference set` against a running OpenClaw sandbox: + # OpenShell route, openclaw.json patch, config hash, no automatic restart, + # and live requests after the switch. + openclaw-inference-switch-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',openclaw-inference-switch-e2e,')) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Run OpenClaw inference switch E2E test + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-openclaw-inference-switch" + NEMOCLAW_RECREATE_SANDBOX: "1" + GITHUB_TOKEN: ${{ github.token }} + run: bash test/e2e/test-openclaw-inference-switch.sh + + - name: Upload install log on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: openclaw-inference-switch-install-log + path: /tmp/nemoclaw-e2e-openclaw-inference-switch-install.log + if-no-files-found: ignore + # ── Network policy E2E ─────────────────────────────────────── # TC-NET-01..07, TC-NET-09: deny-by-default, whitelist, live policy-add, # dry-run, hot-reload, inference exemption, permissive mode, SSRF validation. @@ -1724,10 +1801,12 @@ jobs: sandbox-survival-e2e, issue-2478-crash-loop-recovery-e2e, hermes-e2e, + hermes-inference-switch-e2e, hermes-discord-e2e, hermes-slack-e2e, sandbox-operations-e2e, inference-routing-e2e, + openclaw-inference-switch-e2e, network-policy-e2e, deployment-services-e2e, diagnostics-e2e, @@ -1812,10 +1891,12 @@ jobs: sandbox-survival-e2e, issue-2478-crash-loop-recovery-e2e, hermes-e2e, + hermes-inference-switch-e2e, hermes-discord-e2e, hermes-slack-e2e, sandbox-operations-e2e, inference-routing-e2e, + openclaw-inference-switch-e2e, network-policy-e2e, deployment-services-e2e, diagnostics-e2e, @@ -1948,10 +2029,12 @@ jobs: sandbox-survival-e2e, issue-2478-crash-loop-recovery-e2e, hermes-e2e, + hermes-inference-switch-e2e, hermes-discord-e2e, hermes-slack-e2e, sandbox-operations-e2e, inference-routing-e2e, + openclaw-inference-switch-e2e, network-policy-e2e, deployment-services-e2e, diagnostics-e2e, diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index f26eb8b5d63..bdbd8150be2 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -115,6 +115,12 @@ "endpoint_type": "OpenAI-compatible", "notes": "Uses Google's OpenAI-compatible endpoint" }, + { + "name": "Hermes Provider", + "status": "hermes only", + "endpoint_type": "OpenAI-compatible route", + "notes": "Available when onboarding Hermes Agent through `nemohermes`" + }, { "name": "Local Ollama", "status": "caveated", diff --git a/docs/get-started/quickstart-hermes.md b/docs/get-started/quickstart-hermes.md index 7e4f074998f..a33fdc142c4 100644 --- a/docs/get-started/quickstart-hermes.md +++ b/docs/get-started/quickstart-hermes.md @@ -160,10 +160,11 @@ $ nemohermes my-hermes snapshot create --name before-change $ nemohermes my-hermes rebuild ``` -To change the active model or provider without rebuilding the sandbox, use the OpenShell inference route. +To change the active model or provider without rebuilding the sandbox, use `nemohermes inference set`. +It updates the OpenShell inference route and patches `/sandbox/.hermes/config.yaml` without restarting Hermes. ```console -$ openshell inference set -g nemoclaw --model --provider +$ nemohermes inference set --model --provider ``` To remove the sandbox when you are done, destroy it explicitly. diff --git a/docs/inference/inference-options.md b/docs/inference/inference-options.md index 9fdf757acca..c939db0a352 100644 --- a/docs/inference/inference-options.md +++ b/docs/inference/inference-options.md @@ -48,6 +48,7 @@ NemoClaw uses provider-specific local tokens for those routes, and rebuilds of l | Anthropic | Tested | Native Anthropic | Uses anthropic-messages | | Other Anthropic-compatible endpoint | Tested | Custom Anthropic-compatible | For Claude proxies and compatible gateways | | Google Gemini | Tested | OpenAI-compatible | Uses Google's OpenAI-compatible endpoint | +| Hermes Provider | Hermes only | OpenAI-compatible route | Available when onboarding Hermes Agent through `nemohermes` | | Local Ollama | Caveated | Local Ollama API | Available when Ollama is installed or running on the host | | Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable GPU | | Local vLLM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a server already running on `localhost:8000` | @@ -68,6 +69,7 @@ Experimental local vLLM appears when you opt in and NemoClaw detects either a ru | Anthropic | Routes to the Anthropic Messages API. Set `ANTHROPIC_API_KEY`. | `claude-sonnet-4-6`, `claude-haiku-4-5`, `claude-opus-4-6` | | Other Anthropic-compatible endpoint | Routes to any server that implements the Anthropic Messages API (`/v1/messages`). The wizard prompts for a base URL and model name. Set `COMPATIBLE_ANTHROPIC_API_KEY`. | You provide the model name. | | Google Gemini | Routes to Google's OpenAI-compatible endpoint. NemoClaw prefers `/responses` only when the endpoint proves it can handle tool calling in a way OpenClaw uses; otherwise it falls back to `/chat/completions`. Set `GEMINI_API_KEY`. | `gemini-3.1-pro-preview`, `gemini-3.1-flash-lite-preview`, `gemini-3-flash-preview`, `gemini-2.5-pro`, `gemini-2.5-flash`, `gemini-2.5-flash-lite` | +| Hermes Provider | Routes Hermes Agent through the host OpenShell provider registered by NemoClaw when onboarding Hermes Agent. | Curated Hermes Provider models such as `moonshotai/kimi-k2.6`, `openai/gpt-5.4-mini`, and `z-ai/glm-5.1`. | | Local Ollama | Routes to a local Ollama instance on `localhost:11434`. NemoClaw detects installed models, offers starter models if none are present, pulls and warms the selected model, and validates it. | Selected during onboarding. For more information, refer to [Use a Local Inference Server](use-local-inference.md). | | Model Router | Starts a host-side router on port `4000`, registers it as an OpenAI-compatible provider, and keeps the sandbox pointed at `inference.local`. Set `NEMOCLAW_PROVIDER=routed` for non-interactive setup. | The router pool defines the model names. | diff --git a/docs/inference/switch-inference-providers.md b/docs/inference/switch-inference-providers.md index 4797dce72f0..96e7b3af14b 100644 --- a/docs/inference/switch-inference-providers.md +++ b/docs/inference/switch-inference-providers.md @@ -35,9 +35,12 @@ No restart is required. ## Switch to a Different Model Use `nemoclaw inference set` with the provider and model that match the upstream you want to use. -The command updates the OpenShell inference route and synchronizes the running OpenClaw config so `agents.defaults.model.primary` continues to match the routed model. +The command updates the OpenShell inference route and synchronizes the running agent config. +For OpenClaw, it updates `agents.defaults.model.primary` and the matching provider namespace. +For Hermes, it updates `/sandbox/.hermes/config.yaml` (`model.default`, `model.base_url`, and `model.provider: custom`) without rebuilding or restarting Hermes. Pass `--sandbox ` when you do not want to use the default registered sandbox. +Under `nemohermes`, pass `--sandbox ` when more than one Hermes sandbox is registered. ### NVIDIA Endpoints @@ -75,6 +78,14 @@ $ nemoclaw inference set --provider compatible-endpoint --model $ nemoclaw inference set --provider compatible-anthropic-endpoint --model ``` +### Hermes Provider + +For a NemoClaw-managed Hermes sandbox, use the Hermes alias with the registered Hermes Provider route: + +```console +$ nemohermes inference set --provider hermes-provider --model openai/gpt-5.4-mini +``` + #### Switching from Responses API to Chat Completions If onboarding selected `/v1/responses` but the agent fails at runtime (for @@ -188,7 +199,7 @@ The output includes the active provider, model, and endpoint. - The host keeps provider credentials. - The sandbox continues to use `inference.local`. -- `nemoclaw inference set` patches the selected running OpenClaw sandbox config and recomputes its config hash. +- `nemoclaw inference set` patches the selected running OpenClaw or Hermes sandbox config and recomputes its config hash. - Use `nemoclaw onboard --resume --recreate-sandbox` for build-time settings such as context window, max tokens, reasoning mode, heartbeat cadence, or image contents. - Local Ollama and local vLLM routes use local provider tokens rather than `OPENAI_API_KEY`. Rebuilds of older local-inference sandboxes clear the stale OpenAI credential requirement automatically. diff --git a/docs/reference/cli-selection-guide.md b/docs/reference/cli-selection-guide.md index b3c3edaea0d..8886c082e2b 100644 --- a/docs/reference/cli-selection-guide.md +++ b/docs/reference/cli-selection-guide.md @@ -181,13 +181,19 @@ Approved endpoints are session-scoped unless you also add them to the policy thr ### Change Models or Providers -Use the NemoClaw command for model or provider switches so the OpenShell route and the running OpenClaw config stay consistent: +Use the NemoClaw command for model or provider switches so the OpenShell route and the running agent config stay consistent: ```console $ nemoclaw inference set --provider nvidia-prod --model nvidia/nemotron-3-super-120b-a12b ``` -For a build-time OpenClaw setting change, rerun onboarding so the sandbox configuration is recreated consistently: +For Hermes sandboxes, use the alias; it updates the route and `/sandbox/.hermes/config.yaml` without a rebuild or restart: + +```console +$ nemohermes inference set --provider hermes-provider --model openai/gpt-5.4-mini +``` + +For a build-time agent setting change, rerun onboarding so the sandbox configuration is recreated consistently: ```console $ nemoclaw onboard --resume --recreate-sandbox diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 734c65a5e4b..46c116ab0ba 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -869,17 +869,18 @@ $ nemoclaw status --json ### `nemoclaw inference set` -Switch the active inference provider or model for a NemoClaw-managed OpenClaw sandbox. -The command updates the OpenShell gateway route, patches the selected running OpenClaw config so the agent's primary model identity matches the route, recomputes the config hash, and updates the NemoClaw registry. +Switch the active inference provider or model for a NemoClaw-managed OpenClaw or Hermes sandbox. +The command updates the OpenShell gateway route, patches the selected running agent config so it matches the route, recomputes the config hash, and updates the NemoClaw registry. +For Hermes, the patch updates `/sandbox/.hermes/config.yaml` (`model.default`, `model.base_url`, and `model.provider: custom`) and does not rebuild or restart the gateway. By default, the command syncs the default registered sandbox. -Pass `--sandbox ` to target a different OpenClaw sandbox. +Under the `nemohermes` alias, it uses the registered Hermes sandbox when exactly one exists; otherwise pass `--sandbox ` to target one explicitly. ```console $ nemoclaw inference set --provider --model [--sandbox ] [--no-verify] ``` -Supported provider names are `nvidia-prod`, `nvidia-nim`, `nvidia-router`, `openai-api`, `anthropic-prod`, `compatible-anthropic-endpoint`, `gemini-api`, `compatible-endpoint`, `ollama-local`, and `vllm-local`. +Supported provider names are `nvidia-prod`, `nvidia-nim`, `nvidia-router`, `openai-api`, `anthropic-prod`, `compatible-anthropic-endpoint`, `gemini-api`, `compatible-endpoint`, `hermes-provider`, `ollama-local`, and `vllm-local`. Use `--no-verify` only when OpenShell cannot verify the provider at switch time but you have already confirmed the provider and credential. ### `nemoclaw setup` diff --git a/src/commands/inference/set.ts b/src/commands/inference/set.ts index d13b8700c6f..9f84bb3b9de 100644 --- a/src/commands/inference/set.ts +++ b/src/commands/inference/set.ts @@ -7,7 +7,7 @@ import { withCommandDisplay } from "../../lib/cli/command-display"; export default withCommandDisplay(Command, [ { usage: "nemoclaw inference set", - description: "Switch inference and sync OpenClaw model identity", + description: "Switch inference and sync the running agent config", flags: "--provider --model [--sandbox ] [--no-verify]", group: "Services", scope: "global", diff --git a/src/lib/actions/inference-set.test.ts b/src/lib/actions/inference-set.test.ts index dd398d022ea..1cc39e5ff4b 100644 --- a/src/lib/actions/inference-set.test.ts +++ b/src/lib/actions/inference-set.test.ts @@ -29,6 +29,7 @@ vi.mock("../shields/audit", () => ({ import { type InferenceSetDeps, + patchHermesInferenceConfig, patchOpenClawInferenceConfig, runInferenceSet, } from "./inference-set"; @@ -42,6 +43,15 @@ const OPENCLAW_TARGET: AgentConfigTarget = { sensitiveFiles: ["/sandbox/.openclaw/.config-hash"], }; +const HERMES_TARGET: AgentConfigTarget = { + agentName: "hermes", + configPath: "/sandbox/.hermes/config.yaml", + configDir: "/sandbox/.hermes", + format: "yaml", + configFile: "config.yaml", + sensitiveFiles: ["/sandbox/.hermes/.config-hash", "/sandbox/.hermes/.env"], +}; + function baseSession(overrides: Partial = {}): Session { return { version: 1, @@ -81,6 +91,10 @@ function baseSession(overrides: Partial = {}): Session { function createDeps(options: { config: ConfigObject; entry?: SandboxEntry | null; + entries?: SandboxEntry[]; + defaultSandbox?: string | null; + requestedAgent?: string | null; + target?: AgentConfigTarget; session?: Session | null; openshellStatus?: number; }): InferenceSetDeps & { @@ -96,6 +110,13 @@ function createDeps(options: { getSession: () => Session | null; } { let session = options.session ?? null; + const entries = options.entries ?? [options.entry ?? { name: "alpha", agent: null }]; + const sandboxes = entries.reduce>((acc, entry) => { + acc[entry.name] = entry; + return acc; + }, {}); + const defaultSandbox = + options.defaultSandbox === undefined ? entries[0]?.name ?? null : options.defaultSandbox; const calls = { runOpenshell: vi.fn(() => ({ status: options.openshellStatus ?? 0, stdout: "", stderr: "" })), writeSandboxConfig: vi.fn(), @@ -110,12 +131,14 @@ function createDeps(options: { log: vi.fn(), }; return { - getDefaultSandbox: () => "alpha", - getSandbox: () => options.entry ?? { name: "alpha", agent: null }, + getDefaultSandbox: () => defaultSandbox, + getSandbox: (name: string) => sandboxes[name] ?? null, + listSandboxes: () => ({ sandboxes: entries, defaultSandbox }), updateSandbox: calls.updateSandbox, + getRequestedAgent: () => options.requestedAgent, loadSession: () => session, updateSession: calls.updateSession, - resolveAgentConfig: () => OPENCLAW_TARGET, + resolveAgentConfig: () => options.target ?? OPENCLAW_TARGET, readSandboxConfig: () => options.config, writeSandboxConfig: calls.writeSandboxConfig, recomputeSandboxConfigHash: calls.recomputeSandboxConfigHash, @@ -227,6 +250,45 @@ describe("patchOpenClawInferenceConfig", () => { }); }); +describe("patchHermesInferenceConfig", () => { + it("updates only the Hermes model block for the selected route", () => { + const config: ConfigObject = { + model: { + default: "moonshotai/kimi-k2.6", + provider: "custom", + base_url: "https://old.example/v1", + temperature: 0.2, + }, + models: { + providers: { + inference: { + baseUrl: "https://should-not-change.example/v1", + }, + }, + }, + terminal: { backend: "local" }, + }; + + const result = patchHermesInferenceConfig(config, "hermes-provider", "openai/gpt-5.4-mini"); + + expect(result.changed).toBe(true); + expect(config.model).toEqual({ + default: "openai/gpt-5.4-mini", + provider: "custom", + base_url: "https://inference.local/v1", + temperature: 0.2, + }); + expect(config.models).toEqual({ + providers: { + inference: { + baseUrl: "https://should-not-change.example/v1", + }, + }, + }); + expect(config.terminal).toEqual({ backend: "local" }); + }); +}); + describe("runInferenceSet", () => { it("updates OpenShell, OpenClaw config, registry, and the matching onboard session", async () => { const config: ConfigObject = { @@ -296,15 +358,150 @@ describe("runInferenceSet", () => { }); }); - it("refuses non-OpenClaw sandboxes before changing OpenShell inference", async () => { + it("updates OpenShell, Hermes config.yaml, registry, and the matching onboard session", async () => { + const config: ConfigObject = { + model: { + default: "moonshotai/kimi-k2.6", + provider: "custom", + base_url: "https://inference.local/v1", + }, + terminal: { backend: "local" }, + }; + const deps = createDeps({ + config, + entry: { + name: "hermes", + agent: "hermes", + provider: "hermes-provider", + model: "moonshotai/kimi-k2.6", + }, + defaultSandbox: "hermes", + target: HERMES_TARGET, + session: baseSession({ agent: "hermes", sandboxName: "hermes" }), + }); + + const result = await runInferenceSet( + { + provider: "hermes-provider", + model: "openai/gpt-5.4-mini", + sandboxName: "hermes", + noVerify: true, + }, + deps, + ); + + expect(deps.calls.runOpenshell).toHaveBeenCalledWith( + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "hermes-provider", + "--model", + "openai/gpt-5.4-mini", + "--no-verify", + ], + { ignoreError: true }, + ); + expect(config).toEqual({ + model: { + default: "openai/gpt-5.4-mini", + provider: "custom", + base_url: "https://inference.local/v1", + }, + terminal: { backend: "local" }, + }); + expect(deps.calls.writeSandboxConfig).toHaveBeenCalledTimes(1); + expect(deps.calls.writeSandboxConfig).toHaveBeenCalledWith("hermes", HERMES_TARGET, config); + expect(deps.calls.writeSandboxConfig.mock.calls[0][1].configPath).toBe( + "/sandbox/.hermes/config.yaml", + ); + expect(deps.calls.recomputeSandboxConfigHash).toHaveBeenCalledWith("hermes", HERMES_TARGET); + expect(deps.calls.updateSandbox).toHaveBeenCalledWith("hermes", { + provider: "hermes-provider", + model: "openai/gpt-5.4-mini", + }); + expect(deps.getSession()).toMatchObject({ + provider: "hermes-provider", + model: "openai/gpt-5.4-mini", + endpointUrl: "https://inference.local/v1", + }); + expect(deps.calls.appendAuditEntry).toHaveBeenCalledWith( + expect.objectContaining({ + action: "shields_down", + sandbox: "hermes", + reason: "inference set hermes:hermes-provider:openai/gpt-5.4-mini", + }), + ); + expect(result).toMatchObject({ + sandboxName: "hermes", + provider: "hermes-provider", + model: "openai/gpt-5.4-mini", + primaryModelRef: "inference/openai/gpt-5.4-mini", + providerKey: "inference", + configChanged: true, + sessionUpdated: true, + }); + }); + + it("uses the unambiguous registered Hermes sandbox under the nemohermes alias", async () => { + const config: ConfigObject = { model: {} }; + const deps = createDeps({ + config, + entries: [ + { name: "alpha", agent: "openclaw" }, + { name: "hermes-one", agent: "hermes" }, + ], + defaultSandbox: "alpha", + requestedAgent: "hermes", + target: HERMES_TARGET, + }); + + await runInferenceSet({ provider: "hermes-provider", model: "z-ai/glm-5.1" }, deps); + + expect(deps.calls.writeSandboxConfig).toHaveBeenCalledWith( + "hermes-one", + HERMES_TARGET, + config, + ); + expect(deps.calls.updateSandbox).toHaveBeenCalledWith("hermes-one", { + provider: "hermes-provider", + model: "z-ai/glm-5.1", + }); + }); + + it("requires --sandbox when the nemohermes alias cannot choose one Hermes sandbox", async () => { const deps = createDeps({ config: {}, - entry: { name: "hermes", agent: "hermes" }, + entries: [ + { name: "hermes-one", agent: "hermes" }, + { name: "hermes-two", agent: "hermes" }, + ], + requestedAgent: "hermes", + target: HERMES_TARGET, }); await expect( - runInferenceSet({ provider: "nvidia-prod", model: "nvidia/model-a" }, deps), - ).rejects.toThrow(/currently supports OpenClaw/); + runInferenceSet({ provider: "hermes-provider", model: "z-ai/glm-5.1" }, deps), + ).rejects.toThrow(/Pass --sandbox /); + + expect(deps.calls.runOpenshell).not.toHaveBeenCalled(); + expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); + }); + + it("refuses unsupported agent sandboxes before changing OpenShell inference", async () => { + const deps = createDeps({ + config: {}, + entry: { name: "spark", agent: "spark" }, + }); + + await expect( + runInferenceSet( + { provider: "nvidia-prod", model: "nvidia/model-a", sandboxName: "spark" }, + deps, + ), + ).rejects.toThrow(/supports OpenClaw and Hermes/); expect(deps.calls.runOpenshell).not.toHaveBeenCalled(); expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index 3738d9917d3..33408487912 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -46,7 +46,9 @@ type OpenshellRunResult = Pick, "status" | "stdout" | " export interface InferenceSetDeps { getDefaultSandbox: () => string | null; getSandbox: (name: string) => SandboxEntry | null; + listSandboxes: () => { sandboxes: SandboxEntry[]; defaultSandbox: string | null }; updateSandbox: (name: string, updates: Partial) => boolean; + getRequestedAgent: () => string | null | undefined; loadSession: () => onboardSession.Session | null; updateSession: ( mutator: (session: onboardSession.Session) => onboardSession.Session | void, @@ -83,6 +85,7 @@ const SUPPORTED_PROVIDER_NAMES = [ "compatible-anthropic-endpoint", "gemini-api", "compatible-endpoint", + "hermes-provider", "ollama-local", "vllm-local", ] as const; @@ -91,7 +94,9 @@ function defaultDeps(): InferenceSetDeps { return { getDefaultSandbox: registry.getDefault, getSandbox: registry.getSandbox, + listSandboxes: registry.listSandboxes, updateSandbox: registry.updateSandbox, + getRequestedAgent: () => process.env.NEMOCLAW_AGENT, loadSession: onboardSession.loadSession, updateSession: onboardSession.updateSession, resolveAgentConfig, @@ -120,14 +125,52 @@ function assertSupportedProvider(provider: string, model: string): void { function normalizeSandboxAgent(agentName: string | null | undefined): string { const trimmed = typeof agentName === "string" ? agentName.trim() : ""; - return trimmed || "openclaw"; + return (trimmed || "openclaw").toLowerCase(); } function resolveTargetSandbox( sandboxName: string | null | undefined, - deps: Pick, -): { sandboxName: string; entry: SandboxEntry } { - const targetName = sandboxName?.trim() || deps.getDefaultSandbox(); + deps: Pick< + InferenceSetDeps, + "getDefaultSandbox" | "getSandbox" | "listSandboxes" | "getRequestedAgent" + >, +): { sandboxName: string; entry: SandboxEntry; agentName: string } { + const explicitName = sandboxName?.trim(); + if (explicitName) { + const entry = deps.getSandbox(explicitName); + if (!entry) { + throw new InferenceSetError(`Sandbox '${explicitName}' is not registered.`, 2); + } + return { + sandboxName: explicitName, + entry, + agentName: normalizeSandboxAgent(entry.agent), + }; + } + + if (normalizeSandboxAgent(deps.getRequestedAgent()) === "hermes") { + const hermesSandboxes = deps + .listSandboxes() + .sandboxes.filter((entry) => normalizeSandboxAgent(entry.agent) === "hermes"); + if (hermesSandboxes.length === 1) { + const entry = hermesSandboxes[0]; + return { sandboxName: entry.name, entry, agentName: "hermes" }; + } + if (hermesSandboxes.length === 0) { + throw new InferenceSetError( + "No registered Hermes sandbox found. Pass --sandbox to target a sandbox explicitly.", + 2, + ); + } + throw new InferenceSetError( + `Multiple Hermes sandboxes are registered (${hermesSandboxes + .map((entry) => entry.name) + .join(", ")}). Pass --sandbox to choose one.`, + 2, + ); + } + + const targetName = deps.getDefaultSandbox(); if (!targetName) { throw new InferenceSetError( "No sandbox selected. Pass --sandbox or create a sandbox with nemoclaw onboard.", @@ -139,13 +182,7 @@ function resolveTargetSandbox( if (!entry) { throw new InferenceSetError(`Sandbox '${targetName}' is not registered.`, 2); } - if (normalizeSandboxAgent(entry.agent) !== "openclaw") { - throw new InferenceSetError( - `nemoclaw inference set currently supports OpenClaw sandboxes; '${targetName}' uses '${entry.agent}'.`, - 2, - ); - } - return { sandboxName: targetName, entry }; + return { sandboxName: targetName, entry, agentName: normalizeSandboxAgent(entry.agent) }; } function ensureObject(record: ConfigObject, key: string): ConfigObject { @@ -220,6 +257,21 @@ export function patchOpenClawInferenceConfig( return { changed: before !== JSON.stringify(config), route }; } +export function patchHermesInferenceConfig( + config: ConfigObject, + provider: string, + model: string, +): { changed: boolean; route: SandboxInferenceConfig } { + const before = JSON.stringify(config); + const route = getSandboxInferenceConfig(model, provider); + const modelConfig = ensureObject(config, "model"); + modelConfig.default = model; + modelConfig.base_url = route.inferenceBaseUrl; + modelConfig.provider = "custom"; + + return { changed: before !== JSON.stringify(config), route }; +} + function updateMatchingOnboardSession( sandboxName: string, provider: string, @@ -282,27 +334,29 @@ export async function runInferenceSet( ); } - const { sandboxName } = resolveTargetSandbox(options.sandboxName, deps); + const { sandboxName, agentName } = resolveTargetSandbox(options.sandboxName, deps); + if (agentName !== "openclaw" && agentName !== "hermes") { + throw new InferenceSetError( + `nemoclaw inference set supports OpenClaw and Hermes sandboxes; '${sandboxName}' uses '${agentName}'.`, + 2, + ); + } const target = deps.resolveAgentConfig(sandboxName); - if (target.agentName !== "openclaw") { + const targetAgent = normalizeSandboxAgent(target.agentName); + if (targetAgent !== agentName) { throw new InferenceSetError( - `nemoclaw inference set currently supports OpenClaw configs; '${sandboxName}' uses '${target.agentName}'.`, + `Sandbox '${sandboxName}' is registered as '${agentName}' but resolved config for '${target.agentName}'.`, 2, ); } - const config = deps.readSandboxConfig(sandboxName, target); - const patched = patchOpenClawInferenceConfig( - config, - provider, - model, - getPreferredInferenceApi(config), - ); - deps.log(` Setting OpenShell inference route: ${provider} / ${model}`); - const setResult = deps.runOpenshell(openshellInferenceSetArgs({ provider, model, noVerify: options.noVerify }), { - ignoreError: true, - }); + const setResult = deps.runOpenshell( + openshellInferenceSetArgs({ provider, model, noVerify: options.noVerify }), + { + ignoreError: true, + }, + ); if (setResult.status !== 0) { throw new InferenceSetError( `OpenShell inference route update failed with exit ${setResult.status ?? 1}.`, @@ -310,7 +364,17 @@ export async function runInferenceSet( ); } - deps.log(` Syncing OpenClaw model identity in sandbox '${sandboxName}'...`); + const config = deps.readSandboxConfig(sandboxName, target); + const patched = + agentName === "hermes" + ? patchHermesInferenceConfig(config, provider, model) + : patchOpenClawInferenceConfig(config, provider, model, getPreferredInferenceApi(config)); + + deps.log( + agentName === "hermes" + ? ` Syncing Hermes model route in sandbox '${sandboxName}'...` + : ` Syncing OpenClaw model identity in sandbox '${sandboxName}'...`, + ); deps.writeSandboxConfig(sandboxName, target, config); deps.recomputeSandboxConfigHash(sandboxName, target); @@ -323,10 +387,14 @@ export async function runInferenceSet( action: "shields_down", sandbox: sandboxName, timestamp: new Date().toISOString(), - reason: `inference set openclaw:${provider}:${model}`, + reason: `inference set ${agentName}:${provider}:${model}`, }); - deps.log(` Inference route synced for '${sandboxName}': ${patched.route.primaryModelRef}`); + deps.log( + agentName === "hermes" + ? ` Inference route synced for '${sandboxName}': ${model}` + : ` Inference route synced for '${sandboxName}': ${patched.route.primaryModelRef}`, + ); return { sandboxName, diff --git a/src/lib/commands/inference/set.ts b/src/lib/commands/inference/set.ts index 9d9a77bc0bc..b5045e19e77 100644 --- a/src/lib/commands/inference/set.ts +++ b/src/lib/commands/inference/set.ts @@ -14,7 +14,7 @@ export default class InferenceSetCommand extends NemoClawCommand { static strict = true; static summary = "Switch the NemoClaw inference model"; static description = - "Update the OpenShell inference route and sync the running OpenClaw sandbox model identity."; + "Update the OpenShell inference route and sync the running OpenClaw or Hermes sandbox config."; static usage = [ "inference set --provider --model [--sandbox ] [--no-verify]", ]; @@ -32,7 +32,8 @@ export default class InferenceSetCommand extends NemoClawCommand { required: true, }), sandbox: Flags.string({ - description: "Registered OpenClaw sandbox to sync; defaults to the NemoClaw default sandbox", + description: + "Registered sandbox to sync; defaults to the NemoClaw default sandbox or the unambiguous Hermes sandbox under nemohermes", }), "no-verify": Flags.boolean({ description: "Pass --no-verify through to openshell inference set", diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index b55c0672628..3d67964a3c5 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -10851,7 +10851,7 @@ function printDashboard( console.log(""); console.log(" To change settings later:"); console.log( - ` Model: openshell inference set -g nemoclaw --model --provider `, + ` Model: ${cliName()} inference set --model --provider --sandbox ${sandboxName}`, ); console.log(` Policies: ${cliName()} ${sandboxName} policy-add`); console.log(` Credentials: ${cliName()} credentials reset then ${cliName()} onboard`); diff --git a/src/lib/sandbox-config.ts b/src/lib/sandbox-config.ts index 8a65447a41a..4b69a825237 100644 --- a/src/lib/sandbox-config.ts +++ b/src/lib/sandbox-config.ts @@ -21,10 +21,14 @@ const { isIP } = require("node:net"); const { validateName } = require("./runner"); const { shellQuote } = require("./core/shell-quote"); const { dockerExecFileSync } = require("./adapters/docker/exec"); +const { dockerCapture } = require("./adapters/docker/run"); const credentialFilter: typeof import("./security/credential-filter") = require("./security/credential-filter"); const { stripCredentials, isConfigObject, isConfigValue, isCredentialField } = credentialFilter; const { appendAuditEntry } = require("./shields/audit"); const { isPrivateHostname, isPrivateIp } = require("./private-networks"); +const registry = require("./state/registry") as { + getSandbox?: (name: string) => { openshellDriver?: string | null } | null; +}; type ConfigObject = import("./security/credential-filter").ConfigObject; type ConfigValue = import("./security/credential-filter").ConfigValue; @@ -93,6 +97,79 @@ const DEFAULT_AGENT_CONFIG: AgentConfigTarget = { sensitiveFiles: ["/sandbox/.openclaw/.config-hash"], }; +const HERMES_STRICT_HASH_FILE = "/etc/nemoclaw/hermes.config-hash"; + +// Privileged sandbox exec bypasses the sandbox process's Landlock domain for +// host-initiated config writes. Legacy OpenShell gateways expose the pod via +// K3s/kubectl; Docker-driver gateways expose a sandbox container directly. +function selectDockerDriverSandboxContainer( + sandboxName: string, + openshellDriver: string | null | undefined, + containerNames: string, +): string | null { + if (openshellDriver !== "docker") return null; + const prefix = `openshell-${sandboxName}-`; + const exact = `openshell-${sandboxName}`; + return ( + containerNames + .split("\n") + .map((line: string) => line.trim()) + .find((name: string) => name === exact || name.startsWith(prefix)) || null + ); +} + +function resolveDockerDriverSandboxContainer(sandboxName: string): string | null { + let openshellDriver: string | null | undefined; + try { + openshellDriver = registry.getSandbox?.(sandboxName)?.openshellDriver; + } catch { + return null; + } + + const output = dockerCapture(["ps", "--format", "{{.Names}}"], { ignoreError: true }); + return selectDockerDriverSandboxContainer(sandboxName, openshellDriver, output); +} + +function kubectlExecArgv(sandboxName: string, cmd: string[], stdin = false): string[] { + const args = [ + "exec", + ...(stdin ? ["-i"] : []), + K3S_CONTAINER, + "kubectl", + "exec", + "-n", + "openshell", + sandboxName, + "-c", + "agent", + ...(stdin ? ["-i"] : []), + "--", + ...cmd, + ]; + return args; +} + +function privilegedSandboxExecArgv(sandboxName: string, cmd: string[], stdin = false): string[] { + const dockerDriverContainer = resolveDockerDriverSandboxContainer(sandboxName); + if (dockerDriverContainer) { + return ["exec", ...(stdin ? ["-i"] : []), "--user", "root", dockerDriverContainer, ...cmd]; + } + return kubectlExecArgv(sandboxName, cmd, stdin); +} + +function privilegedSandboxExec( + sandboxName: string, + cmd: string[], + opts: { input?: string | Buffer; timeout?: number } = {}, +): string { + const hasInput = opts.input !== undefined; + return dockerExecFileSync(privilegedSandboxExecArgv(sandboxName, cmd, hasInput), { + input: opts.input, + stdio: hasInput ? ["pipe", "pipe", "pipe"] : ["ignore", "pipe", "pipe"], + timeout: opts.timeout ?? 30000, + }); +} + function resolveAgentConfig(sandboxName: string): AgentConfigTarget { try { const registry = require("./state/registry"); @@ -371,46 +448,14 @@ function writeSandboxConfig( fs.writeFileSync(tmpFile, serializeConfig(config, target.format), { mode: 0o600 }); const content = fs.readFileSync(tmpFile, "utf-8"); - dockerExecFileSync( - [ - "exec", - "-i", - K3S_CONTAINER, - "kubectl", - "exec", - "-n", - "openshell", - sandboxName, - "-c", - "agent", - "-i", - "--", - "sh", - "-c", - `cat > ${shellQuote(target.configPath)}`, - ], - { input: content, stdio: ["pipe", "pipe", "pipe"], timeout: 15000 }, + privilegedSandboxExec( + sandboxName, + ["sh", "-c", `cat > ${shellQuote(target.configPath)}`], + { input: content }, ); try { - dockerExecFileSync( - [ - "exec", - K3S_CONTAINER, - "kubectl", - "exec", - "-n", - "openshell", - sandboxName, - "-c", - "agent", - "--", - "chown", - "sandbox:sandbox", - target.configPath, - ], - { stdio: ["ignore", "pipe", "pipe"], timeout: 15000 }, - ); + privilegedSandboxExec(sandboxName, ["chown", "sandbox:sandbox", target.configPath]); } catch { // Best effort — chown failure is non-fatal. } @@ -424,31 +469,43 @@ function writeSandboxConfig( } } +function buildRecomputeSandboxConfigHashScript(target: AgentConfigTarget): string | null { + if (target.agentName === "hermes") { + const envFile = `${target.configDir}/.env`; + const compatibilityHash = `${target.configDir}/.config-hash`; + const strictHash = shellQuote(HERMES_STRICT_HASH_FILE); + const compatibilityHashQuoted = shellQuote(compatibilityHash); + return [ + `mkdir -p ${shellQuote("/etc/nemoclaw")}`, + `strict_hash=${strictHash}`, + `strict_tmp="\${strict_hash}.tmp.$$"`, + `compat_hash=${compatibilityHashQuoted}`, + `compat_tmp="\${compat_hash}.tmp.$$"`, + `trap 'rm -f "$strict_tmp" "$compat_tmp"' EXIT HUP INT TERM`, + `sha256sum ${shellQuote(target.configPath)} ${shellQuote(envFile)} > "$strict_tmp"`, + `chown root:root "$strict_tmp"`, + `chmod 444 "$strict_tmp"`, + `mv -f "$strict_tmp" "$strict_hash"`, + `cp "$strict_hash" "$compat_tmp"`, + `chown sandbox:sandbox "$compat_tmp"`, + `chmod 600 "$compat_tmp"`, + `mv -f "$compat_tmp" "$compat_hash"`, + "trap - EXIT HUP INT TERM", + ].join(" && "); + } + if (!target.sensitiveFiles?.includes(`${target.configDir}/.config-hash`)) return null; + return [ + `cd ${shellQuote(target.configDir)}`, + `sha256sum ${shellQuote(target.configFile)} > .config-hash`, + "(chown sandbox:sandbox .config-hash 2>/dev/null || true)", + "(chmod 660 .config-hash 2>/dev/null || true)", + ].join(" && "); +} + function recomputeSandboxConfigHash(sandboxName: string, target: AgentConfigTarget): void { - if (!target.sensitiveFiles?.includes(`${target.configDir}/.config-hash`)) return; - dockerExecFileSync( - [ - "exec", - K3S_CONTAINER, - "kubectl", - "exec", - "-n", - "openshell", - sandboxName, - "-c", - "agent", - "--", - "sh", - "-c", - [ - `cd ${shellQuote(target.configDir)}`, - `sha256sum ${shellQuote(target.configFile)} > .config-hash`, - "(chown sandbox:sandbox .config-hash 2>/dev/null || true)", - "(chmod 660 .config-hash 2>/dev/null || true)", - ].join(" && "), - ], - { stdio: ["ignore", "pipe", "pipe"], timeout: 15000 }, - ); + const script = buildRecomputeSandboxConfigHashScript(target); + if (!script) return; + privilegedSandboxExec(sandboxName, ["sh", "-c", script]); } // --------------------------------------------------------------------------- @@ -813,66 +870,9 @@ async function configSet(sandboxName: string, opts: ConfigSetOpts = {}): Promise // Apply change setDotpath(config, opts.key, safeValue); - // Write to temp file in the agent's native format - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-config-")); - const tmpFile = path.join(tmpDir, target.configFile); - fs.writeFileSync(tmpFile, serializeConfig(config, target.format), { mode: 0o600 }); - - // Write config to sandbox via kubectl exec (bypasses Landlock) console.log(` Writing config to sandbox (${target.configPath})...`); - const content = fs.readFileSync(tmpFile, "utf-8"); - dockerExecFileSync( - [ - "exec", - "-i", - K3S_CONTAINER, - "kubectl", - "exec", - "-n", - "openshell", - sandboxName, - "-c", - "agent", - "-i", - "--", - "sh", - "-c", - `cat > ${target.configPath}`, - ], - { input: content, stdio: ["pipe", "pipe", "pipe"], timeout: 15000 }, - ); - - // Fix ownership via kubectl exec (bypasses Landlock) - try { - dockerExecFileSync( - [ - "exec", - K3S_CONTAINER, - "kubectl", - "exec", - "-n", - "openshell", - sandboxName, - "-c", - "agent", - "--", - "chown", - "sandbox:sandbox", - target.configPath, - ], - { stdio: ["ignore", "pipe", "pipe"], timeout: 15000 }, - ); - } catch { - // Best effort — chown failure is non-fatal - } - - // Cleanup temp - try { - fs.unlinkSync(tmpFile); - fs.rmdirSync(tmpDir); - } catch { - // Best effort - } + writeSandboxConfig(sandboxName, target, config); + recomputeSandboxConfigHash(sandboxName, target); // Audit log appendAuditEntry({ @@ -1086,6 +1086,9 @@ export { readSandboxConfig, writeSandboxConfig, recomputeSandboxConfigHash, + buildRecomputeSandboxConfigHashScript, + selectDockerDriverSandboxContainer, + privilegedSandboxExecArgv, extractDotpath, setDotpath, validateConfigDotpath, diff --git a/test/cli.test.ts b/test/cli.test.ts index 94fac49bd27..631d1f3b561 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -518,6 +518,21 @@ describe("CLI dispatch", () => { expect(out).not.toContain("$ nemoclaw list [--json]"); }); + it("nemohermes inference set --help uses alias branding and agent-aware wording", () => { + const out = execSync(`node "${HERMES_CLI}" inference set --help`, { + encoding: "utf-8", + stdio: "pipe", + timeout: execTimeout(), + env: { + ...process.env, + HOME: `/tmp/nemoclaw-cli-test-${Date.now()}`, + }, + }); + expect(out).toContain("$ nemohermes inference set --provider --model "); + expect(out).toContain("[--sandbox ] [--no-verify]"); + expect(out).toMatch(/OpenClaw or Hermes\s+sandbox config/); + }); + it("list --json emits structured empty inventory", () => { const r = run("list --json"); expect(r.code).toBe(0); diff --git a/test/config-set.test.ts b/test/config-set.test.ts index 3da32a487f2..8229560909a 100644 --- a/test/config-set.test.ts +++ b/test/config-set.test.ts @@ -17,6 +17,8 @@ const { rewriteConfigUrlsWithDnsPinning, formatConfigValueForLogs, resolveAgentConfig, + buildRecomputeSandboxConfigHashScript, + selectDockerDriverSandboxContainer, } = require("../dist/lib/sandbox-config"); type MutableScalar = string | number | boolean | null | undefined; @@ -43,6 +45,83 @@ describe("resolveAgentConfig", () => { }); }); +describe("buildRecomputeSandboxConfigHashScript", () => { + it("keeps OpenClaw on the mutable compatibility hash", () => { + const script = buildRecomputeSandboxConfigHashScript({ + agentName: "openclaw", + configPath: "/sandbox/.openclaw/openclaw.json", + configDir: "/sandbox/.openclaw", + format: "json", + configFile: "openclaw.json", + sensitiveFiles: ["/sandbox/.openclaw/.config-hash"], + }); + + expect(script).toContain("cd '/sandbox/.openclaw'"); + expect(script).toContain("sha256sum 'openclaw.json' > .config-hash"); + expect(script).toContain("chown sandbox:sandbox .config-hash"); + expect(script).toContain("chmod 660 .config-hash"); + }); + + it("updates Hermes strict and compatibility hashes with the expected permissions", () => { + const script = buildRecomputeSandboxConfigHashScript({ + agentName: "hermes", + configPath: "/sandbox/.hermes/config.yaml", + configDir: "/sandbox/.hermes", + format: "yaml", + configFile: "config.yaml", + sensitiveFiles: ["/sandbox/.hermes/.config-hash", "/sandbox/.hermes/.env"], + }); + + expect(script).toContain( + "strict_hash='/etc/nemoclaw/hermes.config-hash'", + ); + expect(script).toContain('strict_tmp="${strict_hash}.tmp.$$"'); + expect(script).toContain( + "compat_hash='/sandbox/.hermes/.config-hash'", + ); + expect(script).toContain('compat_tmp="${compat_hash}.tmp.$$"'); + expect(script).toContain('trap \'rm -f "$strict_tmp" "$compat_tmp"\' EXIT HUP INT TERM'); + expect(script).toContain( + 'sha256sum \'/sandbox/.hermes/config.yaml\' \'/sandbox/.hermes/.env\' > "$strict_tmp"', + ); + expect(script).toContain('chown root:root "$strict_tmp"'); + expect(script).toContain('chmod 444 "$strict_tmp"'); + expect(script).toContain('mv -f "$strict_tmp" "$strict_hash"'); + expect(script).toContain('cp "$strict_hash" "$compat_tmp"'); + expect(script).toContain('chown sandbox:sandbox "$compat_tmp"'); + expect(script).toContain('chmod 600 "$compat_tmp"'); + expect(script).toContain('mv -f "$compat_tmp" "$compat_hash"'); + }); +}); + +describe("selectDockerDriverSandboxContainer", () => { + it("returns the exact Docker-driver sandbox container when present", () => { + const selected = selectDockerDriverSandboxContainer( + "demo", + "docker", + "openshell-demo\nopenshell-demo-helper\n", + ); + + expect(selected).toBe("openshell-demo"); + }); + + it("falls back to the generated Docker-driver sandbox container prefix", () => { + const selected = selectDockerDriverSandboxContainer( + "demo", + "docker", + "openshell-other\nopenshell-demo-abc123\n", + ); + + expect(selected).toBe("openshell-demo-abc123"); + }); + + it("does not select a container for legacy gateway sandboxes", () => { + expect( + selectDockerDriverSandboxContainer("demo", "kubernetes", "openshell-demo\n"), + ).toBeNull(); + }); +}); + describe("config set helpers", () => { describe("extractDotpath", () => { it("extracts a top-level key", () => { diff --git a/test/e2e/test-hermes-inference-switch.sh b/test/e2e/test-hermes-inference-switch.sh new file mode 100755 index 00000000000..002bb687ea0 --- /dev/null +++ b/test/e2e/test-hermes-inference-switch.sh @@ -0,0 +1,533 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Hermes inference switch E2E. +# +# Installs NemoClaw with Hermes, switches the running sandbox with +# `nemohermes inference set`, verifies OpenShell and Hermes config state, and +# sends live requests after the switch without restarting Hermes. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key, starts with nvapi-) +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + +# Do not use errexit because this test records pass/fail counts and exits +# explicitly after critical failures or at the final summary. +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + content = c.get('content') or c.get('reasoning_content') or c.get('reasoning') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +hermes_gateway_pid() { + # shellcheck disable=SC2016 # awk runs inside the sandbox. + openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'ps -eo pid=,comm=,args= 2>/dev/null | awk '"'"'$2 != "sh" && $2 != "bash" && $2 != "awk" && $0 ~ /hermes/ && $0 ~ /gateway run/ { print $1; exit }'"'"'' \ + 2>/dev/null || true +} + +get_route_output() { + local output + if output=$(openshell inference get -g nemoclaw 2>&1); then + printf '%s\n' "$output" + return 0 + fi + openshell inference get 2>&1 +} + +strip_ansi() { + python3 -c 'import re, sys; sys.stdout.write(re.sub(r"\x1b\[[0-9;]*m", "", sys.stdin.read()))' +} + +assert_route() { + local output plain_output + if ! output=$(get_route_output); then + fail "OpenShell inference get failed: ${output:0:240}" + return + fi + plain_output=$(printf '%s' "$output" | strip_ansi) + + if grep -Fq "Provider: ${SWITCH_PROVIDER}" <<<"$plain_output" \ + && grep -Fq "Model: ${SWITCH_MODEL}" <<<"$plain_output"; then + pass "OpenShell route points at ${SWITCH_PROVIDER} / ${SWITCH_MODEL}" + else + fail "OpenShell route did not switch to ${SWITCH_PROVIDER} / ${SWITCH_MODEL}: ${plain_output:0:400}" + fi +} + +assert_registry_session() { + local probe + probe=$( + SANDBOX_NAME="$SANDBOX_NAME" EXPECTED_PROVIDER="$SWITCH_PROVIDER" EXPECTED_MODEL="$SWITCH_MODEL" python3 - <<'PY' +import json +import os +from pathlib import Path + +home = Path.home() +name = os.environ["SANDBOX_NAME"] +provider = os.environ["EXPECTED_PROVIDER"] +model = os.environ["EXPECTED_MODEL"] +errors = [] + +registry_path = home / ".nemoclaw" / "sandboxes.json" +try: + registry = json.loads(registry_path.read_text(encoding="utf-8")) + sandbox = (registry.get("sandboxes") or {}).get(name) +except Exception as exc: + sandbox = None + errors.append(f"could not read registry: {exc}") + +if not sandbox: + errors.append(f"sandbox {name} missing from registry") +else: + if sandbox.get("agent") != "hermes": + errors.append(f"registry agent={sandbox.get('agent')!r}") + if sandbox.get("provider") != provider: + errors.append(f"registry provider={sandbox.get('provider')!r}") + if sandbox.get("model") != model: + errors.append(f"registry model={sandbox.get('model')!r}") + +session_path = home / ".nemoclaw" / "onboard-session.json" +try: + session = json.loads(session_path.read_text(encoding="utf-8")) +except Exception as exc: + session = None + errors.append(f"could not read onboard session: {exc}") + +if session is not None: + if not isinstance(session, dict) or not session: + errors.append("onboard session is empty or invalid") + else: + if session.get("sandboxName") != name: + errors.append(f"session sandboxName={session.get('sandboxName')!r}") + if session.get("agent") != "hermes": + errors.append(f"session agent={session.get('agent')!r}") + if session.get("provider") != provider: + errors.append(f"session provider={session.get('provider')!r}") + if session.get("model") != model: + errors.append(f"session model={session.get('model')!r}") + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +PY + ) || { + fail "Registry/session were not updated for switch: ${probe:0:400}" + return + } + pass "Registry and onboard session record the switched Hermes provider/model" +} + +assert_hermes_health() { + local health_response attempt + for attempt in 1 2 3 4 5; do + health_response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- \ + curl -sf --max-time 10 http://localhost:8642/health 2>&1) || true + if grep -qi '"ok"' <<<"$health_response"; then + pass "Hermes health endpoint returns ok" + return + fi + [ "$attempt" -ge 5 ] || sleep 4 + done + fail "Hermes health endpoint did not return ok: ${health_response:0:240}" +} + +assert_hermes_config() { + local config probe + config=$(openshell sandbox exec --name "$SANDBOX_NAME" -- cat /sandbox/.hermes/config.yaml 2>&1) || { + fail "Could not read /sandbox/.hermes/config.yaml: ${config:0:240}" + return + } + + # Keep this parser dependency-free for the E2E runner: it only reads the + # simple model block and should move to PyYAML if nested or multiline values + # become relevant. + probe=$( + CONFIG_TEXT="$config" EXPECTED_MODEL="$SWITCH_MODEL" python3 - <<'PY' +import os +import re + +text = os.environ["CONFIG_TEXT"] +expected = os.environ["EXPECTED_MODEL"] +errors = [] + +model = {} +in_model = False +for line in text.splitlines(): + if re.match(r"^model:\s*$", line): + in_model = True + continue + if in_model and re.match(r"^[A-Za-z0-9_-]+:", line): + break + if in_model: + match = re.match(r"^\s+([A-Za-z0-9_-]+):\s*(.*?)\s*$", line) + if match: + value = match.group(2).strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + model[match.group(1)] = value + +if model.get("default") != expected: + errors.append(f"model.default={model.get('default')!r}") +if model.get("base_url") != "https://inference.local/v1": + errors.append(f"model.base_url={model.get('base_url')!r}") +if model.get("provider") != "custom": + errors.append(f"model.provider={model.get('provider')!r}") + +if re.search(r"(?ms)^models:\s*\n(?:[ \t].*\n)*?[ \t]+providers:", text): + errors.append("OpenClaw-style models.providers block present") + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +PY + ) || { + fail "Hermes config.yaml was not patched correctly: ${probe:0:400}" + return + } + pass "Hermes config.yaml model block uses ${SWITCH_MODEL} via inference.local" +} + +assert_hermes_hashes() { + local strict_check compat_check perms_probe + strict_check=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'sha256sum -c /etc/nemoclaw/hermes.config-hash --status && echo OK' 2>&1 || true) + if grep -qx "OK" <<<"$strict_check"; then + pass "Hermes strict config hash matches config.yaml and .env" + else + fail "Hermes strict config hash check failed: ${strict_check:0:240}" + fi + + compat_check=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'sha256sum -c /sandbox/.hermes/.config-hash --status && echo OK' 2>&1 || true) + if grep -qx "OK" <<<"$compat_check"; then + pass "Hermes compatibility config hash matches config.yaml and .env" + else + fail "Hermes compatibility config hash check failed: ${compat_check:0:240}" + fi + + perms_probe=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + "stat -c '%u %a' /etc/nemoclaw/hermes.config-hash" 2>&1 || true) + if PERMS_PROBE="$perms_probe" python3 - <<'PY'; then +import os +import sys + +parts = os.environ.get("PERMS_PROBE", "").split() +if len(parts) != 2: + raise SystemExit(1) +uid = int(parts[0]) +mode = int(parts[1], 8) +if uid != 0 or mode & 0o222: + raise SystemExit(1) +PY + pass "Hermes strict hash is root-owned and not writable" + else + fail "Hermes strict hash permissions are wrong: ${perms_probe:0:120}" + fi +} + +assert_env_hash_unchanged() { + local after + after=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sha256sum /sandbox/.hermes/.env 2>/dev/null | awk '{print $1}') || true + if [ -n "$ENV_HASH_BEFORE" ] && [ "$after" = "$ENV_HASH_BEFORE" ]; then + pass "Hermes .env was not rewritten by inference set" + else + fail "Hermes .env hash changed during inference set (${ENV_HASH_BEFORE:-missing} -> ${after:-missing})" + fi +} + +check_inference_local() { + local payload payload_arg response rc content attempt last_fail + payload=$(SWITCH_MODEL="$SWITCH_MODEL" python3 -c ' +import json +import os +print(json.dumps({ + "model": os.environ["SWITCH_MODEL"], + "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], + "max_tokens": 100, +})) +') + payload_arg="$(printf '%q' "$payload")" + last_fail="" + + for attempt in 1 2 3; do + rc=0 + response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + "curl -sS --max-time 90 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d $payload_arg" \ + 2>&1) || rc=$? + + if [ "$rc" -ne 0 ]; then + last_fail="curl failed with exit ${rc}: ${response:0:300}" + else + content=$(printf '%s' "$response" | parse_chat_content 2>/dev/null) || content="" + if grep -qi "PONG" <<<"$content"; then + pass "Hermes sandbox inference.local returned PONG with ${SWITCH_MODEL}" + return + fi + last_fail="expected PONG, got ${content:0:300}" + fi + + [ "$attempt" -ge 3 ] || { + info "Hermes inference.local attempt ${attempt}/3 failed: ${last_fail}" + sleep 5 + } + done + + fail "Hermes sandbox inference.local did not work after switch: ${last_fail}" +} + +check_hermes_api_chat() { + local payload payload_arg response rc content remote attempt last_fail + payload=$(SWITCH_MODEL="$SWITCH_MODEL" python3 -c ' +import json +import os +print(json.dumps({ + "model": os.environ["SWITCH_MODEL"], + "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], + "max_tokens": 100, +})) +') + payload_arg="$(printf '%q' "$payload")" + remote="set -a; [ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env; set +a; if [ -n \"\${API_SERVER_KEY:-}\" ]; then curl -sS --max-time 120 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -H \"Authorization: Bearer \${API_SERVER_KEY}\" -d $payload_arg; else curl -sS --max-time 120 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -d $payload_arg; fi" + last_fail="" + + for attempt in 1 2 3; do + rc=0 + response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote" 2>&1) || rc=$? + if [ "$rc" -ne 0 ]; then + last_fail="Hermes API curl failed with exit ${rc}: ${response:0:300}" + else + content=$(printf '%s' "$response" | parse_chat_content 2>/dev/null) || content="" + if grep -qi "PONG" <<<"$content"; then + pass "Hermes API chat works after inference switch" + return + fi + last_fail="expected PONG from Hermes API, got ${content:0:300}; response=${response:0:300}" + fi + + [ "$attempt" -ge 3 ] || { + info "Hermes API chat attempt ${attempt}/3 failed: ${last_fail}" + sleep 5 + } + done + + fail "Hermes API chat did not work after switch: ${last_fail}" +} + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-hermes-inference-switch}" +SWITCH_PROVIDER="${NEMOCLAW_SWITCH_PROVIDER:-nvidia-prod}" +SWITCH_MODEL="${NEMOCLAW_SWITCH_MODEL:-z-ai/glm-5.1}" +INSTALL_LOG="/tmp/nemoclaw-e2e-hermes-inference-switch-install.log" +ENV_HASH_BEFORE="" + +export NEMOCLAW_AGENT="${NEMOCLAW_AGENT:-hermes}" + +# shellcheck source=test/e2e/lib/sandbox-teardown.sh +. "${E2E_DIR}/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e/lib/install-path-refresh.sh +. "${E2E_DIR}/lib/install-path-refresh.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +section "Phase 0: Pre-cleanup" +if command -v nemohermes >/dev/null 2>&1; then + nemohermes "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +elif command -v nemoclaw >/dev/null 2>&1; then + NEMOCLAW_AGENT=hermes nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "Pre-cleanup complete" + +section "Phase 1: Prerequisites" +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + exit 1 +fi + +if [ -n "${NVIDIA_API_KEY:-}" ] && [[ "${NVIDIA_API_KEY}" == nvapi-* ]]; then + pass "NVIDIA_API_KEY is set" +else + fail "NVIDIA_API_KEY not set or invalid" + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ]; then + pass "NEMOCLAW_NON_INTERACTIVE=1" +else + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" = "1" ]; then + pass "Third-party software acceptance is set" +else + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + exit 1 +fi + +section "Phase 2: Install and onboard Hermes" +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" + +info "Running install.sh --non-interactive for Hermes sandbox ${SANDBOX_NAME}..." +bash install.sh --non-interactive --yes-i-accept-third-party-software >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait "$install_pid" +install_exit=$? +kill "$tail_pid" 2>/dev/null || true +wait "$tail_pid" 2>/dev/null || true + +nemoclaw_refresh_install_env +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" +nemoclaw_ensure_local_bin_on_path + +if [ "$install_exit" -eq 0 ]; then + pass "install.sh completed" +else + fail "install.sh failed (exit ${install_exit})" + tail -80 "$INSTALL_LOG" || true + exit 1 +fi + +command -v nemohermes >/dev/null 2>&1 || { + fail "nemohermes not found on PATH" + exit 1 +} +command -v openshell >/dev/null 2>&1 || { + fail "openshell not found on PATH" + exit 1 +} +pass "nemohermes and openshell are on PATH" +assert_hermes_health + +section "Phase 3: Switch inference" +pid_before="$(hermes_gateway_pid)" +ENV_HASH_BEFORE=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sha256sum /sandbox/.hermes/.env 2>/dev/null | awk '{print $1}') || true + +info "Switching Hermes to ${SWITCH_PROVIDER} / ${SWITCH_MODEL} with nemohermes inference set..." +switch_output=$(nemohermes inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" 2>&1) +switch_rc=$? +if [ "$switch_rc" -eq 0 ]; then + pass "nemohermes inference set completed without --sandbox" +else + fail "nemohermes inference set failed (exit ${switch_rc}): ${switch_output:0:500}" + exit 1 +fi + +pid_after="$(hermes_gateway_pid)" +if [ -n "$pid_before" ] && [ -n "$pid_after" ]; then + if [ "$pid_before" = "$pid_after" ]; then + pass "Hermes gateway process stayed running during switch" + else + fail "Hermes gateway process changed during switch (${pid_before} -> ${pid_after})" + fi +else + skip "Could not capture Hermes gateway PID before and after switch" +fi + +assert_hermes_health +assert_route +assert_hermes_config +assert_env_hash_unchanged +assert_hermes_hashes +assert_registry_session + +section "Phase 4: Live requests after switch" +check_inference_local +check_hermes_api_chat + +section "Phase 5: Cleanup" +if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" != "1" ]; then + nemohermes "$SANDBOX_NAME" destroy --yes 2>&1 | tail -3 || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true + + registry_file="${HOME}/.nemoclaw/sandboxes.json" + if [ -f "$registry_file" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$registry_file"; then + fail "Sandbox ${SANDBOX_NAME} still in registry after destroy" + else + pass "Sandbox ${SANDBOX_NAME} removed" + fi +else + skip "Sandbox ${SANDBOX_NAME} kept; removal check skipped" +fi + +echo "" +echo "========================================" +echo " Hermes inference switch E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Hermes inference switch E2E PASSED.\033[0m\n' + exit 0 +fi + +printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" +exit 1 diff --git a/test/e2e/test-openclaw-inference-switch.sh b/test/e2e/test-openclaw-inference-switch.sh new file mode 100755 index 00000000000..05eb033c424 --- /dev/null +++ b/test/e2e/test-openclaw-inference-switch.sh @@ -0,0 +1,463 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# OpenClaw inference switch E2E. +# +# Installs NemoClaw with the default OpenClaw agent, switches the running +# sandbox with `nemoclaw inference set`, verifies OpenShell and OpenClaw config +# state, then sends live requests through inference.local and OpenClaw. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key, starts with nvapi-) +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + +# Do not use errexit because this test records pass/fail counts and exits +# explicitly after critical failures or at the final summary. +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +run_with_timeout() { + local seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$seconds" "$@" + else + "$@" + fi +} + +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + content = c.get('content') or c.get('reasoning_content') or c.get('reasoning') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +openclaw_gateway_pid() { + # shellcheck disable=SC2016 # awk runs inside the sandbox. + openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'ps -eo pid=,comm=,args= 2>/dev/null | awk '"'"'$2 != "sh" && $2 != "bash" && $2 != "awk" && $0 ~ /openclaw/ && $0 ~ /gateway run/ { print $1; exit }'"'"'' \ + 2>/dev/null || true +} + +get_route_output() { + local output + if output=$(openshell inference get -g nemoclaw 2>&1); then + printf '%s\n' "$output" + return 0 + fi + openshell inference get 2>&1 +} + +strip_ansi() { + python3 -c 'import re, sys; sys.stdout.write(re.sub(r"\x1b\[[0-9;]*m", "", sys.stdin.read()))' +} + +assert_route() { + local output plain_output + if ! output=$(get_route_output); then + fail "OpenShell inference get failed: ${output:0:240}" + return + fi + plain_output=$(printf '%s' "$output" | strip_ansi) + + if grep -Fq "Provider: ${SWITCH_PROVIDER}" <<<"$plain_output" \ + && grep -Fq "Model: ${SWITCH_MODEL}" <<<"$plain_output"; then + pass "OpenShell route points at ${SWITCH_PROVIDER} / ${SWITCH_MODEL}" + else + fail "OpenShell route did not switch to ${SWITCH_PROVIDER} / ${SWITCH_MODEL}: ${plain_output:0:400}" + fi +} + +assert_registry_session() { + local probe + probe=$( + SANDBOX_NAME="$SANDBOX_NAME" EXPECTED_PROVIDER="$SWITCH_PROVIDER" EXPECTED_MODEL="$SWITCH_MODEL" python3 - <<'PY' +import json +import os +from pathlib import Path + +home = Path.home() +name = os.environ["SANDBOX_NAME"] +provider = os.environ["EXPECTED_PROVIDER"] +model = os.environ["EXPECTED_MODEL"] +errors = [] + +registry_path = home / ".nemoclaw" / "sandboxes.json" +try: + registry = json.loads(registry_path.read_text(encoding="utf-8")) + sandbox = (registry.get("sandboxes") or {}).get(name) +except Exception as exc: + sandbox = None + errors.append(f"could not read registry: {exc}") + +if not sandbox: + errors.append(f"sandbox {name} missing from registry") +else: + if sandbox.get("provider") != provider: + errors.append(f"registry provider={sandbox.get('provider')!r}") + if sandbox.get("model") != model: + errors.append(f"registry model={sandbox.get('model')!r}") + +session_path = home / ".nemoclaw" / "onboard-session.json" +try: + session = json.loads(session_path.read_text(encoding="utf-8")) +except Exception as exc: + session = None + errors.append(f"could not read onboard session: {exc}") + +if session is not None: + if not isinstance(session, dict) or not session: + errors.append("onboard session is empty or invalid") + else: + if session.get("sandboxName") != name: + errors.append(f"session sandboxName={session.get('sandboxName')!r}") + if session.get("provider") != provider: + errors.append(f"session provider={session.get('provider')!r}") + if session.get("model") != model: + errors.append(f"session model={session.get('model')!r}") + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +PY + ) || { + fail "Registry/session were not updated for switch: ${probe:0:400}" + return + } + pass "Registry and onboard session record the switched provider/model" +} + +assert_openclaw_config() { + local config probe hash_check + config=$(openshell sandbox exec --name "$SANDBOX_NAME" -- cat /sandbox/.openclaw/openclaw.json 2>&1) || { + fail "Could not read /sandbox/.openclaw/openclaw.json: ${config:0:240}" + return + } + + probe=$(EXPECTED_MODEL="$SWITCH_MODEL" python3 -c ' +import json +import os +import sys + +expected = os.environ["EXPECTED_MODEL"] +doc = json.load(sys.stdin) +errors = [] +primary = (((doc.get("agents") or {}).get("defaults") or {}).get("model") or {}).get("primary") +if primary != f"inference/{expected}": + errors.append(f"primary={primary!r}") + +provider = (((doc.get("models") or {}).get("providers") or {}).get("inference") or {}) +if provider.get("baseUrl") != "https://inference.local/v1": + errors.append("baseUrl={!r}".format(provider.get("baseUrl"))) +models = provider.get("models") or [] +if not models or models[0].get("id") != expected: + errors.append("model id={!r}".format(models[0].get("id") if models else None)) +if not models or models[0].get("name") != f"inference/{expected}": + errors.append("model name={!r}".format(models[0].get("name") if models else None)) + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +' <<<"$config" 2>&1) || { + fail "OpenClaw config was not patched correctly: ${probe:0:400}" + return + } + pass "OpenClaw config uses inference/${SWITCH_MODEL}" + + hash_check=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'cd /sandbox/.openclaw && sha256sum -c .config-hash --status && echo OK' 2>&1 || true) + if grep -qx "OK" <<<"$hash_check"; then + pass "OpenClaw config hash matches openclaw.json" + else + fail "OpenClaw config hash check failed: ${hash_check:0:240}" + fi +} + +check_sandbox_inference() { + local payload payload_arg response rc content attempt last_fail + payload=$(SWITCH_MODEL="$SWITCH_MODEL" python3 -c ' +import json +import os +print(json.dumps({ + "model": os.environ["SWITCH_MODEL"], + "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], + "max_tokens": 100, +})) +') + payload_arg="$(printf '%q' "$payload")" + last_fail="" + + for attempt in 1 2 3; do + rc=0 + response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + "curl -sS --max-time 90 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d $payload_arg" \ + 2>&1) || rc=$? + + if [ "$rc" -ne 0 ]; then + last_fail="curl failed with exit ${rc}: ${response:0:300}" + else + content=$(printf '%s' "$response" | parse_chat_content 2>/dev/null) || content="" + if grep -qi "PONG" <<<"$content"; then + pass "Sandbox inference.local returned PONG with ${SWITCH_MODEL}" + return + fi + last_fail="expected PONG, got ${content:0:300}" + fi + + [ "$attempt" -ge 3 ] || { + info "Sandbox inference attempt ${attempt}/3 failed: ${last_fail}" + sleep 5 + } + done + + fail "Sandbox inference.local did not work after switch: ${last_fail}" +} + +check_openclaw_agent_turn() { + local ssh_config session_id raw rc reply + ssh_config="$(mktemp)" + if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + rm -f "$ssh_config" + fail "Could not get SSH config for OpenClaw agent turn" + return + fi + + session_id="e2e-inference-switch-openclaw-$(date +%s)-$$" + rc=0 + raw=$(run_with_timeout 120 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "openclaw agent --agent main --json --thinking off --session-id '${session_id}' -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.'" \ + 2>/dev/null) || rc=$? + rm -f "$ssh_config" + + reply=$(printf '%s' "$raw" | python3 -c ' +import json +import sys +try: + doc = json.load(sys.stdin) +except Exception: + sys.exit(0) +result = doc.get("result") or {} +parts = [] +for payload in result.get("payloads") or []: + if isinstance(payload, dict) and isinstance(payload.get("text"), str): + parts.append(payload["text"]) +print("\n".join(parts)) +' 2>/dev/null) || true + + if [ "$rc" -eq 0 ] && grep -qE '(^|[^0-9])42([^0-9]|$)' <<<"$reply"; then + pass "OpenClaw agent answered through the switched inference route" + else + fail "OpenClaw agent turn failed after switch (exit ${rc}); reply='${reply:0:200}', raw='${raw:0:200}'" + fi +} + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-openclaw-inference-switch}" +SWITCH_PROVIDER="${NEMOCLAW_SWITCH_PROVIDER:-nvidia-prod}" +SWITCH_MODEL="${NEMOCLAW_SWITCH_MODEL:-z-ai/glm-5.1}" +INSTALL_LOG="/tmp/nemoclaw-e2e-openclaw-inference-switch-install.log" + +# shellcheck source=test/e2e/lib/sandbox-teardown.sh +. "${E2E_DIR}/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e/lib/install-path-refresh.sh +. "${E2E_DIR}/lib/install-path-refresh.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +section "Phase 0: Pre-cleanup" +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "Pre-cleanup complete" + +section "Phase 1: Prerequisites" +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + exit 1 +fi + +if [ -n "${NVIDIA_API_KEY:-}" ] && [[ "${NVIDIA_API_KEY}" == nvapi-* ]]; then + pass "NVIDIA_API_KEY is set" +else + fail "NVIDIA_API_KEY not set or invalid" + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ]; then + pass "NEMOCLAW_NON_INTERACTIVE=1" +else + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" = "1" ]; then + pass "Third-party software acceptance is set" +else + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + exit 1 +fi + +section "Phase 2: Install and onboard OpenClaw" +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" + +info "Running install.sh --non-interactive for sandbox ${SANDBOX_NAME}..." +bash install.sh --non-interactive --yes-i-accept-third-party-software >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait "$install_pid" +install_exit=$? +kill "$tail_pid" 2>/dev/null || true +wait "$tail_pid" 2>/dev/null || true + +nemoclaw_refresh_install_env +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" +nemoclaw_ensure_local_bin_on_path + +if [ "$install_exit" -eq 0 ]; then + pass "install.sh completed" +else + fail "install.sh failed (exit ${install_exit})" + tail -80 "$INSTALL_LOG" || true + exit 1 +fi + +command -v nemoclaw >/dev/null 2>&1 || { + fail "nemoclaw not found on PATH" + exit 1 +} +command -v openshell >/dev/null 2>&1 || { + fail "openshell not found on PATH" + exit 1 +} +pass "nemoclaw and openshell are on PATH" + +section "Phase 3: Switch inference" +pid_before="$(openclaw_gateway_pid)" +info "Switching ${SANDBOX_NAME} to ${SWITCH_PROVIDER} / ${SWITCH_MODEL}..." +switch_output=$(nemoclaw inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" --sandbox "$SANDBOX_NAME" 2>&1) +switch_rc=$? +if [ "$switch_rc" -eq 0 ]; then + pass "nemoclaw inference set completed" +else + fail "nemoclaw inference set failed (exit ${switch_rc}): ${switch_output:0:500}" + exit 1 +fi + +pid_after="$(openclaw_gateway_pid)" +if [ -n "$pid_before" ] && [ -n "$pid_after" ]; then + if [ "$pid_before" = "$pid_after" ]; then + pass "OpenClaw gateway process stayed running during switch" + else + fail "OpenClaw gateway process changed during switch (${pid_before} -> ${pid_after})" + fi +else + skip "Could not capture OpenClaw gateway PID before and after switch" +fi + +assert_route +assert_openclaw_config +assert_registry_session + +section "Phase 4: Live requests after switch" +check_sandbox_inference +check_openclaw_agent_turn + +section "Phase 5: Cleanup" +if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" != "1" ]; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tail -3 || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true + + registry_file="${HOME}/.nemoclaw/sandboxes.json" + if [ -f "$registry_file" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$registry_file"; then + fail "Sandbox ${SANDBOX_NAME} still in registry after destroy" + else + pass "Sandbox ${SANDBOX_NAME} removed" + fi +else + skip "Sandbox ${SANDBOX_NAME} kept; removal check skipped" +fi + +echo "" +echo "========================================" +echo " OpenClaw inference switch E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m OpenClaw inference switch E2E PASSED.\033[0m\n' + exit 0 +fi + +printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" +exit 1