Skip to content
1 change: 1 addition & 0 deletions docs/inference/switch-inference-providers.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ An OpenClaw native-Anthropic route and a Hermes OpenAI-frontend route therefore
NemoClaw checks stopped sandboxes because they depend on that route when restarted.
If a registered same-gateway sandbox lacks durable provider or model metadata, or a custom route lacks durable endpoint or API-family metadata, NemoClaw fails closed until you remove and re-onboard that sandbox with complete route metadata.
When a route conflicts, onboarding, runtime switching, and connect-time repair exit non-zero before changing the gateway and name the affected sandboxes.
If the live gateway route still ends up differing from a sandbox's recorded route, for example after a direct `openshell inference set`, `$$nemoclaw <name> status` prints a warning naming both routes and the supported command to realign or adopt the live route.
Align the routes, remove the conflicting sandbox, or onboard it with another `NEMOCLAW_GATEWAY_PORT`.
</Warning>

Expand Down
23 changes: 23 additions & 0 deletions src/lib/actions/sandbox/connect-route-lifecycle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ describe("connectSandbox route lifecycle", () => {
expect(errorOutput).toContain(
"Aligning the gateway to anthropic-prod/claude-sonnet-4-20250514",
);
expect(errorOutput).toContain(
"nemoclaw inference set --provider 'nvidia-prod' --model 'nvidia/nemotron-3-super-120b-a12b' --sandbox 'alpha'",
);
expect(harness.runOpenshellSpy).toHaveBeenCalledWith(
[
"inference",
Expand All @@ -81,6 +84,26 @@ describe("connectSandbox route lifecycle", () => {
);
});

it("shell-quotes hostile route values in drift recovery commands (#3726)", async () => {
const sandboxName = "alpha's box";
const harness = createConnectHarness({
inferenceGetOutput:
"Gateway inference:\n Provider: openai; touch /tmp/pwn\n Model: $(id) model\n",
registryEntry: {
name: sandboxName,
model: "claude-sonnet-4-20250514",
provider: "anthropic-prod",
},
});

await expect(harness.connectSandbox(sandboxName, { probeOnly: true })).resolves.toBeUndefined();

const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n");
expect(errorOutput).toContain(
"nemoclaw inference set --provider 'openai; touch /tmp/pwn' --model '$(id) model' --sandbox 'alpha'\\''s box'",
);
});

it("wires the forced VM DNS monkeypatch into connect route repair", async () => {
vi.stubEnv("NEMOCLAW_FORCE_VM_DNS_MONKEYPATCH", "1");
try {
Expand Down
15 changes: 9 additions & 6 deletions src/lib/actions/sandbox/connect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ import * as agentRuntime from "../../agent/runtime";
import { CLI_NAME } from "../../cli/branding";
import { D, G, R, YW } from "../../cli/terminal-style";
import { spawnExitCode } from "../../core/process-exit";
import { shellQuote } from "../../core/shell-quote";
import { getNamedGatewayLifecycleState } from "../../gateway-runtime-action";
import {
formatInferenceRouteDriftForDisplay,
parseGatewayInference,
planInferenceRouteReconcile,
sanitizeRouteValueForDisplay,
Expand Down Expand Up @@ -733,18 +735,19 @@ function ensureSandboxInferenceRouteUnlocked(
if (plan.kind === "diverged") {
// Shared gateway: re-point loudly (even when quiet) — silent revert was
// #3726. Values sanitized: registry/gateway strings are untrusted.
const liveProvider = sanitizeRouteValueForDisplay(plan.live.provider);
const liveModel = sanitizeRouteValueForDisplay(plan.live.model);
console.error(
` ${YW}Warning: gateway inference route (${liveProvider}/${liveModel}) ` +
`differs from the recorded route for sandbox '${sandboxName}' (${recordedRoute}).${R}`,
const display = formatInferenceRouteDriftForDisplay(
plan.live,
plan.recorded,
`for sandbox '${sandboxName}'`,
);
const { liveProvider, liveModel } = display;
console.error(` ${YW}Warning: ${display.warning}${R}`);
console.error(
` ${YW}Aligning the gateway to ${recordedRoute}. To keep ` +
`${liveProvider}/${liveModel}, set it the supported way:${R}`,
);
console.error(
` ${CLI_NAME} inference set --provider ${liveProvider} --model ${liveModel} --sandbox ${sandboxName}`,
` ${CLI_NAME} inference set --provider ${shellQuote(liveProvider)} --model ${shellQuote(liveModel)} --sandbox ${shellQuote(sandboxName)}`,
);
} else if (!quiet) {
// plan.kind === "repair": empty gateway, genuine repair — quiet-aware.
Expand Down
54 changes: 54 additions & 0 deletions src/lib/actions/sandbox/status-flow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,60 @@ describe("showSandboxStatus flow", () => {
resetStatusFlowModuleCache();
});

it("warns when the live gateway route differs from the sandbox's recorded route (#6315)", async () => {
const harness = createStatusFlowHarness({
currentProvider: "openai",
currentModel: "gpt-5.2",
routeDrift: {
live: { provider: "openai", model: "gpt-5.2" },
recorded: { provider: "nvidia", model: "nvidia/nemotron" },
},
});

await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined();

const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n");
expect(output).toContain(
"Warning: gateway inference route (openai/gpt-5.2) differs from the recorded route for this sandbox (nvidia/nvidia/nemotron).",
);
expect(output).toContain(
"nemoclaw 'alpha' connect realigns the gateway to nvidia/nvidia/nemotron",
);
expect(output).toContain(
"inference set --provider 'openai' --model 'gpt-5.2' --sandbox 'alpha'",
);
});

it("shell-quotes hostile route values in drift recovery commands (#6315)", async () => {
const sandboxName = "alpha's box";
const harness = createStatusFlowHarness({
currentProvider: "openai; touch /tmp/pwn",
currentModel: "$(id) model",
routeDrift: {
live: { provider: "openai; touch /tmp/pwn", model: "$(id) model" },
recorded: { provider: "nvidia", model: "nvidia/nemotron" },
},
sandboxEntry: { name: sandboxName },
});

await expect(harness.showSandboxStatus(sandboxName)).resolves.toBeUndefined();

const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n");
expect(output).toContain("nemoclaw 'alpha'\\''s box' connect realigns the gateway");
expect(output).toContain(
"nemoclaw inference set --provider 'openai; touch /tmp/pwn' --model '$(id) model' --sandbox 'alpha'\\''s box'",
);
});

it("prints no route drift warning when the live route matches the recorded route (#6315)", async () => {
const harness = createStatusFlowHarness();

await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined();

const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n");
expect(output).not.toContain("differs from the recorded route");
});

it("prints the live sandbox, inference, runtime, session, version, and recovery signals", async () => {
const harness = createStatusFlowHarness();

Expand Down
133 changes: 133 additions & 0 deletions src/lib/actions/sandbox/status-snapshot-route-drift.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { afterEach, describe, expect, it, vi } from "vitest";

vi.mock("../../adapters/openshell/runtime", async (importOriginal) => {
const actual = await importOriginal<typeof import("../../adapters/openshell/runtime")>();
return { ...actual, captureOpenshellForStatus: vi.fn() };
});

import { captureOpenshellForStatus } from "../../adapters/openshell/runtime";
import type { SandboxEntry } from "../../state/registry";
import { collectSandboxStatusSnapshot } from "./status-snapshot";

const capture = vi.mocked(captureOpenshellForStatus);

function liveGatewayInference(provider: string, model: string, gatewayName = "nemoclaw"): void {
capture.mockImplementation(async (args) =>
args.join("\0") === ["inference", "get", "-g", gatewayName].join("\0")
? ({
status: 0,
output: `Gateway inference:\n Provider: ${provider}\n Model: ${model}\n`,
} as Awaited<ReturnType<typeof captureOpenshellForStatus>>)
: ({ status: 1, output: "" } as Awaited<ReturnType<typeof captureOpenshellForStatus>>),
);
}

function snapshotDeps(entry: Partial<SandboxEntry> | null) {
return {
suppressInferenceProbe: true,
deps: {
getSandbox: () =>
entry
? ({ name: "alpha", agent: "openclaw", policies: [], ...entry } as SandboxEntry)
: null,
reconcile: async () => ({ state: "present", output: "Phase: Ready" }),
},
};
}

describe("collectSandboxStatusSnapshot route drift", () => {
afterEach(() => {
vi.clearAllMocks();
});

it("reports drift when the live gateway route differs from the recorded route (#6315)", async () => {
liveGatewayInference("openai", "gpt-5.2");

const snapshot = await collectSandboxStatusSnapshot(
"alpha",
snapshotDeps({ provider: "nvidia", model: "nvidia/nemotron" }),
);

expect(snapshot.routeDrift).toEqual({
live: { provider: "openai", model: "gpt-5.2" },
recorded: { provider: "nvidia", model: "nvidia/nemotron" },
});
expect(snapshot.currentProvider).toBe("openai");
expect(snapshot.currentModel).toBe("gpt-5.2");
});

it("reads the sandbox's non-default gateway before computing drift (#6315)", async () => {
liveGatewayInference("openai", "gpt-5.2", "nemoclaw-9090");

const snapshot = await collectSandboxStatusSnapshot(
"alpha",
snapshotDeps({
gatewayPort: 9090,
provider: "nvidia",
model: "nvidia/nemotron",
}),
);

expect(snapshot.routeDrift).toEqual({
live: { provider: "openai", model: "gpt-5.2" },
recorded: { provider: "nvidia", model: "nvidia/nemotron" },
});
expect(snapshot.currentProvider).toBe("openai");
expect(snapshot.currentModel).toBe("gpt-5.2");
});

it("does not fall back to the default gateway for an invalid persisted binding (#6315)", async () => {
liveGatewayInference("openai", "gpt-5.2");

const snapshot = await collectSandboxStatusSnapshot(
"alpha",
snapshotDeps({
gatewayPort: 0,
provider: "nvidia",
model: "nvidia/nemotron",
}),
);

expect(snapshot.routeDrift).toBeNull();
expect(snapshot.currentProvider).toBe("nvidia");
expect(snapshot.currentModel).toBe("nvidia/nemotron");
});

it("reports no drift when the live route matches the recorded route (#6315)", async () => {
liveGatewayInference("nvidia", "nvidia/nemotron");

const snapshot = await collectSandboxStatusSnapshot(
"alpha",
snapshotDeps({ provider: "nvidia", model: "nvidia/nemotron" }),
);

expect(snapshot.routeDrift).toBeNull();
});

it("reports no drift when the live route is unreadable — repair, not divergence (#6315)", async () => {
capture.mockResolvedValue({
status: 1,
output: "",
} as Awaited<ReturnType<typeof captureOpenshellForStatus>>);

const snapshot = await collectSandboxStatusSnapshot(
"alpha",
snapshotDeps({ provider: "nvidia", model: "nvidia/nemotron" }),
);

expect(snapshot.routeDrift).toBeNull();
expect(snapshot.currentProvider).toBe("nvidia");
expect(snapshot.currentModel).toBe("nvidia/nemotron");
});

it("reports no drift when the registry entry has no recorded route (#6315)", async () => {
liveGatewayInference("openai", "gpt-5.2");

const snapshot = await collectSandboxStatusSnapshot("alpha", snapshotDeps({}));

expect(snapshot.routeDrift).toBeNull();
});
});
39 changes: 34 additions & 5 deletions src/lib/actions/sandbox/status-snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@ import {
import { captureOpenshellForStatus, isCommandTimeout } from "../../adapters/openshell/runtime";
import { type AgentDefinition, getAgentRuntimeKind, loadAgent } from "../../agent/defs";
import { withStdoutRedirectedToStderr } from "../../cli/stdout-guard";
import { parseGatewayInference } from "../../inference/config";
import {
type GatewayInference,
parseGatewayInference,
planInferenceRouteReconcile,
type RecordedInferenceRoute,
} from "../../inference/config";
import {
type ProviderHealthProbeOptions,
type ProviderHealthStatus,
Expand All @@ -18,9 +23,11 @@ import {
type DcodeAutoApprovalMode,
normalizeDcodeAutoApprovalMode,
} from "../../onboard/dcode-auto-approval";
import { resolveSandboxGatewayName } from "../../onboard/gateway-binding";
import { redact } from "../../security/redact";
import { parseSandboxPhase } from "../../state/gateway";
import * as registry from "../../state/registry";
import { buildGatewayInferenceGetArgs } from "./connect-inference-gateway";
import { classifyInferenceRouteFailureLabel } from "./connect-inference-route-probe";
import { getSandboxDockerRuntime } from "./docker-health";
import type { SandboxGatewayState } from "./gateway-state";
Expand Down Expand Up @@ -156,12 +163,18 @@ export interface SandboxStatusReport {
dockerPaused: boolean;
}

export interface SandboxStatusRouteDrift {
live: GatewayInference;
recorded: RecordedInferenceRoute;
}

export interface SandboxStatusSnapshot {
sb: registry.SandboxEntry | null;
lookup: SandboxGatewayState;
rpcIssue: OpenShellStateRpcIssue | null;
currentModel: string;
currentProvider: string;
routeDrift: SandboxStatusRouteDrift | null;
inferenceHealth: ProviderHealthStatus | null;
terminalRuntimeHealth: TerminalRuntimeOomProbeResult | null;
}
Expand Down Expand Up @@ -258,11 +271,13 @@ export async function collectSandboxStatusSnapshot(
let liveResult: Awaited<ReturnType<typeof captureOpenshellForStatus>> | null = null;
if (lookup.state === "present") {
try {
liveResult = await (opts.deps?.captureOpenshellForStatusImpl ?? captureOpenshellForStatus)([
"inference",
"get",
]);
const gatewayName = resolveSandboxGatewayName(sb);
liveResult = await (opts.deps?.captureOpenshellForStatusImpl ?? captureOpenshellForStatus)(
buildGatewayInferenceGetArgs(gatewayName),
);
} catch {
// Invalid persisted gateway bindings and failed reads stay fail-closed:
// never substitute the selected/default gateway's inference route.
liveResult = null;
}
}
Expand All @@ -274,6 +289,7 @@ export async function collectSandboxStatusSnapshot(
rpcIssue,
currentModel: "unknown",
currentProvider: "unknown",
routeDrift: null,
inferenceHealth: null,
terminalRuntimeHealth: null,
};
Expand All @@ -282,6 +298,18 @@ export async function collectSandboxStatusSnapshot(
liveResult && !isCommandTimeout(liveResult) ? parseGatewayInference(liveResult.output) : null;
const currentModel = (live && live.model) || (sb && sb.model) || "unknown";
const currentProvider = (live && live.provider) || (sb && sb.provider) || "unknown";
// Status shows the live gateway route when one is readable, which silently
// masks a route another sandbox (or a direct `openshell inference set`)
// moved from under this one — the shared-route trap of #6315. Surface the
// divergence instead of letting the live value pass as this sandbox's own.
const routeDriftPlan =
sb && sb.provider && sb.model
? planInferenceRouteReconcile(live, { provider: sb.provider, model: sb.model })
: null;
const routeDrift =
routeDriftPlan && routeDriftPlan.kind === "diverged"
? { live: routeDriftPlan.live, recorded: routeDriftPlan.recorded }
: null;
// When the caller has already determined that the local stack is failed
// (docker daemon down, sandbox container stopped, dashboard port held),
// skip the provider probe entirely. Without this gate
Expand Down Expand Up @@ -336,6 +364,7 @@ export async function collectSandboxStatusSnapshot(
rpcIssue,
currentModel,
currentProvider,
routeDrift,
inferenceHealth,
terminalRuntimeHealth,
};
Expand Down
Loading
Loading