Skip to content
1 change: 1 addition & 0 deletions docs/inference/switch-inference-providers.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ An OpenClaw native-Anthropic route and a Hermes OpenAI-frontend route therefore
NemoClaw checks stopped sandboxes because they depend on that route when restarted.
If a registered same-gateway sandbox lacks durable provider or model metadata, or a custom route lacks durable endpoint or API-family metadata, NemoClaw fails closed until you remove and re-onboard that sandbox with complete route metadata.
When a route conflicts, onboarding, runtime switching, and connect-time repair exit non-zero before changing the gateway and name the affected sandboxes.
If the live gateway route still ends up differing from a sandbox's recorded route, for example after a direct `openshell inference set`, `$$nemoclaw <name> status` prints a warning naming both routes and the supported command to realign or adopt the live route.
Align the routes, remove the conflicting sandbox, or onboard it with another `NEMOCLAW_GATEWAY_PORT`.
</Warning>

Expand Down
31 changes: 31 additions & 0 deletions src/lib/actions/sandbox/status-flow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,37 @@ describe("showSandboxStatus flow", () => {
resetStatusFlowModuleCache();
});

it("warns when the live gateway route differs from the sandbox's recorded route (#6315)", async () => {
const harness = createStatusFlowHarness({
currentProvider: "openai",
currentModel: "gpt-5.2",
routeDrift: {
live: { provider: "openai", model: "gpt-5.2" },
recorded: { provider: "nvidia", model: "nvidia/nemotron" },
},
});

await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined();

const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n");
expect(output).toContain(
"Warning: gateway inference route (openai/gpt-5.2) differs from the recorded route for this sandbox (nvidia/nvidia/nemotron).",
);
expect(output).toContain(
"'nemoclaw alpha connect' realigns the gateway to nvidia/nvidia/nemotron",
);
expect(output).toContain("inference set --provider openai --model gpt-5.2 --sandbox alpha");
});

it("prints no route drift warning when the live route matches the recorded route (#6315)", async () => {
const harness = createStatusFlowHarness();

await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined();

const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n");
expect(output).not.toContain("differs from the recorded route");
});

it("prints the live sandbox, inference, runtime, session, version, and recovery signals", async () => {
const harness = createStatusFlowHarness();

Expand Down
92 changes: 92 additions & 0 deletions src/lib/actions/sandbox/status-snapshot-route-drift.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { afterEach, describe, expect, it, vi } from "vitest";

vi.mock("../../adapters/openshell/runtime", async (importOriginal) => {
const actual = await importOriginal<typeof import("../../adapters/openshell/runtime")>();
return { ...actual, captureOpenshellForStatus: vi.fn() };
});

import { captureOpenshellForStatus } from "../../adapters/openshell/runtime";
import type { SandboxEntry } from "../../state/registry";
import { collectSandboxStatusSnapshot } from "./status-snapshot";

const capture = vi.mocked(captureOpenshellForStatus);

function liveGatewayInference(provider: string, model: string): void {
capture.mockResolvedValue({
status: 0,
output: `Gateway inference:\n Provider: ${provider}\n Model: ${model}\n`,
} as Awaited<ReturnType<typeof captureOpenshellForStatus>>);
}

function snapshotDeps(entry: Partial<SandboxEntry> | null) {
return {
suppressInferenceProbe: true,
deps: {
getSandbox: () =>
entry
? ({ name: "alpha", agent: "openclaw", policies: [], ...entry } as SandboxEntry)
: null,
reconcile: async () => ({ state: "present", output: "Phase: Ready" }),
},
};
}

describe("collectSandboxStatusSnapshot route drift", () => {
afterEach(() => {
vi.clearAllMocks();
});

it("reports drift when the live gateway route differs from the recorded route (#6315)", async () => {
liveGatewayInference("openai", "gpt-5.2");

const snapshot = await collectSandboxStatusSnapshot(
"alpha",
snapshotDeps({ provider: "nvidia", model: "nvidia/nemotron" }),
);

expect(snapshot.routeDrift).toEqual({
live: { provider: "openai", model: "gpt-5.2" },
recorded: { provider: "nvidia", model: "nvidia/nemotron" },
});
expect(snapshot.currentProvider).toBe("openai");
expect(snapshot.currentModel).toBe("gpt-5.2");
});

it("reports no drift when the live route matches the recorded route (#6315)", async () => {
liveGatewayInference("nvidia", "nvidia/nemotron");

const snapshot = await collectSandboxStatusSnapshot(
"alpha",
snapshotDeps({ provider: "nvidia", model: "nvidia/nemotron" }),
);

expect(snapshot.routeDrift).toBeNull();
});

it("reports no drift when the live route is unreadable — repair, not divergence (#6315)", async () => {
capture.mockResolvedValue({
status: 1,
output: "",
} as Awaited<ReturnType<typeof captureOpenshellForStatus>>);

const snapshot = await collectSandboxStatusSnapshot(
"alpha",
snapshotDeps({ provider: "nvidia", model: "nvidia/nemotron" }),
);

expect(snapshot.routeDrift).toBeNull();
expect(snapshot.currentProvider).toBe("nvidia");
expect(snapshot.currentModel).toBe("nvidia/nemotron");
});

it("reports no drift when the registry entry has no recorded route (#6315)", async () => {
liveGatewayInference("openai", "gpt-5.2");

const snapshot = await collectSandboxStatusSnapshot("alpha", snapshotDeps({}));

expect(snapshot.routeDrift).toBeNull();
});
});
27 changes: 26 additions & 1 deletion src/lib/actions/sandbox/status-snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@ import {
import { captureOpenshellForStatus, isCommandTimeout } from "../../adapters/openshell/runtime";
import { type AgentDefinition, getAgentRuntimeKind, loadAgent } from "../../agent/defs";
import { withStdoutRedirectedToStderr } from "../../cli/stdout-guard";
import { parseGatewayInference } from "../../inference/config";
import {
type GatewayInference,
parseGatewayInference,
planInferenceRouteReconcile,
type RecordedInferenceRoute,
} from "../../inference/config";
import {
type ProviderHealthProbeOptions,
type ProviderHealthStatus,
Expand Down Expand Up @@ -105,12 +110,18 @@ export interface SandboxStatusReport {
dockerPaused: boolean;
}

export interface SandboxStatusRouteDrift {
live: GatewayInference;
recorded: RecordedInferenceRoute;
}

export interface SandboxStatusSnapshot {
sb: registry.SandboxEntry | null;
lookup: SandboxGatewayState;
rpcIssue: OpenShellStateRpcIssue | null;
currentModel: string;
currentProvider: string;
routeDrift: SandboxStatusRouteDrift | null;
inferenceHealth: ProviderHealthStatus | null;
terminalRuntimeHealth: TerminalRuntimeOomProbeResult | null;
}
Expand Down Expand Up @@ -199,6 +210,7 @@ export async function collectSandboxStatusSnapshot(
rpcIssue,
currentModel: "unknown",
currentProvider: "unknown",
routeDrift: null,
inferenceHealth: null,
terminalRuntimeHealth: null,
};
Expand All @@ -207,6 +219,18 @@ export async function collectSandboxStatusSnapshot(
liveResult && !isCommandTimeout(liveResult) ? parseGatewayInference(liveResult.output) : null;
const currentModel = (live && live.model) || (sb && sb.model) || "unknown";
const currentProvider = (live && live.provider) || (sb && sb.provider) || "unknown";
// Status shows the live gateway route when one is readable, which silently
// masks a route another sandbox (or a direct `openshell inference set`)
// moved from under this one — the shared-route trap of #6315. Surface the
// divergence instead of letting the live value pass as this sandbox's own.
const routeDriftPlan =
sb && sb.provider && sb.model
? planInferenceRouteReconcile(live, { provider: sb.provider, model: sb.model })
: null;
const routeDrift =
routeDriftPlan && routeDriftPlan.kind === "diverged"
? { live: routeDriftPlan.live, recorded: routeDriftPlan.recorded }
: null;
// When the caller has already determined that the local stack is failed
// (docker daemon down, sandbox container stopped, dashboard port held),
// skip the provider probe entirely. Without this gate
Expand Down Expand Up @@ -250,6 +274,7 @@ export async function collectSandboxStatusSnapshot(
rpcIssue,
currentModel,
currentProvider,
routeDrift,
inferenceHealth,
terminalRuntimeHealth,
};
Expand Down
33 changes: 32 additions & 1 deletion src/lib/actions/sandbox/status-text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { resolveOpenshell } from "../../adapters/openshell/resolve";
import * as agentRuntime from "../../agent/runtime";
import { CLI_NAME } from "../../cli/branding";
import { D, G, R, RD, YW } from "../../cli/terminal-style";
import { sanitizeRouteValueForDisplay } from "../../inference/config";
import type { ProviderHealthStatus } from "../../inference/health";
import * as nim from "../../inference/nim";
import * as sandboxVersion from "../../sandbox/version";
Expand All @@ -17,7 +18,11 @@ import {
import type { SandboxDockerRuntime } from "./docker-health";
import type { SandboxGatewayState } from "./gateway-state";
import { isSandboxGatewayRunningForStatus } from "./process-recovery";
import type { SandboxStatusAgentInfo, SandboxStatusSnapshot } from "./status-snapshot";
import type {
SandboxStatusAgentInfo,
SandboxStatusRouteDrift,
SandboxStatusSnapshot,
} from "./status-snapshot";

export interface SandboxStatusTextContext
extends Pick<
Expand All @@ -26,6 +31,7 @@ export interface SandboxStatusTextContext
| "lookup"
| "currentModel"
| "currentProvider"
| "routeDrift"
| "inferenceHealth"
| "terminalRuntimeHealth"
> {
Expand Down Expand Up @@ -232,6 +238,30 @@ function printAgentVersion(context: SandboxStatusTextContext, sandbox: SandboxEn
}
}

// The Model/Provider lines above show the live gateway route, which the
// shared per-gateway route lets another sandbox move (#6315). When it no
// longer matches this sandbox's recorded route, say so instead of presenting
// the live value as this sandbox's own; wording mirrors the connect-time
// divergence warning (#3726).
function printInferenceRouteDrift(
drift: SandboxStatusRouteDrift | null,
sandboxName: string,
): void {
if (!drift) return;
const liveProvider = sanitizeRouteValueForDisplay(drift.live.provider);
const liveModel = sanitizeRouteValueForDisplay(drift.live.model);
const recordedRoute = `${sanitizeRouteValueForDisplay(drift.recorded.provider)}/${sanitizeRouteValueForDisplay(drift.recorded.model)}`;
console.log(
` ${YW}Warning: gateway inference route (${liveProvider}/${liveModel}) differs from the recorded route for this sandbox (${recordedRoute}).${R}`,
);
console.log(
` ${YW}'${CLI_NAME} ${sandboxName} connect' realigns the gateway to ${recordedRoute}; to adopt the live route instead:${R}`,
);
console.log(
` ${CLI_NAME} inference set --provider ${liveProvider} --model ${liveModel} --sandbox ${sandboxName}`,
);
}

/** Render registry-backed sandbox details and return any non-fatal degraded outcome. */
export function printSandboxDetails(context: SandboxStatusTextContext): SandboxStatusTextOutcome {
const { sb, currentModel, currentProvider, sandboxName } = context;
Expand All @@ -242,6 +272,7 @@ export function printSandboxDetails(context: SandboxStatusTextContext): SandboxS
console.log(` Sandbox: ${sb.name}`);
console.log(` Model: ${currentModel}`);
console.log(` Provider: ${currentProvider}`);
printInferenceRouteDrift(context.routeDrift, sb.name);
printInferenceStatus(context);
printSandboxGpuStatus(sb);
console.log(
Expand Down
2 changes: 2 additions & 0 deletions src/lib/actions/sandbox/status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ export async function showSandboxStatus(sandboxName: string): Promise<void> {
rpcIssue,
currentModel,
currentProvider,
routeDrift,
inferenceHealth,
terminalRuntimeHealth,
} = snapshot;
Expand Down Expand Up @@ -103,6 +104,7 @@ export async function showSandboxStatus(sandboxName: string): Promise<void> {
lookup,
currentModel,
currentProvider,
routeDrift,
inferenceHealth,
terminalRuntimeHealth,
statusAgent,
Expand Down
3 changes: 3 additions & 0 deletions test/support/status-flow-test-harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { type MockInstance, vi } from "vitest";

import type { SandboxGatewayState } from "../../src/lib/actions/sandbox/gateway-state";
import type { SandboxStatusPreflightResult } from "../../src/lib/actions/sandbox/status-preflight";
import type { SandboxStatusRouteDrift } from "../../src/lib/actions/sandbox/status-snapshot";
import type { ProviderHealthStatus } from "../../src/lib/inference/health";

type ShowSandboxStatus = typeof import("../../src/lib/actions/sandbox/status")["showSandboxStatus"];
Expand Down Expand Up @@ -53,6 +54,7 @@ const baseSandboxEntry = {
export type StatusFlowHarnessOptions = {
currentModel?: string;
currentProvider?: string;
routeDrift?: SandboxStatusRouteDrift | null;
inferenceHealth?: ProviderHealthStatus | null;
lookup?: SandboxGatewayState;
lookupState?: "present" | "missing";
Expand Down Expand Up @@ -135,6 +137,7 @@ export function createStatusFlowHarness(options: StatusFlowHarnessOptions = {}):
rpcIssue: null,
currentModel: options.currentModel ?? "nvidia/nemotron-live",
currentProvider: options.currentProvider ?? "ollama-local",
routeDrift: options.routeDrift ?? null,
inferenceHealth:
options.inferenceHealth === undefined
? {
Expand Down