diff --git a/docs/inference/switch-models.mdx b/docs/inference/switch-models.mdx index 59d77d834df..ef0b0095d47 100644 --- a/docs/inference/switch-models.mdx +++ b/docs/inference/switch-models.mdx @@ -61,10 +61,16 @@ To validate a new model before replacing the current sandbox, onboard it under a ## Account for Shared Gateways OpenShell exposes one inference route per gateway. -Every registered sandbox on the same gateway, including stopped sandboxes, must record the same provider and model. -NemoClaw exits before changing state when the requested model conflicts with another sandbox on the gateway. - -Align the sandboxes to one route, remove a sandbox you no longer need, or onboard a sandbox with another `NEMOCLAW_GATEWAY_PORT`. +NemoClaw records each sandbox's intended model, but only one recorded route is live at a time. +Onboarding a sandbox with another valid model warns immediately before it re-points the shared route. +The warning names each affected sandbox and its recorded provider and model. +When another registered sandbox uses the same provider name, `connect` verifies the custom endpoint and API family when applicable, plus the credential environment-variable name. +It then warns and re-points the route to that sandbox's recorded provider and model. +A different custom endpoint, API family, or credential environment-variable name remains fail-closed because `connect` cannot restore that provider metadata with a provider/model-only route change. +Remove or re-onboard the conflicting sandbox before applying a different provider-global identity. + +`$$nemoclaw status` shows the recorded model and adds an explicit drift warning when another sandbox's route is live. +Runtime `inference set` remains fail-closed when the requested change conflicts with another registered sandbox or when route metadata is incomplete. ## Verify the Change diff --git a/docs/inference/switch-providers.mdx b/docs/inference/switch-providers.mdx index 20c08bf74bd..574dbb030f7 100644 --- a/docs/inference/switch-providers.mdx +++ b/docs/inference/switch-providers.mdx @@ -100,12 +100,17 @@ A rebuild reuses the recorded endpoint and cannot change it. ## Account for Shared Gateways -Every sandbox registered on one OpenShell gateway must use the same provider and model. -Custom routes must also use the same normalized endpoint and API family. -NemoClaw includes stopped sandboxes because they depend on that route when restarted. - -When a route conflicts, NemoClaw exits before changing the gateway, agent configuration, or host registry and names the affected sandboxes. -Align the routes, remove a conflicting sandbox, or onboard it with another `NEMOCLAW_GATEWAY_PORT`. +OpenShell exposes one live inference route per gateway, while NemoClaw records each sandbox's intended provider and model. +Onboarding a sandbox with a different provider or model warns immediately before it re-points the shared route. +When another sandbox uses the same provider name, its custom endpoint, API family, and credential environment-variable name must match. +Onboarding refuses before provider or route mutation when those provider-global fields differ. +The warning names every affected registered sandbox and its recorded provider and model. +Those sandboxes use the new live route until a later onboarding run or, when endpoint, API-family, and credential environment-variable identity already match, `$$nemoclaw connect` warns and re-points its provider and model again. +`connect` refuses a different custom endpoint, API family, or credential environment-variable name because its provider/model-only route change cannot safely reconstruct that metadata. + +`$$nemoclaw status` continues to show that sandbox's recorded provider and model and prints an explicit warning when the live shared route differs. +Incomplete custom-route metadata, provider-global identity conflicts, and invalid gateway bindings remain hard errors because NemoClaw cannot identify or restore those routes safely. +Runtime `inference set` also refuses a change that would leave another registered sandbox with a different recorded route. Do not run `openshell inference set` directly on a shared NemoClaw gateway because that bypasses registry compatibility checks. diff --git a/docs/inference/view-active-inference-route.mdx b/docs/inference/view-active-inference-route.mdx index 200eaf8a11f..095a6cc9a57 100644 --- a/docs/inference/view-active-inference-route.mdx +++ b/docs/inference/view-active-inference-route.mdx @@ -65,7 +65,11 @@ Run the sandbox status command when you also need service, messaging, and infere $$nemoclaw status ``` -The status output includes the configured provider, model, and endpoint with the rest of the sandbox state. +The status output shows the provider and model recorded for that sandbox. +When the gateway's live shared route differs, status also prints an explicit warning with both the live and recorded routes. +The JSON report exposes them separately as `recordedRoute` and `liveRoute` and repeats both under `routeDrift` when they differ. +The existing schema-version `1` `provider` and `model` fields remain live-first when the gateway route is readable and fall back to the recorded route otherwise. +When `routeDrift.canConnect` is `false`, incomplete route metadata, an invalid gateway binding, or a provider-global identity conflict prevents `connect` from restoring the recorded route safely. Use the route verification workflow when you need to prove that an inference request succeeds through the sandbox path. ## Related Topics diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 24b475a72c3..e5785e4eb58 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -819,10 +819,12 @@ If another terminal is already connected to the sandbox, `connect` prints a note An unknown model slug, malformed extra-args JSON, or a gated model (for example `deepseek-r1-distill-70b`) with no `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN` exits non-zero with the same error the installer would emit, before any sandbox readiness probe or SSH attach. Unset the managed-vLLM variable, or fix the value, before retrying. -When the live OpenShell gateway inference route differs from the route recorded in the NemoClaw registry, `connect` checks every registered sandbox on that gateway before attempting a repair. -It realigns the route only when those registry entries are compatible with the requested provider and model. -If another sandbox records a conflicting route, `connect` exits non-zero without changing the gateway and names the affected sandboxes. -Use `$$nemoclaw inference set --provider --model ` to make an intentional compatible route change. +Before reading or changing the live OpenShell gateway inference route, `connect` verifies that every same-name provider has matching custom endpoint and API-family metadata when applicable, plus a matching credential environment-variable name. +When the live route differs from the route recorded for the target sandbox, `connect` then warns and re-points the shared route to that sandbox's recorded provider and model. +This switch lets registered sandboxes with different valid routes time-share the gateway. +Different custom endpoints, API families, or credential environment-variable names, incomplete route metadata, and invalid gateway bindings remain hard errors because a provider/model-only connect-time change cannot reconstruct or prove that identity. +Remove or re-onboard the conflicting sandbox before applying a different provider-global identity. +Use `$$nemoclaw inference set --provider --model ` to make an intentional compatible route change outside the connect flow. Before it opens SSH, `connect` probes `https://inference.local/v1/models` from inside the sandbox with the selected agent's trusted CA and proxy context. HTTP `200` through `499` confirms that the route is reachable. When the probe returns a recognized broken result, `connect` attempts DNS or route repair and verifies the route again. @@ -1160,7 +1162,11 @@ Use this form when you care about a specific sandbox's live OpenShell state, age Do not pass a sandbox name to `$$nemoclaw status`; that command is the global all-sandbox/service overview. Pass `--json` to emit a structured per-sandbox report instead of the text renderer. -The JSON output includes at least `schemaVersion`, `name`, `found`, `agent`, `agentDisplayName`, `agentRuntime`, `dcodeAutoApprovalMode`, `model`, `provider`, `phase`, `gatewayState`, `inferenceHealth`, `rpcIssue`, `hostGpuDetected`, `sandboxGpuEnabled`, `sandboxGpuMode`, `sandboxGpuDevice`, `openshellDriver`, `openshellVersion`, `policies`, `failureLayer`, `terminalRuntimeHealth`, and `dockerPaused`. +The JSON output includes at least `schemaVersion`, `name`, `found`, `agent`, `agentDisplayName`, `agentRuntime`, `dcodeAutoApprovalMode`, `model`, `provider`, `recordedRoute`, `liveRoute`, `routeDrift`, `phase`, `gatewayState`, `inferenceHealth`, `rpcIssue`, `hostGpuDetected`, `sandboxGpuEnabled`, `sandboxGpuMode`, `sandboxGpuDevice`, `openshellDriver`, `openshellVersion`, `policies`, `failureLayer`, `terminalRuntimeHealth`, and `dockerPaused`. +The schema-version `1` `model` and `provider` fields keep their established live-route meaning when the gateway route is readable. +Use `recordedRoute` for the sandbox's durable provider and model and `liveRoute` for the gateway-global route. +When the live shared route differs, text output prints both routes and JSON output sets `routeDrift.live`, `routeDrift.recorded`, and `routeDrift.canConnect`. +When `routeDrift.canConnect` is `false`, `connect` cannot safely restore the recorded route because provider-global identity differs or required route or gateway metadata is incomplete. `openshellDriver` and `openshellVersion` are always strings (falling back to `"unknown"` when the registry has no value), so consumers can rely on `typeof` checks. `failureLayer` is `null` when no preflight failure was detected and otherwise one of `docker_unreachable`, `sandbox_container_stopped`, or `sandbox_dashboard_port_conflict`; when set, `inferenceHealth` is suppressed to `null` so automation does not see a stale remote-provider healthy status during a local outage. `dockerPaused` is `true` when NemoClaw detects that the Docker-driver sandbox container is paused. @@ -2860,8 +2866,17 @@ Run `$$nemoclaw shields down`, apply the inference change, then run `$$ne Each OpenShell gateway exposes one inference route to every sandbox registered on that gateway. Before changing the route, NemoClaw compares the requested provider and model with every same-gateway registry entry, including stopped sandboxes. Custom compatible routes must also have matching normalized endpoint URLs and API families. +Provider-global credential environment-variable names must also match for the same provider name. If a route conflicts or a legacy custom route lacks enough endpoint or API-family metadata to prove compatibility, the command exits non-zero before changing the OpenShell route, agent config, or host registry and names the conflicting sandboxes. -Align those sandboxes to the same route, remove the conflicting sandbox, or onboard it with another `NEMOCLAW_GATEWAY_PORT`. +Align those sandboxes to the same route or remove a conflicting sandbox that you no longer need. + +Onboarding allows provider/model time-sharing but does not replace provider-global configuration used by another registered sandbox. +When another sandbox records a different provider or model, NemoClaw warns immediately before re-pointing the shared route. +For the same provider name, any custom endpoint, API family, and credential environment-variable name must match. +When those provider-global fields differ, onboarding exits before updating the provider or inference route. +The warning names each affected sandbox and its recorded provider and model. +Connecting to a sandbox warns before re-pointing the route to that sandbox's recorded provider and model. +`$$nemoclaw status` shows the recorded route and reports any live-route drift explicitly. ```bash $$nemoclaw inference set --provider --model [--sandbox ] [--no-verify] [--endpoint-url ] [--credential-env ] [--inference-api ] diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 5296e48c7e0..9c7387d47ca 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -356,19 +356,23 @@ $$nemoclaw list `$$nemoclaw list` prints the recorded dashboard URL for each sandbox. OpenShell currently exposes one inference route per gateway. -Sandboxes that share a gateway must record the same provider and model. -Custom compatible routes must also record the same normalized endpoint and API family. -An OpenClaw sandbox using the native Anthropic Messages frontend and a Hermes sandbox using the managed OpenAI frontend need separate gateway ports, even when they use the same custom endpoint and model. -NemoClaw includes stopped sandboxes in this check because they use the gateway route when restarted. - -If onboarding, `inference set`, or connect-time route repair reports a shared-gateway conflict, choose one of these recovery paths: - -- Align every named sandbox to the same route. -- Remove a conflicting sandbox with `$$nemoclaw destroy` when you no longer need it. -- Onboard the sandbox against another gateway port when it needs a different route. - -An aligned group cannot move to a wholly new route one sandbox at a time because the first change would conflict with the remaining registry rows. -Back up each workspace, remove or migrate the peer sandboxes, switch the last sandbox on the original gateway, then re-onboard the peers with that same route or with separate gateway ports. +NemoClaw records each sandbox's intended provider and model, but the gateway can expose only one of those routes at a time. +Onboarding with a different valid route warns immediately before changing the live route. +The warning names every affected registered sandbox and its recorded provider and model. +The affected sandboxes use that route until another onboarding run or `$$nemoclaw connect` warns and re-points the provider and model. +When another sandbox uses the same provider name, its custom endpoint, API family, and credential environment-variable name must match before onboarding or `connect` can re-point the route. +Onboarding exits before provider or route mutation when that provider-global identity differs. +`connect` refuses the same conflict because its provider/model-only route change cannot safely reconstruct that metadata. + +Run `$$nemoclaw status` to compare the sandbox's recorded route with the live shared route. +The normal Model and Provider lines remain the sandbox's recorded configuration. +A separate warning shows the live and recorded routes when they differ. +The JSON status report exposes `recordedRoute` and `liveRoute` explicitly and repeats them in `routeDrift` when they differ. +Schema-version `1` `model` and `provider` remain live-first when the gateway route is readable and fall back to the recorded route otherwise. +When `routeDrift.canConnect` is `false`, the route cannot be restored safely because provider-global identity differs or required route or gateway metadata is incomplete. + +If `inference set` reports a valid shared-route conflict, align the named sandbox records or remove a sandbox you no longer need. +If onboarding or `connect` reports a provider-global identity conflict, align the same-name provider's custom endpoint, API family, and credential environment-variable name across the named sandboxes, or remove a conflicting sandbox you no longer need. If the error names incomplete legacy custom-route metadata, back up and remove the affected sandbox, then re-onboard it with an explicit custom endpoint and API family. For an OpenAI-compatible route, replace the example endpoint, model, and sandbox name in this recovery sequence: @@ -385,16 +389,6 @@ NEMOCLAW_PROVIDER=custom \ If the error names an invalid gateway binding, restore the affected row's known-good `gatewayName` and `gatewayPort` metadata from a trusted backup; otherwise back up and remove the sandbox, then re-onboard it. Do not guess or copy a binding from another sandbox because lifecycle commands use it to select the gateway. -For example, keep the first sandbox on the default gateway and assign a separate gateway to the second sandbox: - -```bash -$$nemoclaw onboard --name first-sandbox -NEMOCLAW_GATEWAY_PORT=8990 $$nemoclaw onboard --name second-sandbox -``` - -NemoClaw derives a separate gateway name and state directory from the second port. -Using a different dashboard port alone does not create a separate inference route. - ## Onboarding diff --git a/src/lib/actions/sandbox/connect-inference-gateway.ts b/src/lib/actions/sandbox/connect-inference-gateway.ts index ad5c259faa6..7cd0ab9782b 100644 --- a/src/lib/actions/sandbox/connect-inference-gateway.ts +++ b/src/lib/actions/sandbox/connect-inference-gateway.ts @@ -1,11 +1,39 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { assertGatewayRouteCompatibility } from "../../inference/gateway-route-compatibility"; +import { + checkGatewayRouteCompatibility, + GatewayRouteConflictError, + isAdvisoryProviderModelRouteConflict, +} from "../../inference/gateway-route-compatibility"; import { LOCAL_INFERENCE_TIMEOUT_SECS } from "../../onboard/env"; import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; +function sandboxGatewayRouteCompatibility( + sandboxName: string, + sb: SandboxEntry, + gatewayName: string, + sandboxes: readonly SandboxEntry[], +) { + return checkGatewayRouteCompatibility({ + gatewayName, + sandboxName, + route: sb, + sandboxes, + }); +} + +export function canSandboxGatewayRouteRealign( + sandboxName: string, + sb: SandboxEntry, + gatewayName: string, + sandboxes: readonly SandboxEntry[] = registry.listSandboxes().sandboxes, +): boolean { + const result = sandboxGatewayRouteCompatibility(sandboxName, sb, gatewayName, sandboxes); + return result.ok || isAdvisoryProviderModelRouteConflict(result); +} + export function buildGatewayInferenceGetArgs(gatewayName: string): string[] { return ["inference", "get", "-g", gatewayName]; } @@ -37,10 +65,13 @@ export function assertSandboxGatewayRouteCompatible( sb: SandboxEntry, gatewayName: string, ): void { - assertGatewayRouteCompatibility({ - gatewayName, + const result = sandboxGatewayRouteCompatibility( sandboxName, - route: sb, - sandboxes: registry.listSandboxes().sandboxes, - }); + sb, + gatewayName, + registry.listSandboxes().sandboxes, + ); + if (!result.ok && !isAdvisoryProviderModelRouteConflict(result)) { + throw new GatewayRouteConflictError(result); + } } diff --git a/src/lib/actions/sandbox/connect-route-containment.test.ts b/src/lib/actions/sandbox/connect-route-containment.test.ts index a95b9545e13..9f816f801b5 100644 --- a/src/lib/actions/sandbox/connect-route-containment.test.ts +++ b/src/lib/actions/sandbox/connect-route-containment.test.ts @@ -74,7 +74,7 @@ describe("connect route containment", () => { expect(repairLegacyDnsProxy).not.toHaveBeenCalled(); }); - it("exits before connect-time route writes when another sandbox conflicts (#6315)", async () => { + it("warns and restores the target route when another sandbox records a different route (#6315)", async () => { const alpha = { name: "alpha", agent: "openclaw", @@ -100,24 +100,92 @@ describe("connect route containment", () => { ], }); - await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( - "process.exit(1)", - ); + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); - expect(harness.ensureLiveSandboxSpy).not.toHaveBeenCalled(); - expect(harness.checkAndRecoverSpy).not.toHaveBeenCalled(); - expect(harness.captureOpenshellSpy).not.toHaveBeenCalled(); - expect(harness.runOpenshellSpy).not.toHaveBeenCalled(); + expect(harness.ensureLiveSandboxSpy).toHaveBeenCalledOnce(); + expect(harness.checkAndRecoverSpy).toHaveBeenCalled(); + expect(harness.captureOpenshellSpy).toHaveBeenCalled(); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + expect.arrayContaining([ + "inference", + "set", + "--provider", + "anthropic-prod", + "--model", + "claude-sonnet-4-20250514", + ]), + expect.any(Object), + ); expect(harness.applyVmDnsMonkeypatchSpy).not.toHaveBeenCalled(); expect(harness.runSetupDnsProxySpy).not.toHaveBeenCalled(); - expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith( - "openshell", - ["sandbox", "connect", "alpha"], + const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain("differs from the recorded route for sandbox 'alpha'"); + expect(errorOutput).toContain( + "Aligning the gateway to anthropic-prod/claude-sonnet-4-20250514", + ); + expect(exitSpy).not.toHaveBeenCalled(); + }); + + it.each([ + [ + "endpoint", + { + endpointUrl: "https://peer.example.test/v1", + preferredInferenceApi: "openai-completions", + }, + ], + [ + "API family", + { + endpointUrl: "https://target.example.test/v1", + preferredInferenceApi: "openai-responses", + }, + ], + ] as const)("refuses a different complete custom %s before route reads, mutation, or target probes (#6315)", async (_difference, peerRoute) => { + const target = { + name: "target", + agent: "openclaw", + gatewayName: "nemoclaw", + gatewayPort: 8080, + provider: "compatible-endpoint", + model: "target/model", + endpointUrl: "https://target.example.test/v1", + preferredInferenceApi: "openai-completions", + } as const; + const harness = createConnectHarness({ + inferenceGetOutput: + "Gateway inference:\n Provider: compatible-endpoint\n Model: target/model\n", + registryEntry: target, + registryEntries: [ + target, + { + name: "peer", + agent: "openclaw", + gatewayName: "nemoclaw", + gatewayPort: 8080, + provider: "compatible-endpoint", + model: "peer/model", + ...peerRoute, + }, + ], + }); + + await expect(harness.connectSandbox("target", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.captureOpenshellSpy).not.toHaveBeenCalledWith( + ["inference", "get", "-g", "nemoclaw"], expect.any(Object), ); - const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); - expect(errorOutput).toContain("stopped-peer"); - expect(errorOutput).toContain("NEMOCLAW_GATEWAY_PORT"); + const targetProbeCalls = harness.captureOpenshellSpy.mock.calls.filter( + ([args]) => Array.isArray(args) && args.join(" ").includes("inference.local/v1/models"), + ); + expect(targetProbeCalls).toHaveLength(0); + expect(harness.runOpenshellSpy).not.toHaveBeenCalled(); + expect(harness.errorSpy).toHaveBeenCalledWith( + expect.stringContaining("Cannot set compatible-endpoint / target/model"), + ); expect(exitSpy).toHaveBeenCalledWith(1); }); @@ -155,18 +223,20 @@ describe("connect route containment", () => { Object.assign(peer!, { provider: "anthropic-prod", model: "claude-new" }); releaseLock(); - await expect(connect).rejects.toThrow("process.exit(1)"); + await expect(connect).resolves.toBeUndefined(); expect(harness.withGatewayRouteMutationLockSpy).toHaveBeenCalledWith( "nemoclaw", expect.any(Function), ); - expect(harness.captureOpenshellSpy).toHaveBeenCalledOnce(); expect(harness.captureOpenshellSpy).toHaveBeenCalledWith( ["inference", "get", "-g", "nemoclaw"], { ignoreError: true, timeout: 15_000 }, ); - expect(harness.runOpenshellSpy).not.toHaveBeenCalled(); - expect(exitSpy).toHaveBeenCalledWith(1); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + expect.arrayContaining(["inference", "set", "--provider", "nvidia-prod"]), + expect.any(Object), + ); + expect(exitSpy).not.toHaveBeenCalled(); }); it("aborts before route reads or repairs when the target changes gateways while waiting", async () => { @@ -303,7 +373,7 @@ describe("connect route containment", () => { expect(exitSpy).toHaveBeenCalledWith(1); }); - it("exits before an endpoint probe when an aligned route conflicts with a stopped sandbox (#6315)", async () => { + it("keeps an aligned route healthy when a stopped sandbox records another valid route (#6315)", async () => { const alpha = { name: "alpha", agent: "openclaw", @@ -329,16 +399,14 @@ describe("connect route containment", () => { ], }); - await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( - "process.exit(1)", - ); + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); const routeProbeCalls = harness.captureOpenshellSpy.mock.calls.filter((call) => JSON.stringify(call[0]).includes("inference.local/v1/models"), ); - expect(routeProbeCalls).toHaveLength(0); + expect(routeProbeCalls).toHaveLength(1); expect(harness.runOpenshellSpy).not.toHaveBeenCalled(); - expect(exitSpy).toHaveBeenCalledWith(1); + expect(exitSpy).not.toHaveBeenCalled(); }); it("scopes every inference read and repair write to the target non-default gateway", async () => { diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 648474a76c9..ed71dae1cb9 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -724,6 +724,9 @@ function ensureSandboxInferenceRouteUnlocked( assertNoOpenShellGatewayEndpointOverride(); const { provider, model } = inference; const gatewayName = resolveSandboxGatewayName(sb); + // The live route exposes only provider/model. Prove the target's durable + // custom endpoint/API identity before any route read, probe, or mutation. + assertSandboxGatewayRouteCompatible(sandboxName, sb, gatewayName); const live = parseGatewayInference( captureOpenshell(buildGatewayInferenceGetArgs(gatewayName), { ignoreError: true, @@ -732,7 +735,6 @@ function ensureSandboxInferenceRouteUnlocked( ); const plan = planInferenceRouteReconcile(live, { provider, model }); if (plan.kind !== "aligned") { - assertSandboxGatewayRouteCompatible(sandboxName, sb, gatewayName); const recordedRoute = `${sanitizeRouteValueForDisplay(provider)}/${sanitizeRouteValueForDisplay(model)}`; if (plan.kind === "diverged") { // Shared gateway: re-point loudly (even when quiet) — silent revert was diff --git a/src/lib/actions/sandbox/status-flow.test.ts b/src/lib/actions/sandbox/status-flow.test.ts index ccabc1c5826..25c1b11321b 100644 --- a/src/lib/actions/sandbox/status-flow.test.ts +++ b/src/lib/actions/sandbox/status-flow.test.ts @@ -26,11 +26,12 @@ describe("showSandboxStatus flow", () => { it("warns when the live gateway route differs from the sandbox's recorded route (#6315)", async () => { const harness = createStatusFlowHarness({ - currentProvider: "openai", - currentModel: "gpt-5.2", + currentProvider: "nvidia", + currentModel: "nvidia/nemotron", routeDrift: { live: { provider: "openai", model: "gpt-5.2" }, recorded: { provider: "nvidia", model: "nvidia/nemotron" }, + canConnect: true, }, }); @@ -46,6 +47,8 @@ describe("showSandboxStatus flow", () => { expect(output).toContain( "inference set --provider 'openai' --model 'gpt-5.2' --sandbox 'alpha'", ); + expect(output).toContain("Model: nvidia/nemotron"); + expect(output).toContain("Provider: nvidia"); }); it("shell-quotes hostile route values in drift recovery commands (#6315)", async () => { @@ -56,6 +59,7 @@ describe("showSandboxStatus flow", () => { routeDrift: { live: { provider: "openai; touch /tmp/pwn", model: "$(id) model" }, recorded: { provider: "nvidia", model: "nvidia/nemotron" }, + canConnect: true, }, sandboxEntry: { name: sandboxName }, }); @@ -69,6 +73,22 @@ describe("showSandboxStatus flow", () => { ); }); + it("does not recommend connect when provider-global identity makes it fail (#6315)", async () => { + const harness = createStatusFlowHarness({ + routeDrift: { + live: { provider: "compatible-endpoint", model: "live/model" }, + recorded: { provider: "compatible-endpoint", model: "recorded/model" }, + canConnect: false, + }, + }); + + await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("cannot be restored with nemoclaw connect"); + expect(output).not.toContain("connect realigns the gateway"); + }); + it("prints no route drift warning when the live route matches the recorded route (#6315)", async () => { const harness = createStatusFlowHarness(); @@ -86,7 +106,7 @@ describe("showSandboxStatus flow", () => { const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); expect(output).toContain("Sandbox-scoped status for 'alpha'"); expect(output).toContain("Sandbox: alpha"); - expect(output).toContain("Model: nvidia/nemotron-live"); + expect(output).toContain("Model: nvidia/nemotron"); expect(output).toContain("Inference: healthy"); expect(output).toContain("Inference (ollama backend):"); expect(output).toContain("Host GPU: yes"); diff --git a/src/lib/actions/sandbox/status-inference.test.ts b/src/lib/actions/sandbox/status-inference.test.ts index d7e40fba453..e535b3dea3c 100644 --- a/src/lib/actions/sandbox/status-inference.test.ts +++ b/src/lib/actions/sandbox/status-inference.test.ts @@ -7,6 +7,8 @@ import { collectSandboxStatusSnapshot, getSandboxStatusInferenceHealth } from ". describe("sandbox status inference.local route health (#6192)", () => { function snapshotDeps(options: { provider?: string; + liveProvider?: string; + liveModel?: string; providerHealth?: ReturnType; providerProbeThrows?: boolean; routeHealth: { @@ -19,13 +21,15 @@ describe("sandbox status inference.local route health (#6192)", () => { }) { const provider = options.provider ?? "nvidia-prod"; const reportInferenceProbeError = vi.fn(); + const sandbox = { + name: "alpha", + agent: "openclaw", + model: "nvidia/nemotron", + provider, + }; return { - getSandbox: () => ({ - name: "alpha", - agent: "openclaw", - model: "nvidia/nemotron", - provider, - }), + getSandbox: () => sandbox, + listSandboxes: () => ({ sandboxes: [sandbox], defaultSandbox: "alpha" }), reconcile: async () => ({ state: "present" as const, output: "Name: alpha\nPhase: Ready\n", @@ -33,7 +37,7 @@ describe("sandbox status inference.local route health (#6192)", () => { captureOpenshellForStatusImpl: async () => ({ status: 0, - output: `Provider: ${provider}\nModel: nvidia/nemotron\n`, + output: `Gateway inference:\n Provider: ${options.liveProvider ?? provider}\n Model: ${options.liveModel ?? "nvidia/nemotron"}\n`, }) as never, probeProviderHealthImpl: vi.fn( options.providerProbeThrows @@ -128,6 +132,33 @@ describe("sandbox status inference.local route health (#6192)", () => { ]); }); + it("probes the live route while status displays the sandbox's recorded route (#6315)", async () => { + const deps = snapshotDeps({ + provider: "nvidia-prod", + liveProvider: "openai-api", + liveModel: "gpt-5.2", + routeHealth: { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: "route reachable", + }, + }); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(snapshot.currentProvider).toBe("nvidia-prod"); + expect(snapshot.currentModel).toBe("nvidia/nemotron"); + expect(snapshot.routeDrift).toEqual({ + live: { provider: "openai-api", model: "gpt-5.2" }, + recorded: { provider: "nvidia-prod", model: "nvidia/nemotron" }, + canConnect: true, + }); + expect(deps.probeProviderHealthImpl).toHaveBeenCalledWith("openai-api", { + model: "gpt-5.2", + }); + }); + it("keeps inference.local authoritative when the upstream diagnostic throws (#6192)", async () => { const deps = snapshotDeps({ providerProbeThrows: true, diff --git a/src/lib/actions/sandbox/status-snapshot-route-drift.test.ts b/src/lib/actions/sandbox/status-snapshot-route-drift.test.ts index a720a9d3fa3..d84166119d1 100644 --- a/src/lib/actions/sandbox/status-snapshot-route-drift.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-route-drift.test.ts @@ -26,13 +26,17 @@ function liveGatewayInference(provider: string, model: string, gatewayName = "ne } function snapshotDeps(entry: Partial | null) { + const sandbox = entry + ? ({ name: "alpha", agent: "openclaw", policies: [], ...entry } as SandboxEntry) + : null; return { suppressInferenceProbe: true, deps: { - getSandbox: () => - entry - ? ({ name: "alpha", agent: "openclaw", policies: [], ...entry } as SandboxEntry) - : null, + getSandbox: () => sandbox, + listSandboxes: () => ({ + sandboxes: sandbox ? [sandbox] : [], + defaultSandbox: sandbox ? sandbox.name : null, + }), reconcile: async () => ({ state: "present", output: "Phase: Ready" }), }, }; @@ -54,9 +58,12 @@ describe("collectSandboxStatusSnapshot route drift", () => { expect(snapshot.routeDrift).toEqual({ live: { provider: "openai", model: "gpt-5.2" }, recorded: { provider: "nvidia", model: "nvidia/nemotron" }, + canConnect: true, }); - expect(snapshot.currentProvider).toBe("openai"); - expect(snapshot.currentModel).toBe("gpt-5.2"); + expect(snapshot.liveRoute).toEqual({ provider: "openai", model: "gpt-5.2" }); + expect(snapshot.recordedRoute).toEqual({ provider: "nvidia", model: "nvidia/nemotron" }); + expect(snapshot.currentProvider).toBe("nvidia"); + expect(snapshot.currentModel).toBe("nvidia/nemotron"); }); it("reads the sandbox's non-default gateway before computing drift (#6315)", async () => { @@ -74,9 +81,10 @@ describe("collectSandboxStatusSnapshot route drift", () => { expect(snapshot.routeDrift).toEqual({ live: { provider: "openai", model: "gpt-5.2" }, recorded: { provider: "nvidia", model: "nvidia/nemotron" }, + canConnect: true, }); - expect(snapshot.currentProvider).toBe("openai"); - expect(snapshot.currentModel).toBe("gpt-5.2"); + expect(snapshot.currentProvider).toBe("nvidia"); + expect(snapshot.currentModel).toBe("nvidia/nemotron"); }); it("does not fall back to the default gateway for an invalid persisted binding (#6315)", async () => { @@ -129,5 +137,49 @@ describe("collectSandboxStatusSnapshot route drift", () => { const snapshot = await collectSandboxStatusSnapshot("alpha", snapshotDeps({})); expect(snapshot.routeDrift).toBeNull(); + expect(snapshot.currentProvider).toBe("unknown"); + expect(snapshot.currentModel).toBe("unknown"); + }); + + it("does not mix partial recorded metadata with the live route (#6315)", async () => { + liveGatewayInference("openai", "gpt-5.2"); + + const snapshot = await collectSandboxStatusSnapshot( + "alpha", + snapshotDeps({ provider: "nvidia" }), + ); + + expect(snapshot.routeDrift).toBeNull(); + expect(snapshot.currentProvider).toBe("nvidia"); + expect(snapshot.currentModel).toBe("unknown"); + }); + + it("does not advertise connect for a legacy custom-provider identity conflict (#6315)", async () => { + liveGatewayInference("compatible-endpoint", "live/model"); + const target = { + provider: "compatible-endpoint", + model: "recorded/model", + endpointUrl: "https://target.example/v1", + credentialEnv: "TARGET_KEY", + preferredInferenceApi: "openai-completions", + } satisfies Partial; + const peer: SandboxEntry = { + name: "peer", + gatewayName: "nemoclaw", + provider: "compatible-endpoint", + model: "peer/model", + endpointUrl: "https://peer.example/v1", + credentialEnv: "PEER_KEY", + preferredInferenceApi: "openai-completions", + }; + const options = snapshotDeps(target); + options.deps.listSandboxes = () => ({ + sandboxes: [options.deps.getSandbox() as SandboxEntry, peer], + defaultSandbox: "alpha", + }); + + const snapshot = await collectSandboxStatusSnapshot("alpha", options); + + expect(snapshot.routeDrift).toMatchObject({ canConnect: false }); }); }); diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index 3bfa975091b..62c9c5a830b 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -27,7 +27,10 @@ import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; import { redact } from "../../security/redact"; import { parseSandboxPhase } from "../../state/gateway"; import * as registry from "../../state/registry"; -import { buildGatewayInferenceGetArgs } from "./connect-inference-gateway"; +import { + buildGatewayInferenceGetArgs, + canSandboxGatewayRouteRealign, +} from "./connect-inference-gateway"; import { classifyInferenceRouteFailureLabel } from "./connect-inference-route-probe"; import { getSandboxDockerRuntime } from "./docker-health"; import type { SandboxGatewayState } from "./gateway-state"; @@ -138,6 +141,9 @@ export interface SandboxStatusReport { agentLoadError?: string; model: string; provider: string; + recordedRoute: RecordedInferenceRoute | null; + liveRoute: GatewayInference | null; + routeDrift: SandboxStatusRouteDrift | null; phase: string | null; gatewayState: string; inferenceHealth: ProviderHealthStatus | null; @@ -166,6 +172,7 @@ export interface SandboxStatusReport { export interface SandboxStatusRouteDrift { live: GatewayInference; recorded: RecordedInferenceRoute; + canConnect: boolean; } export interface SandboxStatusSnapshot { @@ -174,6 +181,8 @@ export interface SandboxStatusSnapshot { rpcIssue: OpenShellStateRpcIssue | null; currentModel: string; currentProvider: string; + recordedRoute: RecordedInferenceRoute | null; + liveRoute: GatewayInference | null; routeDrift: SandboxStatusRouteDrift | null; inferenceHealth: ProviderHealthStatus | null; terminalRuntimeHealth: TerminalRuntimeOomProbeResult | null; @@ -224,6 +233,7 @@ type ProbeTerminalRuntimeHealth = (sandboxName: string) => TerminalRuntimeOomPro interface CollectSandboxStatusSnapshotDeps { getSandbox?: typeof registry.getSandbox; + listSandboxes?: typeof registry.listSandboxes; captureOpenshellForStatusImpl?: typeof captureOpenshellForStatus; probeProviderHealthImpl?: ProbeProviderHealth; probeSandboxInferenceGatewayHealthImpl?: ProbeSandboxInferenceGatewayHealth; @@ -269,9 +279,10 @@ export async function collectSandboxStatusSnapshot( }; } let liveResult: Awaited> | null = null; + let gatewayName: string | null = null; if (lookup.state === "present") { try { - const gatewayName = resolveSandboxGatewayName(sb); + gatewayName = resolveSandboxGatewayName(sb); liveResult = await (opts.deps?.captureOpenshellForStatusImpl ?? captureOpenshellForStatus)( buildGatewayInferenceGetArgs(gatewayName), ); @@ -287,8 +298,10 @@ export async function collectSandboxStatusSnapshot( sb, lookup, rpcIssue, - currentModel: "unknown", - currentProvider: "unknown", + currentModel: (sb && sb.model) || "unknown", + currentProvider: (sb && sb.provider) || "unknown", + recordedRoute: sb?.provider && sb.model ? { provider: sb.provider, model: sb.model } : null, + liveRoute: null, routeDrift: null, inferenceHealth: null, terminalRuntimeHealth: null, @@ -296,19 +309,34 @@ export async function collectSandboxStatusSnapshot( } const live = liveResult && !isCommandTimeout(liveResult) ? parseGatewayInference(liveResult.output) : null; - const currentModel = (live && live.model) || (sb && sb.model) || "unknown"; - const currentProvider = (live && live.provider) || (sb && sb.provider) || "unknown"; - // Status shows the live gateway route when one is readable, which silently - // masks a route another sandbox (or a direct `openshell inference set`) - // moved from under this one — the shared-route trap of #6315. Surface the - // divergence instead of letting the live value pass as this sandbox's own. + const recordedRoute = + sb?.provider && sb.model ? { provider: sb.provider, model: sb.model } : null; + const liveRoute = live ? { provider: live.provider, model: live.model } : null; + // Model/provider are sandbox-scoped status fields, so prefer the durable + // route recorded for this sandbox. The live shared route is shown separately + // as drift instead of being mislabeled as this sandbox's configuration. + const currentModel = sb ? sb.model || "unknown" : (live && live.model) || "unknown"; + const currentProvider = sb ? sb.provider || "unknown" : (live && live.provider) || "unknown"; const routeDriftPlan = sb && sb.provider && sb.model ? planInferenceRouteReconcile(live, { provider: sb.provider, model: sb.model }) : null; const routeDrift = routeDriftPlan && routeDriftPlan.kind === "diverged" - ? { live: routeDriftPlan.live, recorded: routeDriftPlan.recorded } + ? { + live: routeDriftPlan.live, + recorded: routeDriftPlan.recorded, + canConnect: Boolean( + sb && + gatewayName && + canSandboxGatewayRouteRealign( + sandboxName, + sb, + gatewayName, + (opts.deps?.listSandboxes ?? registry.listSandboxes)().sandboxes, + ), + ), + } : null; // When the caller has already determined that the local stack is failed // (docker daemon down, sandbox container stopped, dashboard port held), @@ -321,8 +349,8 @@ export async function collectSandboxStatusSnapshot( providerHealth = maybeGetSandboxStatusInferenceHealth( opts.suppressInferenceProbe === true, lookup.state === "present", - currentProvider, - currentModel, + (live && live.provider) || currentProvider, + (live && live.model) || currentModel, opts.deps?.probeProviderHealthImpl, ); } catch { @@ -364,6 +392,8 @@ export async function collectSandboxStatusSnapshot( rpcIssue, currentModel, currentProvider, + recordedRoute, + liveRoute, routeDrift, inferenceHealth, terminalRuntimeHealth, @@ -398,6 +428,9 @@ async function buildSandboxStatusReport( rpcIssue, currentModel, currentProvider, + recordedRoute, + liveRoute, + routeDrift, inferenceHealth, terminalRuntimeHealth, } = snapshot; @@ -419,8 +452,14 @@ async function buildSandboxStatusReport( agentRuntime: agent.agentRuntime, dcodeAutoApprovalMode: resolveSandboxStatusDcodeAutoApprovalMode(sb), ...(agent.agentLoadError ? { agentLoadError: agent.agentLoadError } : {}), - model: currentModel, - provider: currentProvider, + // Keep schema v1's established live-first fields for existing consumers. + // The explicit route fields separate durable sandbox intent from the one + // gateway-global route without changing those legacy meanings. + model: liveRoute?.model ?? currentModel, + provider: liveRoute?.provider ?? currentProvider, + recordedRoute, + liveRoute, + routeDrift, phase, gatewayState: lookup.state, inferenceHealth, diff --git a/src/lib/actions/sandbox/status-text.ts b/src/lib/actions/sandbox/status-text.ts index 68307b0b40e..a9be7b461a9 100644 --- a/src/lib/actions/sandbox/status-text.ts +++ b/src/lib/actions/sandbox/status-text.ts @@ -249,11 +249,9 @@ function printAgentVersion(context: SandboxStatusTextContext, sandbox: SandboxEn } } -// The Model/Provider lines above show the live gateway route, which the -// shared per-gateway route lets another sandbox move (#6315). When it no -// longer matches this sandbox's recorded route, say so instead of presenting -// the live value as this sandbox's own; wording mirrors the connect-time -// divergence warning (#3726). +// The Model/Provider lines above show this sandbox's recorded route. The live +// shared route can differ after another onboard, so report that drift +// separately; wording mirrors the connect-time divergence warning (#3726). function printInferenceRouteDrift( drift: SandboxStatusRouteDrift | null, sandboxName: string, @@ -266,6 +264,15 @@ function printInferenceRouteDrift( ); const { liveProvider, liveModel, recordedRoute } = display; console.log(` ${YW}Warning: ${display.warning}${R}`); + if (!drift.canConnect) { + console.log( + ` ${YW}The recorded route cannot be restored with ${CLI_NAME} connect while another registered sandbox uses different provider-global endpoint, API-family, or credential identity.${R}`, + ); + console.log( + ` ${YW}Remove or re-onboard the conflicting sandbox before reconnecting '${sandboxName}'.${R}`, + ); + return; + } console.log( ` ${YW}${CLI_NAME} ${shellQuote(sandboxName)} connect realigns the gateway to ${recordedRoute}; to adopt the live route instead:${R}`, ); diff --git a/src/lib/inference/gateway-route-compatibility.test.ts b/src/lib/inference/gateway-route-compatibility.test.ts index c4d3f53c8ee..3680db6404c 100644 --- a/src/lib/inference/gateway-route-compatibility.test.ts +++ b/src/lib/inference/gateway-route-compatibility.test.ts @@ -6,7 +6,10 @@ import type { SandboxEntry } from "../state/registry"; import { checkGatewayRouteCompatibility, formatGatewayRouteConflict, + formatGatewayRouteImpactWarning, type GatewayInferenceRoute, + isAdvisoryGatewayRouteConflict, + isAdvisoryProviderModelRouteConflict, preflightGatewayRouteDiscovery, } from "./gateway-route-compatibility"; @@ -138,7 +141,7 @@ describe("shared gateway inference route compatibility", () => { ).toEqual({ ok: true }); }); - it("blocks provider or model conflicts from every same-gateway registry row (#6315)", () => { + it("classifies valid provider or model differences as advisory and names affected sandboxes (#6315)", () => { const result = check(route("anthropic-prod", "claude-new"), [sandbox("stopped-peer")]); expect(result).toMatchObject({ @@ -148,6 +151,16 @@ describe("shared gateway inference route compatibility", () => { expect(formatGatewayRouteConflict(result as Exclude)).toContain( "Stopped sandboxes are included", ); + expect(isAdvisoryGatewayRouteConflict(result as Exclude)).toBe( + true, + ); + expect( + isAdvisoryProviderModelRouteConflict(result as Exclude), + ).toBe(true); + const warning = formatGatewayRouteImpactWarning(result as Exclude); + expect(warning).toContain("will re-point the one shared inference route"); + expect(warning).toContain("'stopped-peer' (nvidia-prod / nvidia/model-a)"); + expect(warning).toContain("not per sandbox"); }); it("allows different routes on different gateways (#6315)", () => { @@ -223,25 +236,34 @@ describe("shared gateway inference route compatibility", () => { }); }); - it("ignores credential environment differences in route identity (#6315)", () => { - expect( - check( - route("compatible-endpoint", "custom/model", { + it("fails closed before replacing a shared provider credential identity (#6315)", () => { + const result = check( + route("compatible-endpoint", "custom/model", { + endpointUrl: "https://example.test/v1", + preferredInferenceApi: "openai-completions", + credentialEnv: "REQUESTED_KEY", + }), + [ + sandbox("custom-peer", { + provider: "compatible-endpoint", + model: "custom/model", endpointUrl: "https://example.test/v1", preferredInferenceApi: "openai-completions", - credentialEnv: "REQUESTED_KEY", + credentialEnv: "RECORDED_KEY", }), - [ - sandbox("custom-peer", { - provider: "compatible-endpoint", - model: "custom/model", - endpointUrl: "https://example.test/v1", - preferredInferenceApi: "openai-completions", - credentialEnv: "RECORDED_KEY", - }), - ], - ), - ).toEqual({ ok: true }); + ], + ); + + expect(result).toMatchObject({ + ok: false, + conflicts: [{ sandboxName: "custom-peer", reason: "provider-credential" }], + }); + expect(formatGatewayRouteConflict(result as Exclude)).toContain( + "different credential identity", + ); + expect(isAdvisoryGatewayRouteConflict(result as Exclude)).toBe( + false, + ); }); it.each([ @@ -271,6 +293,37 @@ describe("shared gateway inference route compatibility", () => { ); expect(result).toMatchObject({ ok: false, conflicts: [{ reason }] }); + expect(isAdvisoryGatewayRouteConflict(result as Exclude)).toBe( + false, + ); + expect( + isAdvisoryProviderModelRouteConflict(result as Exclude), + ).toBe(false); + }); + + it("does not let a model difference hide a custom endpoint conflict (#6315)", () => { + const result = check( + route("compatible-endpoint", "target/model", { + endpointUrl: "https://target.example.test/v1", + preferredInferenceApi: "openai-completions", + }), + [ + sandbox("custom-peer", { + provider: "compatible-endpoint", + model: "peer/model", + endpointUrl: "https://peer.example.test/v1", + preferredInferenceApi: "openai-completions", + }), + ], + ); + + expect(result).toMatchObject({ + ok: false, + conflicts: [{ sandboxName: "custom-peer", reason: "custom-endpoint" }], + }); + expect( + isAdvisoryProviderModelRouteConflict(result as Exclude), + ).toBe(false); }); it.each([ @@ -305,6 +358,34 @@ describe("shared gateway inference route compatibility", () => { expect(formatGatewayRouteConflict(result as Exclude)).toContain( "remove and re-onboard that sandbox with complete custom-route metadata", ); + expect(isAdvisoryGatewayRouteConflict(result as Exclude)).toBe( + false, + ); + }); + + it("fails closed when a different provider encounters an incomplete custom peer (#6315)", () => { + const result = check(route("anthropic-prod", "claude-new"), [ + sandbox("legacy-custom", { + provider: "compatible-endpoint", + model: "custom/model", + endpointUrl: null, + preferredInferenceApi: "openai-completions", + }), + ]); + + expect(result).toMatchObject({ + ok: false, + conflicts: [ + { + sandboxName: "legacy-custom", + reason: "incomplete-custom-route", + scope: "registered", + }, + ], + }); + expect(isAdvisoryGatewayRouteConflict(result as Exclude)).toBe( + false, + ); }); it("fails closed when a requested custom route has no API metadata or peers (#6315)", () => { @@ -329,6 +410,9 @@ describe("shared gateway inference route compatibility", () => { expect(formatGatewayRouteConflict(result as Exclude)).toContain( "requested custom route lacks durable endpoint or API-family metadata", ); + expect(isAdvisoryGatewayRouteConflict(result as Exclude)).toBe( + false, + ); }); it.each([ diff --git a/src/lib/inference/gateway-route-compatibility.ts b/src/lib/inference/gateway-route-compatibility.ts index 1d96c186214..d44ab6c712f 100644 --- a/src/lib/inference/gateway-route-compatibility.ts +++ b/src/lib/inference/gateway-route-compatibility.ts @@ -30,6 +30,7 @@ export type GatewayRouteConflictReason = | "provider-model" | "custom-endpoint" | "custom-api" + | "provider-credential" | "incomplete-route" | "incomplete-custom-route" | "invalid-gateway-binding"; @@ -38,6 +39,7 @@ export interface GatewayRouteConflict { sandboxName: string; reason: GatewayRouteConflictReason; scope?: "requested" | "registered"; + recordedRoute?: { provider: string; model: string }; } export type GatewayRouteCompatibilityResult = @@ -113,6 +115,13 @@ function customRouteConflict( return null; } +function providerCredentialConflict( + requested: GatewayInferenceRoute, + recorded: GatewayInferenceRoute, +): boolean { + return nonEmptyString(requested.credentialEnv) !== nonEmptyString(recorded.credentialEnv); +} + /** * Constrain read-only route discovery from durable same-gateway registry peers. * Missing requested model/API fields are allowed only when the gateway has no @@ -187,6 +196,7 @@ export function preflightGatewayRouteDiscovery( sandboxName: sandbox.name, reason: "provider-model" as const, scope: "registered" as const, + recordedRoute: configuredRoute(sandbox) ?? undefined, })), }, }; @@ -197,6 +207,7 @@ export function preflightGatewayRouteDiscovery( provider, model: requestedModel ?? recorded.model, endpointUrl: nonEmptyString(request.route.endpointUrl) ?? reference.endpointUrl, + credentialEnv: nonEmptyString(request.route.credentialEnv) ?? reference.credentialEnv, preferredInferenceApi: nonEmptyString(request.route.preferredInferenceApi) ?? reference.preferredInferenceApi, }; @@ -270,17 +281,55 @@ export function checkGatewayRouteCompatibility( continue; } - if (recorded.provider !== requested.provider || recorded.model !== requested.model) { + if ( + CUSTOM_ROUTE_PROVIDERS.has(recorded.provider) && + (!canonicalEndpoint(sandbox.endpointUrl, endpointFlavor(recorded.provider)) || + !normalizedInferenceApi(sandbox.preferredInferenceApi)) + ) { conflicts.push({ sandboxName: sandbox.name, - reason: "provider-model", + reason: "incomplete-custom-route", scope: "registered", + recordedRoute: recorded, }); continue; } - if (CUSTOM_ROUTE_PROVIDERS.has(requested.provider)) { - const reason = customRouteConflict(requested.provider, request.route, sandbox); - if (reason) conflicts.push({ sandboxName: sandbox.name, reason, scope: "registered" }); + + // A provider/model-only mutation cannot safely replace provider-global + // endpoint, API-family, or credential identity. Compare that fingerprint + // first so a simultaneous model difference cannot hide a provider mutation. + if (recorded.provider === requested.provider) { + let providerIdentityConflict = false; + if (CUSTOM_ROUTE_PROVIDERS.has(requested.provider)) { + const reason = customRouteConflict(requested.provider, request.route, sandbox); + if (reason) { + conflicts.push({ + sandboxName: sandbox.name, + reason, + scope: "registered", + recordedRoute: recorded, + }); + providerIdentityConflict = true; + } + } + if (providerCredentialConflict(request.route, sandbox)) { + conflicts.push({ + sandboxName: sandbox.name, + reason: "provider-credential", + scope: "registered", + recordedRoute: recorded, + }); + providerIdentityConflict = true; + } + if (providerIdentityConflict) continue; + } + if (recorded.provider !== requested.provider || recorded.model !== requested.model) { + conflicts.push({ + sandboxName: sandbox.name, + reason: "provider-model", + scope: "registered", + recordedRoute: recorded, + }); } } @@ -299,6 +348,66 @@ function safeDisplay(value: string): string { return value.replace(/[\u0000-\u001f\u007f]/g, "?"); } +const ADVISORY_ROUTE_CONFLICTS = new Set(["provider-model"]); + +/** + * True only for provider/model differences that a route-only mutation can + * reconcile. Provider-global endpoint, API-family, and credential identity + * differences remain hard errors because they require unsafe provider mutation. + */ +export function isAdvisoryGatewayRouteConflict( + result: Exclude, +): boolean { + return ( + result.conflicts.length > 0 && + result.conflicts.every( + (conflict) => conflict.scope !== "requested" && ADVISORY_ROUTE_CONFLICTS.has(conflict.reason), + ) + ); +} + +/** + * True only when a provider/model-only route mutation can safely reconcile the + * conflict. Provider-global endpoint, API-family, and credential changes stay + * fail-closed while another registered sandbox depends on the current identity. + */ +export function isAdvisoryProviderModelRouteConflict( + result: Exclude, +): boolean { + return ( + result.conflicts.length > 0 && + result.conflicts.every( + (conflict) => conflict.scope !== "requested" && conflict.reason === "provider-model", + ) + ); +} + +/** Explain the single-gateway side effect immediately before route mutation. */ +export function formatGatewayRouteImpactWarning( + result: Exclude, +): string { + const affected = [...result.conflicts] + .sort((left, right) => left.sandboxName.localeCompare(right.sandboxName)) + .map((conflict) => { + const name = `'${safeDisplay(conflict.sandboxName)}'`; + return conflict.recordedRoute + ? `${name} (${safeDisplay(conflict.recordedRoute.provider)} / ${safeDisplay(conflict.recordedRoute.model)})` + : name; + }) + .join(", "); + const target = result.sandboxName + ? `Onboarding '${safeDisplay(result.sandboxName)}'` + : "This onboarding run"; + const nextRoute = `${safeDisplay(result.route.provider)} / ${safeDisplay(result.route.model)}`; + return ( + `Warning: ${target} will re-point the one shared inference route on OpenShell gateway ` + + `'${safeDisplay(result.gatewayName)}' to ${nextRoute}. ` + + `Affected registered sandboxes: ${affected}. ` + + `They will use ${nextRoute} until the shared route is changed again. ` + + "OpenShell currently exposes this route per gateway, not per sandbox." + ); +} + export function formatGatewayRouteConflict( result: Exclude, ): string { @@ -325,6 +434,15 @@ export function formatGatewayRouteConflict( const hasInvalidGatewayBinding = result.conflicts.some( (conflict) => conflict.reason === "invalid-gateway-binding", ); + const providerIdentityDifferences = [ + result.conflicts.some((conflict) => conflict.reason === "custom-endpoint") ? "endpoint" : null, + result.conflicts.some((conflict) => conflict.reason === "custom-api") ? "API family" : null, + result.conflicts.some((conflict) => conflict.reason === "provider-credential") + ? "credential identity" + : null, + ].filter((value): value is string => value !== null); + const requiresRegistryRepair = + hasIncompleteCustomRoute || hasIncompleteRoute || hasInvalidGatewayBinding; const detail = [ hasIncompleteCustomRoute ? "At least one custom route lacks durable endpoint or API-family metadata, so compatibility cannot be proven; remove and re-onboard that sandbox with complete custom-route metadata." @@ -335,6 +453,11 @@ export function formatGatewayRouteConflict( hasInvalidGatewayBinding ? "At least one registry row has an invalid gateway binding, so gateway separation cannot be proven; restore its known-good gateway binding or remove and re-onboard that sandbox." : null, + providerIdentityDifferences.length > 0 + ? `At least one registered sandbox uses a different ${providerIdentityDifferences.join( + ", ", + )} for the same shared provider. NemoClaw will not replace provider-global configuration while that sandbox remains registered.` + : null, ] .filter(Boolean) .join(" "); @@ -348,7 +471,9 @@ export function formatGatewayRouteConflict( "Stopped sandboxes are included because they use the same gateway route when restarted. " + (requestedRouteIncomplete ? "Remove and re-onboard the sandbox with complete custom-route metadata." - : "Align the routes, remove the conflicting sandbox, or use another NEMOCLAW_GATEWAY_PORT.") + : requiresRegistryRepair + ? "Repair incomplete registry metadata, or remove and re-onboard the affected sandbox." + : "Align the recorded routes, or remove a conflicting sandbox that is no longer needed.") ); } diff --git a/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts b/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts index 3c330b4c194..378c5695f23 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts @@ -40,15 +40,20 @@ function createDeps() { requiredInferenceApi: null, })), selectionProbe: vi.fn(), + canProbeResult: vi.fn(), + routeConstraints: vi.fn(), setupNim: vi.fn( - async (_gpu, _sandbox, _agent, _recover, _gateway, guard) => { - guard?.({ - provider: fallbackSelection.provider, - model: fallbackSelection.model, - endpointUrl: fallbackSelection.endpointUrl, - credentialEnv: fallbackSelection.credentialEnv, - preferredInferenceApi: fallbackSelection.preferredInferenceApi, - }); + async (_gpu, _sandbox, _agent, _recover, _gateway, guard, canProbeRoute) => { + calls.canProbeResult(canProbeRoute?.(fallbackSelection.provider)); + calls.routeConstraints( + guard?.({ + provider: fallbackSelection.provider, + model: fallbackSelection.model, + endpointUrl: fallbackSelection.endpointUrl, + credentialEnv: fallbackSelection.credentialEnv, + preferredInferenceApi: fallbackSelection.preferredInferenceApi, + }), + ); calls.selectionProbe(); return { ...fallbackSelection }; }, @@ -153,7 +158,7 @@ function resumeOptions( }; } -function rejectRoute( +function reportDifferentRoute( calls: ReturnType["calls"], provider: string, model: string, @@ -178,14 +183,18 @@ function rejectRoute( } describe("provider route containment", () => { - it("rejects a fresh selection before completing its session step or starting inference", async () => { + it("allows fresh selection to continue so setup can issue the mutation warning (#6315)", async () => { const { calls, deps } = createDeps(); - rejectRoute(calls, "nvidia-prod", "nvidia/test"); + reportDifferentRoute(calls, "nvidia-prod", "nvidia/test"); const options = resumeOptions(deps, createSession()); await expect( handleProviderInferenceState({ ...options, resume: false, sandboxName: null }), - ).rejects.toThrow("exit 1"); + ).resolves.toMatchObject({ + sandboxName: "target-sandbox", + provider: "nvidia-prod", + model: "nvidia/test", + }); expect(calls.setupNim).toHaveBeenCalledOnce(); expect(calls.preflightGatewayRouteDiscovery).toHaveBeenCalledWith({ @@ -199,22 +208,62 @@ describe("provider route containment", () => { credentialEnv: "NVIDIA_INFERENCE_API_KEY", }, }); - expect(calls.checkGatewayRouteCompatibility).not.toHaveBeenCalled(); - expect(calls.selectionProbe).not.toHaveBeenCalled(); - expect(calls.recordStepComplete).not.toHaveBeenCalled(); + expect(calls.checkGatewayRouteCompatibility).toHaveBeenCalledWith( + expect.objectContaining({ + route: expect.objectContaining({ provider: "nvidia-prod", model: "nvidia/test" }), + }), + ); + expect(calls.selectionProbe).toHaveBeenCalledOnce(); + expect(calls.canProbeResult).toHaveBeenCalledWith(true); + expect(calls.routeConstraints).toHaveBeenCalledWith({ + requiredModel: null, + requiredEndpointUrl: null, + requiredInferenceApi: null, + }); + expect(calls.recordStepComplete).toHaveBeenCalled(); expect(calls.surfaceReady).not.toHaveBeenCalled(); - expect(calls.setupInference).not.toHaveBeenCalled(); + expect(calls.setupInference).toHaveBeenCalledOnce(); expect(calls.updateSandbox).not.toHaveBeenCalled(); + expect(calls.error).not.toHaveBeenCalled(); + }); + + it("does not constrain fresh selection to a valid peer route (#6315)", async () => { + const { calls, deps } = createDeps(); + calls.preflightGatewayRouteDiscovery.mockReturnValue({ + ok: true, + requiredModel: "peer/model", + requiredEndpointUrl: "https://peer.example.test/v1", + requiredInferenceApi: "openai-completions", + }); + const options = resumeOptions(deps, createSession()); + + await expect( + handleProviderInferenceState({ ...options, resume: false, sandboxName: null }), + ).resolves.toMatchObject({ + provider: "nvidia-prod", + model: "nvidia/test", + }); + + expect(calls.routeConstraints).toHaveBeenCalledWith({ + requiredModel: null, + requiredEndpointUrl: null, + requiredInferenceApi: null, + }); + expect(calls.setupInference).toHaveBeenCalledOnce(); + expect(calls.error).not.toHaveBeenCalled(); }); - it("blocks routed-provider repair before gateway or registry mutation", async () => { + it("allows routed-provider repair across a valid peer-route difference (#6315)", async () => { const session = createSession({ provider: "nvidia-router", model: "router/model" }); session.steps.provider_selection.status = "complete"; const { calls, deps } = createDeps(); - rejectRoute(calls, "nvidia-router", "router/model"); + reportDifferentRoute(calls, "nvidia-router", "router/model"); - await expect(handleProviderInferenceState(resumeOptions(deps, session))).rejects.toThrow( - "exit 1", + await expect(handleProviderInferenceState(resumeOptions(deps, session))).resolves.toMatchObject( + { + provider: "nvidia-router", + model: "router/model", + }, ); expect(calls.checkGatewayRouteCompatibility).toHaveBeenCalledWith({ @@ -224,17 +273,18 @@ describe("provider route containment", () => { provider: "nvidia-router", model: "router/model", endpointUrl: null, + credentialEnv: null, preferredInferenceApi: null, }, }); - expect(calls.reconcileRouter).not.toHaveBeenCalled(); - expect(calls.surfaceReady).not.toHaveBeenCalled(); - expect(calls.reupsertRoutedProvider).not.toHaveBeenCalled(); + expect(calls.reconcileRouter).toHaveBeenCalledOnce(); + expect(calls.surfaceReady).toHaveBeenCalledOnce(); + expect(calls.reupsertRoutedProvider).toHaveBeenCalledOnce(); expect(calls.updateSandbox).not.toHaveBeenCalled(); expect(calls.setupInference).not.toHaveBeenCalled(); }); - it("rechecks routed repair after waiting for the gateway lock", async () => { + it("rechecks routed repair after waiting for the gateway lock (#6315)", async () => { const session = createSession({ provider: "nvidia-router", model: "router/model" }); session.steps.provider_selection.status = "complete"; const { calls, deps } = createDeps(); @@ -254,16 +304,16 @@ describe("provider route containment", () => { const repair = handleProviderInferenceState(resumeOptions(deps, session)); await lockEntered; - rejectRoute(calls, "nvidia-router", "router/model"); + reportDifferentRoute(calls, "nvidia-router", "router/model"); releaseLock(); - await expect(repair).rejects.toThrow("exit 1"); - expect(calls.reconcileRouter).not.toHaveBeenCalled(); - expect(calls.reupsertRoutedProvider).not.toHaveBeenCalled(); + await expect(repair).resolves.toMatchObject({ provider: "nvidia-router" }); + expect(calls.reconcileRouter).toHaveBeenCalledOnce(); + expect(calls.reupsertRoutedProvider).toHaveBeenCalledOnce(); expect(calls.updateSandbox).not.toHaveBeenCalled(); }); - it("blocks compatible-endpoint messaging refresh before endpoint or gateway work", async () => { + it("allows compatible-endpoint refresh to reach the final setup boundary (#6315)", async () => { const session = createSession({ provider: "compatible-endpoint", model: "custom/model", @@ -273,11 +323,11 @@ describe("provider route containment", () => { }); session.steps.provider_selection.status = "complete"; const { calls, deps } = createDeps(); - rejectRoute(calls, "compatible-endpoint", "custom/model"); + reportDifferentRoute(calls, "compatible-endpoint", "custom/model"); await expect( handleProviderInferenceState(resumeOptions(deps, session, ["telegram"])), - ).rejects.toThrow("exit 1"); + ).resolves.toMatchObject({ provider: "compatible-endpoint", model: "custom/model" }); expect(calls.checkGatewayRouteCompatibility).toHaveBeenCalledWith({ gatewayName: "nemoclaw-9090", @@ -286,12 +336,33 @@ describe("provider route containment", () => { provider: "compatible-endpoint", model: "custom/model", endpointUrl: "https://example.test/v1", + credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", }, }); - expect(calls.setupInference).not.toHaveBeenCalled(); - expect(calls.surfaceReady).not.toHaveBeenCalled(); + expect(calls.setupInference).toHaveBeenCalledOnce(); + expect(calls.surfaceReady).toHaveBeenCalledOnce(); expect(calls.updateSandbox).not.toHaveBeenCalled(); - expect(calls.error).toHaveBeenCalledWith(expect.stringContaining("existing-sandbox")); + expect(calls.error).not.toHaveBeenCalled(); + }); + + it("still blocks incomplete registered route metadata (#6315)", async () => { + const session = createSession(); + session.steps.provider_selection.status = "complete"; + const { calls, deps } = createDeps(); + calls.checkGatewayRouteCompatibility.mockReturnValue({ + ok: false, + gatewayName: "nemoclaw-9090", + sandboxName: "target-sandbox", + route: { provider: "nvidia-prod", model: "nvidia/test" }, + conflicts: [{ sandboxName: "broken-sandbox", reason: "incomplete-route" }], + }); + + await expect(handleProviderInferenceState(resumeOptions(deps, session))).rejects.toThrow( + "exit 1", + ); + + expect(calls.error).toHaveBeenCalledWith(expect.stringContaining("broken-sandbox")); + expect(calls.setupInference).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/onboard/machine/handlers/provider-inference-route-containment.ts b/src/lib/onboard/machine/handlers/provider-inference-route-containment.ts index e6ff6327a18..2e881056852 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-route-containment.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-route-containment.ts @@ -7,6 +7,7 @@ import { formatGatewayRouteConflict, type GatewayInferenceRoute, type GatewayRouteDiscoveryConstraints, + isAdvisoryGatewayRouteConflict, } from "../../../inference/gateway-route-compatibility"; export interface ProviderInferenceRouteContainmentDeps { @@ -20,6 +21,14 @@ export type ProviderInferenceProbeRoute = Omit & model: string | null; }; +function unconstrainedGatewayRouteDiscovery(): GatewayRouteDiscoveryConstraints { + return { + requiredModel: null, + requiredEndpointUrl: null, + requiredInferenceApi: null, + }; +} + export function assertProviderInferenceRouteCompatible( deps: ProviderInferenceRouteContainmentDeps, gatewayName: string, @@ -28,12 +37,13 @@ export function assertProviderInferenceRouteCompatible( ): void { const compatibility = deps.checkGatewayRouteCompatibility({ gatewayName, sandboxName, route }); if (!compatibility.ok) { + if (isAdvisoryGatewayRouteConflict(compatibility)) return; deps.error(` Error: ${formatGatewayRouteConflict(compatibility)}`); deps.exitProcess(1); } } -/** Constrain discovery from durable peers, then exact-check complete route identities. */ +/** Reject structurally unsafe peer metadata, then exact-check complete route identities. */ export function guardProviderInferenceRouteSelection( deps: ProviderInferenceRouteContainmentDeps, gatewayName: string, @@ -47,6 +57,9 @@ export function guardProviderInferenceRouteSelection( route: { ...route, model }, }); if (!preflight.ok) { + if (isAdvisoryGatewayRouteConflict(preflight.result)) { + return unconstrainedGatewayRouteDiscovery(); + } deps.error(` Error: ${formatGatewayRouteConflict(preflight.result)}`); deps.exitProcess(1); } @@ -60,5 +73,5 @@ export function guardProviderInferenceRouteSelection( if (model && completeCustomRoute) { assertProviderInferenceRouteCompatible(deps, gatewayName, sandboxName, { ...route, model }); } - return preflight; + return unconstrainedGatewayRouteDiscovery(); } diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index 910f49bf741..99f58434223 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -3,10 +3,11 @@ import { clearAutoDetectedCompatibleContextWindow } from "../../../inference/compatible-endpoint-context"; import { resolveAgentProviderInferenceApi } from "../../../inference/config"; -import type { - CurrentGatewayRouteCompatibilityCheck, - CurrentGatewayRouteDiscoveryPreflight, - GatewayRouteDiscoveryConstraints, +import { + type CurrentGatewayRouteCompatibilityCheck, + type CurrentGatewayRouteDiscoveryPreflight, + type GatewayRouteDiscoveryConstraints, + isAdvisoryGatewayRouteConflict, } from "../../../inference/gateway-route-compatibility"; import type { WebSearchConfig } from "../../../inference/web-search"; import type { HermesAuthMethod, Session, SessionUpdates } from "../../../state/onboard-session"; @@ -375,6 +376,7 @@ export async function handleProviderInferenceState({ provider, model, endpointUrl, + credentialEnv, preferredInferenceApi, }); const recovery = await deps.ensureResumeProviderReady(gatewayName, provider, credentialEnv); @@ -483,8 +485,8 @@ export async function handleProviderInferenceState({ recoverRecordedProvider, gatewayName, (route) => guardProviderInferenceRouteSelection(deps, gatewayName, sandboxName, route), - (provider) => - deps.preflightGatewayRouteDiscovery({ + (provider) => { + const preflight = deps.preflightGatewayRouteDiscovery({ gatewayName, sandboxName, route: { @@ -494,7 +496,9 @@ export async function handleProviderInferenceState({ preferredInferenceApi: null, credentialEnv: null, }, - }).ok, + }); + return preflight.ok || isAdvisoryGatewayRouteConflict(preflight.result); + }, providerRecovery.sessionId, ), ); @@ -538,6 +542,7 @@ export async function handleProviderInferenceState({ provider, model, endpointUrl, + credentialEnv, preferredInferenceApi, }); } @@ -652,6 +657,7 @@ export async function handleProviderInferenceState({ provider: selectedProvider, model: selectedModel, endpointUrl, + credentialEnv, preferredInferenceApi, }); try { @@ -701,6 +707,7 @@ export async function handleProviderInferenceState({ provider: selectedProvider, model: selectedModel, endpointUrl, + credentialEnv, preferredInferenceApi, }); return deps.reserveSandboxInferenceRoute(resumeReservationName, { diff --git a/src/lib/onboard/machine/handlers/sandbox-route-mutation-lock.test.ts b/src/lib/onboard/machine/handlers/sandbox-route-mutation-lock.test.ts index 6308d906346..f565dbefc88 100644 --- a/src/lib/onboard/machine/handlers/sandbox-route-mutation-lock.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-route-mutation-lock.test.ts @@ -6,7 +6,7 @@ import { handleSandboxState } from "./sandbox"; import { baseOptions, createDeps } from "./sandbox-test-fixtures"; describe("sandbox registration route transaction", () => { - it("rechecks compatibility after waiting for the gateway lock and before create", async () => { + it("allows valid peer-route drift after waiting for the gateway lock", async () => { let releaseGateway!: () => void; const gatewayReleased = new Promise((resolve) => { releaseGateway = resolve; @@ -37,16 +37,16 @@ describe("sandbox registration route transaction", () => { expect(checkGatewayRouteCompatibility).not.toHaveBeenCalled(); releaseGateway(); - await expect(onboard).rejects.toThrow("exit 1"); + await expect(onboard).resolves.toMatchObject({ sandboxName: "my-assistant" }); expect(checkGatewayRouteCompatibility).toHaveBeenCalledWith( expect.objectContaining({ gatewayName: "nemoclaw", sandboxName: null }), ); - expect(calls.createSandbox).not.toHaveBeenCalled(); - expect(calls.updateSandbox).not.toHaveBeenCalled(); + expect(calls.createSandbox).toHaveBeenCalledOnce(); + expect(calls.updateSandbox).toHaveBeenCalled(); expect(calls.removeSandbox).not.toHaveBeenCalled(); - expect(calls.startStep).not.toHaveBeenCalled(); - expect(calls.updateSession).not.toHaveBeenCalled(); - expect(calls.error).toHaveBeenCalledWith(expect.stringContaining("peer")); + expect(calls.startStep).toHaveBeenCalled(); + expect(calls.updateSession).toHaveBeenCalled(); + expect(calls.error).not.toHaveBeenCalled(); }); it("holds sandbox, host dashboard, then gateway locks through creation and registration", async () => { diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 9d465f8f445..c7a3d6253d7 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -4,6 +4,8 @@ import { type CurrentGatewayRouteCompatibilityCheck, formatGatewayRouteConflict, + type GatewayRouteCompatibilityResult, + isAdvisoryGatewayRouteConflict, } from "../../../inference/gateway-route-compatibility"; import { parseExplicitWebSearchProvider, @@ -50,6 +52,16 @@ import { type SandboxResumeDecision, } from "./sandbox-resume"; +function isAdvisoryPeerRouteDifference( + result: Exclude, + sandboxName: string, +): boolean { + return ( + isAdvisoryGatewayRouteConflict(result) && + !result.conflicts.some((conflict) => conflict.sandboxName === sandboxName) + ); +} + export interface SandboxStateOptions< Gpu, Agent, @@ -534,9 +546,10 @@ class SandboxStateFlow< credentialEnv: this.options.credentialEnv, }, }); - if (!compatibility.ok) { - this.failGatewayRouteCheck(` Error: ${formatGatewayRouteConflict(compatibility)}`); - } + if (compatibility.ok || isAdvisoryPeerRouteDifference(compatibility, sandboxName)) return; + // The target registry row is the route reservation this transaction owns. + // A changed target is a lost-reservation race, not an advisory peer drift. + this.failGatewayRouteCheck(` Error: ${formatGatewayRouteConflict(compatibility)}`); } private failGatewayRouteCheck(message: string): never { diff --git a/src/lib/onboard/setup-inference-route-containment.test.ts b/src/lib/onboard/setup-inference-route-containment.test.ts index 792e934a999..3764158b1b8 100644 --- a/src/lib/onboard/setup-inference-route-containment.test.ts +++ b/src/lib/onboard/setup-inference-route-containment.test.ts @@ -80,7 +80,7 @@ describe("onboard shared gateway route containment", () => { expect(JSON.stringify(verifyOnboardInferenceSmoke.mock.calls)).not.toContain("10.0.0.8"); }); - it("rejects a conflict before selecting the gateway or mutating provider state (#6315)", async () => { + it("warns once inside the gateway lock before applying a valid conflicting route (#6315)", async () => { const events: string[] = []; const runOpenshell = vi.fn(() => { events.push("openshell"); @@ -91,6 +91,7 @@ describe("onboard shared gateway route containment", () => { const verifyInferenceRoute = vi.fn(); const verifyOnboardInferenceSmoke = vi.fn(); const getGatewayName = vi.fn(() => "nemoclaw-9090"); + const log = vi.fn((message: string) => events.push(`log:${message}`)); const error = vi.fn((message: string) => events.push(`error:${message}`)); const exitProcess = vi.fn((code: number): never => { events.push(`exit:${code}`); @@ -102,8 +103,14 @@ describe("onboard shared gateway route containment", () => { ok: false as const, gatewayName: "nemoclaw-9090", sandboxName: "new-sandbox", - route: { provider: "anthropic-prod", model: "claude-new" }, - conflicts: [{ sandboxName: "stopped-sandbox", reason: "provider-model" as const }], + route: { provider: "router-b", model: "model-b" }, + conflicts: [ + { + sandboxName: "stopped-sandbox", + reason: "provider-model" as const, + recordedRoute: { provider: "router-a", model: "model-a" }, + }, + ], }; }); const setupInference = createSetupInference({ @@ -124,32 +131,100 @@ describe("onboard shared gateway route containment", () => { upsertProvider, verifyInferenceRoute, verifyOnboardInferenceSmoke, + isNonInteractive: () => true, + hermesProviderAuth: { HERMES_PROVIDER_NAME: "hermes-provider" }, + isRoutedInferenceProvider: () => true, + reconcileModelRouter: vi.fn(async () => undefined), + routedInference: { + upsertRoutedProvider: vi.fn(() => ({ + ok: true, + endpointUrl: "http://router-b.test/v1", + result: { ok: true }, + })), + }, + hydrateCredentialEnv: vi.fn(() => "secret"), + redact: (value: string) => value, + compactText: (value: string) => value, + log, error, exitProcess, } as unknown as SetupInferenceDeps); await expect( - setupInference( - "new-sandbox", - "claude-new", - "anthropic-prod", - "https://api.anthropic.com", - "ANTHROPIC_API_KEY", - ), - ).rejects.toThrow("exit 1"); + setupInference("new-sandbox", "model-b", "router-b", "http://router-b.test/v1", "ROUTER_KEY"), + ).resolves.toEqual({ ok: true }); - expect(events.slice(0, 2)).toEqual(["lock", "guard"]); + expect(events.slice(0, 4)).toEqual([ + "lock", + "guard", + expect.stringContaining("error: Warning: Onboarding 'new-sandbox' will re-point"), + "step", + ]); expect(getGatewayName).toHaveBeenCalledOnce(); expect(checkGatewayRouteCompatibility).toHaveBeenCalledWith( expect.objectContaining({ gatewayName: "nemoclaw-9090" }), ); - expect(runOpenshell).not.toHaveBeenCalled(); + expect(runOpenshell).toHaveBeenCalledWith( + expect.arrayContaining(["inference", "set", "--provider", "router-b", "--model", "model-b"]), + { ignoreError: true }, + ); + expect(verifyInferenceRoute).toHaveBeenCalledWith("nemoclaw-9090", "router-b", "model-b"); + expect(verifyOnboardInferenceSmoke).toHaveBeenCalledOnce(); + expect(updateSandbox).toHaveBeenCalledOnce(); + expect(error).toHaveBeenCalledWith(expect.stringContaining("stopped-sandbox")); + expect(exitProcess).not.toHaveBeenCalled(); + }); + + it("fails before provider mutation when endpoint or credential identity differs (#6315)", async () => { + const runOpenshell = vi.fn(() => ({ status: 0 })); + const updateSandbox = vi.fn(() => true); + const upsertProvider = vi.fn(() => ({ ok: true })); + const error = vi.fn(); + const exitProcess = vi.fn((code: number): never => { + throw new Error(`exit ${code}`); + }); + const peer: SandboxEntry = { + name: "existing-custom", + gatewayName: "nemoclaw", + provider: "compatible-endpoint", + model: "model-a", + endpointUrl: "https://endpoint-a.example/v1", + credentialEnv: "KEY_A", + preferredInferenceApi: "openai-completions", + }; + const setupInference = createSetupInference({ + checkGatewayRouteCompatibility: ( + request: Parameters[0], + ) => checkGatewayRouteCompatibility({ ...request, sandboxes: [peer] }), + withSandboxMutationLock: async (_name: string, operation: () => Promise | T) => + await operation(), + withGatewayRouteMutationLock: async (_name: string, operation: () => Promise | T) => + await operation(), + getGatewayName: () => "nemoclaw", + runOpenshell, + updateSandbox, + upsertProvider, + error, + exitProcess, + } as unknown as SetupInferenceDeps); + + await expect( + setupInference( + "new-custom", + "model-b", + "compatible-endpoint", + "https://endpoint-b.example/v1", + "KEY_B", + null, + [], + { preferredInferenceApi: "openai-completions" }, + ), + ).rejects.toThrow("exit 1"); + + expect(error).toHaveBeenCalledWith(expect.stringContaining("provider-global configuration")); expect(upsertProvider).not.toHaveBeenCalled(); - expect(verifyInferenceRoute).not.toHaveBeenCalled(); - expect(verifyOnboardInferenceSmoke).not.toHaveBeenCalled(); + expect(runOpenshell).not.toHaveBeenCalled(); expect(updateSandbox).not.toHaveBeenCalled(); - expect(error).toHaveBeenCalledWith(expect.stringContaining("stopped-sandbox")); - expect(exitProcess).toHaveBeenCalledWith(1); }); it("rechecks recovered-route ownership inside both mutation locks before setup (#6630)", async () => { @@ -205,7 +280,7 @@ describe("onboard shared gateway route containment", () => { expect(exitProcess).toHaveBeenCalledWith(1); }); - it("keeps a pending reservation while async smoke failure blocks another setup (#6315)", async () => { + it("serializes pending setup, then warns and applies the next valid route (#6315)", async () => { const reservations: SandboxEntry[] = []; let rejectSmoke!: (reason?: unknown) => void; const smokePending = new Promise((_resolve, reject) => { @@ -235,8 +310,12 @@ describe("onboard shared gateway route containment", () => { }, ); const runOpenshell = vi.fn(() => ({ status: 0 })); - const verifyOnboardInferenceSmoke = vi.fn(() => smokePending); + const verifyOnboardInferenceSmoke = vi + .fn() + .mockImplementationOnce(() => smokePending) + .mockResolvedValueOnce(undefined); const log = vi.fn(); + const error = vi.fn(); const exitProcess = vi.fn((code: number): never => { throw new Error(`exit ${code}`); }); @@ -275,7 +354,7 @@ describe("onboard shared gateway route containment", () => { redact: (value: string) => value, compactText: (value: string) => value, log, - error: vi.fn(), + error, exitProcess, } as unknown as SetupInferenceDeps); @@ -311,9 +390,9 @@ describe("onboard shared gateway route containment", () => { expect(results).toEqual([ { status: "rejected", reason: expect.objectContaining({ message: "smoke failed" }) }, - { status: "rejected", reason: expect.objectContaining({ message: "exit 1" }) }, + { status: "fulfilled", value: { ok: true } }, ]); - expect(runOpenshell).toHaveBeenCalledTimes(1); + expect(runOpenshell).toHaveBeenCalledTimes(2); expect(updateSandbox).toHaveBeenCalledWith("alpha", { provider: "router-a", model: "model-a", @@ -322,10 +401,15 @@ describe("onboard shared gateway route containment", () => { preferredInferenceApi: null, gatewayName: "nemoclaw", }); - expect(reservations).toHaveLength(1); - expect(updateSandbox).toHaveBeenCalledOnce(); - expect(log).not.toHaveBeenCalledWith(expect.stringContaining("Inference route set")); - expect(exitProcess).toHaveBeenCalledWith(1); + expect(reservations).toHaveLength(2); + expect(updateSandbox).toHaveBeenCalledTimes(2); + expect(error).toHaveBeenCalledWith( + expect.stringContaining("Affected registered sandboxes: 'alpha'"), + ); + expect(log).toHaveBeenCalledWith( + expect.stringContaining("Inference route set: router-b / model-b"), + ); + expect(exitProcess).not.toHaveBeenCalled(); }); it("stamps the owning onboard session on the initial route reservation (#6562)", async () => { diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 0f52e471573..5c62220a576 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -9,6 +9,8 @@ import { import { type CurrentGatewayRouteCompatibilityCheck, formatGatewayRouteConflict, + formatGatewayRouteImpactWarning, + isAdvisoryGatewayRouteConflict, } from "../inference/gateway-route-compatibility"; import { withGatewayRouteMutationLock } from "../inference/gateway-route-mutation-lock"; import { @@ -244,12 +246,16 @@ export function createSetupInference( provider, model, endpointUrl, + credentialEnv, preferredInferenceApi: options.preferredInferenceApi ?? null, }, }); if (!compatibility.ok) { - deps.error(` Error: ${formatGatewayRouteConflict(compatibility)}`); - return deps.exitProcess(1); + if (!isAdvisoryGatewayRouteConflict(compatibility)) { + deps.error(` Error: ${formatGatewayRouteConflict(compatibility)}`); + return deps.exitProcess(1); + } + deps.error(` ${formatGatewayRouteImpactWarning(compatibility)}`); } deps.step(4, 8, "Setting up inference provider"); let endpointPinnedAddresses = options.endpointPinnedAddresses; diff --git a/test/cli/sandbox-status-json.test.ts b/test/cli/sandbox-status-json.test.ts index 251faaec0f2..e50500203c5 100644 --- a/test/cli/sandbox-status-json.test.ts +++ b/test/cli/sandbox-status-json.test.ts @@ -176,6 +176,13 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { found: true, model: "nvidia/nemotron", provider: "nvidia-prod", + recordedRoute: { provider: "configured-provider", model: "configured-model" }, + liveRoute: { provider: "nvidia-prod", model: "nvidia/nemotron" }, + routeDrift: { + live: { provider: "nvidia-prod", model: "nvidia/nemotron" }, + recorded: { provider: "configured-provider", model: "configured-model" }, + canConnect: true, + }, hostGpuDetected: true, sandboxGpuEnabled: true, sandboxGpuMode: "passthrough", @@ -377,8 +384,8 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { const parsed = JSON.parse(r.out); expect(parsed.rpcIssue).toEqual({ kind: "protobuf_mismatch" }); expect(parsed.inferenceHealth).toBeNull(); - expect(parsed.model).toBe("unknown"); - expect(parsed.provider).toBe("unknown"); + expect(parsed.model).toBe("test-model"); + expect(parsed.provider).toBe("nvidia-prod"); }); it("sandbox status --json reports found:false and exits 1 for unknown sandbox via canonical form", () => { diff --git a/test/cli/status-gateway-lifecycle.test.ts b/test/cli/status-gateway-lifecycle.test.ts index ea60c474d1f..28b680b74df 100644 --- a/test/cli/status-gateway-lifecycle.test.ts +++ b/test/cli/status-gateway-lifecycle.test.ts @@ -65,7 +65,7 @@ describe("CLI status gateway lifecycle process contracts", () => { testTimeout(20_000), ); - it("prints healthy inference only after the sandbox and gateway are verified", () => { + it("prints the recorded route and healthy inference after verification (#6315)", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-status-healthy-")); const localBin = path.join(home, "bin"); const markerFile = path.join(home, "openshell-calls"); @@ -134,8 +134,11 @@ describe("CLI status gateway lifecycle process contracts", () => { expect(result.code).toBe(0); expect(result.out).toContain("Sandbox: alpha"); - expect(result.out).toContain("Model: live-model"); + expect(result.out).toContain("Model: configured-model"); expect(result.out).toContain("Provider: nvidia-prod"); + expect(result.out).toContain( + "gateway inference route (nvidia-prod/live-model) differs from the recorded route for this sandbox (nvidia-prod/configured-model)", + ); expect(result.out).toContain("Inference:"); expect(result.out).toContain("healthy"); expect(result.out).not.toContain("not verified"); diff --git a/test/support/status-flow-test-harness.ts b/test/support/status-flow-test-harness.ts index 2147abe284d..180bdc3acef 100644 --- a/test/support/status-flow-test-harness.ts +++ b/test/support/status-flow-test-harness.ts @@ -136,8 +136,16 @@ export function createStatusFlowHarness(options: StatusFlowHarnessOptions = {}): sb: sandboxEntry, lookup, rpcIssue: null, - currentModel: options.currentModel ?? "nvidia/nemotron-live", + currentModel: options.currentModel ?? sandboxEntry.model, currentProvider: options.currentProvider ?? "ollama-local", + recordedRoute: { + provider: sandboxEntry.provider, + model: sandboxEntry.model, + }, + liveRoute: { + provider: options.currentProvider ?? "ollama-local", + model: options.currentModel ?? sandboxEntry.model, + }, routeDrift: options.routeDrift ?? null, inferenceHealth: options.inferenceHealth === undefined