diff --git a/src/lib/actions/inference-set-degraded-state.test.ts b/src/lib/actions/inference-set-degraded-state.test.ts index d2528c03677..552279ee35f 100644 --- a/src/lib/actions/inference-set-degraded-state.test.ts +++ b/src/lib/actions/inference-set-degraded-state.test.ts @@ -2,35 +2,50 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; +import { SandboxConfigError } from "../sandbox/config"; import type { ConfigObject } from "../security/credential-filter"; -import { runInferenceSet } from "./inference-set"; +import { InferenceSetError, runInferenceSet } from "./inference-set"; import { baseSession, createDeps } from "./inference-set.test-support"; describe("runInferenceSet degraded state handling", () => { - it("keeps gateway and registry consistent when the sandbox config read fails", async () => { + it("aborts before mutating any layer when the sandbox config read fails (#6997)", async () => { const deps = createDeps({ config: {}, session: baseSession() }); + // A stopped sandbox surfaces SandboxConfigError from the in-sandbox read — + // the exact path from the issue. deps.calls.readSandboxConfig.mockImplementation(() => { - throw new Error("sandbox config unreadable"); + throw new SandboxConfigError([ + " Cannot read openclaw config (/sandbox/.openclaw/openclaw.json).", + " Is the sandbox running?", + ]); }); - await expect( - runInferenceSet( - { provider: "nvidia-prod", model: "nvidia/nemotron-3-super-120b-a12b", noVerify: true }, - deps, - ), - ).rejects.toThrow("sandbox config unreadable"); + const error = await runInferenceSet( + { provider: "nvidia-prod", model: "nvidia/nemotron-3-super-120b-a12b", noVerify: true }, + deps, + ).then( + () => { + throw new Error("expected runInferenceSet to reject"); + }, + (rejection: unknown) => rejection, + ); - expect(deps.calls.updateSandbox).toHaveBeenCalledWith( - "alpha", - expect.objectContaining({ - provider: "nvidia-prod", - model: "nvidia/nemotron-3-super-120b-a12b", - endpointUrl: null, - credentialEnv: null, - preferredInferenceApi: null, - nimContainer: null, - }), + // Converted to the command-layer error type with an actionable message, so + // the CLI reports cleanly instead of dumping a raw SandboxConfigError stack. + expect(error).toBeInstanceOf(InferenceSetError); + expect((error as Error).message).toMatch(/Is the sandbox running/); + expect((error as Error).message).toMatch(/Start the sandbox and retry/); + + // #6997 core guarantee: the read is a pre-flight gate, so a stopped sandbox + // leaves EVERY mutable layer untouched. Previously the read ran after the + // gateway route and registry were committed, leaving a half-applied switch. + // Assert zero mutation so re-ordering the read back after the mutations + // regresses this test. (The read-only `openshell provider get` probe that + // runs earlier is not a mutation, so filter to the route-SET call.) + const routeSetCalls = deps.calls.captureOpenshell.mock.calls.filter( + (args) => Array.isArray(args[0]) && args[0][0] === "inference" && args[0][1] === "set", ); + expect(routeSetCalls).toHaveLength(0); + expect(deps.calls.updateSandbox).not.toHaveBeenCalled(); expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); expect(deps.calls.restartSandboxGateway).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index b6c5dccf259..9b1275cda94 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -34,6 +34,7 @@ import { recomputeSandboxConfigHash, resolveAgentConfig, rewriteConfigUrlsWithDnsPinning, + SandboxConfigError, seedHermesDashboardConfig, writeSandboxConfig, } from "../sandbox/config"; @@ -573,6 +574,36 @@ function assertHermesCompatibleAnthropicOpenAiProvider( ); } +/** + * Read the in-sandbox agent config, converting a `SandboxConfigError` (the + * in-sandbox config could not be read or parsed — most commonly because the + * sandbox container is stopped) into a clean `InferenceSetError`. Callers use + * this as a pre-flight gate before the gateway route and registry are mutated, + * so an unreadable config aborts the command cleanly instead of crashing with a + * raw stack after a half-applied switch (#6997). + */ +export function readInSandboxConfigOrFail( + deps: Pick, + sandboxName: string, + target: AgentConfigTarget, +): ConfigObject { + try { + return deps.readSandboxConfig(sandboxName, target); + } catch (error) { + if (error instanceof SandboxConfigError) { + const lines = [...error.lines]; + // `readSandboxConfig` also raises this for a corrupt/unparseable config, + // which starting the sandbox would NOT fix — only add the start hint for + // the stopped-sandbox case (the one that asks "Is the sandbox running?"). + if (lines.some((line) => /is the sandbox running/i.test(line))) { + lines.push(" Start the sandbox and retry."); + } + throw new InferenceSetError(lines.join("\n"), error.exitCode); + } + throw error; + } +} + async function runInferenceSetWithoutHostLock( options: InferenceSetOptions, deps: InferenceSetDeps, @@ -750,6 +781,13 @@ async function runInferenceSetWithoutHostLock( deps, ); + // Read the in-sandbox config *before* mutating the gateway route or registry. + // If it is unreadable (e.g. a stopped sandbox), the mutations below would have + // committed the gateway route and registry first (only the in-sandbox layer is + // `rebuild`-recoverable), so gate on the read here to abort cleanly instead of + // leaving a half-applied switch across the three config layers (#6997). + const config = readInSandboxConfigOrFail(deps, sandboxName, target); + deps.log(` Setting OpenShell inference route: ${provider} / ${model}`); const setResult = deps.captureOpenshell( openshellInferenceSetArgs({ @@ -795,7 +833,6 @@ async function runInferenceSetWithoutHostLock( throw new InferenceSetError(`Failed to update NemoClaw registry for sandbox '${sandboxName}'.`); } - const config = deps.readSandboxConfig(sandboxName, target); const previousOpenClawInferenceApi = readPreviousOpenClawInferenceApi(agentName, config); const preferredInferenceApi = explicitPreferredInferenceApi ?? diff --git a/test/inference-set-preflight.test.ts b/test/inference-set-preflight.test.ts new file mode 100644 index 00000000000..f616e3d86f6 --- /dev/null +++ b/test/inference-set-preflight.test.ts @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { InferenceSetError, readInSandboxConfigOrFail } from "../src/lib/actions/inference-set"; +import type { AgentConfigTarget } from "../src/lib/sandbox/config"; +import { SandboxConfigError } from "../src/lib/sandbox/config"; +import type { ConfigObject } from "../src/lib/security/credential-filter"; + +const TARGET = { + agentName: "openclaw", + configPath: "/sandbox/.openclaw/openclaw.json", +} as unknown as AgentConfigTarget; + +describe("readInSandboxConfigOrFail pre-flight gate (#6997)", () => { + it("returns the config when the sandbox is readable", () => { + const config = { model: "old" } as unknown as ConfigObject; + const readSandboxConfig = vi.fn(() => config); + + const result = readInSandboxConfigOrFail({ readSandboxConfig }, "box", TARGET); + + expect(result).toBe(config); + expect(readSandboxConfig).toHaveBeenCalledWith("box", TARGET); + }); + + it("converts a stopped-sandbox SandboxConfigError into an actionable InferenceSetError", () => { + const readSandboxConfig = vi.fn(() => { + throw new SandboxConfigError( + [ + " Cannot read openclaw config (/sandbox/.openclaw/openclaw.json).", + " Is the sandbox running?", + ], + 3, + ); + }); + + let thrown: unknown; + try { + readInSandboxConfigOrFail({ readSandboxConfig }, "box", TARGET); + } catch (error) { + thrown = error; + } + + // Must be the command-layer error type so it is handled cleanly (no raw stack). + expect(thrown).toBeInstanceOf(InferenceSetError); + const err = thrown as InferenceSetError; + // Preserves the original diagnostic lines and the recorded exit code... + expect(err.message).toContain("Is the sandbox running?"); + expect(err.exitCode).toBe(3); + // ...and adds the actionable next step. + expect(err.message).toContain("Start the sandbox and retry"); + }); + + it("omits the start-sandbox hint for a non-stopped config error (e.g. parse failure)", () => { + // A corrupt/unparseable config raises SandboxConfigError too, but starting + // the sandbox would not fix it — the start hint must not be appended. + const readSandboxConfig = vi.fn(() => { + throw new SandboxConfigError([" Failed to parse openclaw config: unexpected token."], 1); + }); + + let thrown: unknown; + try { + readInSandboxConfigOrFail({ readSandboxConfig }, "box", TARGET); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(InferenceSetError); + const err = thrown as InferenceSetError; + expect(err.message).toContain("Failed to parse"); + expect(err.message).not.toContain("Start the sandbox and retry"); + }); + + it("does not swallow unrelated errors", () => { + const boom = new TypeError("unexpected"); + const readSandboxConfig = vi.fn(() => { + throw boom; + }); + + expect(() => readInSandboxConfigOrFail({ readSandboxConfig }, "box", TARGET)).toThrow(boom); + }); +});