diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index ca8d410ef20..8b64b6d931e 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -7,6 +7,7 @@ "src/lib/actions/sandbox/process-recovery.ts": 26, "src/lib/adapters/docker/index.ts": 42, "src/lib/adapters/openshell/client.ts": 21, + "src/lib/adapters/openshell/command-argv.ts": 23, "src/lib/adapters/openshell/resolve.ts": 25, "src/lib/adapters/openshell/runtime.ts": 54, "src/lib/adapters/openshell/timeouts.ts": 33, @@ -43,8 +44,8 @@ "src/lib/actions/sandbox/gateway-state.ts": 21, "src/lib/actions/sandbox/status-snapshot.ts": 19, "src/lib/actions/sandbox/policy-channel.ts": 30, - "src/lib/actions/sandbox/process-recovery.ts": 19, - "src/lib/actions/sandbox/rebuild-pipeline.ts": 29, + "src/lib/actions/sandbox/process-recovery.ts": 20, + "src/lib/actions/sandbox/rebuild-pipeline.ts": 30, "src/lib/actions/sandbox/snapshot.ts": 37, "src/lib/actions/uninstall/run-plan.ts": 25, "src/lib/inference/local.ts": 22, @@ -53,7 +54,8 @@ "src/lib/onboard.ts": 191, "src/lib/onboard/machine/handlers/sandbox.ts": 21, "src/lib/policy/index.ts": 22, - "src/lib/sandbox/config.ts": 19 + "src/lib/sandbox/config.ts": 19, + "src/lib/state/sandbox.ts": 21 } }, "allowedCycles": [], diff --git a/src/lib/actions/sandbox/forward-recovery-declared-ports.test.ts b/src/lib/actions/sandbox/forward-recovery-declared-ports.test.ts index f8a69e6c94f..112f2a384c0 100644 --- a/src/lib/actions/sandbox/forward-recovery-declared-ports.test.ts +++ b/src/lib/actions/sandbox/forward-recovery-declared-ports.test.ts @@ -18,11 +18,13 @@ vi.mock("../../adapters/openshell/forward-service", async (importOriginal) => ({ launchForwardService: mocks.launchForwardService, })); -vi.mock("../../adapters/openshell/resolve", () => ({ +vi.mock("../../adapters/openshell/resolve", async (importOriginal) => ({ + ...(await importOriginal()), resolveOpenshell: () => "/usr/local/bin/openshell", })); -vi.mock("../../adapters/openshell/runtime", () => ({ +vi.mock("../../adapters/openshell/runtime", async (importOriginal) => ({ + ...(await importOriginal()), captureOpenshell: mocks.captureOpenshell, runOpenshell: mocks.runOpenshell, isCommandTimeout: () => false, @@ -68,7 +70,7 @@ beforeEach(() => { mocks.getSessionAgent.mockReturnValue(HERMES_AGENT); }); -describe("ensureDeclaredAgentForwardPortsHealthy", () => { +describe("ensureDeclaredAgentForwardPortsHealthy", { timeout: 30_000 }, () => { it("accepts an already-reachable remote direct service during gateway recovery", async () => { vi.stubEnv("NEMOCLAW_DASHBOARD_BIND", "0.0.0.0"); mocks.getSandbox.mockReturnValue({ @@ -111,9 +113,68 @@ describe("ensureDeclaredAgentForwardPortsHealthy", () => { expect(ensureDeclaredAgentForwardPortsHealthy("beta", 18790)).toBe(true); expect(mocks.launchForwardService).toHaveBeenCalledWith( expect.objectContaining({ localPort: 8643, targetPort: 8643 }), + {}, ); }); + it("pins declared forward inspection and recovery to the selected OpenShell target (#10514)", async () => { + vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "0"); + vi.stubEnv("OPENSHELL_GATEWAY", "hostile-gateway"); + vi.stubEnv("OPENSHELL_WORKSPACE", "hostile-workspace"); + vi.stubEnv("OPENSHELL_LOCAL_TLS_DIR", "/hostile/tls"); + vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://hostile.invalid"); + vi.stubEnv("OPENSHELL_TOKEN", "hostile-token"); + mocks.isLocalForwardReachable.mockReturnValue(false); + mocks.getSandbox.mockReturnValue({ + agent: "hermes", + dashboardPort: 18790, + hermesApiPort: 8643, + }); + mocks.captureOpenshell.mockReturnValue(forwardList([])); + const runtimeSelection = { + gatewayName: "nemoclaw-19080", + workspace: "review-workspace", + localTlsDir: "/authority/tls", + }; + const selectedOptions = expect.objectContaining({ + env: expect.objectContaining({ + OPENSHELL_GATEWAY: runtimeSelection.gatewayName, + OPENSHELL_WORKSPACE: runtimeSelection.workspace, + OPENSHELL_LOCAL_TLS_DIR: runtimeSelection.localTlsDir, + }), + replaceEnv: true, + }); + const { ensureDeclaredAgentForwardPortsHealthy } = await import("./forward-recovery"); + + expect(ensureDeclaredAgentForwardPortsHealthy("beta", 18790, runtimeSelection)).toBe(true); + expect(mocks.captureOpenshell).toHaveBeenCalledWith( + ["forward", "list", "--gateway", runtimeSelection.gatewayName], + selectedOptions, + ); + expect(mocks.launchForwardService).toHaveBeenCalledWith( + expect.objectContaining({ + gatewayName: runtimeSelection.gatewayName, + localPort: 8643, + sandboxName: "beta", + targetPort: 8643, + workspace: runtimeSelection.workspace, + }), + { + sourceEnvironment: expect.objectContaining({ + OPENSHELL_GATEWAY: runtimeSelection.gatewayName, + OPENSHELL_WORKSPACE: runtimeSelection.workspace, + OPENSHELL_LOCAL_TLS_DIR: runtimeSelection.localTlsDir, + }), + }, + ); + const captureEnv = mocks.captureOpenshell.mock.calls[0]?.[1]?.env; + expect(captureEnv).not.toHaveProperty("OPENSHELL_GATEWAY_ENDPOINT"); + expect(captureEnv).not.toHaveProperty("OPENSHELL_TOKEN"); + const sourceEnvironment = mocks.launchForwardService.mock.calls[0]?.[1]?.sourceEnvironment; + expect(sourceEnvironment).not.toHaveProperty("OPENSHELL_GATEWAY_ENDPOINT"); + expect(sourceEnvironment).not.toHaveProperty("OPENSHELL_TOKEN"); + }); + it("keeps the default API port for a sandbox registered without one (#8543)", async () => { mocks.getSandbox.mockReturnValue({ agent: "hermes", dashboardPort: 18789 }); mocks.captureOpenshell.mockReturnValue(forwardList([])); diff --git a/src/lib/actions/sandbox/forward-recovery.ts b/src/lib/actions/sandbox/forward-recovery.ts index bf8c939fdb4..f26329ab463 100644 --- a/src/lib/actions/sandbox/forward-recovery.ts +++ b/src/lib/actions/sandbox/forward-recovery.ts @@ -2,12 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 import { resolveOpenshell } from "../../adapters/openshell/resolve"; +import { + buildSelectedOpenShellSubprocessEnv, + withSelectedOpenShellCommandOptions, +} from "../../adapters/openshell/command-argv"; import { launchForwardService, type ForwardServiceTarget, } from "../../adapters/openshell/forward-service"; import { isLegacySandboxForwardListed } from "../../adapters/openshell/forward-service-migration"; -import { captureOpenshell, runOpenshell } from "../../adapters/openshell/runtime"; +import { + captureOpenshell, + type OpenShellRuntimeSelection, + runOpenshell, +} from "../../adapters/openshell/runtime"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import * as agentRuntime from "../../agent/runtime"; import { DASHBOARD_PORT, HERMES_OPENAI_API_PORT } from "../../core/ports"; @@ -129,27 +137,41 @@ type SandboxForwardRecoveryOptions = { afterSuccess?: () => boolean; beforeStart?: () => boolean; isWsl?: boolean; + runtimeSelection?: OpenShellRuntimeSelection; }; function retireLegacyForwardServiceMigration( sandboxName: string, gatewayName: string, ports: readonly number[], + runtimeSelection?: OpenShellRuntimeSelection, ): number { return retireProductionLegacySandboxForwards(sandboxName, gatewayName, ports, { capture: (gatewayName) => - captureOpenshell(["forward", "list", "--gateway", gatewayName], { - ignoreError: true, - includeStreams: true, - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }), + captureOpenshell( + ["forward", "list", "--gateway", gatewayName], + withSelectedOpenShellCommandOptions( + { + ignoreError: true, + includeStreams: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }, + runtimeSelection, + ), + ), isReachable: isLocalForwardReachable, run: (gatewayName, sandboxName, port) => - runOpenshell(["forward", "stop", String(port), sandboxName, "--gateway", gatewayName], { - ignoreError: true, - stdio: "ignore", - timeout: 30_000, - }), + runOpenshell( + ["forward", "stop", String(port), sandboxName, "--gateway", gatewayName], + withSelectedOpenShellCommandOptions( + { + ignoreError: true, + stdio: "ignore", + timeout: 30_000, + }, + runtimeSelection, + ), + ), }); } @@ -159,11 +181,12 @@ function forwardServiceTarget( sandboxName: string, port: number, expectedBind = "127.0.0.1", + workspace = "default", ): ForwardServiceTarget { return { executable, gatewayName, - workspace: "default", + workspace, sandboxName, localHost: expectedBind === "0.0.0.0" ? ("0.0.0.0" as const) : ("127.0.0.1" as const), localPort: port, @@ -309,13 +332,14 @@ export function ensureSandboxPortForward( (!remoteBindRequested || registry.getSandbox(sandboxName)?.dashboardRemoteBindPrepared === true) && (options.beforeStart?.() ?? true), + runtimeSelection: options.runtimeSelection, }); } /** Probe local reachability for a registered sandbox port without claiming process ownership. */ export function isSandboxForwardHealthy( sandboxName: string, - options: { isWsl?: boolean } = {}, + options: { isWsl?: boolean; runtimeSelection?: OpenShellRuntimeSelection } = {}, ): SandboxForwardHealth { const allInterfaceBindRequired = isRemoteDashboardBindRequested(process.env.NEMOCLAW_DASHBOARD_BIND) || @@ -324,6 +348,7 @@ export function isSandboxForwardHealthy( sandboxName, resolveSandboxDashboardPort(sandboxName), allInterfaceBindRequired ? "0.0.0.0" : "127.0.0.1", + options.runtimeSelection, ); } @@ -331,16 +356,23 @@ export function isSandboxPortForwardHealthy( sandboxName: string, port: number, _expectedBind?: string, + runtimeSelection?: OpenShellRuntimeSelection, ): SandboxForwardHealth { const sandbox = registry.getSandbox(sandboxName); if (!sandbox) return false; if (!isLocalForwardReachable(port)) return false; - const gatewayName = resolveSandboxGatewayName(sandbox); - const listed = captureOpenshell(["forward", "list", "--gateway", gatewayName], { - ignoreError: true, - includeStreams: true, - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }); + const gatewayName = runtimeSelection?.gatewayName ?? resolveSandboxGatewayName(sandbox); + const listed = captureOpenshell( + ["forward", "list", "--gateway", gatewayName], + withSelectedOpenShellCommandOptions( + { + ignoreError: true, + includeStreams: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }, + runtimeSelection, + ), + ); if ( !listed.error && !listed.signal && @@ -360,6 +392,7 @@ export function ensureSandboxPortForwardForPort( forwardTarget?: string; expectedBind?: string; beforeStart?: () => boolean; + runtimeSelection?: OpenShellRuntimeSelection; } = {}, ): boolean { const { @@ -367,6 +400,7 @@ export function ensureSandboxPortForwardForPort( forwardTarget = String(port), expectedBind, beforeStart = () => true, + runtimeSelection, } = options; const acceptSuccessfulForward = () => { let accepted = false; @@ -377,14 +411,19 @@ export function ensureSandboxPortForwardForPort( } return accepted; }; - const forwardHealth = isSandboxPortForwardHealthy(sandboxName, port, expectedBind); + const forwardHealth = isSandboxPortForwardHealthy( + sandboxName, + port, + expectedBind, + runtimeSelection, + ); if (forwardHealth === true) return acceptSuccessfulForward(); if (!beforeStart()) return false; try { const sandbox = registry.getSandbox(sandboxName); if (!sandbox) throw new Error(`Sandbox '${sandboxName}' is not registered`); - const gatewayName = resolveSandboxGatewayName(sandbox); - retireLegacyForwardServiceMigration(sandboxName, gatewayName, [port]); + const gatewayName = runtimeSelection?.gatewayName ?? resolveSandboxGatewayName(sandbox); + retireLegacyForwardServiceMigration(sandboxName, gatewayName, [port], runtimeSelection); const executable = resolveOpenshell(); if (!executable) throw new Error("OpenShell is unavailable"); launchForwardService( @@ -394,7 +433,11 @@ export function ensureSandboxPortForwardForPort( sandboxName, port, expectedBind ?? (forwardTarget.startsWith("0.0.0.0:") ? "0.0.0.0" : "127.0.0.1"), + runtimeSelection?.workspace ?? "default", ), + runtimeSelection + ? { sourceEnvironment: buildSelectedOpenShellSubprocessEnv(runtimeSelection) } + : {}, ); return acceptSuccessfulForward(); } catch (error) { @@ -407,10 +450,15 @@ export function ensureSandboxPortForwardForPort( } } -export function ensureHermesDashboardPortForwardIfEnabled(sandboxName: string): boolean | null { +export function ensureHermesDashboardPortForwardIfEnabled( + sandboxName: string, + runtimeSelection?: OpenShellRuntimeSelection, +): boolean | null { return ensureHermesDashboardPortForward(sandboxName, { - isPortForwardHealthy: isSandboxPortForwardHealthy, - ensurePortForward: ensureSandboxPortForwardForPort, + isPortForwardHealthy: (name, port) => + isSandboxPortForwardHealthy(name, port, undefined, runtimeSelection), + ensurePortForward: (name, port) => + ensureSandboxPortForwardForPort(name, port, { runtimeSelection }), }); } @@ -423,19 +471,25 @@ function getSandboxMessagingHostForward( return getActiveMessagingHostForward(plan); } -export function ensureMessagingHostForwardHealthy(sandboxName: string): boolean | null { +export function ensureMessagingHostForwardHealthy( + sandboxName: string, + runtimeSelection?: OpenShellRuntimeSelection, +): boolean | null { const forward = getSandboxMessagingHostForward(sandboxName); if (!forward) return null; - const health = isSandboxPortForwardHealthy(sandboxName, forward.port); + const health = isSandboxPortForwardHealthy(sandboxName, forward.port, undefined, runtimeSelection); if (health === true) return true; - return ensureSandboxPortForwardForPort(sandboxName, forward.port); + return ensureSandboxPortForwardForPort(sandboxName, forward.port, { runtimeSelection }); } export function recoverMessagingHostForward( sandboxName: string, - { quiet }: { quiet: boolean }, + { + quiet, + runtimeSelection, + }: { quiet: boolean; runtimeSelection?: OpenShellRuntimeSelection }, ): boolean | null { - const recovered = ensureMessagingHostForwardHealthy(sandboxName); + const recovered = ensureMessagingHostForwardHealthy(sandboxName, runtimeSelection); if (!quiet && recovered === false) { console.error(" Messaging webhook port forward could not be re-established."); } @@ -485,6 +539,7 @@ function resolveDeclaredAgentForwardPorts( export function ensureDeclaredAgentForwardPortsHealthy( sandboxName: string, primaryPort: number, + runtimeSelection?: OpenShellRuntimeSelection, ): boolean | null { const agent = agentRuntime.getSessionAgent(sandboxName); if (!agent) return null; @@ -499,9 +554,9 @@ export function ensureDeclaredAgentForwardPortsHealthy( if (ports.length === 0) return null; let allHealthy = true; for (const port of ports) { - const health = isSandboxPortForwardHealthy(sandboxName, port); + const health = isSandboxPortForwardHealthy(sandboxName, port, undefined, runtimeSelection); if (health === true) continue; - if (!ensureSandboxPortForwardForPort(sandboxName, port)) { + if (!ensureSandboxPortForwardForPort(sandboxName, port, { runtimeSelection })) { allHealthy = false; } } @@ -569,9 +624,16 @@ export function resolveSandboxLaunchForwardPorts(sandboxName: string): number[] export function recoverDeclaredAgentForwardPorts( sandboxName: string, recoveryPort: number, - { quiet }: { quiet: boolean }, + { + quiet, + runtimeSelection, + }: { quiet: boolean; runtimeSelection?: OpenShellRuntimeSelection }, ): boolean | null { - const recovered = ensureDeclaredAgentForwardPortsHealthy(sandboxName, recoveryPort); + const recovered = ensureDeclaredAgentForwardPortsHealthy( + sandboxName, + recoveryPort, + runtimeSelection, + ); if (!quiet && recovered === false) { console.error(" One or more agent-declared port forwards could not be re-established."); } diff --git a/src/lib/actions/sandbox/gateway-restart.test.ts b/src/lib/actions/sandbox/gateway-restart.test.ts index a0485efa932..f9fe5f0defe 100644 --- a/src/lib/actions/sandbox/gateway-restart.test.ts +++ b/src/lib/actions/sandbox/gateway-restart.test.ts @@ -185,6 +185,29 @@ describe("restartSandboxGateway — host-mediated gateway restart", () => { ); }); + it("fails closed instead of using host-local supervisor control for a selected runtime", () => { + const deps = baseDeps(); + const { requestGatewaySupervisorAction: hostLocalSupervisorAction, ...nonSupervisorDeps } = + deps; + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + + const result = restartSandboxGateway("alpha", { + quiet: true, + runtimeSelection: { gatewayName: "remote-gateway", workspace: "remote-workspace" }, + deps: nonSupervisorDeps, + }); + + expect(result).toMatchObject({ + ok: false, + failureLayer: "privileged control unavailable", + detail: expect.stringContaining("SELECTED_RUNTIME_SUPERVISOR_UNAVAILABLE"), + }); + expect(hostLocalSupervisorAction).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith( + expect.stringContaining("Failure layer: privileged control unavailable"), + ); + }); + it("prints a bounded sanitized Hermes gateway-log tail after supervisor failure (#8614)", () => { const restore = silenceConsole(); try { diff --git a/src/lib/actions/sandbox/inference-invocation-probe.ts b/src/lib/actions/sandbox/inference-invocation-probe.ts index 83d45daa04d..28507ab8372 100644 --- a/src/lib/actions/sandbox/inference-invocation-probe.ts +++ b/src/lib/actions/sandbox/inference-invocation-probe.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { runOpenshellProviderCommand } from "../../adapters/openshell/provider-command"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import { getSandboxInferenceConfig } from "../../inference/config"; import { validateInferenceResponseBody } from "../../inference/health"; import { MIN_PROBE_REPLY_TOKENS, resolveMaxTokensField } from "../../inference/max-tokens-field"; @@ -17,6 +18,7 @@ import { DCODE_AGENT_NAME } from "./rebuild-dcode-target"; export type SandboxInferenceInvocationInput = { sandboxName: string; gatewayName?: string; + runtimeSelection?: OpenShellRuntimeSelection; agentName?: string | null; provider: string; model: string; @@ -141,6 +143,7 @@ function executeDcodeSandboxInferenceInvocation( try { const result = runOpenshell(buildDcodeSandboxInferenceInvocationArgs(input), { ignoreError: true, + ...(input.runtimeSelection ? { runtimeSelection: input.runtimeSelection } : {}), stdio: ["ignore", "pipe", "pipe"], timeout: timeoutMs, }); @@ -178,9 +181,11 @@ export function probeSandboxInferenceInvocation( result = executeDcodeSandboxInferenceInvocation(input, deps, timeoutMs); } else { const execute = deps.execute ?? executeSandboxExecCommand; - const execOptions: SandboxExecCommandOptions = input.gatewayName - ? { gatewayName: input.gatewayName, allowLocalDockerFallback: false } - : {}; + const execOptions: SandboxExecCommandOptions = { + ...(input.gatewayName ? { gatewayName: input.gatewayName } : {}), + ...(input.runtimeSelection ? { runtimeSelection: input.runtimeSelection } : {}), + allowLocalDockerFallback: false, + }; result = execute( input.sandboxName, buildSandboxInferenceInvocationCommand(input), diff --git a/src/lib/actions/sandbox/messaging-host-forward-lifecycle.ts b/src/lib/actions/sandbox/messaging-host-forward-lifecycle.ts index 7cf04ab991a..d7ac025319e 100644 --- a/src/lib/actions/sandbox/messaging-host-forward-lifecycle.ts +++ b/src/lib/actions/sandbox/messaging-host-forward-lifecycle.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { SandboxMessagingPlan } from "../../messaging"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import { ensureMessagingHostForwardIfConfigured, resolveMessagingHostForward, @@ -14,15 +15,17 @@ import { export function ensureMessagingHostForwardAfterRebuild( sandboxName: string, plan: SandboxMessagingPlan | null | undefined, + runtimeSelection?: OpenShellRuntimeSelection, ): boolean { const forward = resolveMessagingHostForward(plan); if (!forward) return true; - const health = isSandboxPortForwardHealthy(sandboxName, forward.port); + const health = isSandboxPortForwardHealthy(sandboxName, forward.port, undefined, runtimeSelection); if (health === true) return true; return ensureMessagingHostForwardIfConfigured({ sandboxName, plan, - ensureForward: (name, port) => ensureSandboxPortForwardForPort(name, port), + ensureForward: (name, port) => + ensureSandboxPortForwardForPort(name, port, { runtimeSelection }), note: console.log, }); } diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 4e607cdb7da..868cf33ca03 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -3,6 +3,7 @@ import { randomBytes } from "node:crypto"; import { stripAnsi } from "../../adapters/openshell/client"; +import { withSelectedOpenShellCommandOptions } from "../../adapters/openshell/command-argv"; import { buildOpenShellRuntimeSelectionEnv, captureOpenshell, @@ -67,7 +68,7 @@ import { type ManagedGatewayControlCompletion, parseManagedGatewayControlCompletion, printGatewayRestartFailure, - type RestartSandboxGatewayOptions, + type RestartSandboxGatewayOptions as BaseRestartSandboxGatewayOptions, restartSandboxGatewayWithDeps, sandboxAgentName, withUnsupportedHermesPortableGatewayRestartFence, @@ -112,9 +113,12 @@ export type { GatewayRestartFailureLayer, GatewayRestartResult, ManagedGatewayControlCompletion, - RestartSandboxGatewayOptions, } from "./gateway-restart"; +export type RestartSandboxGatewayOptions = BaseRestartSandboxGatewayOptions & { + runtimeSelection?: OpenShellRuntimeSelection; +}; + export { buildSandboxExecMarkedCommand } from "./sandbox-exec-output"; export type { SandboxCommandResult, SandboxExecCommandOptions }; @@ -336,6 +340,20 @@ export function executeGatewaySupervisorAction( return executeGatewaySupervisorActionPinned(sandboxName, action, timeout); } +function refuseHostLocalSupervisorForSelectedRuntime( + _sandboxName: string, + _action: "restart" | "recover" | "probe", + _timeout = 210000, + _expectedContainerId?: string, +): ManagedGatewaySupervisorActionResult { + return { + status: 1, + stdout: "", + stderr: + "PRIVILEGED_CONTROL_UNAVAILABLE\nSELECTED_RUNTIME_SUPERVISOR_UNAVAILABLE: host-local supervisor control is not valid for an explicitly selected OpenShell target", + }; +} + async function executeSandboxExecCommandForStatus( sandboxName: string, command: string, @@ -385,12 +403,22 @@ function parseSandboxGatewayProbe(result: SandboxCommandResult | null): boolean * Fixes #2342 — previously `curl -sf` failed on 401, causing false * "Health Offline" readings. */ -function isSandboxGatewayRunning(sandboxName: string): boolean | null { +function isSandboxGatewayRunning( + sandboxName: string, + runtimeSelection?: OpenShellRuntimeSelection, +): boolean | null { const agent = agentRuntime.getSessionAgent(sandboxName); if (agent && !agentRuntime.hasGatewayRuntime(agent)) return null; const probeUrl = getSandboxHealthProbeUrl(sandboxName); const command = `HTTP_CODE=$(curl -so /dev/null -w '%{http_code}' --max-time 3 ${shellQuote(probeUrl)} 2>/dev/null || echo 000); case "$HTTP_CODE" in 200|401) echo RUNNING ;; *) echo STOPPED ;; esac`; - const execProbe = parseSandboxGatewayProbe(executeSandboxExecCommand(sandboxName, command)); + const execProbe = parseSandboxGatewayProbe( + executeSandboxExecCommand( + sandboxName, + command, + DEFAULT_SANDBOX_EXEC_TIMEOUT_MS, + runtimeSelection ? { runtimeSelection } : {}, + ), + ); if (execProbe !== null) return execProbe; // Built-in OpenClaw and Hermes lifecycle control is host-mediated through @@ -401,7 +429,13 @@ function isSandboxGatewayRunning(sandboxName: string): boolean | null { // their recovery contract is explicitly SSH-owned until manifests can // declare a trusted runtime user/supervisor. if (!agent || agent.name === "openclaw" || agent.name === "hermes") return null; - return parseSandboxGatewayProbe(executeSandboxCommand(sandboxName, command)); + return parseSandboxGatewayProbe( + executeSandboxCommand( + sandboxName, + command, + runtimeSelection ? { runtimeSelection } : DEFAULT_SANDBOX_EXEC_TIMEOUT_MS, + ), + ); } function hasGatewayRecoveryMarker(result: SandboxCommandResult | null): boolean { @@ -739,14 +773,25 @@ function recoverSandboxProcesses( requestPinnedGatewaySupervisorAction = executeGatewaySupervisorActionPinned, relaunchManagedSupervisorSessionImpl = relaunchManagedSupervisorSession, onFailureLayer, + runtimeSelection, }: { quiet?: boolean; requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction; requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction; relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession; onFailureLayer?: (layer: GatewayRestartFailureLayer, detail: string) => void; + runtimeSelection?: OpenShellRuntimeSelection; } = {}, ): SandboxProcessRecovery | null { + const effectiveGatewaySupervisorAction = + runtimeSelection && requestGatewaySupervisorAction === executeGatewaySupervisorAction + ? refuseHostLocalSupervisorForSelectedRuntime + : requestGatewaySupervisorAction; + const effectivePinnedGatewaySupervisorAction = + runtimeSelection && + requestPinnedGatewaySupervisorAction === executeGatewaySupervisorActionPinned + ? refuseHostLocalSupervisorForSelectedRuntime + : requestPinnedGatewaySupervisorAction; const agent = agentRuntime.getSessionAgent(sandboxName); const dashboardPort = resolveSandboxDashboardPort(sandboxName); let persistedAgent: string | null; @@ -784,7 +829,7 @@ function recoverSandboxProcesses( let execResult: ManagedGatewaySupervisorActionResult | null = null; let busyAttempts = 0; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - execResult = requestGatewaySupervisorAction(sandboxName, "recover"); + execResult = effectiveGatewaySupervisorAction(sandboxName, "recover"); const managedControlCompletion = parseManagedGatewayControlCompletion(execResult); if (managedControlCompletion) return { kind: "managed", managedControlCompletion }; if (hasGatewayRecoveryMarker(execResult)) return { kind: "managed" }; @@ -815,24 +860,46 @@ function recoverSandboxProcesses( const relaunch = relaunchManagedSupervisorSessionImpl(sandboxName, { quiet, deps: { - runOpenshell, + runOpenshell: (args, options) => + runOpenshell( + args, + withSelectedOpenShellCommandOptions(options ?? {}, runtimeSelection), + ), runCaptureOpenshell: (args, options) => - captureOpenshell(args, { - ignoreError: true, - includeStderr: true, - killProcessTreeOnTimeout: options?.killProcessTreeOnTimeout === true, - killSignal: options?.killSignal === "SIGKILL" ? "SIGKILL" : undefined, - timeout: - typeof options?.timeout === "number" ? options.timeout : OPENSHELL_PROBE_TIMEOUT_MS, - }).output, + captureOpenshell( + args, + withSelectedOpenShellCommandOptions( + { + ignoreError: true, + includeStderr: true, + killProcessTreeOnTimeout: options?.killProcessTreeOnTimeout === true, + killSignal: options?.killSignal === "SIGKILL" ? "SIGKILL" : undefined, + timeout: + typeof options?.timeout === "number" + ? options.timeout + : OPENSHELL_PROBE_TIMEOUT_MS, + }, + runtimeSelection, + ), + ).output, confirmMissingSupervisor: (containerId) => isExactlyManagedControlMarker( - requestPinnedGatewaySupervisorAction(sandboxName, "probe", 210000, containerId), + effectivePinnedGatewaySupervisorAction( + sandboxName, + "probe", + 210000, + containerId, + ), "SUPERVISOR_NOT_RUNNING", ), restartRestoredManagedGateway: (containerId) => { const restarted = parseManagedGatewayControlCompletion( - requestPinnedGatewaySupervisorAction(sandboxName, "restart", 210000, containerId), + effectivePinnedGatewaySupervisorAction( + sandboxName, + "restart", + 210000, + containerId, + ), ); if (restarted?.disposition !== "ok") return false; return waitForRecoveredSandboxGateway(sandboxName, { @@ -840,10 +907,11 @@ function recoverSandboxProcesses( initialManagedHealthPassed: true, requireManagedProbe: true, timeoutSeconds: gatewayRecoveryTimeoutSeconds(agent), + runtimeSelection, managedProbeImpl: (name) => confirmRecoveredSandboxGatewayManaged(name, { requestGatewaySupervisorActionImpl: (name, action) => - requestPinnedGatewaySupervisorAction(name, action, 210000, containerId), + effectivePinnedGatewaySupervisorAction(name, action, 210000, containerId), }), }); }, @@ -884,7 +952,13 @@ function recoverSandboxProcesses( // Non-Hermes custom manifests do not yet declare a supported host-side // runtime user. Recover them over SSH so the launch inherits the sandbox // login user instead of creating root-owned agent state under /sandbox. - return recoveredSsh(executeSandboxCommand(sandboxName, agentScript)); + return recoveredSsh( + executeSandboxCommand( + sandboxName, + agentScript, + runtimeSelection ? { runtimeSelection } : DEFAULT_SANDBOX_EXEC_TIMEOUT_MS, + ), + ); } return null; @@ -892,9 +966,12 @@ function recoverSandboxProcesses( export function restartSandboxGateway( sandboxName: string, - { quiet = false, deps = {} }: RestartSandboxGatewayOptions = {}, + { quiet = false, deps = {}, runtimeSelection }: RestartSandboxGatewayOptions = {}, ): GatewayRestartResult { return withUnsupportedHermesPortableGatewayRestartFence(sandboxName, () => { + const defaultSupervisorAction = runtimeSelection + ? refuseHostLocalSupervisorForSelectedRuntime + : executeGatewaySupervisorAction; return withMcpLifecycleLockSync(sandboxName, () => restartSandboxGatewayWithDeps(sandboxName, { quiet, @@ -902,25 +979,44 @@ export function restartSandboxGateway( getSessionAgent: agentRuntime.getSessionAgent, getSandbox: registry.getSandbox, resolveSandboxDashboardPort, - requestGatewaySupervisorAction: executeGatewaySupervisorAction, - executeSandboxExecCommand, + requestGatewaySupervisorAction: defaultSupervisorAction, + executeSandboxExecCommand: (name, command, timeout) => + executeSandboxExecCommand( + name, + command, + timeout, + runtimeSelection ? { runtimeSelection } : {}, + ), waitForRecoveredSandboxGateway: (name, options) => waitForRecoveredSandboxGateway(name, { ...options, initialManagedHealthPassed: true, + runtimeSelection, timeoutSeconds: gatewayRecoveryTimeoutSeconds(agentRuntime.getSessionAgent(name)), managedProbeImpl: (sandboxName) => confirmRecoveredSandboxGatewayManaged(sandboxName, { requestGatewaySupervisorActionImpl: - deps.requestGatewaySupervisorAction ?? executeGatewaySupervisorAction, + deps.requestGatewaySupervisorAction ?? defaultSupervisorAction, }), }), - ensureSandboxPortForward, - ensureHermesDashboardPortForwardIfEnabled, - recoverMessagingHostForward, - recoverDeclaredAgentForwardPorts, + ensureSandboxPortForward: (name) => + ensureSandboxPortForward(name, { runtimeSelection }), + ensureHermesDashboardPortForwardIfEnabled: (name) => + ensureHermesDashboardPortForwardIfEnabled(name, runtimeSelection), + recoverMessagingHostForward: (name, options) => + recoverMessagingHostForward(name, { ...options, runtimeSelection }), + recoverDeclaredAgentForwardPorts: (name, recoveryPort, options) => + recoverDeclaredAgentForwardPorts(name, recoveryPort, { + ...options, + runtimeSelection, + }), printGatewayWedgeDiagnostics, - inspectHermesMcpReconciliationRefusal, + inspectHermesMcpReconciliationRefusal: (name) => + inspectHermesMcpReconciliationRefusal( + name, + undefined, + runtimeSelection, + ), ...deps, }, }), @@ -1034,6 +1130,7 @@ type RecreatedSandboxOpenShellReadyOptions = { nowImpl?: () => number; sleepImpl?: (seconds: number) => void; timeoutSeconds?: number; + runtimeSelection?: OpenShellRuntimeSelection; }; function recreatedSandboxOpenShellReadinessFailureDetail( @@ -1134,12 +1231,18 @@ function waitForRecreatedSandboxOpenShellReadyResult( ready: false, }; } - const result = capture(["sandbox", "exec", "--name", sandboxName, "--", "true"], { - ignoreError: true, - includeStderr: true, - includeStreams: true, - timeout: Math.max(1, Math.min(OPENSHELL_PROBE_TIMEOUT_MS, remainingMs)), - }); + const result = capture( + ["sandbox", "exec", "--name", sandboxName, "--", "true"], + withSelectedOpenShellCommandOptions( + { + ignoreError: true, + includeStderr: true, + includeStreams: true, + timeout: Math.max(1, Math.min(OPENSHELL_PROBE_TIMEOUT_MS, remainingMs)), + }, + options.runtimeSelection, + ), + ); if (result.status === 0 && !result.error) return { ready: true }; const openshellError = normalizeOpenshellStructuredError(String(result.stderr ?? "")); if (openshellError) lastOpenshellError = openshellError; @@ -1304,9 +1407,12 @@ export function waitForRecoveredSandboxGateway( quiet?: boolean; timeoutSeconds?: number; requireManagedProbe?: boolean; + runtimeSelection?: OpenShellRuntimeSelection; } = {}, ): boolean { - const probe = options.probeImpl ?? isSandboxGatewayRunning; + const probe = + options.probeImpl ?? + ((name: string) => isSandboxGatewayRunning(name, options.runtimeSelection)); const managedProbe = options.managedProbeImpl ?? (options.probeImpl ? null : confirmRecoveredSandboxGatewayManaged); const sleep = options.sleepImpl ?? sleepSeconds; @@ -1430,6 +1536,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( isWsl: isWslOverride, onRecoveryFailureLayer, probeTiming, + runtimeSelection, }: { quiet?: boolean; requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction; @@ -1440,10 +1547,20 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( isWsl?: boolean; onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null, detail?: string) => void; probeTiming?: ProcessRecoveryProbeTiming; + runtimeSelection?: OpenShellRuntimeSelection; } = {}, ) { const measure = (stage: "processes" | "forward", operation: () => T): T => probeTiming ? probeTiming.measure(stage, operation) : operation(); + const effectiveGatewaySupervisorAction = + runtimeSelection && requestGatewaySupervisorAction === executeGatewaySupervisorAction + ? refuseHostLocalSupervisorForSelectedRuntime + : requestGatewaySupervisorAction; + const effectivePinnedGatewaySupervisorAction = + runtimeSelection && + requestPinnedGatewaySupervisorAction === executeGatewaySupervisorActionPinned + ? refuseHostLocalSupervisorForSelectedRuntime + : requestPinnedGatewaySupervisorAction; const recoveryAgent = agentRuntime.getSessionAgent(sandboxName); const recoveryDisplayName = recoveryAgentDisplayName(sandboxName, recoveryAgent); if (recoveryAgent && !agentRuntime.hasGatewayRuntime(recoveryAgent)) { @@ -1455,7 +1572,9 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( runtime: "terminal" as const, }; } - const running = measure("processes", () => isSandboxGatewayRunningImpl(sandboxName)); + const running = measure("processes", () => + isSandboxGatewayRunningImpl(sandboxName, runtimeSelection), + ); if (running === null) { return { checked: false, wasRunning: null, recovered: false, forwardRecovered: false }; } @@ -1464,7 +1583,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( const enforcement = enforceHermesSecretBoundaryOnRunningGateway( sandboxName, recoveryAgent, - requestGatewaySupervisorAction, + effectiveGatewaySupervisorAction, ); if (enforcement?.refused) { return { @@ -1476,7 +1595,12 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( secretBoundaryReason: enforcement.reason, }; } - const mcpRefusal = processRecoveryMcpReconciliationRefusal(sandboxName, true); + const mcpRefusal = processRecoveryMcpReconciliationRefusal( + sandboxName, + true, + undefined, + runtimeSelection, + ); if (mcpRefusal) return mcpRefusal; } if (running) { @@ -1484,7 +1608,10 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( // owned by another sandbox. Probe and re-establish only when // necessary so the live-and-healthy path stays a no-op. const forwardHealthy = measure("forward", () => - isSandboxForwardHealthy(sandboxName, { isWsl: isWslOverride }), + isSandboxForwardHealthy(sandboxName, { + isWsl: isWslOverride, + runtimeSelection, + }), ); if (forwardHealthy === false) { if (!quiet) { @@ -1493,17 +1620,18 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( console.log(" Re-establishing..."); } const forwardRecovered = measure("forward", () => - ensureSandboxPortForward(sandboxName, { isWsl: isWslOverride }), + ensureSandboxPortForward(sandboxName, { isWsl: isWslOverride, runtimeSelection }), ); const dashboardForwardRecovered = measure("forward", () => - ensureHermesDashboardPortForwardIfEnabled(sandboxName), + ensureHermesDashboardPortForwardIfEnabled(sandboxName, runtimeSelection), ); const messagingForwardRecovered = measure("forward", () => - recoverMessagingHostForward(sandboxName, { quiet }), + recoverMessagingHostForward(sandboxName, { quiet, runtimeSelection }), ); const declaredForwardsRecovered = measure("forward", () => recoverDeclaredAgentForwardPorts(sandboxName, recoveryPort, { quiet, + runtimeSelection, }), ); const auxiliaryResults = [ @@ -1556,13 +1684,13 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( }; } const dashboardForwardRecovered = measure("forward", () => - ensureHermesDashboardPortForwardIfEnabled(sandboxName), + ensureHermesDashboardPortForwardIfEnabled(sandboxName, runtimeSelection), ); const messagingForwardRecovered = measure("forward", () => - recoverMessagingHostForward(sandboxName, { quiet }), + recoverMessagingHostForward(sandboxName, { quiet, runtimeSelection }), ); const declaredForwardsRecovered = measure("forward", () => - recoverDeclaredAgentForwardPorts(sandboxName, recoveryPort, { quiet }), + recoverDeclaredAgentForwardPorts(sandboxName, recoveryPort, { quiet, runtimeSelection }), ); const auxiliaryResults = [ { label: "the Hermes dashboard host forward", recovered: dashboardForwardRecovered }, @@ -1607,9 +1735,10 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( const recovery = measure("processes", () => recoverSandboxProcesses(sandboxName, { quiet, - requestGatewaySupervisorAction, - requestPinnedGatewaySupervisorAction, + requestGatewaySupervisorAction: effectiveGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction: effectivePinnedGatewaySupervisorAction, relaunchManagedSupervisorSessionImpl, + runtimeSelection, onFailureLayer: (layer, detail) => { managedRecoveryFailureLayer = layer; managedRecoveryFailureDetail = detail; @@ -1626,8 +1755,8 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( const relaunch = recovery.kind === "relaunched" ? recovery.relaunch : null; const requestManagedProbe = relaunch ? (name: string, action: "restart" | "recover" | "probe", timeout = 210000) => - requestPinnedGatewaySupervisorAction(name, action, timeout, relaunch.containerId) - : requestGatewaySupervisorAction; + effectivePinnedGatewaySupervisorAction(name, action, timeout, relaunch.containerId) + : effectiveGatewaySupervisorAction; const relaunchedManagedHealth = { failure: null as ReturnType | null, }; @@ -1667,6 +1796,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( quiet, initialManagedHealthPassed: recovery.kind === "managed", requireManagedProbe: recovery.kind === "relaunched", + runtimeSelection, // A legacy keepalive relaunch starts a new OpenClaw container. The // #10153 failure exhausted the ordinary 30-second health budget // during that full recreation. Give only this OpenClaw transition @@ -1706,7 +1836,14 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( const rollbackUnconfirmed = recoveryFailureDetail !== gatewayWaitFailureDetail; if (!quiet) { console.error(" Gateway process started but is not responding."); - printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand); + printGatewayWedgeDiagnostics(sandboxName, (name, command) => + executeSandboxExecCommand( + name, + command, + DEFAULT_SANDBOX_EXEC_TIMEOUT_MS, + runtimeSelection ? { runtimeSelection } : {}, + ), + ); console.error(" Check /tmp/gateway.log inside the sandbox for details."); if (rollbackUnconfirmed) { console.error( @@ -1741,6 +1878,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( beforeProbe: relaunch ? (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null : undefined, + runtimeSelection, }; const readiness = waitForRecreatedSandboxOpenShellReadyImpl === waitForRecreatedSandboxOpenShellReady @@ -1791,13 +1929,19 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( ); if (finalizationFailure) return finalizationFailure; } - const mcpRefusal = processRecoveryMcpReconciliationRefusal(sandboxName, false); + const mcpRefusal = processRecoveryMcpReconciliationRefusal( + sandboxName, + false, + undefined, + runtimeSelection, + ); if (mcpRefusal) return mcpRefusal; const forwardRecovered = measure("forward", () => ensureSandboxPortForward(sandboxName, { afterSuccess: confirmRelaunchedManagedHealthForForward ?? undefined, beforeStart: confirmRelaunchedManagedHealthForForward ?? undefined, isWsl: isWslOverride, + runtimeSelection, }), ); if (!forwardRecovered && relaunchedManagedHealth.failure) { @@ -1814,13 +1958,13 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( }; } const dashboardForwardRecovered = measure("forward", () => - ensureHermesDashboardPortForwardIfEnabled(sandboxName), + ensureHermesDashboardPortForwardIfEnabled(sandboxName, runtimeSelection), ); const messagingForwardRecovered = measure("forward", () => - recoverMessagingHostForward(sandboxName, { quiet }), + recoverMessagingHostForward(sandboxName, { quiet, runtimeSelection }), ); const declaredForwardsRecovered = measure("forward", () => - recoverDeclaredAgentForwardPorts(sandboxName, recoveryPort, { quiet }), + recoverDeclaredAgentForwardPorts(sandboxName, recoveryPort, { quiet, runtimeSelection }), ); const auxiliaryResults = [ { label: "the Hermes dashboard host forward", recovered: dashboardForwardRecovered }, @@ -1890,6 +2034,7 @@ export function checkAndRecoverSandboxProcesses( isWsl?: boolean; onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null, detail?: string) => void; probeTiming?: ProcessRecoveryProbeTiming; + runtimeSelection?: OpenShellRuntimeSelection; } = {}, ) { return withMcpLifecycleLockSync(sandboxName, () => diff --git a/src/lib/actions/sandbox/rebuild-backup-phase.ts b/src/lib/actions/sandbox/rebuild-backup-phase.ts index 12ad3571f78..88a91717b5a 100644 --- a/src/lib/actions/sandbox/rebuild-backup-phase.ts +++ b/src/lib/actions/sandbox/rebuild-backup-phase.ts @@ -9,6 +9,7 @@ import { isPolicyObservationError, PolicyObservationError, } from "../../adapters/openshell/policy-state"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import { formatOpenShellPolicyRecoveryAction } from "../../gateway-start-guidance"; import type { WebSearchConfig } from "../../inference/web-search"; import type { SandboxMessagingPlan } from "../../messaging"; @@ -44,6 +45,7 @@ export interface RebuildBackupPhaseInput { webSearchConfig: WebSearchConfig | null; log: RebuildLog; bail: RebuildBail; + runtimeSelection?: OpenShellRuntimeSelection; } export interface RebuildBackupPhaseResult { @@ -62,12 +64,17 @@ function bailForUnsafeOpenClawPluginProvenance(input: RebuildBackupPhaseInput): return input.bail("Custom-image OpenClaw plugin provenance is unavailable."); } -export function captureRebuildPolicyDocument(sandboxName: string, gatewayName: string): string { +export function captureRebuildPolicyDocument( + sandboxName: string, + gatewayName: string, + runtimeSelection?: OpenShellRuntimeSelection, +): string { let policy: string; try { policy = captureRecordedSandboxBasePolicy( sandboxName, "capture the live policy before sandbox replacement", + runtimeSelection, ); } catch (error) { if (!isPolicyObservationError(error)) throw error; @@ -136,7 +143,11 @@ export function runRebuildBackupPhase( const capturedPolicy = input.staleRecovery || preparedRetainedPolicy ? null - : captureRebuildPolicyDocument(input.sandboxName, input.gatewayName); + : captureRebuildPolicyDocument( + input.sandboxName, + input.gatewayName, + input.runtimeSelection, + ); let backupManifest = preparedRecoveryManifest ?? backupStateForRebuild( @@ -200,7 +211,8 @@ export function runRebuildBackupPhase( }; } const policy = - capturedPolicy ?? captureRebuildPolicyDocument(input.sandboxName, input.gatewayName); + capturedPolicy ?? + captureRebuildPolicyDocument(input.sandboxName, input.gatewayName, input.runtimeSelection); if (backupManifest && !retainedPolicy) { try { backupManifest = writeRebuildPolicyHandoff(backupManifest, policy); diff --git a/src/lib/actions/sandbox/rebuild-config-hash.test.ts b/src/lib/actions/sandbox/rebuild-config-hash.test.ts index 1b24b4ab524..22f8e305d48 100644 --- a/src/lib/actions/sandbox/rebuild-config-hash.test.ts +++ b/src/lib/actions/sandbox/rebuild-config-hash.test.ts @@ -7,13 +7,72 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as processRecovery from "./process-recovery"; +import { + refreshMutableOpenClawConfigHashAfterPostRestoreWrites, + verifyFinalMutableOpenClawConfigHash, +} from "./rebuild-config-hash"; import { buildRefreshMutableOpenClawConfigHashCommand, buildVerifyMutableOpenClawConfigHashCommand, } from "./rebuild-config-hash-command"; +const runtimeSelection = { + gatewayName: "recorded-gateway", + workspace: "default", + localTlsDir: "/authority/tls", +} as const; + +describe("OpenClaw rebuild config hash target selection", () => { + beforeEach(() => { + vi.stubEnv("OPENSHELL_GATEWAY", "hostile-gateway"); + vi.stubEnv("OPENSHELL_WORKSPACE", "hostile-workspace"); + vi.stubEnv("OPENSHELL_LOCAL_TLS_DIR", "/hostile/tls"); + vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://hostile.invalid"); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + + it("refreshes the config hash on the selected target instead of the ambient target (#10514)", () => { + const execute = vi.spyOn(processRecovery, "executeSandboxCommand").mockReturnValue({ + status: 0, + stdout: "", + stderr: "", + }); + + expect( + refreshMutableOpenClawConfigHashAfterPostRestoreWrites("alpha", vi.fn(), runtimeSelection), + ).toBe(true); + expect(execute).toHaveBeenCalledExactlyOnceWith( + "alpha", + buildRefreshMutableOpenClawConfigHashCommand(), + { runtimeSelection }, + ); + expect(process.env.OPENSHELL_GATEWAY).toBe("hostile-gateway"); + }); + + it("verifies the config hash on the selected target instead of the ambient target (#10514)", () => { + const execute = vi.spyOn(processRecovery, "executeSandboxCommand").mockReturnValue({ + status: 0, + stdout: "", + stderr: "", + }); + + expect(verifyFinalMutableOpenClawConfigHash("alpha", vi.fn(), runtimeSelection)).toBe(true); + expect(execute).toHaveBeenCalledExactlyOnceWith( + "alpha", + buildVerifyMutableOpenClawConfigHashCommand(), + { runtimeSelection }, + ); + expect(process.env.OPENSHELL_GATEWAY).toBe("hostile-gateway"); + }); +}); + function sha256Hex(filePath: string): string { return createHash("sha256").update(fs.readFileSync(filePath)).digest("hex"); } diff --git a/src/lib/actions/sandbox/rebuild-config-hash.ts b/src/lib/actions/sandbox/rebuild-config-hash.ts index fd2c2ce288b..8b14bfae80f 100644 --- a/src/lib/actions/sandbox/rebuild-config-hash.ts +++ b/src/lib/actions/sandbox/rebuild-config-hash.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime"; import { R, YW } from "../../cli/terminal-style"; import { redact } from "../../security/redact"; import { executeSandboxCommand } from "./process-recovery"; @@ -14,8 +15,13 @@ export { buildRefreshMutableOpenClawConfigHashCommand }; export function refreshMutableOpenClawConfigHashAfterPostRestoreWrites( sandboxName: string, log: (msg: string) => void, + runtimeSelection?: OpenShellRuntimeSelection, ): boolean { - const result = executeSandboxCommand(sandboxName, buildRefreshMutableOpenClawConfigHashCommand()); + const result = runtimeSelection + ? executeSandboxCommand(sandboxName, buildRefreshMutableOpenClawConfigHashCommand(), { + runtimeSelection, + }) + : executeSandboxCommand(sandboxName, buildRefreshMutableOpenClawConfigHashCommand()); if (result && result.status === 0) { log("Mutable OpenClaw config hash refreshed after post-restore config writes"); return true; @@ -31,8 +37,13 @@ export function refreshMutableOpenClawConfigHashAfterPostRestoreWrites( export function verifyFinalMutableOpenClawConfigHash( sandboxName: string, log: (msg: string) => void, + runtimeSelection?: OpenShellRuntimeSelection, ): boolean { - const result = executeSandboxCommand(sandboxName, buildVerifyMutableOpenClawConfigHashCommand()); + const result = runtimeSelection + ? executeSandboxCommand(sandboxName, buildVerifyMutableOpenClawConfigHashCommand(), { + runtimeSelection, + }) + : executeSandboxCommand(sandboxName, buildVerifyMutableOpenClawConfigHashCommand()); if (result && result.status === 0) { log("Final mutable OpenClaw config hash verified after post-restore finalization"); return true; diff --git a/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts b/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts index 2ee4c632308..6dbc3398991 100644 --- a/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts +++ b/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { WebSearchConfig } from "../../inference/web-search"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import type { DcodeAutoApprovalMode } from "../../onboard/dcode-auto-approval"; import type { Session } from "../../state/onboard-session"; import type { ToolDisclosure } from "../../tool-disclosure"; @@ -19,7 +20,11 @@ import type { RebuildAgentBaseImageOptions, RebuildSandboxEntry } from "./rebuil import type { RebuildResumeConfig } from "./rebuild-resume-config"; type DcodeRebuildOrchestratorDeps = { - checkGatewaySchema(sandboxName: string, bail: DcodeRebuildPreflightBail): boolean; + checkGatewaySchema( + sandboxName: string, + bail: DcodeRebuildPreflightBail, + runtimeSelection?: OpenShellRuntimeSelection, + ): boolean; preflightCredentials( sandboxName: string, entry: RebuildSandboxEntry, @@ -48,7 +53,7 @@ export type DcodeRebuildOrchestrator = { readonly preparedReplacement: PreparedDcodeReplacement | null; run(action: () => Promise): Promise; runSync(action: () => T): T; - preflightCredentials(): Promise; + preflightCredentials(runtimeSelection?: OpenShellRuntimeSelection): Promise; prepareImage( resumeConfig: RebuildResumeConfig, webSearchConfig: WebSearchConfig | null, @@ -57,6 +62,7 @@ export type DcodeRebuildOrchestrator = { skipLiveRoute: boolean, gatewayPort: number, baseImageOptions?: RebuildAgentBaseImageOptions, + runtimeSelection?: OpenShellRuntimeSelection, ): Promise; revalidateBeforeDelete( resumeConfig: RebuildResumeConfig, @@ -64,6 +70,7 @@ export type DcodeRebuildOrchestrator = { dcodeAutoApprovalMode: DcodeAutoApprovalMode, skipLiveRoute: boolean, gatewayPort: number, + runtimeSelection?: OpenShellRuntimeSelection, ): Promise; checkAtDeleteEdge( resumeConfig: RebuildResumeConfig, @@ -71,6 +78,7 @@ export type DcodeRebuildOrchestrator = { dcodeAutoApprovalMode: DcodeAutoApprovalMode, skipLiveRoute: boolean, gatewayPort: number, + runtimeSelection?: OpenShellRuntimeSelection, ): Promise<{ ok: true } | { ok: false; message: string; code?: number }>; clearManagedCustomDockerfile(session: Session): void; storedDockerfile(sessionMatchesSandbox: boolean, session: Session | null): string | null; @@ -135,15 +143,21 @@ export function createDcodeRebuildOrchestrator( }, run, runSync, - preflightCredentials: () => + preflightCredentials: (runtimeSelection) => run(async () => { if (scope.enabled) { if ( - !(await ensureDcodeRebuildTargetGatewaySelected(sandboxName, entry, log, scope.bail)) + !(await ensureDcodeRebuildTargetGatewaySelected( + sandboxName, + entry, + log, + scope.bail, + runtimeSelection, + )) ) { return false; } - if (!deps.checkGatewaySchema(sandboxName, scope.bail)) return false; + if (!deps.checkGatewaySchema(sandboxName, scope.bail, runtimeSelection)) return false; } return deps.preflightCredentials(sandboxName, entry, log, scope.bail); }), @@ -155,6 +169,7 @@ export function createDcodeRebuildOrchestrator( skipLiveRoute, gatewayPort, baseImageOptions, + runtimeSelection, ) => run(async () => { if (!scope.enabled) { @@ -171,7 +186,9 @@ export function createDcodeRebuildOrchestrator( gatewayPort, log, bail: scope.bail, - checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, scope.bail), + checkGatewaySchema: (selection) => + deps.checkGatewaySchema(sandboxName, scope.bail, selection), + runtimeSelection, }); } const replacement = await prepareDcodeReplacementBeforeMutation({ @@ -186,7 +203,9 @@ export function createDcodeRebuildOrchestrator( baseImageOptions, log, bail: scope.bail, - checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, scope.bail), + checkGatewaySchema: (selection) => + deps.checkGatewaySchema(sandboxName, scope.bail, selection), + runtimeSelection, }); if (!replacement) { scope.cleanup(); @@ -201,6 +220,7 @@ export function createDcodeRebuildOrchestrator( dcodeAutoApprovalMode, skipLiveRoute, gatewayPort, + runtimeSelection, ) => run(async () => { if (!scope.enabled) return true; @@ -215,7 +235,9 @@ export function createDcodeRebuildOrchestrator( gatewayPort, log, bail: scope.bail, - checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, scope.bail), + checkGatewaySchema: (selection) => + deps.checkGatewaySchema(sandboxName, scope.bail, selection), + runtimeSelection, }); } const replacement = scope.preparedReplacement; @@ -230,7 +252,9 @@ export function createDcodeRebuildOrchestrator( gatewayPort, log, bail: scope.bail, - checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, scope.bail), + checkGatewaySchema: (selection) => + deps.checkGatewaySchema(sandboxName, scope.bail, selection), + runtimeSelection, replacement, }); }), @@ -240,6 +264,7 @@ export function createDcodeRebuildOrchestrator( dcodeAutoApprovalMode, skipLiveRoute, gatewayPort, + runtimeSelection, ) => { if (!scope.enabled) return { ok: true }; const replacement = scope.preparedReplacement; @@ -261,7 +286,9 @@ export function createDcodeRebuildOrchestrator( gatewayPort, log, bail: capturedBail, - checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, capturedBail), + checkGatewaySchema: (selection) => + deps.checkGatewaySchema(sandboxName, capturedBail, selection), + runtimeSelection, }) : revalidateDcodeReplacementAtMutationEdge({ sandboxName, @@ -273,7 +300,9 @@ export function createDcodeRebuildOrchestrator( gatewayPort, log, bail: capturedBail, - checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, capturedBail), + checkGatewaySchema: (selection) => + deps.checkGatewaySchema(sandboxName, capturedBail, selection), + runtimeSelection, replacement: replacement!, })); if (!valid) { diff --git a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts index cd71f7c049a..8f5e5c82f0a 100644 --- a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts @@ -4,6 +4,7 @@ import { isDeepStrictEqual } from "node:util"; import { dockerImageInspectFormat, dockerRmi } from "../../adapters/docker"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import type { TrustedRemoteBaseImageOverride } from "../../agent/base-image"; import { loadAgent } from "../../agent/defs"; import { @@ -13,7 +14,7 @@ import { pinTrustedAgentRemoteBaseImageOverrideForOperation, } from "../../agent/onboard"; import { RD as _RD, R } from "../../cli/terminal-style"; -import { recoverNamedGatewayRuntime } from "../../gateway-runtime-action"; +import { snapshotOpenShellEnv } from "../../gateway-runtime-action"; import * as nim from "../../inference/nim"; import type { WebSearchConfig } from "../../inference/web-search"; import type { DcodeAutoApprovalMode } from "../../onboard/dcode-auto-approval"; @@ -38,7 +39,11 @@ import { type ResolvedDcodeRebuildTarget, resolveDcodeRebuildTarget, } from "./rebuild-dcode-target"; -import type { RebuildAgentBaseImageOptions, RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import { + ensureRebuildTargetGatewaySelected, + type RebuildAgentBaseImageOptions, + type RebuildSandboxEntry, +} from "./rebuild-flow-helpers"; import { disposePreparedDcodeRebuildImage, type PreparedDcodeRebuildImage, @@ -78,7 +83,8 @@ export type DcodeReplacementPreflightInput = { gatewayPort?: number; log(message: string): void; bail: DcodeRebuildPreflightBail; - checkGatewaySchema(): boolean; + checkGatewaySchema(runtimeSelection?: OpenShellRuntimeSelection): boolean; + runtimeSelection?: OpenShellRuntimeSelection; }; export type DcodeReplacementPreparationInput = DcodeReplacementPreflightInput & { @@ -102,9 +108,8 @@ export function createDcodeRebuildPreflightScope( bail: DcodeRebuildPreflightBail, env: NodeJS.ProcessEnv = process.env, ): DcodeRebuildPreflightScope { - const previousOpenshellGateway = env.OPENSHELL_GATEWAY; + const restoreOpenShellEnv = snapshotOpenShellEnv(env); let preparedReplacement: PreparedDcodeReplacement | null = null; - let gatewayRestored = false; let cleaned = false; const cleanup = () => { if (!enabled || cleaned) return; @@ -115,11 +120,7 @@ export function createDcodeRebuildPreflightScope( console.warn(" Warning: temporary DCode rebuild inputs could not be fully removed."); } } finally { - if (!gatewayRestored) { - gatewayRestored = true; - if (previousOpenshellGateway === undefined) delete env.OPENSHELL_GATEWAY; - else env.OPENSHELL_GATEWAY = previousOpenshellGateway; - } + restoreOpenShellEnv(); cleaned = disposed; } }; @@ -172,33 +173,14 @@ export async function ensureDcodeRebuildTargetGatewaySelected( entry: RebuildSandboxEntry, log: (message: string) => void, bail: DcodeRebuildPreflightBail, + runtimeSelection?: OpenShellRuntimeSelection, ): Promise { - let gatewayName: string; try { - gatewayName = resolveSandboxGatewayName(entry); + resolveSandboxGatewayName(entry); } catch (error) { fail(error instanceof Error ? error.message : String(error), bail); } - - const recovery = await recoverNamedGatewayRuntime({ - gatewayName, - recoverableStates: ["missing_named", "named_unhealthy", "named_unreachable", "connected_other"], - }); - const beforeState = recovery.before?.state ?? "unknown"; - const afterState = recovery.after?.state ?? "unknown"; - if (!recovery.recovered || afterState !== "healthy_named") { - console.error(""); - console.error( - ` ${_RD}Rebuild preflight failed:${R} could not select the target gateway '${gatewayName}'.`, - ); - console.error(` Gateway state before: ${beforeState}; after: ${afterState}.`); - console.error(" Sandbox is untouched — no data was lost."); - bail(`Could not select healthy gateway '${gatewayName}' for sandbox '${sandboxName}'`); - return false; - } - process.env.OPENSHELL_GATEWAY = gatewayName; - log(`Pinned rebuild subprocesses to target gateway '${gatewayName}'`); - return true; + return ensureRebuildTargetGatewaySelected(sandboxName, entry, log, bail, runtimeSelection); } function resolveTarget( @@ -218,11 +200,13 @@ function requireInferenceRoute( sandboxName: string, target: ResolvedDcodeRebuildTarget, bail: DcodeRebuildPreflightBail, + runtimeSelection?: OpenShellRuntimeSelection, ): void { const result = probeSandboxInferenceInvocation({ sandboxName, agentName: target.agent, ...target, + ...(runtimeSelection ? { runtimeSelection } : {}), }); if (!result.ok) { fail( @@ -513,6 +497,7 @@ export async function prepareDcodeReplacementBeforeMutation( gatewayPort, log, bail, + runtimeSelection, } = input; let buildContext: PreparedDcodeRebuildImage | null = null; let pinnedBase: PinnedDcodeBaseImage | null = null; @@ -527,7 +512,7 @@ export async function prepareDcodeReplacementBeforeMutation( const session = loadMatchingDcodeSession(sandboxName); const target = resolveTarget(entry, resumeConfig, bail, gatewayPort); - if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail); + if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail, runtimeSelection); pinnedBase = resolvePinnedDcodeBaseImage(bail, input.baseImageOptions); const sandboxGpuConfig = getRecordedGpuConfig(sandboxName, entry, session); @@ -552,11 +537,19 @@ export async function prepareDcodeReplacementBeforeMutation( if (!imageResult.ok) fail(imageResult.detail, bail); buildContext = imageResult.prepared; - if (!(await ensureDcodeRebuildTargetGatewaySelected(sandboxName, entry, log, bail))) { + if ( + !(await ensureDcodeRebuildTargetGatewaySelected( + sandboxName, + entry, + log, + bail, + runtimeSelection, + )) + ) { return null; } - if (!input.checkGatewaySchema()) return null; - if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail); + if (!input.checkGatewaySchema(runtimeSelection)) return null; + if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail, runtimeSelection); requireCurrentTarget(sandboxName, entry, target, resumeConfig, bail, gatewayPort); if (!verifyPreparedDcodeRebuildImage(buildContext) || !pinnedBase.verify()) { fail("the prepared DCode replacement inputs changed during preflight", bail); @@ -583,8 +576,17 @@ export async function prepareDcodeReplacementBeforeMutation( export async function revalidateDcodeReplacementAtMutationEdge( input: DcodeReplacementPreflightInput & { replacement: PreparedDcodeReplacement }, ): Promise { - const { sandboxName, entry, resumeConfig, skipLiveRoute, gatewayPort, log, bail, replacement } = - input; + const { + sandboxName, + entry, + resumeConfig, + skipLiveRoute, + gatewayPort, + log, + bail, + replacement, + runtimeSelection, + } = input; const target = resolveTarget(entry, resumeConfig, bail, gatewayPort); if (replacement.gatewayName !== target.gatewayName) { fail("the prepared DCode gateway changed before deletion", bail); @@ -595,11 +597,19 @@ export async function revalidateDcodeReplacementAtMutationEdge( if (replacement.dcodeAutoApprovalMode !== input.dcodeAutoApprovalMode) { fail("the prepared DCode auto-approval mode changed before deletion", bail); } - if (!(await ensureDcodeRebuildTargetGatewaySelected(sandboxName, entry, log, bail))) { + if ( + !(await ensureDcodeRebuildTargetGatewaySelected( + sandboxName, + entry, + log, + bail, + runtimeSelection, + )) + ) { return false; } - if (!input.checkGatewaySchema()) return false; - if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail); + if (!input.checkGatewaySchema(runtimeSelection)) return false; + if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail, runtimeSelection); requireCurrentTarget(sandboxName, entry, target, resumeConfig, bail, gatewayPort); if (!replacement.verify()) { fail("the prepared DCode replacement inputs changed before deletion", bail); @@ -615,13 +625,30 @@ export async function revalidateDcodeReplacementAtMutationEdge( export async function revalidateManagedDcodeWorkloadAtMutationEdge( input: DcodeReplacementPreflightInput, ): Promise { - const { sandboxName, entry, resumeConfig, skipLiveRoute, gatewayPort, log, bail } = input; + const { + sandboxName, + entry, + resumeConfig, + skipLiveRoute, + gatewayPort, + log, + bail, + runtimeSelection, + } = input; const target = resolveTarget(entry, resumeConfig, bail, gatewayPort); - if (!(await ensureDcodeRebuildTargetGatewaySelected(sandboxName, entry, log, bail))) { + if ( + !(await ensureDcodeRebuildTargetGatewaySelected( + sandboxName, + entry, + log, + bail, + runtimeSelection, + )) + ) { return false; } - if (!input.checkGatewaySchema()) return false; - if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail); + if (!input.checkGatewaySchema(runtimeSelection)) return false; + if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail, runtimeSelection); requireCurrentTarget(sandboxName, entry, target, resumeConfig, bail, gatewayPort); return true; } diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index 1524f78ecca..ad063c62588 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -26,6 +26,7 @@ const mocks = vi.hoisted(() => ({ runOpenshell: vi.fn( ( _args: string[], + _options?: Record, ): { status: number | null; stdout: string; @@ -224,6 +225,7 @@ describe("rebuild destroy phase", () => { false, true, expect.any(Function), + undefined, ); }); @@ -283,8 +285,22 @@ describe("rebuild destroy phase", () => { expect(onDeleted).toHaveBeenCalledOnce(); }); - it("pins deletion to the recorded gateway when ambient selection changes (#7062)", async () => { + it("pins deletion and the delete-edge user-file probe when ambient selection changes (#10514)", async () => { vi.stubEnv("OPENSHELL_GATEWAY", "nemoclaw-29080"); + vi.stubEnv("OPENSHELL_WORKSPACE", "hostile-workspace"); + vi.stubEnv("OPENSHELL_LOCAL_TLS_DIR", "/hostile/tls"); + vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://hostile.invalid"); + const runtimeSelection = { + gatewayName: "nemoclaw-19080", + workspace: "default", + localTlsDir: "/authority/tls", + }; + mocks.prepareMcpForRebuild.mockResolvedValue({ + entries: [], + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + runtimeSelection, + }); mocks.getSandbox.mockReturnValue({ name: "alpha", agent: "openclaw", @@ -304,6 +320,7 @@ describe("rebuild destroy phase", () => { recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, + runtimeSelection, log: vi.fn(), bail: vi.fn((message: string): never => { throw new Error(message); @@ -313,8 +330,59 @@ describe("rebuild destroy phase", () => { expect(mocks.runOpenshell).toHaveBeenCalledWith( ["sandbox", "delete", "-g", "nemoclaw-19080", "alpha"], - expect.objectContaining({ ignoreError: true }), + expect.objectContaining({ + ignoreError: true, + replaceEnv: true, + env: expect.objectContaining({ + OPENSHELL_GATEWAY: "nemoclaw-19080", + OPENSHELL_WORKSPACE: "default", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + }), + }), + ); + expect(mocks.warnUnpreservedUserManagedFiles).toHaveBeenCalledWith( + "alpha", + expect.any(Function), + runtimeSelection, + ); + const deleteOptions = mocks.runOpenshell.mock.calls.find( + ([args]) => args[0] === "sandbox" && args[1] === "delete", + )?.[1] as { env?: Record } | undefined; + expect(deleteOptions?.env).not.toHaveProperty("OPENSHELL_GATEWAY_ENDPOINT"); + expect(mocks.captureOpenshell).toHaveBeenCalledWith( + ["sandbox", "get", "-g", "nemoclaw-19080", "alpha"], + expect.objectContaining({ + replaceEnv: true, + env: expect.objectContaining({ + OPENSHELL_GATEWAY: "nemoclaw-19080", + OPENSHELL_WORKSPACE: "default", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + }), + }), + ); + }); + + it("refuses deletion when the frozen OpenShell target does not match (#10514)", async () => { + await expect( + runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, + staleRecovery: false, + recreateJournal: stubRecreateJournal(), + backupManifest: null, + force: true, + runtimeSelection: { gatewayName: "nemoclaw-19080", workspace: "default" }, + log: vi.fn(), + bail: vi.fn((message: string): never => { + throw new Error(message); + }), + onDeleted: vi.fn(), + }), + ).rejects.toThrow( + "Rebuild delete target gateway 'nemoclaw' does not match recorded OpenShell gateway 'nemoclaw-19080'. NemoClaw did not delete the original sandbox. Restore recorded gateway 'nemoclaw-19080', confirm it is healthy, then retry.", ); + + expectNoSandboxDelete(mocks.runOpenshell); }); it.each([ @@ -376,6 +444,7 @@ describe("rebuild destroy phase", () => { "alpha", [{ server: "github" }], [], + undefined, ); expect(mocks.removeSandboxRegistryEntryWithReceipt).not.toHaveBeenCalled(); expect(mocks.stopNimContainer).not.toHaveBeenCalled(); @@ -414,7 +483,12 @@ describe("rebuild destroy phase", () => { expect(revalidateBeforeDelete).toHaveBeenCalledOnce(); expect(mocks.runOpenshell).not.toHaveBeenCalled(); expect(mocks.removeSandboxRegistryEntryWithReceipt).not.toHaveBeenCalled(); - expect(mocks.reattachMcpAfterDeleteFailure).toHaveBeenCalledWith("alpha", [], []); + expect(mocks.reattachMcpAfterDeleteFailure).toHaveBeenCalledWith( + "alpha", + [], + [], + undefined, + ); expect(mocks.stopNimContainer).not.toHaveBeenCalled(); expect(mocks.stopNimContainerByName).not.toHaveBeenCalled(); }); @@ -454,7 +528,12 @@ describe("rebuild destroy phase", () => { expect(revalidateBeforeDelete.mock.invocationCallOrder[0]).toBeLessThan( mocks.runOpenshell.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, ); - expect(mocks.reattachMcpAfterDeleteFailure).toHaveBeenCalledWith("alpha", [], []); + expect(mocks.reattachMcpAfterDeleteFailure).toHaveBeenCalledWith( + "alpha", + [], + [], + undefined, + ); expect(mocks.teardownSandboxDashboardForward).not.toHaveBeenCalled(); expect(mocks.restoreSandboxLaunchForwards).not.toHaveBeenCalled(); expect(mocks.removeSandboxRegistryEntryWithReceipt).not.toHaveBeenCalled(); @@ -1024,6 +1103,7 @@ describe("rebuild destroy phase", () => { "alpha", [{ providerName: "nemoclaw-mcp-alpha-github" }], [{ server: "github" }], + undefined, ); expect(mocks.runOpenshell).not.toHaveBeenCalledWith( ["sandbox", "delete", "-g", "nemoclaw", "alpha"], diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts index dedbad6db9d..e6d7c7a0a82 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { buildSelectedOpenShellSubprocessEnv } from "../../adapters/openshell/command-argv"; import { captureOpenshell, runOpenshell } from "../../adapters/openshell/runtime"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import { G, R } from "../../cli/terminal-style"; import { waitUntil } from "../../core/wait"; @@ -41,10 +43,13 @@ export interface RebuildDestroyPhaseInput { log: RebuildLog; bail: RebuildBail; force?: boolean; + runtimeSelection?: OpenShellRuntimeSelection; validateAfterMcpPreparation?: ( preparation: McpRebuildPreparation, ) => Promise; - validateAtDeleteEdge?: () => RebuildDeleteValidationResult; + validateAtDeleteEdge?: ( + runtimeSelection?: OpenShellRuntimeSelection, + ) => RebuildDeleteValidationResult; cleanupDockerOrphanAfterDelete?: () => void; onDeleted: () => void; onDeleteStateAmbiguous?: () => void; @@ -79,6 +84,7 @@ interface RebuildDeleteAbsenceDeps { }; now?: () => number; sleep?: (milliseconds: number) => void; + runtimeSelection?: OpenShellRuntimeSelection; } const REBUILD_DELETE_ABSENCE_MAX_ATTEMPTS = 20; @@ -124,6 +130,9 @@ export function waitForRebuildDeleteAbsence( log: RebuildLog, deps: RebuildDeleteAbsenceDeps = {}, ): boolean { + if (deps.runtimeSelection && deps.runtimeSelection.gatewayName !== gatewayName) { + throw new Error("Rebuild delete gateway does not match the frozen OpenShell target."); + } const now = deps.now ?? Date.now; const deadlineMs = now() + OPENSHELL_PROBE_TIMEOUT_MS; const captureSandboxGet = @@ -134,6 +143,12 @@ export function waitForRebuildDeleteAbsence( includeStderr: true, includeStreams: true, timeout: timeoutMs, + ...(deps.runtimeSelection + ? { + env: buildSelectedOpenShellSubprocessEnv(deps.runtimeSelection), + replaceEnv: true, + } + : {}), }); return probe; }); @@ -180,6 +195,7 @@ function reconcileFailedSandboxDelete( sandboxName: string, sandboxEntry: RebuildSandboxEntry, log: RebuildLog, + runtimeSelection?: OpenShellRuntimeSelection, ): PostDeleteReconciliation { let gatewayName: string; try { @@ -188,6 +204,10 @@ function reconcileFailedSandboxDelete( log("Post-delete reconciliation could not resolve the recorded sandbox gateway."); return { state: "ambiguous", phase: null, status: null }; } + if (runtimeSelection && runtimeSelection.gatewayName !== gatewayName) { + log("Post-delete reconciliation target does not match the frozen OpenShell target."); + return { state: "ambiguous", phase: null, status: null }; + } let probe: ReturnType; try { @@ -195,6 +215,12 @@ function reconcileFailedSandboxDelete( ignoreError: true, stdio: ["ignore", "pipe", "pipe"], timeout: OPENSHELL_PROBE_TIMEOUT_MS, + ...(runtimeSelection + ? { + env: buildSelectedOpenShellSubprocessEnv(runtimeSelection), + replaceEnv: true, + } + : {}), }); } catch { log(`Post-delete reconciliation could not query recorded gateway '${gatewayName}'.`); @@ -275,6 +301,7 @@ export async function runRebuildDestroyPhase( staleRecovery, input.force === true, bail, + input.runtimeSelection, ); return preparation; }, @@ -283,7 +310,9 @@ export async function runRebuildDestroyPhase( // fingerprints match the registry. Probe afterward so a Deep Agents // user `.mcp.json` is not confused with the separate managed projection. // This can block on SSH, so it must finish before the final DCode check. - if (!staleRecovery) warnUnpreservedUserManagedFiles(sandboxName, log); + if (!staleRecovery) { + warnUnpreservedUserManagedFiles(sandboxName, log, preparation.runtimeSelection); + } if (validateAfterMcpPreparation) { let validation: RebuildDeleteValidationResult; try { @@ -301,6 +330,7 @@ export async function runRebuildDestroyPhase( sandboxName, preparation.detachedProviderEntries, preparation.scrubbedAdapterEntries, + preparation.runtimeSelection, ); bail( mcpRecoveryFailure @@ -319,6 +349,24 @@ export async function runRebuildDestroyPhase( if (!mcpPreparation) return null; const rebuildDetachedMcpProviderEntries = mcpPreparation.detachedProviderEntries; const rebuildScrubbedMcpAdapterEntries = mcpPreparation.scrubbedAdapterEntries; + const rebuildMcpRuntimeSelection = input.runtimeSelection ?? mcpPreparation.runtimeSelection; + if ( + rebuildMcpRuntimeSelection && + rebuildMcpRuntimeSelection.gatewayName !== deleteTarget.gatewayName + ) { + const mcpRecoveryFailure = await reattachMcpAfterDeleteFailure( + sandboxName, + rebuildDetachedMcpProviderEntries, + rebuildScrubbedMcpAdapterEntries, + rebuildMcpRuntimeSelection, + ); + bail( + mcpRecoveryFailure + ? `Rebuild delete target gateway '${deleteTarget.gatewayName}' does not match recorded OpenShell gateway '${rebuildMcpRuntimeSelection.gatewayName}'. NemoClaw did not delete the original sandbox. MCP provider recovery also failed: ${mcpRecoveryFailure}. Restore recorded gateway '${rebuildMcpRuntimeSelection.gatewayName}', confirm it is healthy, then retry.` + : `Rebuild delete target gateway '${deleteTarget.gatewayName}' does not match recorded OpenShell gateway '${rebuildMcpRuntimeSelection.gatewayName}'. NemoClaw did not delete the original sandbox. Restore recorded gateway '${rebuildMcpRuntimeSelection.gatewayName}', confirm it is healthy, then retry.`, + ); + return null; + } // Exec-unavailable recovery deliberately made no MCP mutation during // preparation. Re-prove target, policy, provider, and registry state while @@ -337,6 +385,7 @@ export async function runRebuildDestroyPhase( sandboxName, rebuildDetachedMcpProviderEntries, rebuildScrubbedMcpAdapterEntries, + rebuildMcpRuntimeSelection, ); const detail = error instanceof Error ? error.message : String(error); bail( @@ -355,6 +404,7 @@ export async function runRebuildDestroyPhase( sandboxName, rebuildDetachedMcpProviderEntries, rebuildScrubbedMcpAdapterEntries, + rebuildMcpRuntimeSelection, ); bail( mcpRecoveryFailure @@ -367,7 +417,7 @@ export async function runRebuildDestroyPhase( if (validateAtDeleteEdge) { let validation: RebuildDeleteValidationResult; try { - validation = validateAtDeleteEdge(); + validation = validateAtDeleteEdge(rebuildMcpRuntimeSelection); } catch (error) { const detail = error instanceof Error ? error.message : String(error); log(`Unexpected delete-edge validation failure: ${redactFull(detail)}`); @@ -381,6 +431,7 @@ export async function runRebuildDestroyPhase( sandboxName, rebuildDetachedMcpProviderEntries, rebuildScrubbedMcpAdapterEntries, + rebuildMcpRuntimeSelection, ); bail( mcpRecoveryFailure @@ -403,6 +454,7 @@ export async function runRebuildDestroyPhase( sandboxName, rebuildDetachedMcpProviderEntries, rebuildScrubbedMcpAdapterEntries, + rebuildMcpRuntimeSelection, ); const detail = error instanceof Error ? error.message : String(error); bail( @@ -423,12 +475,23 @@ export async function runRebuildDestroyPhase( : runOpenshell(["sandbox", "delete", "-g", gatewayName, sandboxName], { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], + ...(rebuildMcpRuntimeSelection + ? { + env: buildSelectedOpenShellSubprocessEnv(rebuildMcpRuntimeSelection), + replaceEnv: true, + } + : {}), }); const alreadyGone = deleteResult === null || getSandboxDeleteOutcome(deleteResult).alreadyGone; if (deleteResult) log(`Delete result: exit=${deleteResult.status}, alreadyGone=${alreadyGone}`); let deletionConfirmed = alreadyGone; if (deleteResult && deleteResult.status !== 0) { - const reconciledDelete = reconcileFailedSandboxDelete(sandboxName, input.sandboxEntry, log); + const reconciledDelete = reconcileFailedSandboxDelete( + sandboxName, + input.sandboxEntry, + log, + rebuildMcpRuntimeSelection, + ); if (reconciledDelete.state === "deleted") { log("Delete returned nonzero, but exact post-delete state confirms sandbox removal."); deletionConfirmed = true; @@ -441,6 +504,7 @@ export async function runRebuildDestroyPhase( sandboxName, rebuildDetachedMcpProviderEntries, rebuildScrubbedMcpAdapterEntries, + rebuildMcpRuntimeSelection, ); if (mcpRecoveryFailure) { console.error( @@ -479,7 +543,9 @@ export async function runRebuildDestroyPhase( return null; } } - deletionConfirmed ||= waitForRebuildDeleteAbsence(sandboxName, gatewayName, log); + deletionConfirmed ||= waitForRebuildDeleteAbsence(sandboxName, gatewayName, log, { + runtimeSelection: rebuildMcpRuntimeSelection, + }); if (!deletionConfirmed) { console.error( " Sandbox delete was accepted, but OpenShell did not confirm that the sandbox is absent.", diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts index cf323378778..4dcb2f29bf3 100644 --- a/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; +import { restoreEnvBulk } from "../../../../test/helpers/env-test-helpers"; import * as dockerImage from "../../adapters/docker/image"; import * as agentDefs from "../../agent/defs"; import * as agentOnboard from "../../agent/onboard"; @@ -65,20 +66,30 @@ function makeBail(): (msg: string, code?: number) => never { } describe("rebuild target gateway preflight", () => { - const priorGateway = process.env.OPENSHELL_GATEWAY; + const priorOpenShellEnv = { + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY, + OPENSHELL_GATEWAY_ENDPOINT: process.env.OPENSHELL_GATEWAY_ENDPOINT, + OPENSHELL_LOCAL_TLS_DIR: process.env.OPENSHELL_LOCAL_TLS_DIR, + OPENSHELL_TOKEN: process.env.OPENSHELL_TOKEN, + OPENSHELL_WORKSPACE: process.env.OPENSHELL_WORKSPACE, + }; afterEach(() => { vi.restoreAllMocks(); - switch (priorGateway) { - case undefined: - delete process.env.OPENSHELL_GATEWAY; - break; - default: - process.env.OPENSHELL_GATEWAY = priorGateway; - } + restoreEnvBulk(priorOpenShellEnv); }); it("health-checks and pins the sandbox's persisted gateway", async () => { + process.env.OPENSHELL_GATEWAY = "hostile-gateway"; + process.env.OPENSHELL_GATEWAY_ENDPOINT = "https://hostile.invalid"; + process.env.OPENSHELL_LOCAL_TLS_DIR = "/hostile/tls"; + process.env.OPENSHELL_TOKEN = "hostile-token"; + process.env.OPENSHELL_WORKSPACE = "hostile-workspace"; + const runtimeSelection = { + gatewayName: "nemoclaw-19080", + localTlsDir: "/authority/tls", + workspace: "default", + }; const recover = vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockResolvedValue({ recovered: true, before: { state: "connected_other", status: "", gatewayInfo: "", activeGateway: null }, @@ -92,11 +103,54 @@ describe("rebuild target gateway preflight", () => { { name: "alpha", gatewayName: "nemoclaw-19080", gatewayPort: 19080 }, () => undefined, makeBail(), + runtimeSelection, + ), + ).resolves.toBe(true); + + expect(recover).toHaveBeenCalledWith({ + gatewayName: "nemoclaw-19080", + runtimeSelection, + }); + expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw-19080"); + expect(process.env.OPENSHELL_WORKSPACE).toBe("default"); + expect(process.env.OPENSHELL_LOCAL_TLS_DIR).toBe("/authority/tls"); + expect(process.env.OPENSHELL_GATEWAY_ENDPOINT).toBeUndefined(); + expect(process.env.OPENSHELL_TOKEN).toBeUndefined(); + }); + + it("keeps non-MCP ambient selectors while pinning the recorded gateway (#10514)", async () => { + process.env.OPENSHELL_GATEWAY = "hostile-gateway"; + process.env.OPENSHELL_GATEWAY_ENDPOINT = "https://hostile.invalid"; + process.env.OPENSHELL_LOCAL_TLS_DIR = "/hostile/tls"; + process.env.OPENSHELL_TOKEN = "hostile-token"; + process.env.OPENSHELL_WORKSPACE = "hostile-workspace"; + const recover = vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockResolvedValue({ + recovered: true, + before: { state: "connected_other", status: "", gatewayInfo: "", activeGateway: null }, + after: { + state: "healthy_named", + status: "", + gatewayInfo: "", + activeGateway: "nemoclaw-19080", + }, + attempted: true, + }); + + await expect( + ensureRebuildTargetGatewaySelected( + "alpha", + { name: "alpha", gatewayName: "nemoclaw-19080", gatewayPort: 19080 }, + vi.fn(), + makeBail(), ), ).resolves.toBe(true); expect(recover).toHaveBeenCalledWith({ gatewayName: "nemoclaw-19080" }); expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw-19080"); + expect(process.env.OPENSHELL_GATEWAY_ENDPOINT).toBe("https://hostile.invalid"); + expect(process.env.OPENSHELL_LOCAL_TLS_DIR).toBe("/hostile/tls"); + expect(process.env.OPENSHELL_TOKEN).toBe("hostile-token"); + expect(process.env.OPENSHELL_WORKSPACE).toBe("hostile-workspace"); }); it("fails closed when the target gateway cannot become healthy", async () => { @@ -844,7 +898,7 @@ describe("warnUnpreservedUserManagedFiles", () => { warnUnpreservedUserManagedFiles("alpha", () => undefined); expect(probeSpy).toHaveBeenCalledOnce(); - expect(probeSpy).toHaveBeenCalledWith("alpha"); + expect(probeSpy).toHaveBeenCalledWith("alpha", undefined); const warnLines = warnSpy.mock.calls.map((args: unknown[]) => String(args[0])); expect( diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.ts index dde27b49786..f95979b8eec 100644 --- a/src/lib/actions/sandbox/rebuild-flow-helpers.ts +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.ts @@ -3,6 +3,11 @@ import { dockerRmi } from "../../adapters/docker/image"; import { printOpenShellStateRpcIssue } from "../../adapters/openshell/gateway-drift"; +import { + replaceOpenShellRuntimeSelectionEnv, + snapshotOpenShellEnv, + type OpenShellRuntimeSelection, +} from "../../adapters/openshell/runtime-selection"; import { loadAgent } from "../../agent/defs"; import { bindLocalAgentBaseImageHandoffToResolution, @@ -49,6 +54,7 @@ import { import * as snapshotBackup from "./snapshot/backup-authority"; export { removeStaleRebuildDockerOrphan }; +export { replaceOpenShellRuntimeSelectionEnv, snapshotOpenShellEnv }; export type RebuildSandboxEntry = SandboxEntry & { agents?: unknown[] }; @@ -136,9 +142,19 @@ export async function ensureRebuildTargetGatewaySelected( sb: RebuildSandboxEntry, log: (message: string) => void, bail: (message: string, code?: number) => never, + runtimeSelection?: OpenShellRuntimeSelection, ): Promise { const gatewayName = resolveSandboxGatewayName(sb); - const recovery = await recoverNamedGatewayRuntime({ gatewayName }); + if (runtimeSelection && runtimeSelection.gatewayName !== gatewayName) { + bail( + `OpenShell runtime selection '${runtimeSelection.gatewayName}' does not match recorded gateway '${gatewayName}'`, + ); + return false; + } + const recovery = await recoverNamedGatewayRuntime({ + gatewayName, + ...(runtimeSelection ? { runtimeSelection } : {}), + }); if (!recovery.recovered || recovery.after.state !== "healthy_named") { console.error(""); console.error( @@ -151,7 +167,8 @@ export async function ensureRebuildTargetGatewaySelected( bail(`Could not select healthy gateway '${gatewayName}' for sandbox '${sandboxName}'`); return false; } - process.env.OPENSHELL_GATEWAY = gatewayName; + if (runtimeSelection) replaceOpenShellRuntimeSelectionEnv(process.env, runtimeSelection); + else process.env.OPENSHELL_GATEWAY = gatewayName; log(`Pinned rebuild subprocesses to target gateway '${gatewayName}'`); return true; } @@ -539,10 +556,11 @@ export function backupSandboxStateForRebuild( export function warnUnpreservedUserManagedFiles( sandboxName: string, log: (msg: string) => void, + runtimeSelection?: OpenShellRuntimeSelection, ): void { let probe: userManagedFilesProbe.UserManagedFilesProbe; try { - probe = userManagedFilesProbe.probeUserManagedFiles(sandboxName); + probe = userManagedFilesProbe.probeUserManagedFiles(sandboxName, runtimeSelection); } catch (err) { const message = err instanceof Error ? err.message : String(err); log(`User-managed file probe errored: ${message}`); diff --git a/src/lib/actions/sandbox/rebuild-flow-lifecycle.test.ts b/src/lib/actions/sandbox/rebuild-flow-lifecycle.test.ts index 1d0ca8a7120..90a7f04da2a 100644 --- a/src/lib/actions/sandbox/rebuild-flow-lifecycle.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow-lifecycle.test.ts @@ -633,6 +633,38 @@ describe("rebuildSandbox flow: lifecycle", () => { }); }); + it("rebuilds prepared-only MCP state without claiming runtime authority", async () => { + const preparedMcpEntry = { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["GITHUB_TOKEN"], + policyName: "mcp-bridge-github", + addedAt: "2026-06-01T00:00:00.000Z", + addState: "prepared" as const, + }; + const harness = createRebuildFlowHarness({ + sandboxEntry: { + mcp: { bridges: { github: preparedMcpEntry } }, + }, + mcpPreparation: { + entries: [], + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes", "--force"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.prepareMcpBridgesForRebuildSpy).toHaveBeenCalledWith( + "alpha", + ); + expect(harness.onboardSpy).toHaveBeenCalledOnce(); + }); + it("keeps the journaled row when replacement creation fails (#7734)", async () => { const harness = createRebuildFlowHarness({ onboard: () => { diff --git a/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts b/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts index eaf47775078..d3b22f35edc 100644 --- a/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts +++ b/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts @@ -119,6 +119,27 @@ describe("rebuild gateway drift preflight", () => { expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled(); }); + it("binds gateway schema preflight to the frozen runtime target (#10514)", () => { + const runtimeSelection = { + gatewayName: "nemoclaw", + localTlsDir: "/authority/tls", + workspace: "default", + }; + + expect( + checkRebuildGatewaySchemaPreflight( + "alpha", + makeSandboxEntry(), + bail, + runtimeSelection, + ), + ).toBe(true); + expect(gatewayDrift.detectOpenShellStateRpcPreflightIssue).toHaveBeenCalledWith({ + gatewayName: "nemoclaw", + runtimeSelection, + }); + }); + it("prints the safe-abort diagnostic before bailing on gateway schema drift (#7794)", () => { vi.mocked(gatewayDrift.detectOpenShellStateRpcPreflightIssue).mockReturnValue(driftIssue); const nonThrowingBail = vi.fn(); diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts index b16b7cd8bcd..5a1f5a5a6c2 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { loadAgent } from "../../agent/defs"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import { type InferenceEndpointSource, normalizeInferenceEndpointSource, @@ -120,6 +121,7 @@ export type RebuildRecreateOnboardOpts = { controlUiPort: number | null; targetGatewayName: string; targetGatewayPort: number; + runtimeSelection?: OpenShellRuntimeSelection; onboardLockAlreadyHeld: true; deferProcessExit: true; /** Target fingerprint of the replacement journal opened before deletion. */ diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts index abbef103048..d035a4fe171 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { CLI_NAME } from "../../cli/branding"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime"; import { isDirectSandboxFallbackUnavailableError } from "../../sandbox/privileged-exec"; import type { GatewayRestartResult } from "./gateway-restart"; import { @@ -101,16 +102,17 @@ type GatewayRecoveryObservation = { interface HermesPostRestoreGatewayDeps { checkAndRecoverSandboxProcesses?: ( sandboxName: string, - options: { quiet: boolean }, + options: { quiet: boolean; runtimeSelection?: OpenShellRuntimeSelection }, ) => GatewayRecoveryObservation; restartSandboxGateway?: ( sandboxName: string, - options: { quiet: boolean }, + options: { quiet: boolean; runtimeSelection?: OpenShellRuntimeSelection }, ) => GatewayRestartResult; observeHermesCronReplacement?: ( sandboxName: string, originalIdentity: HermesCronRestoreIdentity, ) => HermesCronRestoreIdentity; + runtimeSelection?: OpenShellRuntimeSelection; } export interface HermesPostRestoreGatewayVerification { @@ -169,7 +171,10 @@ export function restartHermesGatewayAfterStateRestore( ): HermesPostRestoreGatewayRestartState { if (agentName !== "hermes") return "not-applicable"; const restart = deps.restartSandboxGateway ?? restartSandboxGateway; - const result = restart(sandboxName, { quiet: true }); + const result = restart(sandboxName, { + quiet: true, + ...(deps.runtimeSelection ? { runtimeSelection: deps.runtimeSelection } : {}), + }); if (result.ok) return "restarted"; const mcpRestoreCanSupersede = result.failureLayer === "MCP reconciliation refusal" && @@ -237,7 +242,10 @@ function verifyHermesGatewayAfterStateRestoreImpl( // later iteration must observe it both before and after health. } } - const observation: GatewayRecoveryObservation = checkAndRecover(sandboxName, { quiet: true }); + const observation: GatewayRecoveryObservation = checkAndRecover(sandboxName, { + quiet: true, + ...(deps.runtimeSelection ? { runtimeSelection: deps.runtimeSelection } : {}), + }); if ( observation.forwardRecoveryFailed === true || observation.secretBoundaryRefused === true || diff --git a/src/lib/actions/sandbox/rebuild-mcp-phase.test.ts b/src/lib/actions/sandbox/rebuild-mcp-phase.test.ts index bd559c2d7f6..87479b7bbea 100644 --- a/src/lib/actions/sandbox/rebuild-mcp-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-mcp-phase.test.ts @@ -6,11 +6,18 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ executeSandboxCommand: vi.fn(), executeSandboxExecCommand: vi.fn(), + getMcpProviderInspectionRuntimeSelection: vi.fn(), + getSandbox: vi.fn(), prepareAbsent: vi.fn(), prepareExecUnavailable: vi.fn(), prepareLive: vi.fn(), })); +vi.mock("../../state/registry", async (importOriginal) => ({ + ...(await importOriginal()), + getSandbox: mocks.getSandbox, +})); + vi.mock("./mcp-bridge", () => ({ prepareMcpBridgesForAbsentSandboxRebuild: mocks.prepareAbsent, prepareMcpBridgesForExecUnavailableRebuild: mocks.prepareExecUnavailable, @@ -24,7 +31,18 @@ vi.mock("./process-recovery", () => ({ executeSandboxExecCommand: mocks.executeSandboxExecCommand, })); -import { prepareMcpForRebuild, printMcpRebuildRetryCommand } from "./rebuild-mcp-phase"; +vi.mock("./mcp-bridge-provider", async (importOriginal) => ({ + ...(await importOriginal()), + getMcpProviderInspectionRuntimeSelection: mocks.getMcpProviderInspectionRuntimeSelection, +})); + +import { + prepareMcpForRebuild, + printMcpRebuildRetryCommand, + resolveMcpPreparationRuntimeSelection, +} from "./rebuild-mcp-phase"; + +const runtimeSelection = { gatewayName: "nemoclaw", workspace: "default" }; const emptyPreparation = { entries: [], @@ -40,6 +58,10 @@ describe("forced rebuild MCP preparation", () => { beforeEach(() => { vi.clearAllMocks(); vi.spyOn(console, "error").mockImplementation(() => undefined); + mocks.getSandbox.mockReturnValue({ + mcp: { bridges: { github: { addState: undefined } } }, + }); + mocks.getMcpProviderInspectionRuntimeSelection.mockReturnValue(runtimeSelection); mocks.executeSandboxCommand.mockReturnValue({ status: 0, stdout: "", stderr: "" }); mocks.executeSandboxExecCommand.mockReturnValue({ status: 0, stdout: "", stderr: "" }); mocks.prepareAbsent.mockResolvedValue(emptyPreparation); @@ -59,9 +81,12 @@ describe("forced rebuild MCP preparation", () => { expect(mocks.executeSandboxExecCommand).toHaveBeenCalledWith("alpha", ":", undefined, { allowLocalDockerFallback: false, + runtimeSelection, + }); + expect(mocks.executeSandboxCommand).toHaveBeenCalledWith("alpha", ":", { + runtimeSelection, }); - expect(mocks.executeSandboxCommand).toHaveBeenCalledWith("alpha", ":"); - expect(mocks.prepareExecUnavailable).toHaveBeenCalledWith("alpha"); + expect(mocks.prepareExecUnavailable).toHaveBeenCalledWith("alpha", runtimeSelection); expect(mocks.prepareAbsent).not.toHaveBeenCalled(); expect(mocks.prepareLive).not.toHaveBeenCalled(); }); @@ -76,11 +101,14 @@ describe("forced rebuild MCP preparation", () => { emptyPreparation, ); - expect(mocks.executeSandboxCommand).toHaveBeenCalledWith("alpha", ":"); + expect(mocks.executeSandboxCommand).toHaveBeenCalledWith("alpha", ":", { + runtimeSelection, + }); expect(mocks.executeSandboxExecCommand).toHaveBeenCalledWith("alpha", ":", undefined, { allowLocalDockerFallback: false, + runtimeSelection, }); - expect(mocks.prepareExecUnavailable).toHaveBeenCalledWith("alpha"); + expect(mocks.prepareExecUnavailable).toHaveBeenCalledWith("alpha", runtimeSelection); expect(mocks.prepareLive).not.toHaveBeenCalled(); expect(mocks.prepareAbsent).not.toHaveBeenCalled(); }); @@ -101,8 +129,9 @@ describe("forced rebuild MCP preparation", () => { expect(mocks.executeSandboxExecCommand).toHaveBeenCalledWith("alpha", ":", undefined, { allowLocalDockerFallback: false, + runtimeSelection, }); - expect(mocks.prepareExecUnavailable).toHaveBeenCalledWith("alpha"); + expect(mocks.prepareExecUnavailable).toHaveBeenCalledWith("alpha", runtimeSelection); expect(mocks.prepareLive).not.toHaveBeenCalled(); }); @@ -122,7 +151,7 @@ describe("forced rebuild MCP preparation", () => { emptyPreparation, ); - expect(mocks.prepareExecUnavailable).toHaveBeenCalledWith("alpha"); + expect(mocks.prepareExecUnavailable).toHaveBeenCalledWith("alpha", runtimeSelection); expect(mocks.prepareLive).not.toHaveBeenCalled(); expect(mocks.prepareAbsent).not.toHaveBeenCalled(); }, @@ -138,10 +167,13 @@ describe("forced rebuild MCP preparation", () => { "Failed to preserve MCP bridges before rebuild: generated policy drifted", ); - expect(mocks.prepareLive).toHaveBeenCalledWith("alpha"); - expect(mocks.executeSandboxCommand).toHaveBeenCalledWith("alpha", ":"); + expect(mocks.prepareLive).toHaveBeenCalledWith("alpha", runtimeSelection); + expect(mocks.executeSandboxCommand).toHaveBeenCalledWith("alpha", ":", { + runtimeSelection, + }); expect(mocks.executeSandboxExecCommand).toHaveBeenCalledWith("alpha", ":", undefined, { allowLocalDockerFallback: false, + runtimeSelection, }); expect(mocks.prepareAbsent).not.toHaveBeenCalled(); }); @@ -176,7 +208,7 @@ describe("forced rebuild MCP preparation", () => { expect(mocks.executeSandboxExecCommand).not.toHaveBeenCalled(); expect(mocks.executeSandboxCommand).not.toHaveBeenCalled(); - expect(mocks.prepareLive).toHaveBeenCalledWith("alpha"); + expect(mocks.prepareLive).toHaveBeenCalledWith("alpha", runtimeSelection); expect(mocks.prepareExecUnavailable).not.toHaveBeenCalled(); expect(mocks.prepareAbsent).not.toHaveBeenCalled(); }); @@ -196,6 +228,26 @@ describe("forced rebuild MCP preparation", () => { expect(mocks.prepareExecUnavailable).not.toHaveBeenCalled(); expect(mocks.prepareLive).not.toHaveBeenCalled(); }); + + it("does not resolve or probe a runtime target for prepared-only MCP state", async () => { + mocks.getSandbox.mockReturnValue({ + mcp: { bridges: { github: { addState: "prepared" } } }, + }); + const bail = vi.fn((message: string): never => { + throw new Error(message); + }); + + expect(resolveMcpPreparationRuntimeSelection("alpha")).toBeUndefined(); + await expect(prepareMcpForRebuild("alpha", false, true, bail)).resolves.toEqual( + emptyPreparation, + ); + + expect(mocks.getMcpProviderInspectionRuntimeSelection).not.toHaveBeenCalled(); + expect(mocks.executeSandboxCommand).not.toHaveBeenCalled(); + expect(mocks.executeSandboxExecCommand).not.toHaveBeenCalled(); + expect(mocks.prepareExecUnavailable).not.toHaveBeenCalled(); + expect(mocks.prepareLive).toHaveBeenCalledWith("alpha"); + }); }); describe("MCP rebuild retry guidance", () => { diff --git a/src/lib/actions/sandbox/rebuild-mcp-phase.ts b/src/lib/actions/sandbox/rebuild-mcp-phase.ts index 1d8edc32829..fa812cc4f07 100644 --- a/src/lib/actions/sandbox/rebuild-mcp-phase.ts +++ b/src/lib/actions/sandbox/rebuild-mcp-phase.ts @@ -14,20 +14,49 @@ import { reattachMcpProvidersAfterRebuildAbort, restoreMcpBridgesAfterRebuild, } from "./mcp-bridge"; +import { getMcpProviderInspectionRuntimeSelection } from "./mcp-bridge-provider"; import { executeSandboxCommand, executeSandboxExecCommand } from "./process-recovery"; import type { RebuildBail } from "./rebuild-credential-preflight"; import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import type { McpProviderInspectionRuntimeSelection } from "./mcp-bridge-provider"; export type McpRebuildPreparation = Awaited>; -function canExecuteMcpPreparation(sandboxName: string): boolean { +export function getMcpPreparationRuntimeSelection( + sandbox: RebuildSandboxEntry, +): ReturnType { + return getMcpProviderInspectionRuntimeSelection(sandbox); +} + +/** Prepared-only MCP adds own no external runtime state and need no target authority. */ +export function mcpRebuildRequiresRuntimeSelection(sandbox: RebuildSandboxEntry): boolean { + return Object.values(sandbox.mcp?.bridges ?? {}).some((entry) => entry.addState !== "prepared"); +} + +export function resolveMcpPreparationRuntimeSelection( + sandboxName: string, +): ReturnType | undefined { + const sandbox = registry.getSandbox(sandboxName); + if (!sandbox || !mcpRebuildRequiresRuntimeSelection(sandbox)) return undefined; + try { + return getMcpPreparationRuntimeSelection(sandbox); + } catch { + return undefined; + } +} + +function canExecuteMcpPreparation( + sandboxName: string, + runtimeSelection: ReturnType, +): boolean { // Live MCP preparation uses both transports: SSH-backed adapter // inspection/mutation and OpenShell-mediated adapter/provider operations. // Prove both before any mutation. A direct Docker fallback would not prove // that the OpenShell transport itself can run. - const sshProbe = executeSandboxCommand(sandboxName, ":"); + const sshProbe = executeSandboxCommand(sandboxName, ":", { runtimeSelection }); const execProbe = executeSandboxExecCommand(sandboxName, ":", undefined, { allowLocalDockerFallback: false, + runtimeSelection, }); return sshProbe !== null && sshProbe.status === 0 && execProbe !== null && execProbe.status === 0; } @@ -37,7 +66,13 @@ export async function prepareMcpForRebuild( staleRecovery: boolean, force: boolean, bail: RebuildBail, + frozenRuntimeSelection?: McpProviderInspectionRuntimeSelection, ): Promise { + const sandbox = staleRecovery ? undefined : registry.getSandbox(sandboxName); + const requiresRuntimeSelection = sandbox ? mcpRebuildRequiresRuntimeSelection(sandbox) : false; + const runtimeSelection = + frozenRuntimeSelection ?? + (staleRecovery ? undefined : resolveMcpPreparationRuntimeSelection(sandboxName)); // invalidState: OpenShell still reports a live sandbox, but the // side-effect-free `:` command cannot cross every transport required by live // MCP preparation. Every nonzero result is non-authoritative, so interpreting @@ -50,10 +85,15 @@ export async function prepareMcpForRebuild( // nonzero results through this exact force-only branch. // removalCondition: remove this fallback only when OpenShell exposes an // attested read-only adapter snapshot that is safe without sandbox transport. - if (force && !staleRecovery && !canExecuteMcpPreparation(sandboxName)) { + if ( + force && + !staleRecovery && + requiresRuntimeSelection && + (!runtimeSelection || !canExecuteMcpPreparation(sandboxName, runtimeSelection)) + ) { console.error(` ${YW}⚠${R} MCP transport probe failed; --force using host-side MCP recovery`); try { - return await prepareMcpBridgesForExecUnavailableRebuild(sandboxName); + return await prepareMcpBridgesForExecUnavailableRebuild(sandboxName, runtimeSelection); } catch (error) { bail( `Failed to preserve MCP bridges before rebuild (--force host-side recovery): ${error instanceof Error ? error.message : String(error)}`, @@ -64,8 +104,12 @@ export async function prepareMcpForRebuild( try { return await (staleRecovery - ? prepareMcpBridgesForAbsentSandboxRebuild(sandboxName) - : prepareMcpBridgesForRebuild(sandboxName)); + ? runtimeSelection + ? prepareMcpBridgesForAbsentSandboxRebuild(sandboxName, runtimeSelection) + : prepareMcpBridgesForAbsentSandboxRebuild(sandboxName) + : runtimeSelection + ? prepareMcpBridgesForRebuild(sandboxName, runtimeSelection) + : prepareMcpBridgesForRebuild(sandboxName)); } catch (error) { bail( `Failed to preserve MCP bridges before rebuild: ${error instanceof Error ? error.message : String(error)}`, @@ -78,9 +122,15 @@ export async function reattachMcpAfterDeleteFailure( sandboxName: string, entries: McpRebuildPreparation["detachedProviderEntries"], scrubbedAdapterEntries: McpRebuildPreparation["scrubbedAdapterEntries"], + runtimeSelection?: McpRebuildPreparation["runtimeSelection"], ): Promise { try { - await reattachMcpProvidersAfterRebuildAbort(sandboxName, entries, scrubbedAdapterEntries); + await reattachMcpProvidersAfterRebuildAbort( + sandboxName, + entries, + scrubbedAdapterEntries, + runtimeSelection, + ); return undefined; } catch (error) { return error instanceof Error ? error.message : String(error); @@ -147,11 +197,16 @@ export function printMcpRebuildRetryCommand( export async function restoreMcpAfterRebuild( sandboxName: string, entries: McpRebuildPreparation["entries"], + runtimeSelection?: McpRebuildPreparation["runtimeSelection"], ): Promise { if (entries.length === 0) return true; console.log(" Restoring MCP bridges..."); try { - await restoreMcpBridgesAfterRebuild(sandboxName, entries); + if (runtimeSelection) { + await restoreMcpBridgesAfterRebuild(sandboxName, entries, runtimeSelection); + } else { + await restoreMcpBridgesAfterRebuild(sandboxName, entries); + } console.log(` ${G}✓${R} MCP bridges restored`); return true; } catch { diff --git a/src/lib/actions/sandbox/rebuild-messaging-phase.ts b/src/lib/actions/sandbox/rebuild-messaging-phase.ts index 3253f23cef5..b69b464de21 100644 --- a/src/lib/actions/sandbox/rebuild-messaging-phase.ts +++ b/src/lib/actions/sandbox/rebuild-messaging-phase.ts @@ -1,7 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { runOpenshell } from "../../adapters/openshell/runtime"; +import { + buildSelectedOpenShellSubprocessEnv, + type OpenShellRuntimeSelection, + runOpenshell, +} from "../../adapters/openshell/runtime"; import { RD as _RD, G, R } from "../../cli/terminal-style"; import { MessagingSetupApplier } from "../../messaging/applier/setup-applier"; import type { @@ -45,19 +49,31 @@ export async function stageRebuildMessagingPlanOrBail( } } -const runMessagingOpenshell: MessagingOpenShellRunner = (args, options = {}) => - runOpenshell([...args], { - env: options.env as NodeJS.ProcessEnv | undefined, - ignoreError: options.ignoreError, - input: options.input, - stdio: options.stdio as never, - }); +function createRunMessagingOpenshell( + runtimeSelection?: OpenShellRuntimeSelection, +): MessagingOpenShellRunner { + return (args, options = {}) => + runOpenshell([...args], { + env: runtimeSelection + ? buildSelectedOpenShellSubprocessEnv( + runtimeSelection, + options.env ? { ...options.env } : undefined, + ) + : (options.env as NodeJS.ProcessEnv | undefined), + replaceEnv: runtimeSelection ? true : undefined, + ignoreError: options.ignoreError, + input: options.input, + stdio: options.stdio as never, + }); +} export function finalizePendingMessagingRemovalsAfterRestore( plan: SandboxMessagingPlan | null, log: (message: string) => void, + runtimeSelection?: OpenShellRuntimeSelection, ): SandboxMessagingPlan | null { if (!plan) return null; + const runMessagingOpenshell = createRunMessagingOpenshell(runtimeSelection); const pendingRemovals = plan.channels.filter( (channel) => channel.pendingRemoval === true, ); @@ -97,6 +113,7 @@ export async function reapplyMessagingManifestAfterOpenClawDoctor( sandboxName: string, plan: SandboxMessagingPlan | null, log: (message: string) => void, + runtimeSelection?: OpenShellRuntimeSelection, ): Promise { if (!plan || plan.agent !== "openclaw") { log("Messaging manifest reapply skipped: no OpenClaw messaging plan"); @@ -104,6 +121,7 @@ export async function reapplyMessagingManifestAfterOpenClawDoctor( } log("Reapplying messaging manifest render and post-agent-install hooks after doctor"); + const runMessagingOpenshell = createRunMessagingOpenshell(runtimeSelection); const result = await MessagingSetupApplier.applyAgentConfigAtOpenShell(plan, { runOpenshell: runMessagingOpenshell, runHook: (request) => hookOutputsFromBuildSteps(plan, request), diff --git a/src/lib/actions/sandbox/rebuild-messaging-removal.test.ts b/src/lib/actions/sandbox/rebuild-messaging-removal.test.ts index 388d2387758..8bbb485067a 100644 --- a/src/lib/actions/sandbox/rebuild-messaging-removal.test.ts +++ b/src/lib/actions/sandbox/rebuild-messaging-removal.test.ts @@ -1,14 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import YAML from "yaml"; import type { SandboxMessagingPlan } from "../../messaging/manifest"; const mocks = vi.hoisted(() => ({ runOpenshell: vi.fn() })); -vi.mock("../../adapters/openshell/runtime", () => ({ +vi.mock("../../adapters/openshell/runtime", async (importOriginal) => ({ + ...(await importOriginal()), runOpenshell: mocks.runOpenshell, })); @@ -66,6 +67,10 @@ describe("post-restore messaging removal", () => { }); }); + afterEach(() => { + vi.unstubAllEnvs(); + }); + it("applies the config tombstone and retires it before Hermes restart", () => { const finalized = finalizePendingMessagingRemovalsAfterRestore(removalPlan(), vi.fn()); @@ -85,4 +90,42 @@ describe("post-restore messaging removal", () => { const finalized = finalizePendingMessagingRemovalsAfterRestore(removalPlan(), vi.fn()); expect(finalized?.channels).toEqual([]); }); + + it("pins every post-restore config operation to the rebuild target (#10514)", () => { + vi.stubEnv("OPENSHELL_GATEWAY", "ambient-gateway"); + vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://ambient.invalid"); + vi.stubEnv("OPENSHELL_GATEWAY_INSECURE", "true"); + vi.stubEnv("OPENSHELL_LOCAL_TLS_DIR", "/ambient/tls"); + vi.stubEnv("OPENSHELL_TOKEN", "ambient-token"); + vi.stubEnv("OPENSHELL_WORKSPACE", "ambient-workspace"); + const runtimeSelection = { + gatewayName: "recorded-gateway", + workspace: "default", + localTlsDir: "/authority/tls", + }; + + finalizePendingMessagingRemovalsAfterRestore(removalPlan(), vi.fn(), runtimeSelection); + + expect(mocks.runOpenshell).toHaveBeenCalled(); + const selectedTargets = mocks.runOpenshell.mock.calls.map(([, options]) => ({ + endpoint: options.env.OPENSHELL_GATEWAY_ENDPOINT, + gateway: options.env.OPENSHELL_GATEWAY, + insecure: options.env.OPENSHELL_GATEWAY_INSECURE, + replaceEnv: options.replaceEnv, + tlsDir: options.env.OPENSHELL_LOCAL_TLS_DIR, + token: options.env.OPENSHELL_TOKEN, + workspace: options.env.OPENSHELL_WORKSPACE, + })); + expect(selectedTargets).toEqual( + new Array(selectedTargets.length).fill({ + endpoint: undefined, + gateway: "recorded-gateway", + insecure: undefined, + replaceEnv: true, + tlsDir: "/authority/tls", + token: undefined, + workspace: "default", + }), + ); + }); }); diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index d8a32c718f3..c4360f62124 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import type { RebuildSandboxOptions } from "../../domain/lifecycle/options"; import { normalizeRebuildSandboxOptions } from "../../domain/lifecycle/options"; import { BRAVE_API_KEY_ENV, TAVILY_API_KEY_ENV } from "../../inference/web-search"; @@ -34,7 +35,9 @@ import { REBUILD_HERMES_DASHBOARD_ENV_KEYS } from "./rebuild-durable-config"; import { disposeRebuildAgentBaseImagePreflight, removeStaleRebuildDockerOrphan, + snapshotOpenShellEnv, } from "./rebuild-flow-helpers"; +import { mcpRebuildRequiresRuntimeSelection } from "./rebuild-mcp-phase"; import { stageMessagingManifestPlanForRebuild } from "./rebuild-messaging-phase"; import { type HermesCronRestoreIdentity, @@ -118,11 +121,11 @@ export async function rebuildSandbox( { allowStateRecord: true }, ); assertSandboxRebuildCommandAvailable(sandboxName); + const restoreOpenShellEnv = snapshotOpenShellEnv(); const scopedEnvKeys = [ BRAVE_API_KEY_ENV, TAVILY_API_KEY_ENV, MESSAGING_SETUP_APPLIER_ENV_KEY, - "OPENSHELL_GATEWAY", DOCKER_GPU_PATCH_NETWORK_ENV, ...REBUILD_HERMES_DASHBOARD_ENV_KEYS, ...MESSAGING_CHANNEL_CONFIG_ENV_KEYS, @@ -136,6 +139,7 @@ export async function rebuildSandbox( removedImmutabilityMigration.stateRecord !== null, ); } finally { + restoreOpenShellEnv(); for (const key of scopedEnvKeys) delete process.env[key]; Object.assign( process.env, @@ -236,6 +240,15 @@ async function rebuildSandboxUnlocked( recoveryManifest = preDeleteRecovery.manifest; recoveryRegistrySnapshot = preDeleteRecovery.registrySnapshot; const activeRecoveryTransaction = onboardSession.loadSession()?.checkpoint?.sandboxRecreate; + const mcpEntries = Object.values(sandboxEntry.mcp?.bridges ?? {}); + const mcpRuntimeSelectionRequired = mcpRebuildRequiresRuntimeSelection(sandboxEntry); + const mcpRuntimeSelection = mcpRuntimeSelectionRequired + ? recreateOptions.runtimeSelection + : undefined; + if (mcpRuntimeSelectionRequired && !mcpRuntimeSelection) { + bail("MCP rebuild preflight did not retain its recorded OpenShell runtime target."); + return; + } const openRecreateJournal = () => { const expectedGatewayAuthority = recreateOptions.rebuildGatewayAuthority; if (!expectedGatewayAuthority) { @@ -251,6 +264,7 @@ async function rebuildSandboxUnlocked( expectedGatewayAuthority, agentName: rebuildAgent || "openclaw", targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent(recreateOptions), + ...(mcpRuntimeSelection ? { resolveRuntimeSelection: () => mcpRuntimeSelection } : {}), log, onAuthorityRefusal: (lines) => bail(lines.join("\n")), }); @@ -383,6 +397,7 @@ async function rebuildSandboxUnlocked( webSearchConfig: durableConfig.webSearchConfig, log, bail, + ...(mcpRuntimeSelection ? { runtimeSelection: mcpRuntimeSelection } : {}), }); if (!backup) return; rebuildPolicySourcePath = backup.policySourcePath; @@ -411,9 +426,13 @@ async function rebuildSandboxUnlocked( return false; } }; - const capturePolicyHandoff = (): boolean => { + const capturePolicyHandoff = (runtimeSelection?: OpenShellRuntimeSelection): boolean => { return publishPolicyHandoff( - captureRebuildPolicyDocument(sandboxName, recreateOptions.targetGatewayName), + captureRebuildPolicyDocument( + sandboxName, + recreateOptions.targetGatewayName, + runtimeSelection, + ), ); }; @@ -476,6 +495,7 @@ async function rebuildSandboxUnlocked( durableConfig.dcodeAutoApprovalMode, recoveryRecreate, recreateOptions.targetGatewayPort, + recreateOptions.runtimeSelection, )) ) { return; @@ -492,6 +512,10 @@ async function rebuildSandboxUnlocked( const recreateJournal = openRecreateJournal(); if (!recreateJournal) return; + if (mcpRuntimeSelectionRequired && !recreateJournal.runtimeSelection) { + bail("The rebuild journal did not retain its recorded OpenShell runtime target."); + return; + } recreateOptions.rebuildGatewayAuthority = recreateJournal.gatewayAuthority; const rebuildRecoveryIdentity = { sandboxName, @@ -571,6 +595,9 @@ async function rebuildSandboxUnlocked( targetAgentType: rebuildAgent || "openclaw", targetImageIsCustom: Boolean(fromDockerfile), backupManifest: recoveryBackup, + ...(recreateJournal.runtimeSelection + ? { runtimeSelection: recreateJournal.runtimeSelection } + : {}), log, }); const postRestoreVerification = await runRebuildPostRestorePhase({ @@ -578,7 +605,10 @@ async function rebuildSandboxUnlocked( targetAgentName: rebuildAgent || "openclaw", messagingPlan, backupManifest: recoveryBackup, - mcpEntries: Object.values(sandboxEntry.mcp?.bridges ?? {}), + mcpEntries, + ...(recreateJournal.runtimeSelection + ? { mcpRuntimeSelection: recreateJournal.runtimeSelection } + : {}), restoreSucceeded: restored.restoreSucceeded, preparedBackupRecovery: true, versionCheck, @@ -613,6 +643,9 @@ async function rebuildSandboxUnlocked( recreateJournal, backupManifest: backup.backupManifest, force: normalized.force, + ...(recreateJournal.runtimeSelection + ? { runtimeSelection: recreateJournal.runtimeSelection } + : {}), log, bail, validateAfterMcpPreparation: async (preparation) => { @@ -642,6 +675,7 @@ async function rebuildSandboxUnlocked( providerReconfigure.provider, log, "Delete-edge", + preparation.runtimeSelection, ) : "missing"; if (providerReconfigure && providerRegistration !== "missing") { @@ -659,9 +693,10 @@ async function rebuildSandboxUnlocked( durableConfig.dcodeAutoApprovalMode, recoveryRecreate, recreateOptions.targetGatewayPort, + preparation.runtimeSelection, ); }, - validateAtDeleteEdge: () => { + validateAtDeleteEdge: (runtimeSelection) => { const validation = revalidateManagedWorkloadRebuildBeforeDelete( sandboxName, @@ -679,7 +714,7 @@ async function rebuildSandboxUnlocked( // prepared recovery manifest, so there is no live policy to recapture. if (staleRecovery) return validation; try { - return capturePolicyHandoff() + return capturePolicyHandoff(runtimeSelection) ? validation : { ok: false, @@ -745,6 +780,9 @@ async function rebuildSandboxUnlocked( targetAgentType: rebuildAgent || "openclaw", targetImageIsCustom: Boolean(fromDockerfile), backupManifest: backup.backupManifest, + ...(mcpPreparation.runtimeSelection + ? { runtimeSelection: mcpPreparation.runtimeSelection } + : {}), log, }); let hermesCronRestoreIdentity: HermesCronRestoreIdentity | undefined; @@ -781,6 +819,7 @@ async function rebuildSandboxUnlocked( messagingPlan, backupManifest: backup.backupManifest, mcpEntries: mcpPreparation.entries, + mcpRuntimeSelection: mcpPreparation.runtimeSelection, restoreSucceeded: restored.restoreSucceeded, hermesCronRestoreIdentity, preparedBackupRecovery, diff --git a/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts b/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts index 6e1e6d77a33..8ecaea6fc54 100644 --- a/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts @@ -117,6 +117,7 @@ describe("rebuild post-restore phase", () => { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); }); function input() { @@ -159,6 +160,47 @@ describe("rebuild post-restore phase", () => { ); }); + it("reuses the MCP rebuild target for every post-restore sandbox command (#10514)", async () => { + vi.stubEnv("OPENSHELL_GATEWAY", "hostile-gateway"); + vi.stubEnv("OPENSHELL_WORKSPACE", "hostile-workspace"); + vi.stubEnv("OPENSHELL_LOCAL_TLS_DIR", "/hostile/tls"); + vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://hostile.invalid"); + const runtimeSelection = { + gatewayName: "recorded-gateway", + workspace: "default", + localTlsDir: "/authority/tls", + }; + const args = { ...input(), mcpRuntimeSelection: runtimeSelection }; + + await runRebuildPostRestorePhase(args); + + expect(processRecovery.executeSandboxExecCommand).toHaveBeenCalledWith( + "alpha", + "openclaw doctor --fix", + 300_000, + { allowLocalDockerFallback: false, runtimeSelection }, + ); + expect(rebuildMessaging.reapplyMessagingManifestAfterOpenClawDoctor).toHaveBeenCalledWith( + "alpha", + null, + args.log, + runtimeSelection, + ); + expect(sessionModels.reconcileStalePinnedSessionModelsAfterRebuild).toHaveBeenCalledWith( + "alpha", + args.log, + runtimeSelection, + ); + expect( + rebuildConfigHash.refreshMutableOpenClawConfigHashAfterPostRestoreWrites, + ).toHaveBeenCalledExactlyOnceWith("alpha", args.log, runtimeSelection); + expect(vi.mocked(rebuildConfigHash.verifyFinalMutableOpenClawConfigHash).mock.calls).toEqual([ + ["alpha", args.log, runtimeSelection], + ["alpha", args.log, runtimeSelection], + ]); + expect(process.env.OPENSHELL_GATEWAY).toBe("hostile-gateway"); + }); + it("does not record a final hash without trusted doctor completion (#9946)", async () => { vi.mocked(processRecovery.executeSandboxExecCommand).mockReturnValue(null); const args = input(); diff --git a/src/lib/actions/sandbox/rebuild-post-restore-phase.ts b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts index 89cd4e3e8aa..dfc4b3900ab 100644 --- a/src/lib/actions/sandbox/rebuild-post-restore-phase.ts +++ b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts @@ -82,6 +82,7 @@ export interface RebuildPostRestorePhaseInput { messagingPlan: SandboxMessagingPlan | null; backupManifest: RebuildBackupManifest; mcpEntries: McpRebuildPreparation["entries"]; + mcpRuntimeSelection?: McpRebuildPreparation["runtimeSelection"]; restoreSucceeded: boolean; hermesCronRestoreIdentity?: HermesCronRestoreIdentity; preparedBackupRecovery: boolean; @@ -119,6 +120,7 @@ export async function runRebuildPostRestorePhase( messagingPlan, backupManifest, mcpEntries, + mcpRuntimeSelection, restoreSucceeded, hermesCronRestoreIdentity, preparedBackupRecovery, @@ -166,7 +168,10 @@ export async function runRebuildPostRestorePhase( sandboxName, "openclaw doctor --fix", OPENCLAW_DOCTOR_TIMEOUT_MS, - { allowLocalDockerFallback: false }, + { + allowLocalDockerFallback: false, + ...(mcpRuntimeSelection ? { runtimeSelection: mcpRuntimeSelection } : {}), + }, ); log(`doctor --fix: exit=${doctorResult?.status ?? "unverified"}`); if (doctorResult === null) { @@ -185,10 +190,15 @@ export async function runRebuildPostRestorePhase( // #7102: clear stale per-session pinned models left over from an // `inference set` before this rebuild, while the gateway is still down. - reconcileStalePinnedSessionModelsAfterRebuild(sandboxName, log); + reconcileStalePinnedSessionModelsAfterRebuild(sandboxName, log, mcpRuntimeSelection); try { - await reapplyMessagingManifestAfterOpenClawDoctor(sandboxName, messagingPlan, log); + await reapplyMessagingManifestAfterOpenClawDoctor( + sandboxName, + messagingPlan, + log, + mcpRuntimeSelection, + ); } catch (error) { log( `Messaging manifest reapply failed: ${error instanceof Error ? error.message : String(error)}`, @@ -227,6 +237,7 @@ export async function runRebuildPostRestorePhase( const finalizedMessagingPlan = finalizePendingMessagingRemovalsAfterRestore( effectiveMessagingPlan, log, + mcpRuntimeSelection, ); if (finalizedMessagingPlan !== effectiveMessagingPlan && finalizedMessagingPlan) { if ( @@ -254,15 +265,22 @@ export async function runRebuildPostRestorePhase( const hermesGatewayRestartState = restartHermesGatewayAfterStateRestore( sandboxName, targetAgentName, + mcpRuntimeSelection ? { runtimeSelection: mcpRuntimeSelection } : {}, ); - const mcpBridgeRestoreUnverified = !(await restoreMcpAfterRebuild(sandboxName, mcpEntries)); + const mcpBridgeRestoreUnverified = !(await restoreMcpAfterRebuild( + sandboxName, + mcpEntries, + mcpRuntimeSelection, + )); if (targetAgentName === "openclaw" && mcpBridgeRestoreUnverified) { mutableConfigHashRefreshUnverified = true; } else if (targetAgentName === "openclaw") { log("Refreshing mutable OpenClaw config hash after MCP restoration"); - if (!refreshMutableOpenClawConfigHashAfterPostRestoreWrites(sandboxName, log)) { + if ( + !refreshMutableOpenClawConfigHashAfterPostRestoreWrites(sandboxName, log, mcpRuntimeSelection) + ) { mutableConfigHashRefreshUnverified = true; - } else if (!verifyFinalMutableOpenClawConfigHash(sandboxName, log)) { + } else if (!verifyFinalMutableOpenClawConfigHash(sandboxName, log, mcpRuntimeSelection)) { finalMutableConfigHashUnverified = true; } } @@ -272,12 +290,14 @@ export async function runRebuildPostRestorePhase( targetAgentName, hermesGatewayRestartState, hermesCronRestoreIdentity, + mcpRuntimeSelection ? { runtimeSelection: mcpRuntimeSelection } : {}, ) : { state: verifyHermesGatewayAfterStateRestore( sandboxName, targetAgentName, hermesGatewayRestartState, + mcpRuntimeSelection ? { runtimeSelection: mcpRuntimeSelection } : {}, ), replacementIdentity: undefined, }; @@ -363,14 +383,20 @@ export async function runRebuildPostRestorePhase( }); log(`Registry updated: agentVersion=${agentDef.expectedVersion}`); - if (!ensureMessagingHostForwardAfterRebuild(sandboxName, effectiveMessagingPlan)) { + if ( + !ensureMessagingHostForwardAfterRebuild( + sandboxName, + effectiveMessagingPlan, + mcpRuntimeSelection, + ) + ) { messagingHostForwardUnverified = true; } if ( targetAgentName === "openclaw" && !mcpBridgeRestoreUnverified && !mutableConfigHashRefreshUnverified && - !verifyFinalMutableOpenClawConfigHash(sandboxName, log) + !verifyFinalMutableOpenClawConfigHash(sandboxName, log, mcpRuntimeSelection) ) { finalMutableConfigHashUnverified = true; } diff --git a/src/lib/actions/sandbox/rebuild-preflight-guards.ts b/src/lib/actions/sandbox/rebuild-preflight-guards.ts index 38922802347..48b637ae3c2 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-guards.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-guards.ts @@ -5,6 +5,7 @@ import { detectOpenShellStateRpcPreflightIssue, printOpenShellStateRpcIssue, } from "../../adapters/openshell/gateway-drift"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import { CLI_NAME } from "../../cli/branding"; import { checkGatewayRouteCompatibility, @@ -269,9 +270,17 @@ export function checkRebuildGatewaySchemaPreflight( sandboxName: string, sb: RebuildSandboxEntry, bail: RebuildBail, + runtimeSelection?: OpenShellRuntimeSelection, ): boolean { + const gatewayName = resolveSandboxGatewayName(sb); + if (runtimeSelection && runtimeSelection.gatewayName !== gatewayName) { + return bail( + `Rebuild gateway schema target '${gatewayName}' does not match the frozen OpenShell target '${runtimeSelection.gatewayName}'.`, + ); + } const issue = detectOpenShellStateRpcPreflightIssue({ - gatewayName: resolveSandboxGatewayName(sb), + gatewayName, + ...(runtimeSelection ? { runtimeSelection } : {}), }); if (issue) { printOpenShellStateRpcIssue(issue, { diff --git a/src/lib/actions/sandbox/rebuild-preflight-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-phase.ts index c01b4caca6a..0f875ab79f1 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-phase.ts @@ -50,7 +50,11 @@ import { type RebuildRoutePreflightReceipt, runRebuildGatewayIntentPreflight, } from "./rebuild-preflight-guards"; -import { prepareRebuildTargetPreflights } from "./rebuild-preflight-target-phase"; +import { + pinRebuildTargetGatewayForReadiness, + prepareRebuildTargetPreflights, + resolveRebuildMcpRuntimeSelection, +} from "./rebuild-preflight-target-phase"; import { disposePreparedBuildContext } from "./rebuild-prepared-image-context"; import { type RebuildSandboxExecutionOptions, @@ -138,7 +142,6 @@ export async function runRebuildPreflightPhase( const sandboxEntry = getRebuildSandboxEntryOrBail(sandboxName, bail); if (!sandboxEntry) return null; if (blockRebuildOnRetainedSandboxRecovery(sandboxName, bail)) return null; - const activeSessionCount = countActiveSandboxSessionsForRebuild(sandboxName); // #6376: refuse a stuck MCP destroy transaction up front — before backup, // image prep, or the old-sandbox delete. The only MCP marker check used to // live inside the destroy phase, which runs AFTER the backup phase, so a @@ -203,10 +206,15 @@ export async function runRebuildPreflightPhase( return null; } const agentName = getRebuildAgentDisplayName(sandboxName); + const mcpRuntimeSelection = resolveRebuildMcpRuntimeSelection(sandboxEntry, bail); + if (mcpRuntimeSelection) { + pinRebuildTargetGatewayForReadiness(sandboxName, sandboxEntry, log, mcpRuntimeSelection); + } + const activeSessionCount = countActiveSandboxSessionsForRebuild(sandboxName); const versionCheck = await runRebuildGatewayIntentPreflight({ checkGatewaySchema: () => isDcodeRebuildAgent(rebuildAgent) || - checkRebuildGatewaySchemaPreflight(sandboxName, sandboxEntry, bail), + checkRebuildGatewaySchemaPreflight(sandboxName, sandboxEntry, bail, mcpRuntimeSelection), confirmIntent: () => confirmRebuildIntent( sandboxName, @@ -231,8 +239,13 @@ export async function runRebuildPreflightPhase( log, bail, deps: { - checkGatewaySchema: (name, scopedBail) => - checkRebuildGatewaySchemaPreflight(name, expectedSandboxEntry, scopedBail), + checkGatewaySchema: (name, scopedBail, runtimeSelection) => + checkRebuildGatewaySchemaPreflight( + name, + expectedSandboxEntry, + scopedBail, + runtimeSelection, + ), preflightCredentials: (_name, entry, scopedLog, scopedBail) => preflightRebuildCredentials(entry, scopedLog, scopedBail), // Non-DCode rebuilds stay on the existing typed base-image preflight. @@ -262,6 +275,7 @@ export async function runRebuildPreflightPhase( requestedDcodeAutoApprovalMode, requestedObservabilityEnabled, allowLegacyManagedImageRecovery, + mcpRuntimeSelection, // A validated prepared backup is the only path allowed to reconstruct // a missing gateway provider and route during recreate. The exact // endpoint, credential, image, and registry checks still run before @@ -297,6 +311,7 @@ export async function runRebuildPreflightPhase( { resolutionHint: preparedTarget.recreateOptions.baseImageResolutionHint, }, + preparedTarget.recreateOptions.runtimeSelection, ); if (!imageReady) return null; if (!preparedTarget.recreateOptions.managedWorkloadRebuild) { diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase-orchestration.test.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase-orchestration.test.ts index ab4fcdeb885..359a6fecf61 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase-orchestration.test.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase-orchestration.test.ts @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ bail: vi.fn(), + getMcpPreparationRuntimeSelection: vi.fn(), preflightAuthoritativeOnboardRuntime: vi.fn(async (..._args: unknown[]) => false), prepareManagedWorkloadRebuildHandoff: vi.fn(), prepareSandboxWorkloadSourceFromRebuildHandoff: vi.fn(), @@ -15,6 +16,11 @@ const mocks = vi.hoisted(() => ({ stageManagedWorkloadRebuildProfile: vi.fn(), })); +vi.mock("./rebuild-mcp-phase", async (importOriginal) => ({ + ...(await importOriginal()), + getMcpPreparationRuntimeSelection: mocks.getMcpPreparationRuntimeSelection, +})); + vi.mock("../../onboard/workload/rebuild", async (importOriginal) => ({ ...(await importOriginal()), prepareManagedWorkloadRebuildHandoff: mocks.prepareManagedWorkloadRebuildHandoff, @@ -49,16 +55,25 @@ vi.mock("./rebuild-messaging-conflict-preflight", () => ({ })); import { managedRebuildProfileDependencies } from "./agents/managed-workload-rebuild-profile"; +import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; import { prepareRebuildTargetPreflights } from "./rebuild-preflight-target-phase"; describe("prepareRebuildTargetPreflights", () => { beforeEach(() => { vi.clearAllMocks(); + mocks.getMcpPreparationRuntimeSelection.mockReturnValue({ + gatewayName: "nemoclaw", + localTlsDir: "/authority/tls", + workspace: "default", + }); mocks.prepareManagedWorkloadRebuildHandoff.mockResolvedValue(null); mocks.preflightAuthoritativeOnboardRuntime.mockResolvedValue(false); }); - async function prepareN1xTarget(endpointSource: "onboard" | "inference-set") { + async function prepareN1xTarget( + endpointSource: "onboard" | "inference-set", + mcp: { bridges: Record } | null = null, + ) { const resumeConfig = { provider: "vllm-local", model: "nvidia/Qwen3.6-35B-A3B-NVFP4", @@ -100,13 +115,16 @@ describe("prepareRebuildTargetPreflights", () => { model: resumeConfig.model, endpointUrl: "http://host.openshell.internal:8000/v1", endpointSource, + mcp, } as never, rebuildAgent: "openclaw", autoYes: true, log: vi.fn(), bail: mocks.bail as never, }); - return mocks.preflightAuthoritativeOnboardRuntime.mock.calls[0]?.[2]; + return mocks.preflightAuthoritativeOnboardRuntime.mock.calls[0]?.[2] as + | RebuildRecreateOnboardOpts + | undefined; } it("resolves the Ollama context window through target preparation", async () => { @@ -195,4 +213,20 @@ describe("prepareRebuildTargetPreflights", () => { expect(readinessOptions).not.toHaveProperty("allowDeferredN1xManagedVllm"); }); + + it("freezes one MCP runtime target before authoritative readiness (#10514)", async () => { + const runtimeSelection = { + gatewayName: "nemoclaw", + localTlsDir: "/authority/tls", + workspace: "default", + }; + mocks.getMcpPreparationRuntimeSelection.mockReturnValue(runtimeSelection); + + const readinessOptions = await prepareN1xTarget("onboard", { + bridges: { github: { server: "github" } }, + }); + + expect(mocks.getMcpPreparationRuntimeSelection).toHaveBeenCalledOnce(); + expect(readinessOptions?.runtimeSelection).toBe(runtimeSelection); + }); }); diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts index 86b7a30144c..8fcccc0e62a 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { restoreEnvBulk } from "../../../../test/helpers/env-test-helpers"; import type { ProviderRecoveryReceipt, RegistryInferenceRoute, @@ -10,36 +11,93 @@ import type { import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; import { pinRebuildTargetGatewayForReadiness, + resolveRebuildMcpRuntimeSelection, runRebuildGatewayRecoveryAfterReadiness, stageRebuildBaseImageResolutionHandoff, stageRegistryProviderRecoveryReceipt, } from "./rebuild-preflight-target-phase"; -const originalGateway = process.env.OPENSHELL_GATEWAY; +const originalOpenShellEnv = { + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY, + OPENSHELL_GATEWAY_ENDPOINT: process.env.OPENSHELL_GATEWAY_ENDPOINT, + OPENSHELL_LOCAL_TLS_DIR: process.env.OPENSHELL_LOCAL_TLS_DIR, + OPENSHELL_TOKEN: process.env.OPENSHELL_TOKEN, + OPENSHELL_WORKSPACE: process.env.OPENSHELL_WORKSPACE, +}; afterEach(() => { - originalGateway === undefined - ? Reflect.deleteProperty(process.env, "OPENSHELL_GATEWAY") - : (process.env.OPENSHELL_GATEWAY = originalGateway); + restoreEnvBulk(originalOpenShellEnv); }); describe("rebuild readiness gateway pin", () => { + it("does not require runtime authority for prepared-only MCP state", () => { + const bail = vi.fn((message: string): never => { + throw new Error(message); + }); + + expect( + resolveRebuildMcpRuntimeSelection( + { + mcp: { + bridges: { + github: { addState: "prepared" }, + }, + }, + } as never, + bail, + ), + ).toBeUndefined(); + expect(bail).not.toHaveBeenCalled(); + }); + it("pins the recorded target without selecting or recovering a gateway (#7411)", () => { const log = vi.fn(); + process.env.OPENSHELL_GATEWAY = "hostile-gateway"; + process.env.OPENSHELL_GATEWAY_ENDPOINT = "https://hostile.invalid"; + process.env.OPENSHELL_LOCAL_TLS_DIR = "/hostile/tls"; + process.env.OPENSHELL_TOKEN = "hostile-token"; + process.env.OPENSHELL_WORKSPACE = "hostile-workspace"; + const runtimeSelection = { gatewayName: "nemoclaw-9443", workspace: "default" }; expect( pinRebuildTargetGatewayForReadiness( "alpha", { gatewayName: "nemoclaw-9443", gatewayPort: 9443 } as never, log, + runtimeSelection, ), ).toBe("nemoclaw-9443"); expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw-9443"); + expect(process.env.OPENSHELL_WORKSPACE).toBe("default"); + expect(process.env.OPENSHELL_GATEWAY_ENDPOINT).toBeUndefined(); + expect(process.env.OPENSHELL_LOCAL_TLS_DIR).toBeUndefined(); + expect(process.env.OPENSHELL_TOKEN).toBeUndefined(); expect(log).toHaveBeenCalledWith( "Pinned rebuild readiness probes for 'alpha' to target gateway 'nemoclaw-9443'", ); }); + it("keeps non-MCP ambient selectors while pinning the recorded gateway (#10514)", () => { + process.env.OPENSHELL_GATEWAY = "hostile-gateway"; + process.env.OPENSHELL_GATEWAY_ENDPOINT = "https://hostile.invalid"; + process.env.OPENSHELL_LOCAL_TLS_DIR = "/hostile/tls"; + process.env.OPENSHELL_TOKEN = "hostile-token"; + process.env.OPENSHELL_WORKSPACE = "hostile-workspace"; + + expect( + pinRebuildTargetGatewayForReadiness( + "alpha", + { gatewayName: "nemoclaw-9443", gatewayPort: 9443 } as never, + vi.fn(), + ), + ).toBe("nemoclaw-9443"); + expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw-9443"); + expect(process.env.OPENSHELL_GATEWAY_ENDPOINT).toBe("https://hostile.invalid"); + expect(process.env.OPENSHELL_LOCAL_TLS_DIR).toBe("/hostile/tls"); + expect(process.env.OPENSHELL_TOKEN).toBe("hostile-token"); + expect(process.env.OPENSHELL_WORKSPACE).toBe("hostile-workspace"); + }); + it("does not select or recover the gateway when readiness rejects (#7411)", async () => { const afterReadiness = vi.fn(); const recoverGateway = vi.fn(async () => true); diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts index 0c415187ce8..496005d3755 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { randomUUID } from "node:crypto"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import { CLI_NAME } from "../../cli/branding"; import type { SandboxMessagingPlan } from "../../messaging"; import { isSandboxBaseImageRefreshRequested } from "../../onboard/base-image-resolution-flow"; @@ -36,10 +37,15 @@ import { ensureRebuildAgentBaseImage, ensureRebuildTargetGatewaySelected, pinRebuildAgentBaseImageForRecreate, + replaceOpenShellRuntimeSelectionEnv, type RebuildAgentBaseImagePreflight, type RebuildSandboxEntry, } from "./rebuild-flow-helpers"; import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; +import { + getMcpPreparationRuntimeSelection, + mcpRebuildRequiresRuntimeSelection, +} from "./rebuild-mcp-phase"; import { preflightRebuildMessagingConflicts } from "./rebuild-messaging-conflict-preflight"; import { stageRebuildMessagingPlanOrBail } from "./rebuild-messaging-phase"; import { @@ -93,14 +99,36 @@ export interface RebuildPreparedTarget { routePreflightReceipt: RebuildRoutePreflightReceipt; } +/** Freeze the MCP-bearing rebuild on the recorded OpenShell target before live probes. */ +export function resolveRebuildMcpRuntimeSelection( + sandboxEntry: RebuildSandboxEntry, + bail: RebuildBail, +): OpenShellRuntimeSelection | undefined { + if (!mcpRebuildRequiresRuntimeSelection(sandboxEntry)) return undefined; + try { + return getMcpPreparationRuntimeSelection(sandboxEntry); + } catch (error) { + return bail( + `Could not bind MCP rebuild preflight to the recorded OpenShell target: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + /** Pin read-only rebuild probes without selecting, starting, or repairing a gateway. */ export function pinRebuildTargetGatewayForReadiness( sandboxName: string, sandboxEntry: RebuildSandboxEntry, log: RebuildLog, + runtimeSelection?: OpenShellRuntimeSelection, ): string { const gatewayName = getPersistedSandboxTargetGatewayName(sandboxEntry); - process.env.OPENSHELL_GATEWAY = gatewayName; + if (runtimeSelection && runtimeSelection.gatewayName !== gatewayName) { + throw new Error( + `OpenShell runtime selection '${runtimeSelection.gatewayName}' does not match recorded gateway '${gatewayName}'.`, + ); + } + if (runtimeSelection) replaceOpenShellRuntimeSelectionEnv(process.env, runtimeSelection); + else process.env.OPENSHELL_GATEWAY = gatewayName; log(`Pinned rebuild readiness probes for '${sandboxName}' to target gateway '${gatewayName}'`); return gatewayName; } @@ -154,6 +182,7 @@ export async function prepareRebuildTargetPreflights(args: { requestedObservabilityEnabled?: boolean; allowLegacyManagedImageRecovery?: boolean; preparedBackupRecovery?: boolean; + mcpRuntimeSelection?: OpenShellRuntimeSelection; log: RebuildLog; bail: RebuildBail; }): Promise { @@ -167,11 +196,14 @@ export async function prepareRebuildTargetPreflights(args: { requestedObservabilityEnabled, allowLegacyManagedImageRecovery, preparedBackupRecovery, + mcpRuntimeSelection: frozenMcpRuntimeSelection, log, bail, } = args; + const mcpRuntimeSelection = + frozenMcpRuntimeSelection ?? resolveRebuildMcpRuntimeSelection(sandboxEntry, bail); hydrateMessagingConfigForRebuild(sandboxName, log); - pinRebuildTargetGatewayForReadiness(sandboxName, sandboxEntry, log); + pinRebuildTargetGatewayForReadiness(sandboxName, sandboxEntry, log, mcpRuntimeSelection); const targetConfig = prepareRebuildTargetConfig( sandboxName, @@ -198,6 +230,7 @@ export async function prepareRebuildTargetPreflights(args: { bail, ); if (!recreateOptions) return null; + if (mcpRuntimeSelection) recreateOptions.runtimeSelection = mcpRuntimeSelection; let managedWorkloadRebuildCatalog: Awaited< ReturnType > = null; @@ -297,10 +330,13 @@ export async function prepareRebuildTargetPreflights(args: { }, resumeConfig.registryInferenceRoute, ), - recoverGateway: () => ensureRebuildTargetGatewaySelected(sandboxName, sandboxEntry, log, bail), + recoverGateway: () => + ensureRebuildTargetGatewaySelected(sandboxName, sandboxEntry, log, bail, mcpRuntimeSelection), }); if (!gatewayRecovered) return null; - if (!checkRebuildGatewaySchemaPreflight(sandboxName, sandboxEntry, bail)) return null; + if (!checkRebuildGatewaySchemaPreflight(sandboxName, sandboxEntry, bail, mcpRuntimeSelection)) { + return null; + } const rebuildsDcodeSandbox = isDcodeRebuildAgent(rebuildAgent); const rebuildsManagedWorkload = recreateOptions.managedWorkloadRebuild !== undefined; diff --git a/src/lib/actions/sandbox/rebuild-provider-preflight.test.ts b/src/lib/actions/sandbox/rebuild-provider-preflight.test.ts index 801bdb03c54..1850c971f57 100644 --- a/src/lib/actions/sandbox/rebuild-provider-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-provider-preflight.test.ts @@ -2,12 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, describe, expect, it, vi } from "vitest"; +import * as openshellRuntime from "../../adapters/openshell/runtime"; import type { GatewayProviderMetadata } from "../../onboard/gateway-provider-metadata"; import { canRecreateMissingRebuildGatewayProvider, checkRebuildGatewayCredentialReuseOrBail, checkRebuildGatewayProviderOrBail, classifyRebuildGatewayProviderRegistration, + inspectRebuildGatewayProviderRegistration, shouldVerifyRebuildGatewayProvider, } from "./rebuild-provider-preflight"; import type { RebuildResumeConfig } from "./rebuild-resume-config"; @@ -48,6 +50,7 @@ const throwingBail = (message: string): never => { }; afterEach(() => { + vi.unstubAllEnvs(); vi.restoreAllMocks(); }); @@ -192,6 +195,48 @@ describe("classifyRebuildGatewayProviderRegistration", () => { }); }); +describe("inspectRebuildGatewayProviderRegistration", () => { + it("pins the delete-edge lookup to the frozen target under hostile ambient selectors (#10514)", () => { + vi.stubEnv("OPENSHELL_GATEWAY", "hostile-gateway"); + vi.stubEnv("OPENSHELL_WORKSPACE", "hostile-workspace"); + vi.stubEnv("OPENSHELL_LOCAL_TLS_DIR", "/hostile/tls"); + vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://hostile.invalid"); + const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ + status: 1, + stdout: "", + stderr: "provider not found", + } as never); + const runtimeSelection = { + gatewayName: "recorded-gateway", + workspace: "default", + localTlsDir: "/authority/tls", + }; + + expect( + inspectRebuildGatewayProviderRegistration( + "compatible-endpoint", + vi.fn(), + "Delete-edge", + runtimeSelection, + ), + ).toBe("missing"); + + expect(runOpenshell).toHaveBeenCalledWith( + ["provider", "get", "compatible-endpoint"], + expect.objectContaining({ + replaceEnv: true, + env: expect.objectContaining({ + OPENSHELL_GATEWAY: "recorded-gateway", + OPENSHELL_WORKSPACE: "default", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + }), + }), + ); + const env = runOpenshell.mock.calls[0]?.[1]?.env as Record; + expect(env).not.toHaveProperty("OPENSHELL_GATEWAY_ENDPOINT"); + }); +}); + describe("checkRebuildGatewayCredentialReuseOrBail", () => { it("accepts an exact complete registry route and gateway provider identity", () => { expect( diff --git a/src/lib/actions/sandbox/rebuild-provider-preflight.ts b/src/lib/actions/sandbox/rebuild-provider-preflight.ts index 26489a176b2..14c4b0053c2 100644 --- a/src/lib/actions/sandbox/rebuild-provider-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-provider-preflight.ts @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import { runOpenshell } from "../../adapters/openshell/runtime"; +import { buildSelectedOpenShellSubprocessEnv } from "../../adapters/openshell/command-argv"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import { RD as _RD, R } from "../../cli/terminal-style"; import { hasBedrockRuntimeAwsAuthEnv, @@ -85,10 +87,17 @@ export function inspectRebuildGatewayProviderRegistration( provider: string, log: (msg: string) => void, phase = "Preflight", + runtimeSelection?: OpenShellRuntimeSelection, ): RebuildGatewayProviderRegistration { const result = runOpenshell(["provider", "get", provider], { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], + ...(runtimeSelection + ? { + env: buildSelectedOpenShellSubprocessEnv(runtimeSelection), + replaceEnv: true, + } + : {}), }); const registration = classifyRebuildGatewayProviderRegistration(result, provider); log( diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts index e0db2c06f9c..70a71e7fd30 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts @@ -608,6 +608,72 @@ describe("rebuild replacement journal", () => { expect(resumed.id).toBe(first.id); }); + it("pins an interrupted replacement observation to its recorded OpenShell target (#10514)", () => { + open(); + const runtimeSelection = { + gatewayName: "nemoclaw-9090", + workspace: "default", + localTlsDir: "/authority/tls", + }; + const resolveRuntimeSelection = vi.fn(() => runtimeSelection); + mocks.captureOpenshell.mockClear(); + + const resumed = openRebuildRecreateJournal({ + target: NON_DEFAULT_TARGET, + expectedGatewayAuthority: STANDALONE_GATEWAY_AUTHORITY, + agentName: "langchain-deepagents-code", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent(recreateOptions), + log: vi.fn(), + resolveRuntimeSelection, + }); + + expect(resolveRuntimeSelection).toHaveBeenCalledOnce(); + expect(resumed.runtimeSelection).toEqual(runtimeSelection); + expect(mocks.captureOpenshell).toHaveBeenCalledWith( + ["sandbox", "get", "-g", "nemoclaw-9090", "alpha"], + expect.objectContaining({ + replaceEnv: true, + env: expect.objectContaining({ + OPENSHELL_GATEWAY: "nemoclaw-9090", + OPENSHELL_WORKSPACE: "default", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + }), + }), + ); + }); + + it("pins the first replacement observation to its recorded OpenShell target (#10514)", () => { + const runtimeSelection = { + gatewayName: "nemoclaw-9090", + workspace: "default", + localTlsDir: "/authority/tls", + }; + const resolveRuntimeSelection = vi.fn(() => runtimeSelection); + + const journal = openRebuildRecreateJournal({ + target: NON_DEFAULT_TARGET, + expectedGatewayAuthority: STANDALONE_GATEWAY_AUTHORITY, + agentName: "langchain-deepagents-code", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent(recreateOptions), + log: vi.fn(), + resolveRuntimeSelection, + }); + + expect(resolveRuntimeSelection).toHaveBeenCalledOnce(); + expect(journal.runtimeSelection).toEqual(runtimeSelection); + expect(mocks.captureOpenshell).toHaveBeenCalledWith( + ["sandbox", "get", "-g", "nemoclaw-9090", "alpha"], + expect.objectContaining({ + replaceEnv: true, + env: expect.objectContaining({ + OPENSHELL_GATEWAY: "nemoclaw-9090", + OPENSHELL_WORKSPACE: "default", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + }), + }), + ); + }); + it("retires the journal of a proven replacement instead of deleting it again (#7734)", () => { const first = open(); proveReplacement(first.targetGeneration); diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.ts index b34ce2f7aea..da247607daf 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.ts @@ -5,6 +5,7 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import { checkpointGatewayAuthority, gatewayOwnerFromCheckpoint, @@ -533,6 +534,7 @@ export interface RebuildRecreateJournal { readonly gatewayAuthority: CheckpointGatewayAuthority; readonly targetGeneration: string; readonly targetIntentFingerprint: string; + readonly runtimeSelection?: OpenShellRuntimeSelection; beginDelete(): RebuildRecreateSourcePresence; confirmDeleted(): void; completeAcceptedTarget(): void; @@ -600,6 +602,8 @@ export interface OpenRebuildRecreateJournalInput { readonly targetIntentFingerprint: string; readonly log: (message: string) => void; readonly observe?: RebuildSandboxObserver; + readonly runtimeSelection?: OpenShellRuntimeSelection; + readonly resolveRuntimeSelection?: () => OpenShellRuntimeSelection; /** * Invoked with ready-to-print lines when gateway authority cannot be * revalidated, so the command layer can fail cleanly (#8103). @@ -611,7 +615,12 @@ export function openRebuildRecreateJournal( input: OpenRebuildRecreateJournalInput, ): RebuildRecreateJournal { const { target, agentName, targetIntentFingerprint, log } = input; - const observe = input.observe ?? observeRebuildSandbox; + const observeTarget = ( + runtimeSelection = input.runtimeSelection, + ): ReturnType => + input.observe + ? input.observe(target) + : observeRebuildSandbox(target, undefined, runtimeSelection); // Authority revalidation runs before the destroy phase. Handing the refusal // to the caller lets rebuild report the migration and its remedy instead of // crashing with a Node stack trace (#8103). The dedicated rebuild resolver @@ -639,6 +648,9 @@ export function openRebuildRecreateJournal( throw error; } const gatewayAuthority = checkpointGatewayAuthority(authority); + const runtimeSelection = input.resolveRuntimeSelection + ? input.resolveRuntimeSelection() + : input.runtimeSelection; const owned = ownSandboxRecreateTransaction({ sessionStore: { loadSession: onboardSession.loadSession, @@ -650,7 +662,7 @@ export function openRebuildRecreateJournal( gatewayPort: target.gatewayPort, targetIntentFingerprint, readRegistryEntry: () => registry.getSandbox(target.sandboxName), - observe: () => observe(target), + observe: () => observeTarget(runtimeSelection), decorateCheckpoint: (current, checkpoint, now) => ({ ...checkpoint, machineState: current.machine.state, @@ -704,6 +716,7 @@ export function openRebuildRecreateJournal( gatewayAuthority, targetGeneration: transaction.targetGeneration, targetIntentFingerprint: transaction.targetIntentFingerprint, + ...(runtimeSelection ? { runtimeSelection } : {}), beginDelete: () => { const begun = beginSandboxRecreateDelete({ sessionStore: { @@ -716,14 +729,14 @@ export function openRebuildRecreateJournal( targetIntentFingerprint, revalidateGatewayAuthority, readRegistryEntry: () => registry.getSandbox(target.sandboxName), - observe: () => observe(target), + observe: () => observeTarget(runtimeSelection), }); currentTransaction = begun.transaction; phase = currentTransaction.phase; return begun.sourcePresence; }, confirmDeleted: () => { - if (observe(target).state !== "missing") { + if (observeTarget(runtimeSelection).state !== "missing") { throw new Error( `Cannot continue sandbox '${target.sandboxName}' replacement: OpenShell still reports the journaled source after delete.`, ); diff --git a/src/lib/actions/sandbox/rebuild-recreate-phase.ts b/src/lib/actions/sandbox/rebuild-recreate-phase.ts index ea100668b5e..c0ff898c1af 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-phase.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-phase.ts @@ -273,6 +273,9 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): try { await rebuildOnboardDependencies.onboard({ ...recreateOptions, + ...(recreateJournal.runtimeSelection + ? { runtimeSelection: recreateJournal.runtimeSelection } + : {}), ...(preparedBackupRecovery ? { allowRemovedImmutabilityStateRecord: true } : {}), rebuildGatewayAuthority, rebuildPolicySourcePath, diff --git a/src/lib/actions/sandbox/rebuild-restore-phase.test.ts b/src/lib/actions/sandbox/rebuild-restore-phase.test.ts index 7c754d4a999..994eb0769a5 100644 --- a/src/lib/actions/sandbox/rebuild-restore-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-restore-phase.test.ts @@ -77,6 +77,40 @@ describe("rebuild filesystem restore", () => { ); }); + it("carries the frozen OpenShell target into fresh-plugin discovery and SSH restore reads (#10514)", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const restore = vi + .spyOn(snapshotRestore, "restoreRecreatedSandboxStateWithManagedAuthority") + .mockReturnValue({ + success: true, + restoredDirs: [], + restoredFiles: [], + failedDirs: [], + failedFiles: [], + }); + const runtimeSelection = { + gatewayName: "nemoclaw-8081", + localTlsDir: "/authority/tls", + workspace: "default", + }; + + runRebuildRestorePhase({ + sandboxName: "alpha", + targetAgentType: "openclaw", + targetImageIsCustom: false, + backupManifest, + runtimeSelection, + log: vi.fn(), + }); + + expect(restore).toHaveBeenCalledWith( + "alpha", + backupManifest, + { targetAgentType: "openclaw", runtimeSelection }, + { getSandbox: expect.any(Function) }, + ); + }); + it("migrates restored Hermes dashboard state into its current profile", () => { vi.spyOn(console, "log").mockImplementation(() => undefined); vi.spyOn(snapshotRestore, "restoreRecreatedSandboxStateWithManagedAuthority").mockReturnValue({ diff --git a/src/lib/actions/sandbox/rebuild-restore-phase.ts b/src/lib/actions/sandbox/rebuild-restore-phase.ts index 79c01047851..70ec9c9362b 100644 --- a/src/lib/actions/sandbox/rebuild-restore-phase.ts +++ b/src/lib/actions/sandbox/rebuild-restore-phase.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection"; import { G, R, YW } from "../../cli/terminal-style"; import * as sandboxConfig from "../../sandbox/config"; import { load as loadRegistry } from "../../state/registry/persistence"; @@ -14,6 +15,7 @@ export interface RebuildRestorePhaseInput { targetImageIsCustom: boolean; backupManifest: RebuildBackupManifest; reconcileManagedDcodeObservability?: boolean; + runtimeSelection?: OpenShellRuntimeSelection; log: RebuildLog; } @@ -23,7 +25,14 @@ export interface RebuildRestorePhaseResult { /** Restore sandbox files. The replacement already received the captured live OpenShell policy. */ export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): RebuildRestorePhaseResult { - const { sandboxName, targetAgentType, targetImageIsCustom, backupManifest, log } = input; + const { + sandboxName, + targetAgentType, + targetImageIsCustom, + backupManifest, + runtimeSelection, + log, + } = input; let restoreSucceeded = true; if (backupManifest) { console.log(""); @@ -34,6 +43,7 @@ export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): Rebuild { targetAgentType, ...(targetImageIsCustom ? { allowCustomImageWholeStateFileRestore: true } : {}), + ...(runtimeSelection ? { runtimeSelection } : {}), }, { getSandbox: (name) => loadRegistry().sandboxes[name] ?? null }, ); diff --git a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts index 4b5f88c68d4..ed52d5daa6a 100644 --- a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts +++ b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts @@ -24,6 +24,7 @@ import * as registry from "../../state/registry"; import * as sandboxState from "../../state/sandbox"; import * as sandboxSession from "../../state/sandbox-session"; import * as destroy from "./destroy"; +import * as mcpBridgeProvider from "./mcp-bridge-provider"; import { rebuildSandbox } from "./rebuild"; import * as rebuildImagePreflight from "./rebuild-custom-image-preflight"; import { rebuildOnboardDependencies } from "./rebuild-onboard-dependencies"; @@ -203,6 +204,10 @@ describe("rebuild resume snapshot repair", () => { } as never), vi.spyOn(registry, "updateSandbox").mockReturnValue(true), vi.spyOn(registry, "listSandboxes").mockReturnValue({ sandboxes: [] } as never), + vi.spyOn(mcpBridgeProvider, "getMcpProviderInspectionRuntimeSelection").mockReturnValue({ + gatewayName: "nemoclaw", + workspace: "default", + }), vi.spyOn(rebuildRoutePreflight, "commitRebuildRoutePreflight").mockReturnValue({ ok: true, receipt: { diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.test.ts b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts index 0b549a78e15..3592792193f 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.test.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts @@ -199,10 +199,16 @@ describe("preflightRebuildTargetRuntime GPU route", () => { describe("authoritative rebuild readiness", () => { it("passes recorded managed-vLLM intent to the pre-delete readiness gate (#9292)", async () => { const authority = { checkpoint: "gateway-authority" }; + const runtimeSelection = { + gatewayName: "nemoclaw", + localTlsDir: "/authority/tls", + workspace: "default", + }; mocks.preflightAuthoritativeRebuildTarget.mockResolvedValue(authority); const recreateOptions = { ...RECREATE_OPTIONS, allowDeferredN1xManagedVllm: true, + runtimeSelection, } as RebuildRecreateOnboardOpts; const bail = vi.fn((message: string): never => { throw new Error(message); @@ -222,6 +228,7 @@ describe("authoritative rebuild readiness", () => { allowDeferredN1xManagedVllm: true, provider: "vllm-local", model: "test-model", + runtimeSelection, sandboxName: "alpha", }), ); diff --git a/src/lib/actions/sandbox/reconcile-session-models.test.ts b/src/lib/actions/sandbox/reconcile-session-models.test.ts index f1bcb98a097..77978d01ed3 100644 --- a/src/lib/actions/sandbox/reconcile-session-models.test.ts +++ b/src/lib/actions/sandbox/reconcile-session-models.test.ts @@ -295,6 +295,27 @@ describe("reconcileStalePinnedSessionModelsAfterRebuild", () => { ); }); + it("reuses the rebuild target for every restored session read and write (#10514)", () => { + executeSandboxCommandMock + .mockReturnValueOnce({ status: 0, stdout: config, stderr: "" }) + .mockReturnValueOnce({ status: 0, stdout: staleStore, stderr: "" }) + .mockReturnValueOnce({ status: 0, stdout: "", stderr: "" }); + const runtimeSelection = { + gatewayName: "recorded-gateway", + workspace: "default", + localTlsDir: "/authority/tls", + }; + + reconcileStalePinnedSessionModelsAfterRebuild("alpha", vi.fn(), runtimeSelection); + + expect(executeSandboxCommandMock).toHaveBeenCalledTimes(3); + expect(executeSandboxCommandMock.mock.calls.map((call) => call[2])).toEqual([ + { runtimeSelection }, + { runtimeSelection }, + { runtimeSelection }, + ]); + }); + it("stops when the restored config has no primary model (#7102)", () => { executeSandboxCommandMock.mockReturnValueOnce({ status: 0, diff --git a/src/lib/actions/sandbox/reconcile-session-models.ts b/src/lib/actions/sandbox/reconcile-session-models.ts index 3a20f18fd3c..39be8602e33 100644 --- a/src/lib/actions/sandbox/reconcile-session-models.ts +++ b/src/lib/actions/sandbox/reconcile-session-models.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { createHash } from "node:crypto"; +import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime"; import { shellQuote } from "../../core/shell-quote"; import { MANAGED_PROVIDER_ID } from "../../inference/config"; import { isSafeModelId } from "../../validation"; @@ -221,8 +222,25 @@ export function reconcilePinnedSessionModels( }; } -function readPrimaryModelRef(sandboxName: string): string | null { - const res = executeSandboxCommand(sandboxName, `cat ${OPENCLAW_CONFIG_PATH} 2>/dev/null`); +function executeReconcileCommand( + sandboxName: string, + command: string, + runtimeSelection?: OpenShellRuntimeSelection, +) { + return runtimeSelection + ? executeSandboxCommand(sandboxName, command, { runtimeSelection }) + : executeSandboxCommand(sandboxName, command); +} + +function readPrimaryModelRef( + sandboxName: string, + runtimeSelection?: OpenShellRuntimeSelection, +): string | null { + const res = executeReconcileCommand( + sandboxName, + `cat ${OPENCLAW_CONFIG_PATH} 2>/dev/null`, + runtimeSelection, + ); if (!res || res.status !== 0 || !res.stdout.trim()) return null; try { const config = JSON.parse(res.stdout) as { @@ -252,14 +270,19 @@ function readPrimaryModelRef(sandboxName: string): string | null { export function reconcileStalePinnedSessionModelsAfterRebuild( sandboxName: string, log: RebuildLog, + runtimeSelection?: OpenShellRuntimeSelection, ): void { - const primary = readPrimaryModelRef(sandboxName); + const primary = readPrimaryModelRef(sandboxName, runtimeSelection); if (!primary) { log("Session model reconcile skipped: could not read agents.defaults.model.primary"); return; } const sessionsPath = defaultAgentSessionsPath(DEFAULT_AGENT_ID); - const readResult = executeSandboxCommand(sandboxName, `cat ${sessionsPath} 2>/dev/null`); + const readResult = executeReconcileCommand( + sandboxName, + `cat ${sessionsPath} 2>/dev/null`, + runtimeSelection, + ); if (!readResult || readResult.status !== 0 || !readResult.stdout.trim()) { log(`Session model reconcile skipped: no session store at ${sessionsPath}`); return; @@ -269,9 +292,10 @@ export function reconcileStalePinnedSessionModelsAfterRebuild( log("Session model reconcile: no stale pinned session models"); return; } - const writeResult = executeSandboxCommand( + const writeResult = executeReconcileCommand( sandboxName, buildSessionStoreReplaceCommand(sessionsPath, reconciled.content, readResult.stdout), + runtimeSelection, ); if (!writeResult || writeResult.status !== 0) { log( diff --git a/src/lib/adapters/openshell/forward-service.test.ts b/src/lib/adapters/openshell/forward-service.test.ts index 9a4d12b727d..2de4e61ca73 100644 --- a/src/lib/adapters/openshell/forward-service.test.ts +++ b/src/lib/adapters/openshell/forward-service.test.ts @@ -39,6 +39,12 @@ describe("OpenShell forward service", () => { ]); }); + it("builds the direct ForwardTcp command for a selected non-default workspace", () => { + expect(buildForwardServiceArgs({ ...target, workspace: "review-workspace" })).toContain( + "review-workspace", + ); + }); + it("detaches the OpenShell child and waits for its local port", () => { const unref = vi.fn(); const spawnDetached = vi.fn(() => ({ unref })); diff --git a/src/lib/adapters/openshell/forward-service.ts b/src/lib/adapters/openshell/forward-service.ts index fe4f0f75519..8f78fe81983 100644 --- a/src/lib/adapters/openshell/forward-service.ts +++ b/src/lib/adapters/openshell/forward-service.ts @@ -15,7 +15,7 @@ const sleepBuffer = new Int32Array(new SharedArrayBuffer(4)); export interface ForwardServiceTarget { readonly executable: string; readonly gatewayName: string; - readonly workspace: "default"; + readonly workspace: string; readonly sandboxName: string; readonly localHost: "127.0.0.1" | "0.0.0.0"; readonly localPort: number; @@ -56,8 +56,8 @@ export function validateForwardServiceTarget(target: ForwardServiceTarget): Forw if (!isCanonicalNemoClawGatewayName(target.gatewayName)) { throw new Error("OpenShell forward service gateway must be a canonical NemoClaw gateway"); } - if (target.workspace !== "default") { - throw new Error("OpenShell forward service workspace must be default"); + if (!isValidName(target.workspace)) { + throw new Error("OpenShell forward service workspace is invalid"); } if (!isValidName(target.sandboxName)) { throw new Error("OpenShell forward service sandbox name is invalid"); diff --git a/src/lib/adapters/openshell/gateway-drift.test.ts b/src/lib/adapters/openshell/gateway-drift.test.ts index 3008874cdeb..214e29ddf3f 100644 --- a/src/lib/adapters/openshell/gateway-drift.test.ts +++ b/src/lib/adapters/openshell/gateway-drift.test.ts @@ -23,6 +23,7 @@ describe("OpenShell gateway drift preflight", () => { afterEach(() => { for (const spy of spies) spy.mockRestore(); spies = []; + vi.unstubAllEnvs(); }); it("parses OpenShell cluster image versions", () => { @@ -120,6 +121,71 @@ describe("OpenShell gateway drift preflight", () => { expect(isGatewayClusterActiveForGateway("nemoclaw", { expectedGatewayPort: 9090 })).toBe(false); }); + it("pins gateway health probes to the frozen OpenShell target (#10514)", () => { + vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://hostile.example.invalid"); + vi.stubEnv("OPENSHELL_LOCAL_TLS_DIR", "/hostile/tls"); + vi.stubEnv("OPENSHELL_TOKEN", "hostile-token"); + vi.stubEnv("OPENSHELL_WORKSPACE", "hostile-workspace"); + const openshellRuntime = requireDist("./runtime.js"); + const docker = requireDist("../docker/inspect.js"); + const captureOpenshell = vi + .spyOn(openshellRuntime, "captureOpenshell") + .mockReturnValueOnce({ + status: 0, + output: "Server Status\n\n Gateway: nemoclaw-9090\n Status: Connected", + }) + .mockReturnValue({ + status: 0, + output: + "Gateway Info\n\n Gateway: nemoclaw-9090\n Gateway endpoint: https://127.0.0.1:9090", + }); + const inspectContainer = vi + .spyOn(docker, "dockerContainerInspectFormat") + .mockReturnValueOnce("true") + .mockReturnValue('{"30051/tcp":[{"HostIp":"0.0.0.0","HostPort":"9090"}]}'); + spies.push( + captureOpenshell, + inspectContainer, + ); + const runtimeSelection = { + gatewayName: "nemoclaw-9090", + localTlsDir: "/authority/tls", + workspace: "default", + }; + + expect( + isGatewayClusterActiveForGateway(runtimeSelection.gatewayName, { + expectedGatewayPort: 9090, + runtimeSelection, + }), + ).toBe(true); + const selectedProbeOptions = expect.objectContaining({ + env: expect.objectContaining({ + OPENSHELL_GATEWAY: "nemoclaw-9090", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + OPENSHELL_WORKSPACE: "default", + }), + replaceEnv: true, + }); + expect(captureOpenshell).toHaveBeenNthCalledWith(1, ["status"], selectedProbeOptions); + expect(captureOpenshell).toHaveBeenNthCalledWith( + 2, + ["gateway", "info", "-g", "nemoclaw-9090"], + selectedProbeOptions, + ); + expect(captureOpenshell).toHaveBeenNthCalledWith( + 3, + ["gateway", "info"], + selectedProbeOptions, + ); + const firstProbeOptions = captureOpenshell.mock.calls[0]?.[1] as + | { env?: Record } + | undefined; + const selectedEnv = firstProbeOptions?.env; + expect(selectedEnv).not.toHaveProperty("OPENSHELL_GATEWAY_ENDPOINT"); + expect(selectedEnv).not.toHaveProperty("OPENSHELL_TOKEN"); + }); + it("ignores stale cluster containers whose published port is not the active gateway endpoint", () => { const openshellRuntime = requireDist("./runtime.js"); const docker = requireDist("../docker/inspect.js"); diff --git a/src/lib/adapters/openshell/gateway-drift.ts b/src/lib/adapters/openshell/gateway-drift.ts index 9f1d33bb9df..254302b8dcc 100644 --- a/src/lib/adapters/openshell/gateway-drift.ts +++ b/src/lib/adapters/openshell/gateway-drift.ts @@ -30,6 +30,8 @@ import { } from "./client"; import { resolveOpenshell } from "./resolve"; import { captureOpenshell, getInstalledOpenshellVersionOrNull } from "./runtime"; +import { buildSelectedOpenShellSubprocessEnv } from "./command-argv"; +import type { OpenShellRuntimeSelection } from "./runtime-selection"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "./timeouts"; export type { GatewayReuseState, ManagedGatewayEndpointBinding }; @@ -99,6 +101,7 @@ export function isHostProcessGatewayDrift( export type GatewayDriftOptions = { gatewayName?: string; deps?: GatewayDriftDeps; + runtimeSelection?: OpenShellRuntimeSelection; timeoutMs?: number; }; @@ -215,19 +218,34 @@ export function isGatewayClusterActiveForGateway( gatewayName = DEFAULT_GATEWAY_NAME, { expectedGatewayPort, + runtimeSelection, timeoutMs = OPENSHELL_PROBE_TIMEOUT_MS, - }: { expectedGatewayPort?: number; timeoutMs?: number } = {}, + }: { + expectedGatewayPort?: number; + runtimeSelection?: OpenShellRuntimeSelection; + timeoutMs?: number; + } = {}, ): boolean { + if (runtimeSelection && runtimeSelection.gatewayName !== gatewayName) return false; + const selectedOptions = runtimeSelection + ? { + env: buildSelectedOpenShellSubprocessEnv(runtimeSelection), + replaceEnv: true, + } + : {}; const status = captureOpenshell(["status"], { ignoreError: true, + ...selectedOptions, timeout: timeoutMs, }); const gatewayInfo = captureOpenshell(["gateway", "info", "-g", gatewayName], { ignoreError: true, + ...selectedOptions, timeout: timeoutMs, }); const activeGatewayInfo = captureOpenshell(["gateway", "info"], { ignoreError: true, + ...selectedOptions, timeout: timeoutMs, }); if (!isGatewayHealthy(status.output, gatewayInfo.output, activeGatewayInfo.output, gatewayName)) { @@ -255,6 +273,7 @@ export function isGatewayClusterActiveForGateway( export function getGatewayClusterImageDrift({ gatewayName = DEFAULT_GATEWAY_NAME, deps = {}, + runtimeSelection, timeoutMs = OPENSHELL_PROBE_TIMEOUT_MS, }: GatewayDriftOptions = {}): GatewayClusterImageDrift | null { if (isGatewayDriftPreflightDisabled(deps)) { @@ -268,7 +287,7 @@ export function getGatewayClusterImageDrift({ deps.isGatewayClusterActive?.(gatewayName) ?? (typeof deps.getGatewayClusterImageRef === "function" ? true - : isGatewayClusterActiveForGateway(gatewayName, { timeoutMs })); + : isGatewayClusterActiveForGateway(gatewayName, { runtimeSelection, timeoutMs })); if (!clusterActive) { return null; } @@ -412,6 +431,7 @@ export function getHostProcessGatewayRuntimeOrNull({ export function getGatewayHostProcessDrift({ gatewayName = DEFAULT_GATEWAY_NAME, deps = {}, + runtimeSelection, timeoutMs = OPENSHELL_PROBE_TIMEOUT_MS, }: GatewayDriftOptions = {}): GatewayHostProcessDrift | null { if (isGatewayDriftPreflightDisabled(deps)) { @@ -431,7 +451,7 @@ export function getGatewayHostProcessDrift({ if (clusterImage) { const clusterActive = deps.isGatewayClusterActive?.(gatewayName) ?? - isGatewayClusterActiveForGateway(gatewayName, { timeoutMs }); + isGatewayClusterActiveForGateway(gatewayName, { runtimeSelection, timeoutMs }); if (clusterActive) { return null; } @@ -466,6 +486,7 @@ export function observeOpenShellGatewayVersionCompatibility({ source, gatewayName = DEFAULT_GATEWAY_NAME, deps = {}, + runtimeSelection, timeoutMs = OPENSHELL_PROBE_TIMEOUT_MS, }: GatewayVersionCompatibilityOptions): GatewayVersionCompatibility { if (isGatewayDriftPreflightDisabled(deps)) return "unknown"; @@ -478,7 +499,7 @@ export function observeOpenShellGatewayVersionCompatibility({ if (source === "legacy-cluster") { const clusterActive = deps.isGatewayClusterActive?.(gatewayName) ?? - isGatewayClusterActiveForGateway(gatewayName, { timeoutMs }); + isGatewayClusterActiveForGateway(gatewayName, { runtimeSelection, timeoutMs }); if (!clusterActive) return "unknown"; const clusterImage = typeof deps.getGatewayClusterImageRef === "function" @@ -503,13 +524,24 @@ export function observeOpenShellGatewayVersionCompatibility({ export function detectOpenShellStateRpcPreflightIssue({ gatewayName = DEFAULT_GATEWAY_NAME, deps = {}, + runtimeSelection, timeoutMs = OPENSHELL_PROBE_TIMEOUT_MS, }: GatewayDriftOptions = {}): OpenShellStateRpcIssue | null { - const imageDrift = getGatewayClusterImageDrift({ gatewayName, deps, timeoutMs }); + const imageDrift = getGatewayClusterImageDrift({ + gatewayName, + deps, + runtimeSelection, + timeoutMs, + }); if (imageDrift) { return { kind: "image_drift", drift: imageDrift }; } - const hostDrift = getGatewayHostProcessDrift({ gatewayName, deps, timeoutMs }); + const hostDrift = getGatewayHostProcessDrift({ + gatewayName, + deps, + runtimeSelection, + timeoutMs, + }); if (hostDrift) { return { kind: "host_process_drift", drift: hostDrift }; } @@ -521,6 +553,7 @@ export function detectOpenShellStateRpcResultIssue( { gatewayName = DEFAULT_GATEWAY_NAME, deps = {}, + runtimeSelection, timeoutMs = OPENSHELL_PROBE_TIMEOUT_MS, }: GatewayDriftOptions = {}, ): OpenShellStateRpcIssue | null { @@ -532,8 +565,8 @@ export function detectOpenShellStateRpcResultIssue( kind: "protobuf_mismatch", drift: isGatewayDriftPreflightDisabled(deps) ? null - : (getGatewayClusterImageDrift({ gatewayName, deps, timeoutMs }) ?? - getGatewayHostProcessDrift({ gatewayName, deps, timeoutMs })), + : (getGatewayClusterImageDrift({ gatewayName, deps, runtimeSelection, timeoutMs }) ?? + getGatewayHostProcessDrift({ gatewayName, deps, runtimeSelection, timeoutMs })), output, }; } diff --git a/src/lib/onboard/authoritative-rebuild-target.test.ts b/src/lib/onboard/authoritative-rebuild-target.test.ts index 9d05d258482..6ba1a34de50 100644 --- a/src/lib/onboard/authoritative-rebuild-target.test.ts +++ b/src/lib/onboard/authoritative-rebuild-target.test.ts @@ -6,6 +6,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { authoritativeRebuildSandboxFlowOptions, authoritativeRebuildRuntimePreflightOptions, + beginAuthoritativeRebuildRuntimeSelectionScope, type AuthoritativeRebuildTargetDeps, type AuthoritativeRebuildPreflightOptions, preflightAuthoritativeRebuildTarget, @@ -152,6 +153,48 @@ describe("authoritative rebuild gateway binding", () => { }); }); +describe("authoritative rebuild OpenShell runtime selection", () => { + it("replaces hostile ambient selectors for inner onboard and restores them (#10514)", () => { + const env: NodeJS.ProcessEnv = { + PATH: "/usr/bin", + OPENSHELL_GATEWAY: "hostile-gateway", + OPENSHELL_GATEWAY_AUTH_TOKEN: "hostile-token", + OPENSHELL_GATEWAY_ENDPOINT: "https://hostile.invalid", + OPENSHELL_LOCAL_TLS_DIR: "/hostile/tls", + OPENSHELL_WORKSPACE: "hostile-workspace", + }; + const previous = { ...env }; + const restore = beginAuthoritativeRebuildRuntimeSelectionScope( + { + authoritativeResumeConfig: true, + onboardLockAlreadyHeld: true, + recreateSandbox: true, + resume: true, + targetGatewayName: "nemoclaw-8081", + targetGatewayPort: 8081, + runtimeSelection: { + gatewayName: "nemoclaw-8081", + localTlsDir: "/authority/tls", + workspace: "default", + }, + }, + env, + ); + + expect(env).toMatchObject({ + PATH: "/usr/bin", + OPENSHELL_GATEWAY: "nemoclaw-8081", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + OPENSHELL_WORKSPACE: "default", + }); + expect(env.OPENSHELL_GATEWAY_AUTH_TOKEN).toBeUndefined(); + expect(env.OPENSHELL_GATEWAY_ENDPOINT).toBeUndefined(); + + restore(); + expect(env).toEqual(previous); + }); +}); + describe("prepared provider reconfiguration handoff", () => { const providerTarget = { sandboxName: "alpha", diff --git a/src/lib/onboard/authoritative-rebuild-target.ts b/src/lib/onboard/authoritative-rebuild-target.ts index c08cee58c06..aeda139f451 100644 --- a/src/lib/onboard/authoritative-rebuild-target.ts +++ b/src/lib/onboard/authoritative-rebuild-target.ts @@ -2,6 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { findDashboardForwardOwner } from "./dashboard-port"; +import { + replaceOpenShellRuntimeSelectionEnv, + snapshotOpenShellEnv, + type OpenShellRuntimeSelection, +} from "../adapters/openshell/runtime-selection"; import { resolveGatewayName } from "./gateway-binding"; import type { InferenceRouteState } from "./inference-route"; import { assertDashboardPortNotReserved } from "./preflight-ports"; @@ -34,9 +39,53 @@ export type AuthoritativeGatewayOptions = Pick< "authoritativeResumeConfig" | "targetGatewayName" | "targetGatewayPort" | "onboardLockAlreadyHeld" >; +type AuthoritativeRuntimeSelectionOptions = AuthoritativeGatewayOptions & + Pick; + +function beginOpenShellRuntimeSelectionEnvScope( + runtimeSelection: OpenShellRuntimeSelection, + env: NodeJS.ProcessEnv, +): () => void { + const restore = snapshotOpenShellEnv(env); + replaceOpenShellRuntimeSelectionEnv(env, runtimeSelection); + return restore; +} + +/** Keep every OpenShell child in an inner rebuild onboard on its frozen target. */ +export function beginAuthoritativeRebuildRuntimeSelectionScope( + opts: AuthoritativeRuntimeSelectionOptions, + env: NodeJS.ProcessEnv = process.env, +): () => void { + const runtimeSelection = opts.runtimeSelection; + if (!runtimeSelection) return () => undefined; + const gateway = resolveAuthoritativeOnboardGatewayBinding(opts); + if ( + opts.authoritativeResumeConfig !== true || + opts.resume !== true || + opts.recreateSandbox !== true || + opts.onboardLockAlreadyHeld !== true || + !gateway + ) { + throw new Error( + "An OpenShell runtime selection may be supplied only for a locked authoritative rebuild resume.", + ); + } + if (runtimeSelection.gatewayName !== gateway.name) { + throw new Error( + `OpenShell runtime selection '${runtimeSelection.gatewayName}' does not match authoritative gateway '${gateway.name}'.`, + ); + } + return beginOpenShellRuntimeSelectionEnvScope(runtimeSelection, env); +} + export type AuthoritativeRebuildPreflightOptions = Pick< OnboardOptions, - "sandboxGpu" | "sandboxGpuDevice" | "noGpu" | "controlUiPort" | "allowDeferredN1xManagedVllm" + | "sandboxGpu" + | "sandboxGpuDevice" + | "noGpu" + | "controlUiPort" + | "allowDeferredN1xManagedVllm" + | "runtimeSelection" > & { authoritativeResumeConfig: true; /** Internal prepared-backup recovery defers route repair to authoritative onboard. */ @@ -102,6 +151,7 @@ export type AuthoritativeRebuildTarget = { model: string; targetGatewayName: string; controlUiPort: number | null; + runtimeSelection?: OpenShellRuntimeSelection; }; /** Validate the one-shot authority to reconstruct a provider during a locked rebuild resume. */ @@ -221,11 +271,23 @@ export async function preflightAuthoritativeRebuildTarget( deps: AuthoritativeRebuildTargetDeps, ): Promise { const env = deps.env ?? process.env; - const previousGateway = env.OPENSHELL_GATEWAY; const fail = (message: string): never => { throw new Error(message); }; - env.OPENSHELL_GATEWAY = target.targetGatewayName; + const runtimeSelection = target.runtimeSelection; + if (runtimeSelection && runtimeSelection.gatewayName !== target.targetGatewayName) { + fail( + `OpenShell runtime selection '${runtimeSelection.gatewayName}' does not match authoritative gateway '${target.targetGatewayName}'.`, + ); + } + const previousGateway = env.OPENSHELL_GATEWAY; + const restoreRuntimeSelection = runtimeSelection + ? beginOpenShellRuntimeSelectionEnvScope(runtimeSelection, env) + : () => { + if (previousGateway === undefined) delete env.OPENSHELL_GATEWAY; + else env.OPENSHELL_GATEWAY = previousGateway; + }; + if (!runtimeSelection) env.OPENSHELL_GATEWAY = target.targetGatewayName; try { if (!deps.resolveBaselinePolicy(target.sandboxName)) { fail(`Could not read the baseline policy for sandbox '${target.sandboxName}'.`); @@ -261,7 +323,6 @@ export async function preflightAuthoritativeRebuildTarget( // sandbox unavailable first; the natural OpenShell lifecycle releases its // port. The replacement launch then refuses any port that stayed occupied. } finally { - if (previousGateway === undefined) delete env.OPENSHELL_GATEWAY; - else env.OPENSHELL_GATEWAY = previousGateway; + restoreRuntimeSelection(); } } diff --git a/src/lib/onboard/entry-options.ts b/src/lib/onboard/entry-options.ts index a25f813cac9..acd09476bf0 100644 --- a/src/lib/onboard/entry-options.ts +++ b/src/lib/onboard/entry-options.ts @@ -4,6 +4,7 @@ import { isNonInteractiveEnv } from "../core/non-interactive"; import { getNameValidationGuidance } from "../name-validation"; export { enforceRemovedImmutabilityMigrationBoundary } from "../state/migrations/removed-immutability"; +import { beginAuthoritativeRebuildRuntimeSelectionScope } from "./authoritative-rebuild-target"; import { cliDisplayName } from "./branding"; import { canonicalPlaceholderKeys, @@ -11,6 +12,7 @@ import { parseExtraPlaceholderKeys, } from "./extra-placeholder-keys"; import { RESERVED_SANDBOX_NAMES } from "./sandbox-agent"; +import type { OnboardOptions } from "./types"; import { requireStationExpressResumeIntent, type StationExpressSessionLike, @@ -74,11 +76,19 @@ interface DefaultRunEntryState { } type NonInteractiveEntryOptions = { nonInteractive?: boolean }; -type ResumableEntryOptions = NonInteractiveEntryOptions & { - resume?: boolean; - fresh?: boolean; - apfInterceptorRequested?: boolean | null; -}; +type ResumableEntryOptions = Pick< + OnboardOptions, + | "apfInterceptorRequested" + | "authoritativeResumeConfig" + | "fresh" + | "nonInteractive" + | "onboardLockAlreadyHeld" + | "recreateSandbox" + | "resume" + | "runtimeSelection" + | "targetGatewayName" + | "targetGatewayPort" +>; const PROVIDER_INTENT_ENV_KEYS = [ "NEMOCLAW_PROVIDER", @@ -256,7 +266,12 @@ export function wrapOnboard( options?.apfInterceptorRequested === true, process.env, ); - await run(options); + const restoreRuntimeSelection = beginAuthoritativeRebuildRuntimeSelectionScope(options ?? {}); + try { + await run(options); + } finally { + restoreRuntimeSelection(); + } }; return wrapStationExpressOnboard( withNonInteractiveEnvironment(guardProviderlessInput), diff --git a/src/lib/onboard/gateway/registration.ts b/src/lib/onboard/gateway/registration.ts index 7cc72033468..c02ecaf9d57 100644 --- a/src/lib/onboard/gateway/registration.ts +++ b/src/lib/onboard/gateway/registration.ts @@ -48,8 +48,7 @@ export function createGatewayRegistration(deps: GatewayRegistrationDeps): Gatewa const runCaptureOpenshell: GatewayRegistrationDeps["runCaptureOpenshell"] = ( args, options = {}, - ) => - deps.runCaptureOpenshell(args, { ...options, ...runtimeOptions }); + ) => deps.runCaptureOpenshell(args, { ...options, ...runtimeOptions }); const runOpenshell: GatewayRegistrationDeps["runOpenshell"] = (args, options = {}) => deps.runOpenshell(args, { ...options, ...runtimeOptions }); const runQuietOpenshell = (args: string[]) => diff --git a/src/lib/onboard/sandbox-recreate-probe.ts b/src/lib/onboard/sandbox-recreate-probe.ts index 1290ef64fb1..ad0e66af54b 100644 --- a/src/lib/onboard/sandbox-recreate-probe.ts +++ b/src/lib/onboard/sandbox-recreate-probe.ts @@ -8,6 +8,8 @@ import { stripAnsi, } from "../adapters/openshell/client"; import { captureOpenshell } from "../adapters/openshell/runtime"; +import { buildSelectedOpenShellSubprocessEnv } from "../adapters/openshell/command-argv"; +import type { OpenShellRuntimeSelection } from "../adapters/openshell/runtime-selection"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts"; import { parseSandboxPhase } from "../state/gateway"; import { @@ -97,12 +99,24 @@ export function observeSandboxPresenceOnGateway( export function observeSandboxOnGateway( target: SandboxRecreateTarget, capture: SandboxRecreateCapture = captureOpenshell, + runtimeSelection?: OpenShellRuntimeSelection, ): SandboxRecreateObservation { + if (runtimeSelection && runtimeSelection.gatewayName !== target.gatewayName) { + throw new Error( + `Cannot journal sandbox '${target.sandboxName}' replacement: selected gateway does not match the recorded target.`, + ); + } const probe = capture(["sandbox", "get", "-g", target.gatewayName, target.sandboxName], { ignoreError: true, includeStderr: true, includeStreams: true, timeout: OPENSHELL_PROBE_TIMEOUT_MS, + ...(runtimeSelection + ? { + env: buildSelectedOpenShellSubprocessEnv(runtimeSelection), + replaceEnv: true, + } + : {}), }); const stdout = String(probe.stdout ?? (probe.status === 0 ? probe.output : "")).trim(); const combined = `${stdout}\n${String(probe.stderr ?? probe.output ?? "")}`.trim(); diff --git a/src/lib/onboard/types.ts b/src/lib/onboard/types.ts index c8a36bd1cec..c8e8b63deb3 100644 --- a/src/lib/onboard/types.ts +++ b/src/lib/onboard/types.ts @@ -140,6 +140,8 @@ export type OnboardOptions = { targetGatewayName?: string | null; /** Internal authoritative rebuild target; must match targetGatewayName. */ targetGatewayPort?: number | null; + /** Exact OpenShell client target frozen by the outer rebuild transaction. */ + runtimeSelection?: import("../adapters/openshell/runtime-selection").OpenShellRuntimeSelection; /** Internal rebuild handoff: the outer destructive lifecycle owns the onboard lock. */ onboardLockAlreadyHeld?: boolean; /** Internal command handoff: propagate an exit request after onboarding restores its scopes. */ diff --git a/src/lib/state/openclaw-config-restore-input.ts b/src/lib/state/openclaw-config-restore-input.ts index f518b8f017e..5c8c6b17e58 100644 --- a/src/lib/state/openclaw-config-restore-input.ts +++ b/src/lib/state/openclaw-config-restore-input.ts @@ -20,6 +20,7 @@ export type OpenClawConfigRestoreInputResult = export interface OpenClawConfigRestoreFromSandboxOptions { backupContents: Buffer; dir: string; + env?: NodeJS.ProcessEnv; freshImagePluginInstalls?: readonly OpenClawImagePluginInstall[]; log?: (message: string) => void; previousImagePluginInstalls?: readonly OpenClawImagePluginInstall[]; @@ -47,9 +48,11 @@ function readCurrentOpenClawConfig( dir: string, specPath: string, log: (message: string) => void, + env?: NodeJS.ProcessEnv, ): Buffer | null { const command = buildOpenClawConfigReadCommand(dir, specPath); const result = spawnSync("ssh", [...sshArgs, command], { + ...(env ? { env } : {}), stdio: ["ignore", "pipe", "pipe"], timeout: 120000, maxBuffer: 256 * 1024 * 1024, @@ -91,6 +94,7 @@ export function buildOpenClawConfigRestoreInput( export function buildOpenClawConfigRestoreInputFromSandbox({ backupContents, dir, + env, freshImagePluginInstalls, log = () => {}, previousImagePluginInstalls, @@ -117,7 +121,7 @@ export function buildOpenClawConfigRestoreInputFromSandbox({ } return buildOpenClawConfigRestoreInput( backupContents, - readCurrentOpenClawConfig(sshArgs, dir, specPath, log), + readCurrentOpenClawConfig(sshArgs, dir, specPath, log, env), { freshImagePluginInstalls, previousImagePluginInstalls }, ); } diff --git a/src/lib/state/openclaw-plugin-restore.ts b/src/lib/state/openclaw-plugin-restore.ts index d78ee0d8dbf..7193f629608 100644 --- a/src/lib/state/openclaw-plugin-restore.ts +++ b/src/lib/state/openclaw-plugin-restore.ts @@ -49,6 +49,7 @@ export type CompleteOpenClawImagePluginInstall = Omit; backupExtensionDirs: string[]; + discoverFreshPluginInstalls?: boolean; freshConfig: Record; freshPluginInstalls: OpenClawImagePluginInstall[]; previousPluginInstalls?: OpenClawImagePluginInstall[]; + runtimeSelection?: { + gatewayName: string; + localTlsDir?: string; + workspace: string; + }; }): { cleanupCommand: string | undefined; freshMarkers: Record; + openshellInvocations: Array<{ args: string[]; env: Record }>; restore: ReturnType; restoredConfig: Record; + sshInvocations: Array<{ cmd: string; env: Record }>; staleUserExtensionExists: boolean; userExtensionMarker: string; } { @@ -50,6 +58,7 @@ function runRestoreScenario(options: { const extensionsDir = path.join(openclawDir, "extensions"); const backupPath = path.join(fixture, "backup"); const backupExtensionsDir = path.join(backupPath, "extensions"); + const openshellLog = path.join(fixture, "openshell-log.jsonl"); const sshLog = path.join(fixture, "ssh-log.jsonl"); const freshExtensionDirs = [ "nemoclaw", @@ -101,7 +110,10 @@ function runRestoreScenario(options: { writeExecutable( openshell, `#!/usr/bin/env node +const fs = require("node:fs"); const args = process.argv.slice(2); +const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => name.startsWith("OPENSHELL_"))); +fs.appendFileSync(${JSON.stringify(openshellLog)}, JSON.stringify({ args, env }) + "\\n"); if (args[0] === "sandbox" && args[1] === "ssh-config") { process.stdout.write("Host openshell-alpha\\n HostName 127.0.0.1\\n User sandbox\\n"); } @@ -117,7 +129,8 @@ const { spawnSync } = require("node:child_process"); const cmd = process.argv[process.argv.length - 1] || ""; const openclawDir = ${JSON.stringify(openclawDir)}; const extensionsDir = ${JSON.stringify(extensionsDir)}; -fs.appendFileSync(${JSON.stringify(sshLog)}, JSON.stringify({ cmd }) + "\\n"); +const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => name.startsWith("OPENSHELL_"))); +fs.appendFileSync(${JSON.stringify(sshLog)}, JSON.stringify({ cmd, env }) + "\\n"); function readStdin() { const chunks = []; for (;;) { @@ -135,6 +148,10 @@ if (cmd.includes("${OPENCLAW_DIR}/extensions") && cmd.includes("-exec rm -rf")) } process.exit(0); } +if (cmd.includes("installed_plugin_index")) { + process.stdout.write(JSON.stringify({ version: 1, installRecords: {}, loadPaths: [] })); + process.exit(0); +} if (cmd.includes("tar --no-same-owner -xf -")) { const result = spawnSync("tar", ["--no-same-owner", "-xf", "-", "-C", openclawDir], { input: readStdin(), @@ -159,13 +176,22 @@ process.exit(1); process.env.PATH = `${binDir}:${previousPath ?? ""}`; const restore = restoreRecreatedSandboxState("alpha", backupPath, { targetAgentType: "openclaw", - freshOpenClawImagePluginInstalls: options.freshPluginInstalls, + ...(options.discoverFreshPluginInstalls + ? {} + : { freshOpenClawImagePluginInstalls: options.freshPluginInstalls }), + ...(options.runtimeSelection ? { runtimeSelection: options.runtimeSelection } : {}), }); - const loggedCommands = fs + const sshInvocations = fs .readFileSync(sshLog, "utf8") .trim() .split("\n") - .map((line) => JSON.parse(line).cmd as string); + .map((line) => JSON.parse(line) as { cmd: string; env: Record }); + const loggedCommands = sshInvocations.map(({ cmd }) => cmd); + const openshellInvocations = fs + .readFileSync(openshellLog, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as { args: string[]; env: Record }); return { cleanupCommand: loggedCommands.find((command) => command.includes("-exec rm -rf")), @@ -175,8 +201,10 @@ process.exit(1); fs.readFileSync(path.join(extensionsDir, name, "marker.txt"), "utf8"), ]), ), + openshellInvocations, restore, restoredConfig: JSON.parse(fs.readFileSync(path.join(openclawDir, "openclaw.json"), "utf8")), + sshInvocations, staleUserExtensionExists: fs.existsSync(path.join(extensionsDir, "stale-user-extension")), userExtensionMarker: fs.readFileSync( path.join(extensionsDir, "user-extension", "marker.txt"), @@ -201,46 +229,95 @@ function expectSuccessfulRestore(result: ReturnType): expect(result.userExtensionMarker).toBe("restored\n"); } -describe("recreated OpenClaw state restore", () => { +describe("recreated OpenClaw state restore", { timeout: 30_000 }, () => { + it("pins plugin discovery and restore SSH to the frozen OpenShell target (#10514)", () => { + const previousEnv = { + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY, + OPENSHELL_GATEWAY_ENDPOINT: process.env.OPENSHELL_GATEWAY_ENDPOINT, + OPENSHELL_GATEWAY_INSECURE: process.env.OPENSHELL_GATEWAY_INSECURE, + OPENSHELL_LOCAL_TLS_DIR: process.env.OPENSHELL_LOCAL_TLS_DIR, + OPENSHELL_TOKEN: process.env.OPENSHELL_TOKEN, + OPENSHELL_WORKSPACE: process.env.OPENSHELL_WORKSPACE, + }; + process.env.OPENSHELL_GATEWAY = "hostile-gateway"; + process.env.OPENSHELL_GATEWAY_ENDPOINT = "https://hostile.example.invalid"; + process.env.OPENSHELL_GATEWAY_INSECURE = "1"; + process.env.OPENSHELL_LOCAL_TLS_DIR = "/hostile/tls"; + process.env.OPENSHELL_TOKEN = "hostile-token"; + process.env.OPENSHELL_WORKSPACE = "hostile-workspace"; + try { + const result = runRestoreScenario({ + backupConfig: { plugins: { entries: {} } }, + backupExtensionDirs: [], + discoverFreshPluginInstalls: true, + freshConfig: { plugins: { entries: {} } }, + freshPluginInstalls: [], + previousPluginInstalls: [], + runtimeSelection: { + gatewayName: "nemoclaw-9090", + localTlsDir: "/authority/tls", + workspace: "default", + }, + }); + + expectSuccessfulRestore(result); + expect(result.openshellInvocations.length).toBeGreaterThan(0); + expect(result.sshInvocations.length).toBeGreaterThan(1); + const invocationEnvironments = [...result.openshellInvocations, ...result.sshInvocations].map( + ({ env }) => env, + ); + expect(invocationEnvironments).toEqual( + new Array(invocationEnvironments.length).fill({ + OPENSHELL_GATEWAY: "nemoclaw-9090", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + OPENSHELL_WORKSPACE: "default", + }), + ); + } finally { + restoreEnvBulk(previousEnv); + } + }); + it.each([ { provenance: "missing legacy", previousPluginInstalls: undefined }, { provenance: "known-empty", previousPluginInstalls: [] }, - ])("restores config and extensions with $provenance previous provenance", ({ - previousPluginInstalls, - }) => { - const weather = imageInstall("weather", "weather"); - const result = runRestoreScenario({ - previousPluginInstalls, - freshPluginInstalls: [weather], - backupExtensionDirs: ["weather"], - backupConfig: { - gateway: { auth: { token: "stale-token" } }, - mcpServers: { filesystem: { command: "npx" } }, - plugins: { entries: { "user-plugin": { enabled: true } } }, - }, - freshConfig: { - gateway: { auth: { token: "fresh-token" } }, - plugins: { - entries: { weather: { enabled: true, config: { revision: "fresh" } } }, - load: { paths: weather.loadPaths }, + ])( + "restores config and extensions with $provenance previous provenance", + ({ previousPluginInstalls }) => { + const weather = imageInstall("weather", "weather"); + const result = runRestoreScenario({ + previousPluginInstalls, + freshPluginInstalls: [weather], + backupExtensionDirs: ["weather"], + backupConfig: { + gateway: { auth: { token: "stale-token" } }, + mcpServers: { filesystem: { command: "npx" } }, + plugins: { entries: { "user-plugin": { enabled: true } } }, }, - }, - }); + freshConfig: { + gateway: { auth: { token: "fresh-token" } }, + plugins: { + entries: { weather: { enabled: true, config: { revision: "fresh" } } }, + load: { paths: weather.loadPaths }, + }, + }, + }); - expectSuccessfulRestore(result); - expect(result.freshMarkers).toEqual({ - nemoclaw: "fresh-nemoclaw\n", - weather: "fresh-weather\n", - }); - expect(result.restoredConfig.gateway.auth.token).toBe("fresh-token"); - expect(result.restoredConfig.mcpServers.filesystem.command).toBe("npx"); - expect(result.restoredConfig.plugins.entries).toEqual({ - "user-plugin": { enabled: true }, - weather: { enabled: true, config: { revision: "fresh" } }, - }); - expect(result.cleanupCommand).toContain("! -name 'nemoclaw'"); - expect(result.cleanupCommand).toContain("! -name 'weather'"); - }); + expectSuccessfulRestore(result); + expect(result.freshMarkers).toEqual({ + nemoclaw: "fresh-nemoclaw\n", + weather: "fresh-weather\n", + }); + expect(result.restoredConfig.gateway.auth.token).toBe("fresh-token"); + expect(result.restoredConfig.mcpServers.filesystem.command).toBe("npx"); + expect(result.restoredConfig.plugins.entries).toEqual({ + "user-plugin": { enabled: true }, + weather: { enabled: true, config: { revision: "fresh" } }, + }); + expect(result.cleanupCommand).toContain("! -name 'nemoclaw'"); + expect(result.cleanupCommand).toContain("! -name 'weather'"); + }, + ); it("uses fresh primary-model routing during an ordinary sandbox re-create (#7011)", () => { const result = runRestoreScenario({ diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 3af01a7e231..eeb860a36fb 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -39,7 +39,9 @@ import { isOpenShellSandboxPolicyCredentialFree, resolveOpenshellSandboxSshHost, } from "../adapters/openshell/client.js"; +import { buildSelectedOpenShellSubprocessEnv } from "../adapters/openshell/command-argv.js"; import { resolveOpenshell } from "../adapters/openshell/resolve.js"; +import type { OpenShellRuntimeSelection } from "../adapters/openshell/runtime-selection.js"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts.js"; import type { AgentStateFile } from "../agent/defs.js"; import { loadAgent } from "../agent/defs.js"; @@ -271,6 +273,8 @@ export interface RecreatedSandboxRestoreOptions extends SnapshotRestoreOptions { allowCustomImageWholeStateFileRestore?: true; /** Pre-captured baseline avoids a second remote read during onboarding finalization. */ freshOpenClawImagePluginInstalls?: readonly OpenClawImagePluginInstall[]; + /** Exact OpenShell target frozen by the enclosing rebuild transaction. */ + runtimeSelection?: OpenShellRuntimeSelection; } interface InternalRestoreOptions { @@ -278,6 +282,7 @@ interface InternalRestoreOptions { allowCustomImageWholeStateFileRestore?: true; discoverFreshOpenClawImagePluginInstalls?: true; freshOpenClawImagePluginInstalls?: readonly OpenClawImagePluginInstall[]; + runtimeSelection?: OpenShellRuntimeSelection; authority?: SnapshotRestoreAuthority; validateBeforeMutation?: () => void; } @@ -716,6 +721,18 @@ export function getSshConfig( return result.output; } +function selectedSshConfigOptions( + runtimeSelection?: OpenShellRuntimeSelection, +): Parameters[1] { + return runtimeSelection + ? { + env: buildSelectedOpenShellSubprocessEnv(runtimeSelection), + gatewayName: runtimeSelection.gatewayName, + replaceEnv: true, + } + : undefined; +} + export function sshArgs(configFile: string, sandboxName: string): string[] { const sshHost = resolveOpenshellSandboxSshHost(sandboxName, readFileSync(configFile, "utf8")); if (sshHost === null) { @@ -2052,6 +2069,7 @@ export function restoreRecreatedSandboxState( ? { discoverFreshOpenClawImagePluginInstalls: true } : {}), freshOpenClawImagePluginInstalls: options.freshOpenClawImagePluginInstalls, + ...(options.runtimeSelection ? { runtimeSelection: options.runtimeSelection } : {}), ...(options.authority ? { authority: options.authority } : {}), ...(options.validateBeforeMutation ? { validateBeforeMutation: options.validateBeforeMutation } @@ -2065,6 +2083,9 @@ function restoreSandboxStateInternal( options: InternalRestoreOptions, ): RestoreResult { _log(`restoreSandboxState: sandbox=${sandboxName}, backupPath=${backupPath}`); + const selectedSshEnv = options.runtimeSelection + ? buildSelectedOpenShellSubprocessEnv(options.runtimeSelection) + : undefined; const manifest = readManifest(backupPath); if (!manifest) { _log("FAILED: Could not read rebuild-manifest.json"); @@ -2238,7 +2259,12 @@ function restoreSandboxStateInternal( } else if (options.discoverFreshOpenClawImagePluginInstalls === true) { const discovery = discoverFreshOpenClawImagePluginInstalls( sandboxName, - { getSshConfig, sshArgs }, + { + ...(selectedSshEnv ? { env: selectedSshEnv } : {}), + getSshConfig: (name) => + getSshConfig(name, selectedSshConfigOptions(options.runtimeSelection)), + sshArgs, + }, targetAgent.configPaths.dir, ); if (!discovery.ok) { @@ -2261,7 +2287,10 @@ function restoreSandboxStateInternal( } _log("Getting SSH config for restore"); - const sshConfig = getSshConfig(sandboxName); + const sshConfig = getSshConfig( + sandboxName, + selectedSshConfigOptions(options.runtimeSelection), + ); if (!sshConfig) { _log("FAILED: Could not get SSH config for restore"); return { @@ -2367,6 +2396,7 @@ function restoreSandboxStateInternal( ); _log(`Cleaning target dirs before restore: ${rmCmd}`); const rmResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), rmCmd], { + ...(selectedSshEnv ? { env: selectedSshEnv } : {}), stdio: ["ignore", "pipe", "pipe"], timeout: 30000, }); @@ -2390,6 +2420,7 @@ function restoreSandboxStateInternal( if (restoreTar !== undefined) { const extractCmd = `tar --no-same-owner -xf - -C ${shellQuote(dir)}`; const sshResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), extractCmd], { + ...(selectedSshEnv ? { env: selectedSshEnv } : {}), input: restoreTar, stdio: ["pipe", "pipe", "pipe"], timeout: 120000, @@ -2405,6 +2436,7 @@ function restoreSandboxStateInternal( const chownCmd = `chown -R sandbox:sandbox -- ${restoredPaths.map(shellQuote).join(" ")} 2>/dev/null || true`; _log(`Best-effort ownership repair: ${chownCmd}`); const chownResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), chownCmd], { + ...(selectedSshEnv ? { env: selectedSshEnv } : {}), stdio: ["ignore", "pipe", "pipe"], timeout: 30000, }); @@ -2428,6 +2460,7 @@ function restoreSandboxStateInternal( "ssh", [...sshArgs(configFile, sandboxName), usabilityCmd], { + ...(selectedSshEnv ? { env: selectedSshEnv } : {}), stdio: ["ignore", "pipe", "pipe"], timeout: 30000, }, @@ -2464,6 +2497,7 @@ function restoreSandboxStateInternal( _log, configFreshOpenClawImagePluginInstalls, previousOpenClawImagePluginInstalls, + selectedSshEnv, ) ) { restoredFiles.push(spec.path); diff --git a/src/lib/state/state-file-restore.ts b/src/lib/state/state-file-restore.ts index 5219ef8ab67..fa5c277d66e 100644 --- a/src/lib/state/state-file-restore.ts +++ b/src/lib/state/state-file-restore.ts @@ -147,6 +147,7 @@ export function restoreStateFile( log: (message: string) => void, freshImagePluginInstalls?: readonly OpenClawImagePluginInstall[], previousImagePluginInstalls?: readonly OpenClawImagePluginInstall[], + env?: NodeJS.ProcessEnv, ): boolean { const localPath = path.join(backupPath, spec.path); if (!existsSync(localPath)) return true; @@ -161,6 +162,7 @@ export function restoreStateFile( const result = buildOpenClawConfigRestoreInputFromSandbox({ backupContents, dir, + env, freshImagePluginInstalls, log, previousImagePluginInstalls, @@ -185,6 +187,7 @@ export function restoreStateFile( if (input === null) return false; const result = spawnSync("ssh", [...sshArgs, command], { + ...(env ? { env } : {}), input, stdio: ["pipe", "pipe", "pipe"], timeout: 120000, diff --git a/src/lib/state/user-managed-files-probe.test.ts b/src/lib/state/user-managed-files-probe.test.ts index 721cd571b99..ac441de53fe 100644 --- a/src/lib/state/user-managed-files-probe.test.ts +++ b/src/lib/state/user-managed-files-probe.test.ts @@ -13,12 +13,16 @@ type SandboxStateModule = typeof import("./sandbox"); type RegistryModule = typeof import("./registry"); type DefsModule = typeof import("../agent/defs"); type ProbeModule = typeof import("./user-managed-files-probe"); +type OpenShellClientModule = typeof import("../adapters/openshell/client"); +type OpenShellResolveModule = typeof import("../adapters/openshell/resolve"); const requireDist = createRequire(import.meta.url); const sandboxStatePath = "./sandbox.js"; const registryPath = "./registry.js"; const defsPath = "../agent/defs.js"; const probePath = "./user-managed-files-probe.js"; +const openshellClientPath = "../adapters/openshell/client.js"; +const openshellResolvePath = "../adapters/openshell/resolve.js"; function loadProbe(): ProbeModule { delete require.cache[requireDist.resolve(probePath)]; @@ -37,6 +41,14 @@ function loadDefs(): DefsModule { return requireDist(defsPath); } +function loadOpenShellClient(): OpenShellClientModule { + return requireDist(openshellClientPath); +} + +function loadOpenShellResolve(): OpenShellResolveModule { + return requireDist(openshellResolvePath); +} + function makeFakeAgent(declared: string[]): ReturnType { return { name: "fake-agent", @@ -70,12 +82,14 @@ function makeFakeAgent(declared: string[]): ReturnType describe("probeUserManagedFiles", () => { let recordedArgs: string[][]; + let recordedOptions: Array>; let spawnSpy: ReturnType; let tempSshFiles: Set; let originalMkdtempSync: typeof fs.mkdtempSync; beforeEach(() => { recordedArgs = []; + recordedOptions = []; tempSshFiles = new Set(); const sandboxState = loadSandboxState(); const registry = loadRegistry(); @@ -99,6 +113,7 @@ describe("probeUserManagedFiles", () => { }); afterEach(() => { + vi.unstubAllEnvs(); vi.restoreAllMocks(); for (const dir of tempSshFiles) { try { @@ -113,10 +128,12 @@ describe("probeUserManagedFiles", () => { spawnSpy = vi.spyOn(child_process, "spawnSync").mockImplementation((( command: string, args?: readonly string[], + options?: Record, ) => { const argList = Array.isArray(args) ? [...args] : []; const sshCall = command === "ssh" && argList.length > 0; sshCall && recordedArgs.push(argList); + sshCall && recordedOptions.push(options ?? {}); return { status, signal: null, @@ -145,6 +162,44 @@ describe("probeUserManagedFiles", () => { expect(probeCmd).not.toContain("/sandbox/.fake/"); }); + it("pins SSH config and ProxyCommand environment to the frozen target (#10514)", () => { + vi.stubEnv("OPENSHELL_GATEWAY", "hostile-gateway"); + vi.stubEnv("OPENSHELL_WORKSPACE", "hostile-workspace"); + vi.stubEnv("OPENSHELL_LOCAL_TLS_DIR", "/hostile/tls"); + vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://hostile.invalid"); + const runtimeSelection = { + gatewayName: "recorded-gateway", + workspace: "default", + localTlsDir: "/authority/tls", + }; + const sandboxState = loadSandboxState(); + const getSshConfig = vi.mocked(sandboxState.getSshConfig); + stubSpawnSync(".env\n", 0); + const { probeUserManagedFiles } = loadProbe(); + + probeUserManagedFiles("alpha", runtimeSelection); + + expect(getSshConfig).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + gatewayName: "recorded-gateway", + replaceEnv: true, + env: expect.objectContaining({ + OPENSHELL_GATEWAY: "recorded-gateway", + OPENSHELL_WORKSPACE: "default", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + }), + }), + ); + const env = recordedOptions[0]?.env as Record; + expect(env).toMatchObject({ + OPENSHELL_GATEWAY: "recorded-gateway", + OPENSHELL_WORKSPACE: "default", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + }); + expect(env).not.toHaveProperty("OPENSHELL_GATEWAY_ENDPOINT"); + }); + it("supports nested declared files relative to the sandbox root", () => { const defs = loadDefs(); vi.spyOn(defs, "loadAgent").mockImplementation(() => makeFakeAgent([".hermes/.env"])); @@ -255,3 +310,50 @@ describe("probeUserManagedFiles", () => { expect(result.existing).toEqual([]); }); }); + +describe("getSshConfig frozen target", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("uses replacement environment, workspace, TLS, and gateway for SSH config (#10514)", () => { + vi.stubEnv("OPENSHELL_GATEWAY", "hostile-gateway"); + vi.stubEnv("OPENSHELL_WORKSPACE", "hostile-workspace"); + vi.stubEnv("OPENSHELL_LOCAL_TLS_DIR", "/hostile/tls"); + vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://hostile.invalid"); + vi.spyOn(loadOpenShellResolve(), "resolveOpenshell").mockReturnValue("/usr/bin/openshell"); + const capture = vi + .spyOn(loadOpenShellClient(), "captureSandboxSshConfigCommand") + .mockReturnValue({ status: 0, output: "Host openshell-alpha.default\n" }); + + const runtimeOptions = { + gatewayName: "recorded-gateway", + replaceEnv: true, + env: { + OPENSHELL_GATEWAY: "recorded-gateway", + OPENSHELL_WORKSPACE: "default", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + }, + }; + expect(loadSandboxState().getSshConfig("alpha", runtimeOptions)).toContain( + "openshell-alpha.default", + ); + + expect(capture).toHaveBeenCalledWith( + "/usr/bin/openshell", + "alpha", + expect.objectContaining({ + gatewayName: "recorded-gateway", + replaceEnv: true, + env: expect.objectContaining({ + OPENSHELL_GATEWAY: "recorded-gateway", + OPENSHELL_WORKSPACE: "default", + OPENSHELL_LOCAL_TLS_DIR: "/authority/tls", + }), + }), + ); + const env = capture.mock.calls[0]?.[2]?.env as Record; + expect(env).not.toHaveProperty("OPENSHELL_GATEWAY_ENDPOINT"); + }); +}); diff --git a/src/lib/state/user-managed-files-probe.ts b/src/lib/state/user-managed-files-probe.ts index f89ab8658e3..4f61f56a3cc 100644 --- a/src/lib/state/user-managed-files-probe.ts +++ b/src/lib/state/user-managed-files-probe.ts @@ -4,6 +4,8 @@ import { spawnSync } from "child_process"; import { loadAgent } from "../agent/defs.js"; +import { buildSelectedOpenShellSubprocessEnv } from "../adapters/openshell/command-argv.js"; +import type { OpenShellRuntimeSelection } from "../adapters/openshell/runtime-selection.js"; import { shellQuote } from "../runner.js"; import { createTempSshConfig } from "../sandbox/temp-ssh-config.js"; @@ -23,7 +25,10 @@ function _log(msg: string): void { if (_verbose()) console.error(` [user-managed-files-probe ${new Date().toISOString()}] ${msg}`); } -export function probeUserManagedFiles(sandboxName: string): UserManagedFilesProbe { +export function probeUserManagedFiles( + sandboxName: string, + runtimeSelection?: OpenShellRuntimeSelection, +): UserManagedFilesProbe { const sb = registry.getSandbox(sandboxName); const agentName = sb?.agent || "openclaw"; const agent = loadAgent(agentName); @@ -34,7 +39,19 @@ export function probeUserManagedFiles(sandboxName: string): UserManagedFilesProb `sandbox=${sandboxName}, agent=${agentName}, declared=[${declared.join(",")}], base=${USER_MANAGED_FILES_BASE}`, ); - const sshConfig = getSshConfig(sandboxName); + const selectedEnv = runtimeSelection + ? buildSelectedOpenShellSubprocessEnv(runtimeSelection) + : undefined; + const sshConfig = getSshConfig( + sandboxName, + runtimeSelection + ? { + env: selectedEnv, + gatewayName: runtimeSelection.gatewayName, + replaceEnv: true, + } + : undefined, + ); if (!sshConfig) { _log("no SSH config — cannot probe declared user-managed files"); throw new Error( @@ -54,6 +71,7 @@ export function probeUserManagedFiles(sandboxName: string): UserManagedFilesProb .join("; ") + " 2>/dev/null"; const result = spawnSync("ssh", [...sshArgs(configFile, sandboxName), probeCmd], { encoding: "utf-8", + ...(selectedEnv ? { env: selectedEnv } : {}), stdio: ["ignore", "pipe", "pipe"], timeout: 30000, }); diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 9879e3b1fc8..c180db8ca0b 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -41,6 +41,8 @@ export const gatewayTeardownAuthority = requireDist( ) as typeof import("../../src/lib/onboard/gateway-teardown-authority"); export const hermesProviderAuth = requireDist("../../hermes-provider-auth.js"); export const mcpBridge = requireDist("./mcp-bridge.js"); +export const mcpBridgeProvider = requireDist("./mcp-bridge-provider.js"); +export const mcpBridgeProviderInspection = requireDist("./mcp-bridge-provider-inspection.js"); export const messaging = requireDist("../../messaging/index.js"); export const messagingHostForwardLifecycle = requireDist("./messaging-host-forward-lifecycle.js"); export const mutableConfigPerms = requireDist("../../sandbox/mutable-config-perms.js"); diff --git a/test/mcp/mcp-destroy-lifecycle.test.ts b/test/mcp/mcp-destroy-lifecycle.test.ts index 4984227e5d4..a6a78f9e652 100644 --- a/test/mcp/mcp-destroy-lifecycle.test.ts +++ b/test/mcp/mcp-destroy-lifecycle.test.ts @@ -49,6 +49,7 @@ const testState = vi.hoisted(() => { removePreset: vi.fn(), runOpenshell: vi.fn(), runOpenshellProviderCommand: vi.fn(), + runtimeSelection: { gatewayName: "nemoclaw", workspace: "default" }, stopNimContainer: vi.fn(), stopNimContainerByName: vi.fn(), warnUnpreservedUserManagedFiles: vi.fn(), @@ -981,8 +982,11 @@ describe("authenticated MCP sandbox destroy lifecycle", () => { expect(testState.executeSandboxExecCommand).toHaveBeenCalledOnce(); expect(testState.executeSandboxExecCommand).toHaveBeenCalledWith("alpha", ":", undefined, { allowLocalDockerFallback: false, + runtimeSelection: testState.runtimeSelection, + }); + expect(testState.executeSandboxCommand).toHaveBeenCalledWith("alpha", ":", { + runtimeSelection: testState.runtimeSelection, }); - expect(testState.executeSandboxCommand).toHaveBeenCalledWith("alpha", ":"); expect(testState.runOpenshell).toHaveBeenCalledWith( ["sandbox", "delete", "-g", "nemoclaw", "alpha"], expect.any(Object),