diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 689d7d22278..3d7cac3e955 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -85,11 +85,12 @@ import { type GatewayRestartFailureLayer, type ManagedGatewayControlCompletion, resolveSandboxDashboardPort, + waitForManagedGatewaySupervisor, } from "./process-recovery"; import { runTerminalAgentConnectProbe } from "./terminal-connect-probe"; import { applyOpenShellVmDnsMonkeypatch, shouldApplyVmDnsMonkeypatch } from "./vm-dns-monkeypatch"; -export { runConnectAutoPairApprovalPass }; +export { runConnectAutoPairApprovalPass, waitForManagedGatewaySupervisor }; export type SandboxConnectOptions = { probeOnly?: boolean; diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index 2037a957c3e..3d984c1f0d3 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -58,6 +58,9 @@ function harness(overrides: Partial = {}) { const restoreStartupState = vi.fn>( () => SUCCESSFUL_RECOVERY, ); + const waitForManagedGatewaySupervisor = vi.fn< + NonNullable + >(() => false); const log = vi.fn<(message: string) => void>(); const runtimeProviders = createRuntimeProviderBundleRegistry([ [ @@ -76,6 +79,7 @@ function harness(overrides: Partial = {}) { getSandbox, runtimeProviders, restoreStartupState, + waitForManagedGatewaySupervisor, verifyGateway, log, ...overrides, @@ -90,6 +94,7 @@ function harness(overrides: Partial = {}) { printDockerRuntimeDownGuidance, recoverDockerDriverSandbox, restoreStartupState, + waitForManagedGatewaySupervisor, verifyGateway, }; } @@ -164,6 +169,125 @@ describe("startSandbox", () => { ); }); + it("waits for a transient managed supervisor before repeating full startup recovery (#8726)", async () => { + const h = harness(); + h.restoreStartupState + .mockReturnValueOnce({ + ...FAILED_RECOVERY, + recoveryFailureLayer: "supervisor not running", + recoveryFailureDetail: "SUPERVISOR_NOT_RUNNING", + }) + .mockReturnValueOnce(SUCCESSFUL_RECOVERY); + h.waitForManagedGatewaySupervisor.mockReturnValue(true); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(0); + expect(h.restoreStartupState).toHaveBeenCalledTimes(2); + expect(h.waitForManagedGatewaySupervisor).toHaveBeenCalledOnce(); + expect(h.waitForManagedGatewaySupervisor).toHaveBeenCalledWith("my-sandbox"); + expect(h.verifyGateway).toHaveBeenCalledOnce(); + expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan( + h.waitForManagedGatewaySupervisor.mock.invocationCallOrder[0], + ); + expect(h.waitForManagedGatewaySupervisor.mock.invocationCallOrder[0]).toBeLessThan( + h.restoreStartupState.mock.invocationCallOrder[1], + ); + expect(h.restoreStartupState.mock.invocationCallOrder[1]).toBeLessThan( + h.verifyGateway.mock.invocationCallOrder[0], + ); + }); + + it("preserves the first recovery failure when the managed supervisor remains absent (#8726)", async () => { + const h = harness(); + h.restoreStartupState.mockReturnValue({ + ...FAILED_RECOVERY, + recoveryFailureLayer: "supervisor not running", + recoveryFailureDetail: "SUPERVISOR_NOT_RUNNING", + }); + + await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow( + "supervisor not running: SUPERVISOR_NOT_RUNNING", + ); + expect(h.restoreStartupState).toHaveBeenCalledOnce(); + expect(h.waitForManagedGatewaySupervisor).toHaveBeenCalledOnce(); + expect(h.verifyGateway).not.toHaveBeenCalled(); + }); + + it("preserves the first recovery failure when the managed supervisor wait throws (#8726)", async () => { + const h = harness(); + h.restoreStartupState.mockReturnValue({ + ...FAILED_RECOVERY, + recoveryFailureLayer: "supervisor not running", + recoveryFailureDetail: "SUPERVISOR_NOT_RUNNING", + }); + h.waitForManagedGatewaySupervisor.mockImplementation(() => { + throw new Error("managed supervisor probe failed"); + }); + + await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow( + "supervisor not running: SUPERVISOR_NOT_RUNNING", + ); + expect(h.restoreStartupState).toHaveBeenCalledOnce(); + expect(h.waitForManagedGatewaySupervisor).toHaveBeenCalledOnce(); + expect(h.verifyGateway).not.toHaveBeenCalled(); + }); + + it("fails closed when recovery still fails after the managed supervisor appears (#8726)", async () => { + const h = harness(); + const missingSupervisor = { + ...FAILED_RECOVERY, + recoveryFailureLayer: "supervisor not running" as const, + recoveryFailureDetail: "SUPERVISOR_NOT_RUNNING", + }; + h.restoreStartupState.mockReturnValue(missingSupervisor); + h.waitForManagedGatewaySupervisor.mockReturnValue(true); + + await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow( + "supervisor not running: SUPERVISOR_NOT_RUNNING", + ); + expect(h.restoreStartupState).toHaveBeenCalledTimes(2); + expect(h.waitForManagedGatewaySupervisor).toHaveBeenCalledOnce(); + expect(h.verifyGateway).not.toHaveBeenCalled(); + }); + + it.each([ + ["definitive supervisor failure", "supervisor unavailable", "SUPERVISOR_UNAVAILABLE"], + [ + "unclassified missing-supervisor output", + "supervisor not running", + "prefix SUPERVISOR_NOT_RUNNING suffix", + ], + ] as const)("does not wait after a %s (#8726)", async (_label, layer, detail) => { + const h = harness(); + h.restoreStartupState.mockReturnValue({ + ...FAILED_RECOVERY, + recoveryFailureLayer: layer, + recoveryFailureDetail: detail, + }); + + await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow(detail); + expect(h.restoreStartupState).toHaveBeenCalledOnce(); + expect(h.waitForManagedGatewaySupervisor).not.toHaveBeenCalled(); + expect(h.verifyGateway).not.toHaveBeenCalled(); + }); + + it("keeps successful legacy supervisor relaunch recovery free of a settling wait (#8726)", async () => { + const h = harness(); + h.restoreStartupState.mockReturnValue({ + ...SUCCESSFUL_RECOVERY, + wasRunning: false, + recovered: true, + }); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(0); + expect(h.restoreStartupState).toHaveBeenCalledOnce(); + expect(h.waitForManagedGatewaySupervisor).not.toHaveBeenCalled(); + expect(h.verifyGateway).toHaveBeenCalledOnce(); + }); + it.each([ [ "openclaw", diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index aeb74de90b6..1a197771925 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -54,10 +54,22 @@ export interface SandboxStartDeps { getSandbox?: typeof registry.getSandbox; runtimeProviders?: RuntimeProviderBundleRegistry; restoreStartupState?: (sandboxName: string) => SandboxStartupRecoveryResult; + waitForManagedGatewaySupervisor?: (sandboxName: string) => boolean; verifyGateway?: (sandboxName: string) => Promise; log?: (message: string) => void; } +function isMissingManagedSupervisorStartupFailure( + check: SandboxStartupRecoveryResult, + failure: string, +): boolean { + return ( + failure === "supervisor not running: SUPERVISOR_NOT_RUNNING" && + check.recoveryFailureLayer === "supervisor not running" && + check.recoveryFailureDetail === "SUPERVISOR_NOT_RUNNING" + ); +} + function startupRecoveryFailure(check: SandboxStartupRecoveryResult): string | null { if (!check.checked) return "managed agent gateway inspection did not complete"; if ("runtime" in check && check.runtime === "terminal") return null; @@ -132,7 +144,27 @@ export async function startSandbox( } catch (error) { throw preservedSandboxRecoveryError(name, error); } - const failure = startupRecoveryFailure(recovery); + let failure = startupRecoveryFailure(recovery); + if (failure && isMissingManagedSupervisorStartupFailure(recovery, failure)) { + let supervisorReady = false; + try { + const waitForSupervisor = + deps.waitForManagedGatewaySupervisor ?? + (require("./connect") as typeof import("./connect")).waitForManagedGatewaySupervisor; + supervisorReady = waitForSupervisor(name); + } catch { + // Preserve the authoritative first recovery failure when the bounded + // read-only settling probe itself cannot complete. + } + if (supervisorReady) { + try { + recovery = restoreStartupState(name); + } catch (error) { + throw preservedSandboxRecoveryError(name, error); + } + failure = startupRecoveryFailure(recovery); + } + } if (failure) throw preservedSandboxRecoveryError(name, failure); log(" Checking gateway health and host forwards…"); await (deps.verifyGateway ?? verifyGateway)(name);