Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/lib/actions/sandbox/connect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,11 +85,12 @@ import {
type GatewayRestartFailureLayer,
type ManagedGatewayControlCompletion,
resolveSandboxDashboardPort,
waitForManagedGatewaySupervisor,
} from "./process-recovery";
import { runTerminalAgentConnectProbe } from "./terminal-connect-probe";
import { applyOpenShellVmDnsMonkeypatch, shouldApplyVmDnsMonkeypatch } from "./vm-dns-monkeypatch";

export { runConnectAutoPairApprovalPass };
export { runConnectAutoPairApprovalPass, waitForManagedGatewaySupervisor };

export type SandboxConnectOptions = {
probeOnly?: boolean;
Expand Down
124 changes: 124 additions & 0 deletions src/lib/actions/sandbox/start.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ function harness(overrides: Partial<SandboxStartDeps> = {}) {
const restoreStartupState = vi.fn<NonNullable<SandboxStartDeps["restoreStartupState"]>>(
() => SUCCESSFUL_RECOVERY,
);
const waitForManagedGatewaySupervisor = vi.fn<
NonNullable<SandboxStartDeps["waitForManagedGatewaySupervisor"]>
>(() => false);
const log = vi.fn<(message: string) => void>();
const runtimeProviders = createRuntimeProviderBundleRegistry([
[
Expand All @@ -76,6 +79,7 @@ function harness(overrides: Partial<SandboxStartDeps> = {}) {
getSandbox,
runtimeProviders,
restoreStartupState,
waitForManagedGatewaySupervisor,
verifyGateway,
log,
...overrides,
Expand All @@ -90,6 +94,7 @@ function harness(overrides: Partial<SandboxStartDeps> = {}) {
printDockerRuntimeDownGuidance,
recoverDockerDriverSandbox,
restoreStartupState,
waitForManagedGatewaySupervisor,
verifyGateway,
};
}
Expand Down Expand Up @@ -164,6 +169,125 @@ describe("startSandbox", () => {
);
});

it("waits for a transient managed supervisor before repeating full startup recovery (#8726)", async () => {
const h = harness();
h.restoreStartupState
.mockReturnValueOnce({
...FAILED_RECOVERY,
recoveryFailureLayer: "supervisor not running",
recoveryFailureDetail: "SUPERVISOR_NOT_RUNNING",
})
.mockReturnValueOnce(SUCCESSFUL_RECOVERY);
h.waitForManagedGatewaySupervisor.mockReturnValue(true);

const result = await startSandbox("my-sandbox", h.deps);

expect(result.exitCode).toBe(0);
expect(h.restoreStartupState).toHaveBeenCalledTimes(2);
expect(h.waitForManagedGatewaySupervisor).toHaveBeenCalledOnce();
expect(h.waitForManagedGatewaySupervisor).toHaveBeenCalledWith("my-sandbox");
expect(h.verifyGateway).toHaveBeenCalledOnce();
expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan(
h.waitForManagedGatewaySupervisor.mock.invocationCallOrder[0],
);
expect(h.waitForManagedGatewaySupervisor.mock.invocationCallOrder[0]).toBeLessThan(
h.restoreStartupState.mock.invocationCallOrder[1],
);
expect(h.restoreStartupState.mock.invocationCallOrder[1]).toBeLessThan(
h.verifyGateway.mock.invocationCallOrder[0],
);
});

it("preserves the first recovery failure when the managed supervisor remains absent (#8726)", async () => {
const h = harness();
h.restoreStartupState.mockReturnValue({
...FAILED_RECOVERY,
recoveryFailureLayer: "supervisor not running",
recoveryFailureDetail: "SUPERVISOR_NOT_RUNNING",
});

await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow(
"supervisor not running: SUPERVISOR_NOT_RUNNING",
);
expect(h.restoreStartupState).toHaveBeenCalledOnce();
expect(h.waitForManagedGatewaySupervisor).toHaveBeenCalledOnce();
expect(h.verifyGateway).not.toHaveBeenCalled();
});

it("preserves the first recovery failure when the managed supervisor wait throws (#8726)", async () => {
const h = harness();
h.restoreStartupState.mockReturnValue({
...FAILED_RECOVERY,
recoveryFailureLayer: "supervisor not running",
recoveryFailureDetail: "SUPERVISOR_NOT_RUNNING",
});
h.waitForManagedGatewaySupervisor.mockImplementation(() => {
throw new Error("managed supervisor probe failed");
});

await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow(
"supervisor not running: SUPERVISOR_NOT_RUNNING",
);
expect(h.restoreStartupState).toHaveBeenCalledOnce();
expect(h.waitForManagedGatewaySupervisor).toHaveBeenCalledOnce();
expect(h.verifyGateway).not.toHaveBeenCalled();
});

it("fails closed when recovery still fails after the managed supervisor appears (#8726)", async () => {
const h = harness();
const missingSupervisor = {
...FAILED_RECOVERY,
recoveryFailureLayer: "supervisor not running" as const,
recoveryFailureDetail: "SUPERVISOR_NOT_RUNNING",
};
h.restoreStartupState.mockReturnValue(missingSupervisor);
h.waitForManagedGatewaySupervisor.mockReturnValue(true);

await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow(
"supervisor not running: SUPERVISOR_NOT_RUNNING",
);
expect(h.restoreStartupState).toHaveBeenCalledTimes(2);
expect(h.waitForManagedGatewaySupervisor).toHaveBeenCalledOnce();
expect(h.verifyGateway).not.toHaveBeenCalled();
});

it.each([
["definitive supervisor failure", "supervisor unavailable", "SUPERVISOR_UNAVAILABLE"],
[
"unclassified missing-supervisor output",
"supervisor not running",
"prefix SUPERVISOR_NOT_RUNNING suffix",
],
] as const)("does not wait after a %s (#8726)", async (_label, layer, detail) => {
const h = harness();
h.restoreStartupState.mockReturnValue({
...FAILED_RECOVERY,
recoveryFailureLayer: layer,
recoveryFailureDetail: detail,
});

await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow(detail);
expect(h.restoreStartupState).toHaveBeenCalledOnce();
expect(h.waitForManagedGatewaySupervisor).not.toHaveBeenCalled();
expect(h.verifyGateway).not.toHaveBeenCalled();
});

it("keeps successful legacy supervisor relaunch recovery free of a settling wait (#8726)", async () => {
const h = harness();
h.restoreStartupState.mockReturnValue({
...SUCCESSFUL_RECOVERY,
wasRunning: false,
recovered: true,
});

const result = await startSandbox("my-sandbox", h.deps);

expect(result.exitCode).toBe(0);
expect(h.restoreStartupState).toHaveBeenCalledOnce();
expect(h.waitForManagedGatewaySupervisor).not.toHaveBeenCalled();
expect(h.verifyGateway).toHaveBeenCalledOnce();
});

it.each([
[
"openclaw",
Expand Down
34 changes: 33 additions & 1 deletion src/lib/actions/sandbox/start.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,22 @@ export interface SandboxStartDeps {
getSandbox?: typeof registry.getSandbox;
runtimeProviders?: RuntimeProviderBundleRegistry;
restoreStartupState?: (sandboxName: string) => SandboxStartupRecoveryResult;
waitForManagedGatewaySupervisor?: (sandboxName: string) => boolean;
verifyGateway?: (sandboxName: string) => Promise<void>;
log?: (message: string) => void;
}

function isMissingManagedSupervisorStartupFailure(
check: SandboxStartupRecoveryResult,
failure: string,
): boolean {
return (
failure === "supervisor not running: SUPERVISOR_NOT_RUNNING" &&
check.recoveryFailureLayer === "supervisor not running" &&
check.recoveryFailureDetail === "SUPERVISOR_NOT_RUNNING"
);
}

function startupRecoveryFailure(check: SandboxStartupRecoveryResult): string | null {
if (!check.checked) return "managed agent gateway inspection did not complete";
if ("runtime" in check && check.runtime === "terminal") return null;
Expand Down Expand Up @@ -132,7 +144,27 @@ export async function startSandbox(
} catch (error) {
throw preservedSandboxRecoveryError(name, error);
}
const failure = startupRecoveryFailure(recovery);
let failure = startupRecoveryFailure(recovery);
if (failure && isMissingManagedSupervisorStartupFailure(recovery, failure)) {
let supervisorReady = false;
try {
const waitForSupervisor =
deps.waitForManagedGatewaySupervisor ??
(require("./connect") as typeof import("./connect")).waitForManagedGatewaySupervisor;
supervisorReady = waitForSupervisor(name);
} catch {
// Preserve the authoritative first recovery failure when the bounded
// read-only settling probe itself cannot complete.
}
if (supervisorReady) {
try {
recovery = restoreStartupState(name);
} catch (error) {
throw preservedSandboxRecoveryError(name, error);
}
failure = startupRecoveryFailure(recovery);
}
}
if (failure) throw preservedSandboxRecoveryError(name, failure);
log(" Checking gateway health and host forwards…");
await (deps.verifyGateway ?? verifyGateway)(name);
Expand Down
Loading