diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index bfab0cdd343..23b02cff418 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -119,6 +119,78 @@ describe("recreated sandbox OpenShell readiness", () => { expect(captureOpenshellImpl).toHaveBeenCalledOnce(); expect(sleeps).toEqual([3]); }); + + it("retries an inconclusive managed guard within the readiness deadline", () => { + const captureOpenshellImpl = vi.fn(() => ({ + status: 0, + output: "", + stdout: "", + stderr: "", + })); + const beforeProbe = vi.fn().mockReturnValueOnce(null).mockReturnValueOnce(true); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 6, + }), + ).toBe(true); + expect(beforeProbe).toHaveBeenCalledTimes(2); + expect(captureOpenshellImpl).toHaveBeenCalledOnce(); + expect(sleeps).toEqual([3]); + }); + + it("fails closed on a definitive managed guard failure without probing OpenShell", () => { + const captureOpenshellImpl = vi.fn(() => ({ + status: 0, + output: "", + stdout: "", + stderr: "", + })); + const beforeProbe = vi.fn(() => false); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 6, + }), + ).toBe(false); + expect(beforeProbe).toHaveBeenCalledOnce(); + expect(captureOpenshellImpl).not.toHaveBeenCalled(); + expect(sleeps).toEqual([]); + }); + + it("fails when the managed guard stays inconclusive until the deadline", () => { + const captureOpenshellImpl = vi.fn(() => ({ + status: 0, + output: "", + stdout: "", + stderr: "", + })); + const beforeProbe = vi.fn(() => null); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 6, + }), + ).toBe(false); + expect(beforeProbe).toHaveBeenCalledTimes(3); + expect(captureOpenshellImpl).not.toHaveBeenCalled(); + expect(sleeps).toEqual([3, 3]); + }); }); describe("confirmRecoveredSandboxGatewayManaged scope", () => { diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 9024fc530cb..c1ab3dd11b5 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -601,23 +601,47 @@ function isExactlyRetryableOpenshellSandboxNotReady( ); } +type RecreatedSandboxOpenShellReadinessFailure = + | "managed-health-definitive-failure" + | "managed-health-inconclusive-timeout" + | "openshell-readiness-failure"; + +type RecreatedSandboxOpenShellReadinessResult = + | { ready: true } + | { failure: RecreatedSandboxOpenShellReadinessFailure; ready: false }; + +type RecreatedSandboxOpenShellReadyOptions = { + captureOpenshellImpl?: typeof captureOpenshell; + beforeProbe?: (timeoutMs: number) => boolean | null; + intervalSeconds?: number; + nowImpl?: () => number; + sleepImpl?: (seconds: number) => void; + timeoutSeconds?: number; +}; + +function recreatedSandboxOpenShellReadinessFailureDetail( + failure: RecreatedSandboxOpenShellReadinessFailure, +): string { + switch (failure) { + case "managed-health-definitive-failure": + return "the recreated sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; + case "managed-health-inconclusive-timeout": + return "the recreated sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; + case "openshell-readiness-failure": + return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; + } +} + /** * Wait until OpenShell has re-registered a directly recreated sandbox as * ready. This probe deliberately has no direct-Docker or SSH fallback: it is * proving control-plane readiness, not authorizing the already completed * replacement-container recovery. */ -export function waitForRecreatedSandboxOpenShellReady( +function waitForRecreatedSandboxOpenShellReadyResult( sandboxName: string, - options: { - captureOpenshellImpl?: typeof captureOpenshell; - beforeProbe?: (timeoutMs: number) => boolean; - intervalSeconds?: number; - nowImpl?: () => number; - sleepImpl?: (seconds: number) => void; - timeoutSeconds?: number; - } = {}, -): boolean { + options: RecreatedSandboxOpenShellReadyOptions = {}, +): RecreatedSandboxOpenShellReadinessResult { const capture = options.captureOpenshellImpl ?? captureOpenshell; const now = options.nowImpl ?? Date.now; const sleep = options.sleepImpl ?? sleepSeconds; @@ -637,25 +661,56 @@ export function waitForRecreatedSandboxOpenShellReady( for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const preGuardRemainingMs = deadlineMs - now(); - if (attempt > 1 && preGuardRemainingMs <= 0) return false; + if (attempt > 1 && preGuardRemainingMs <= 0) { + return { failure: "managed-health-inconclusive-timeout", ready: false }; + } const guardBudgetMs = Math.max(1, Math.min(OPENSHELL_PROBE_TIMEOUT_MS, preGuardRemainingMs)); - if (options.beforeProbe?.(guardBudgetMs) === false) return false; + const guardResult = options.beforeProbe?.(guardBudgetMs); + if (guardResult === false) { + return { failure: "managed-health-definitive-failure", ready: false }; + } + if (guardResult === null) { + if (attempt === maxAttempts) { + return { failure: "managed-health-inconclusive-timeout", ready: false }; + } + const postGuardRemainingMs = deadlineMs - now(); + if (postGuardRemainingMs <= 0) { + return { failure: "managed-health-inconclusive-timeout", ready: false }; + } + sleep(Math.min(intervalSeconds * 1000, postGuardRemainingMs) / 1000); + continue; + } const remainingMs = deadlineMs - now(); - if (attempt > 1 && remainingMs <= 0) return false; + if (attempt > 1 && remainingMs <= 0) { + return { failure: "openshell-readiness-failure", ready: false }; + } const result = capture(["sandbox", "exec", "--name", sandboxName, "--", "true"], { ignoreError: true, includeStderr: true, includeStreams: true, timeout: Math.max(1, Math.min(OPENSHELL_PROBE_TIMEOUT_MS, remainingMs)), }); - if (result.status === 0 && !result.error) return true; - if (!isExactlyRetryableOpenshellSandboxNotReady(result)) return false; - if (attempt === maxAttempts) return false; + if (result.status === 0 && !result.error) return { ready: true }; + if (!isExactlyRetryableOpenshellSandboxNotReady(result)) { + return { failure: "openshell-readiness-failure", ready: false }; + } + if (attempt === maxAttempts) { + return { failure: "openshell-readiness-failure", ready: false }; + } const postProbeRemainingMs = deadlineMs - now(); - if (postProbeRemainingMs <= 0) return false; + if (postProbeRemainingMs <= 0) { + return { failure: "openshell-readiness-failure", ready: false }; + } sleep(Math.min(intervalSeconds * 1000, postProbeRemainingMs) / 1000); } - return false; + return { failure: "openshell-readiness-failure", ready: false }; +} + +export function waitForRecreatedSandboxOpenShellReady( + sandboxName: string, + options: RecreatedSandboxOpenShellReadyOptions = {}, +): boolean { + return waitForRecreatedSandboxOpenShellReadyResult(sandboxName, options).ready; } function gatewayRecoveryTimeoutSeconds( @@ -1055,20 +1110,22 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( let relaunchedIdentityRejected = false; const confirmRelaunchedManagedHealth = relaunch ? (timeout = OPENSHELL_PROBE_TIMEOUT_MS) => { - let confirmed = false; try { - confirmed = - confirmRecoveredSandboxGatewayManaged(sandboxName, { - requestGatewaySupervisorActionImpl: (name, action) => - requestManagedProbe(name, action, timeout), - }) === true; + const confirmed = confirmRecoveredSandboxGatewayManaged(sandboxName, { + requestGatewaySupervisorActionImpl: (name, action) => + requestManagedProbe(name, action, timeout), + }); + if (confirmed === false) relaunchedIdentityRejected = true; + return confirmed; } catch { - confirmed = false; + relaunchedIdentityRejected = true; + return false; } - relaunchedIdentityRejected ||= !confirmed; - return confirmed; } : null; + const confirmRelaunchedManagedHealthForForward = relaunch + ? () => confirmRelaunchedManagedHealth?.() === true + : null; // Wait for gateway to bind its HTTP port before declaring success. The // recovered process can be alive before the OpenAI-compatible API is ready. let gatewayReady = false; @@ -1134,28 +1191,38 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } } } - if ( - relaunch && - !waitForRecreatedSandboxOpenShellReadyImpl(sandboxName, { - beforeProbe: (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) === true, - timeoutSeconds: gatewayRecoveryTimeoutSeconds(recoveryAgent), - }) - ) { + const readinessFailureDetail = relaunch + ? (() => { + const readinessOptions: RecreatedSandboxOpenShellReadyOptions = { + beforeProbe: (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null, + timeoutSeconds: gatewayRecoveryTimeoutSeconds(recoveryAgent), + }; + const readiness = + waitForRecreatedSandboxOpenShellReadyImpl === waitForRecreatedSandboxOpenShellReady + ? waitForRecreatedSandboxOpenShellReadyResult(sandboxName, readinessOptions) + : waitForRecreatedSandboxOpenShellReadyImpl(sandboxName, readinessOptions) + ? ({ ready: true } as const) + : ({ failure: "openshell-readiness-failure", ready: false } as const); + return readiness.ready + ? null + : recreatedSandboxOpenShellReadinessFailureDetail(readiness.failure); + })() + : null; + if (readinessFailureDetail) { return { checked: true, wasRunning: false, recovered: true, forwardRecovered: false, forwardRecoveryFailed: true, - forwardRecoveryFailureDetail: - "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started", + forwardRecoveryFailureDetail: readinessFailureDetail, }; } const mcpRefusal = processRecoveryMcpReconciliationRefusal(sandboxName, false); if (mcpRefusal) return mcpRefusal; const forwardRecovered = ensureSandboxPortForward(sandboxName, { - afterSuccess: confirmRelaunchedManagedHealth ?? undefined, - beforeStart: confirmRelaunchedManagedHealth ?? undefined, + afterSuccess: confirmRelaunchedManagedHealthForForward ?? undefined, + beforeStart: confirmRelaunchedManagedHealthForForward ?? undefined, isWsl: isWslOverride, }); if (!forwardRecovered && relaunchedIdentityRejected) { diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index ef262ce6aef..d16bef438a6 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -212,6 +212,94 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(finalize).toHaveBeenCalledWith(true); }); + it("retries a busy pinned managed probe before starting the replacement forward", () => { + mockOpenClawSandbox("busy-recovered-box"); + vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", "0"); + vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", "1"); + vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", "0"); + vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "0"); + const finalize = vi.fn((supervisorReady: boolean) => + supervisorReady + ? { backupRemoved: true, rolledBack: false } + : { backupRemoved: false, rolledBack: true }, + ); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => + action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, + ); + const acceptedProbe = { + status: 0, + stdout: "GATEWAY_PID=4242\n", + stderr: "", + }; + const requestPinnedGatewaySupervisorAction = vi + .fn() + .mockReturnValueOnce(acceptedProbe) + .mockReturnValueOnce({ status: 1, stdout: "", stderr: "SUPERVISOR_BUSY" }) + .mockReturnValue(acceptedProbe); + let forwardStarted = false; + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockImplementation(() => forwardStarted); + const captureOpenshell = vi + .spyOn(openshellRuntime, "captureOpenshell") + .mockImplementation((args) => { + const command = args.join(" "); + const responses = { + "sandbox exec --name busy-recovered-box -- true": () => ({ + status: 0, + output: "", + stdout: "", + stderr: "", + }), + "forward list": () => ({ + status: 0, + output: forwardStarted + ? "SANDBOX BIND PORT PID STATUS\nbusy-recovered-box 127.0.0.1 18789 12345 running" + : "SANDBOX BIND PORT PID STATUS", + }), + }; + return ( + responses[command as keyof typeof responses]?.() ?? { + status: 1, + output: "", + stdout: "", + stderr: "unexpected openshell command", + } + ); + }); + const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell").mockImplementation((args) => { + forwardStarted ||= args.join(" ") === "forward start --background 18789 busy-recovered-box"; + return { status: 0 } as never; + }); + + const result = checkAndRecoverSandboxProcesses("busy-recovered-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }); + expect(requestPinnedGatewaySupervisorAction).toHaveBeenCalledTimes(5); + expect(captureOpenshell).toHaveBeenCalledWith( + ["sandbox", "exec", "--name", "busy-recovered-box", "--", "true"], + expect.objectContaining({ ignoreError: true }), + ); + expect(finalize).toHaveBeenCalledWith(true); + expect(runOpenshell).toHaveBeenCalledWith( + ["forward", "start", "--background", "18789", "busy-recovered-box"], + expect.objectContaining({ ignoreError: true }), + ); + }); + it("retains a healthy replacement but does not start a forward when OpenShell stays unready", () => { mockOpenClawSandbox("unready-box"); setImmediateRecoveryPolling(); @@ -259,6 +347,54 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(runOpenshell).not.toHaveBeenCalled(); }); + it("reports a definitive managed health failure separately from OpenShell readiness", () => { + mockOpenClawSandbox("managed-failed-box"); + setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "SUPERVISOR_NOT_RUNNING", + })); + const acceptedProbe = { + status: 0, + stdout: "GATEWAY_PID=4242\n", + stderr: "", + }; + const requestPinnedGatewaySupervisorAction = vi + .fn() + .mockReturnValueOnce(acceptedProbe) + .mockReturnValue({ + status: 1, + stdout: "", + stderr: "SUPERVISOR_UNAVAILABLE", + }); + const captureOpenshell = vi.spyOn(openshellRuntime, "captureOpenshell"); + + const result = checkAndRecoverSandboxProcesses("managed-failed-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: expect.stringContaining("failed the managed health guard"), + }); + expect(finalize).toHaveBeenCalledWith(true); + expect(captureOpenshell).not.toHaveBeenCalled(); + }); + it("rejects a healthy forward when the replacement identity changes after readiness", () => { mockOpenClawSandbox("drifted-box"); vi.mocked(agentRuntime.getSessionAgent).mockReturnValue({