diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 7b83588ce1c..4c5ca21b8b7 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -28,6 +28,9 @@ const OPENSHELL_SUPERVISOR_RELAY_CHANNEL_TIMED_OUT_STDERR = `Error: × code: ' │ relay failed: status: DeadlineExceeded, message: \\"relay channel timed │ out\\", details: [], metadata: MetadataMap { headers: {} }" `; +const OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR = `Error: × status: Unavailable, message: "relay + │ channel dropped", details: [], metadata: MetadataMap { headers: {} } +`; const OPENSHELL_RELAY_TARGET_NOT_FOUND_STDERR = `Error: × code: 'The service is currently unavailable', message: "No such file │ or directory (os error 2)" `; @@ -139,6 +142,33 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3]); }); + it("retries when OpenShell drops the replacement supervisor's reverse relay", () => { + const captureOpenshellImpl = vi + .fn() + .mockReturnValueOnce({ + status: 1, + output: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR.trim(), + stdout: "", + stderr: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR, + }) + .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" }); + const beforeProbe = vi.fn(() => true); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 30, + }), + ).toBe(true); + expect(beforeProbe).toHaveBeenCalledTimes(2); + expect(captureOpenshellImpl).toHaveBeenCalledTimes(2); + expect(sleeps).toEqual([3]); + }); + it.each([ OPENSHELL_RELAY_TARGET_NOT_FOUND_STDERR, OPENSHELL_RELAY_TARGET_REFUSED_STDERR, diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index ee0258b5de0..013cfa5adeb 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -587,9 +587,11 @@ function readNonNegativeNumberEnv(name: string, fallback: number): number { const OPENSHELL_SANDBOX_NOT_READY = `Error: code: 'The system is not in a state required for the operation's execution', message: "sandbox is not ready"`; const OPENSHELL_SERVICE_UNAVAILABLE = "code: 'The service is currently unavailable'"; +const OPENSHELL_STATUS_UNAVAILABLE = "status: Unavailable"; const OPENSHELL_RELAY_OPEN_TIMED_OUT = 'message: "relay open timed out"'; const OPENSHELL_SUPERVISOR_RELAY_DEADLINE = "supervisor relay failed: status: DeadlineExceeded"; const OPENSHELL_RELAY_CHANNEL_TIMED_OUT = "relay channel timed out"; +const OPENSHELL_RELAY_CHANNEL_DROPPED = 'message: "relay channel dropped"'; const OPENSHELL_RELAY_TARGET_NOT_FOUND = 'message: "No such file or directory (os error 2)"'; const OPENSHELL_RELAY_TARGET_REFUSED = 'message: "Connection refused (os error 111)"'; @@ -633,6 +635,10 @@ function isRetryableOpenshellReRegistrationState( error.includes(OPENSHELL_SERVICE_UNAVAILABLE) && error.includes(OPENSHELL_SUPERVISOR_RELAY_DEADLINE) && error.includes(OPENSHELL_RELAY_CHANNEL_TIMED_OUT); + const relayChannelDropped = + (error.includes(OPENSHELL_SERVICE_UNAVAILABLE) || + error.includes(OPENSHELL_STATUS_UNAVAILABLE)) && + error.includes(OPENSHELL_RELAY_CHANNEL_DROPPED); const relayTargetUnavailable = error.includes(OPENSHELL_SERVICE_UNAVAILABLE) && (error.includes(OPENSHELL_RELAY_TARGET_NOT_FOUND) || @@ -640,6 +646,7 @@ function isRetryableOpenshellReRegistrationState( return ( sessionUnavailable || relayChannelTimedOut || + relayChannelDropped || relayTargetUnavailable || error.includes(OPENSHELL_RELAY_OPEN_TIMED_OUT) ); @@ -652,7 +659,11 @@ type RecreatedSandboxOpenShellReadinessFailure = type RecreatedSandboxOpenShellReadinessResult = | { ready: true } - | { failure: RecreatedSandboxOpenShellReadinessFailure; ready: false }; + | { + failure: RecreatedSandboxOpenShellReadinessFailure; + openshellError?: string; + ready: false; + }; type RecreatedSandboxOpenShellReadyOptions = { captureOpenshellImpl?: typeof captureOpenshell; @@ -665,15 +676,19 @@ type RecreatedSandboxOpenShellReadyOptions = { function recreatedSandboxOpenShellReadinessFailureDetail( failure: RecreatedSandboxOpenShellReadinessFailure, + openshellError?: string, ): string { - switch (failure) { - case "managed-health-definitive-failure": - return "the recreated sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; - case "managed-health-inconclusive-timeout": - return "the recreated sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; - case "openshell-readiness-failure": - return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; - } + const detail = (() => { + switch (failure) { + case "managed-health-definitive-failure": + return "the recreated sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; + case "managed-health-inconclusive-timeout": + return "the recreated sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; + case "openshell-readiness-failure": + return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; + } + })(); + return openshellError ? `${detail} Last OpenShell readiness error: ${openshellError}` : detail; } /** @@ -704,6 +719,7 @@ function waitForRecreatedSandboxOpenShellReadyResult( intervalSeconds > 0 ? Math.max(1, Math.floor(timeoutSeconds / intervalSeconds) + 1) : Math.max(1, Math.floor(timeoutSeconds) + 1); + let lastOpenshellError: string | undefined; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const preGuardRemainingMs = deadlineMs - now(); @@ -728,7 +744,11 @@ function waitForRecreatedSandboxOpenShellReadyResult( } const remainingMs = deadlineMs - now(); if (attempt > 1 && remainingMs <= 0) { - return { failure: "openshell-readiness-failure", ready: false }; + return { + failure: "openshell-readiness-failure", + openshellError: lastOpenshellError, + ready: false, + }; } const result = capture(["sandbox", "exec", "--name", sandboxName, "--", "true"], { ignoreError: true, @@ -737,23 +757,41 @@ function waitForRecreatedSandboxOpenShellReadyResult( timeout: Math.max(1, Math.min(OPENSHELL_PROBE_TIMEOUT_MS, remainingMs)), }); if (result.status === 0 && !result.error) return { ready: true }; + const openshellError = normalizeOpenshellStructuredError(String(result.stderr ?? "")); + if (openshellError) lastOpenshellError = openshellError; // This probe executes only `true`, so an OpenShell process timeout has no // mutation outcome to reconcile. Treat that exact timeout as inconclusive // and retry behind the pinned managed-health guard on the next iteration. // All other unexpected OpenShell failures remain definitive. if (!isRetryableOpenshellReRegistrationState(result) && !isCommandTimeout(result)) { - return { failure: "openshell-readiness-failure", ready: false }; + return { + failure: "openshell-readiness-failure", + openshellError: lastOpenshellError, + ready: false, + }; } if (attempt === maxAttempts) { - return { failure: "openshell-readiness-failure", ready: false }; + return { + failure: "openshell-readiness-failure", + openshellError: lastOpenshellError, + ready: false, + }; } const postProbeRemainingMs = deadlineMs - now(); if (postProbeRemainingMs <= 0) { - return { failure: "openshell-readiness-failure", ready: false }; + return { + failure: "openshell-readiness-failure", + openshellError: lastOpenshellError, + ready: false, + }; } sleep(Math.min(intervalSeconds * 1000, postProbeRemainingMs) / 1000); } - return { failure: "openshell-readiness-failure", ready: false }; + return { + failure: "openshell-readiness-failure", + openshellError: lastOpenshellError, + ready: false, + }; } export function waitForRecreatedSandboxOpenShellReady( @@ -1255,7 +1293,10 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( : ({ failure: "openshell-readiness-failure", ready: false } as const); return readiness.ready ? null - : recreatedSandboxOpenShellReadinessFailureDetail(readiness.failure); + : recreatedSandboxOpenShellReadinessFailureDetail( + readiness.failure, + "openshellError" in readiness ? readiness.openshellError : undefined, + ); })() : null; if (readinessFailureDetail) { diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index af899316aad..cb0fe7ac52e 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -9,6 +9,10 @@ import * as openshellRuntime from "../src/lib/adapters/openshell/runtime.ts"; import * as agentRuntime from "../src/lib/agent/runtime.ts"; import * as registry from "../src/lib/state/registry.ts"; +const OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR = `Error: × status: Unavailable, message: "relay + │ channel dropped", details: [], metadata: MetadataMap { headers: {} } +`; + afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); @@ -351,6 +355,65 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(runOpenshell).not.toHaveBeenCalled(); }); + it("reports the last structured OpenShell error when readiness times out", () => { + mockOpenClawSandbox("relay-dropped-box"); + setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "SUPERVISOR_NOT_RUNNING", + })); + const requestPinnedGatewaySupervisorAction = vi.fn(() => ({ + status: 0, + stdout: "GATEWAY_PID=4242\n", + stderr: "", + })); + const timeoutError = Object.assign(new Error("timed out"), { code: "ETIMEDOUT" }); + const captureOpenshell = vi + .spyOn(openshellRuntime, "captureOpenshell") + .mockReturnValueOnce({ + status: 1, + output: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR.trim(), + stdout: "", + stderr: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR, + }) + .mockReturnValue({ + status: null, + output: "", + stdout: "", + stderr: "", + error: timeoutError, + }); + const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell"); + + const result = checkAndRecoverSandboxProcesses("relay-dropped-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: expect.stringContaining( + 'Last OpenShell readiness error: Error: status: Unavailable, message: "relay channel dropped"', + ), + }); + expect(captureOpenshell).toHaveBeenCalled(); + expect(finalize).toHaveBeenCalledWith(true); + expect(runOpenshell).not.toHaveBeenCalled(); + }); + it("reports a definitive managed health failure separately from OpenShell readiness", () => { mockOpenClawSandbox("managed-failed-box"); setImmediateRecoveryPolling();