From 4b1e9a7ce48cbfa26e1fb964e6e48bcaa96a7d21 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 24 Jul 2026 14:37:30 -0700 Subject: [PATCH 1/4] fix(sandbox): retry dropped OpenShell relay during recovery Signed-off-by: Senthil Ravichandran --- .../actions/sandbox/process-recovery.test.ts | 30 +++++++++++++++++++ src/lib/actions/sandbox/process-recovery.ts | 5 ++++ 2 files changed, 35 insertions(+) diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 7b83588ce1c..38f24870ebf 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -28,6 +28,9 @@ const OPENSHELL_SUPERVISOR_RELAY_CHANNEL_TIMED_OUT_STDERR = `Error: × code: ' │ relay failed: status: DeadlineExceeded, message: \\"relay channel timed │ out\\", details: [], metadata: MetadataMap { headers: {} }" `; +const OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR = `Error: × code: 'The service is currently unavailable', message: "relay + │ channel dropped" +`; const OPENSHELL_RELAY_TARGET_NOT_FOUND_STDERR = `Error: × code: 'The service is currently unavailable', message: "No such file │ or directory (os error 2)" `; @@ -139,6 +142,33 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3]); }); + it("retries when OpenShell drops the replacement supervisor's reverse relay", () => { + const captureOpenshellImpl = vi + .fn() + .mockReturnValueOnce({ + status: 1, + output: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR.trim(), + stdout: "", + stderr: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR, + }) + .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" }); + const beforeProbe = vi.fn(() => true); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 30, + }), + ).toBe(true); + expect(beforeProbe).toHaveBeenCalledTimes(2); + expect(captureOpenshellImpl).toHaveBeenCalledTimes(2); + expect(sleeps).toEqual([3]); + }); + it.each([ OPENSHELL_RELAY_TARGET_NOT_FOUND_STDERR, OPENSHELL_RELAY_TARGET_REFUSED_STDERR, diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index ee0258b5de0..e66ba723ccd 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -590,6 +590,7 @@ const OPENSHELL_SERVICE_UNAVAILABLE = "code: 'The service is currently unavailab const OPENSHELL_RELAY_OPEN_TIMED_OUT = 'message: "relay open timed out"'; const OPENSHELL_SUPERVISOR_RELAY_DEADLINE = "supervisor relay failed: status: DeadlineExceeded"; const OPENSHELL_RELAY_CHANNEL_TIMED_OUT = "relay channel timed out"; +const OPENSHELL_RELAY_CHANNEL_DROPPED = 'message: "relay channel dropped"'; const OPENSHELL_RELAY_TARGET_NOT_FOUND = 'message: "No such file or directory (os error 2)"'; const OPENSHELL_RELAY_TARGET_REFUSED = 'message: "Connection refused (os error 111)"'; @@ -633,6 +634,9 @@ function isRetryableOpenshellReRegistrationState( error.includes(OPENSHELL_SERVICE_UNAVAILABLE) && error.includes(OPENSHELL_SUPERVISOR_RELAY_DEADLINE) && error.includes(OPENSHELL_RELAY_CHANNEL_TIMED_OUT); + const relayChannelDropped = + error.includes(OPENSHELL_SERVICE_UNAVAILABLE) && + error.includes(OPENSHELL_RELAY_CHANNEL_DROPPED); const relayTargetUnavailable = error.includes(OPENSHELL_SERVICE_UNAVAILABLE) && (error.includes(OPENSHELL_RELAY_TARGET_NOT_FOUND) || @@ -640,6 +644,7 @@ function isRetryableOpenshellReRegistrationState( return ( sessionUnavailable || relayChannelTimedOut || + relayChannelDropped || relayTargetUnavailable || error.includes(OPENSHELL_RELAY_OPEN_TIMED_OUT) ); From 93feb7d5462996caa8429988b1191b87f4fa3ac1 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 24 Jul 2026 15:15:30 -0700 Subject: [PATCH 2/4] fix(sandbox): match streamed relay drop status Signed-off-by: Senthil Ramamoorthy --- src/lib/actions/sandbox/process-recovery.test.ts | 4 ++-- src/lib/actions/sandbox/process-recovery.ts | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 38f24870ebf..4c5ca21b8b7 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -28,8 +28,8 @@ const OPENSHELL_SUPERVISOR_RELAY_CHANNEL_TIMED_OUT_STDERR = `Error: × code: ' │ relay failed: status: DeadlineExceeded, message: \\"relay channel timed │ out\\", details: [], metadata: MetadataMap { headers: {} }" `; -const OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR = `Error: × code: 'The service is currently unavailable', message: "relay - │ channel dropped" +const OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR = `Error: × status: Unavailable, message: "relay + │ channel dropped", details: [], metadata: MetadataMap { headers: {} } `; const OPENSHELL_RELAY_TARGET_NOT_FOUND_STDERR = `Error: × code: 'The service is currently unavailable', message: "No such file │ or directory (os error 2)" diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index e66ba723ccd..742c785a4c4 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -587,6 +587,7 @@ function readNonNegativeNumberEnv(name: string, fallback: number): number { const OPENSHELL_SANDBOX_NOT_READY = `Error: code: 'The system is not in a state required for the operation's execution', message: "sandbox is not ready"`; const OPENSHELL_SERVICE_UNAVAILABLE = "code: 'The service is currently unavailable'"; +const OPENSHELL_STATUS_UNAVAILABLE = "status: Unavailable"; const OPENSHELL_RELAY_OPEN_TIMED_OUT = 'message: "relay open timed out"'; const OPENSHELL_SUPERVISOR_RELAY_DEADLINE = "supervisor relay failed: status: DeadlineExceeded"; const OPENSHELL_RELAY_CHANNEL_TIMED_OUT = "relay channel timed out"; @@ -635,7 +636,8 @@ function isRetryableOpenshellReRegistrationState( error.includes(OPENSHELL_SUPERVISOR_RELAY_DEADLINE) && error.includes(OPENSHELL_RELAY_CHANNEL_TIMED_OUT); const relayChannelDropped = - error.includes(OPENSHELL_SERVICE_UNAVAILABLE) && + (error.includes(OPENSHELL_SERVICE_UNAVAILABLE) || + error.includes(OPENSHELL_STATUS_UNAVAILABLE)) && error.includes(OPENSHELL_RELAY_CHANNEL_DROPPED); const relayTargetUnavailable = error.includes(OPENSHELL_SERVICE_UNAVAILABLE) && From 48f65b673482a8e8a5c36cb8b8dc6ff6c292ab8f Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 24 Jul 2026 15:27:12 -0700 Subject: [PATCH 3/4] fix(sandbox): report final OpenShell readiness error Signed-off-by: Senthil Ramamoorthy --- src/lib/actions/sandbox/process-recovery.ts | 63 ++++++++++++++++----- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 742c785a4c4..29286cf95c8 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -659,7 +659,11 @@ type RecreatedSandboxOpenShellReadinessFailure = type RecreatedSandboxOpenShellReadinessResult = | { ready: true } - | { failure: RecreatedSandboxOpenShellReadinessFailure; ready: false }; + | { + failure: RecreatedSandboxOpenShellReadinessFailure; + openshellError?: string; + ready: false; + }; type RecreatedSandboxOpenShellReadyOptions = { captureOpenshellImpl?: typeof captureOpenshell; @@ -672,15 +676,19 @@ type RecreatedSandboxOpenShellReadyOptions = { function recreatedSandboxOpenShellReadinessFailureDetail( failure: RecreatedSandboxOpenShellReadinessFailure, + openshellError?: string, ): string { - switch (failure) { - case "managed-health-definitive-failure": - return "the recreated sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; - case "managed-health-inconclusive-timeout": - return "the recreated sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; - case "openshell-readiness-failure": - return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; - } + const detail = (() => { + switch (failure) { + case "managed-health-definitive-failure": + return "the recreated sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; + case "managed-health-inconclusive-timeout": + return "the recreated sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; + case "openshell-readiness-failure": + return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; + } + })(); + return openshellError ? `${detail} Last OpenShell readiness error: ${openshellError}` : detail; } /** @@ -711,6 +719,7 @@ function waitForRecreatedSandboxOpenShellReadyResult( intervalSeconds > 0 ? Math.max(1, Math.floor(timeoutSeconds / intervalSeconds) + 1) : Math.max(1, Math.floor(timeoutSeconds) + 1); + let lastOpenshellError: string | undefined; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const preGuardRemainingMs = deadlineMs - now(); @@ -735,7 +744,11 @@ function waitForRecreatedSandboxOpenShellReadyResult( } const remainingMs = deadlineMs - now(); if (attempt > 1 && remainingMs <= 0) { - return { failure: "openshell-readiness-failure", ready: false }; + return { + failure: "openshell-readiness-failure", + openshellError: lastOpenshellError, + ready: false, + }; } const result = capture(["sandbox", "exec", "--name", sandboxName, "--", "true"], { ignoreError: true, @@ -744,23 +757,40 @@ function waitForRecreatedSandboxOpenShellReadyResult( timeout: Math.max(1, Math.min(OPENSHELL_PROBE_TIMEOUT_MS, remainingMs)), }); if (result.status === 0 && !result.error) return { ready: true }; + lastOpenshellError = normalizeOpenshellStructuredError(String(result.stderr ?? "")); // This probe executes only `true`, so an OpenShell process timeout has no // mutation outcome to reconcile. Treat that exact timeout as inconclusive // and retry behind the pinned managed-health guard on the next iteration. // All other unexpected OpenShell failures remain definitive. if (!isRetryableOpenshellReRegistrationState(result) && !isCommandTimeout(result)) { - return { failure: "openshell-readiness-failure", ready: false }; + return { + failure: "openshell-readiness-failure", + openshellError: lastOpenshellError, + ready: false, + }; } if (attempt === maxAttempts) { - return { failure: "openshell-readiness-failure", ready: false }; + return { + failure: "openshell-readiness-failure", + openshellError: lastOpenshellError, + ready: false, + }; } const postProbeRemainingMs = deadlineMs - now(); if (postProbeRemainingMs <= 0) { - return { failure: "openshell-readiness-failure", ready: false }; + return { + failure: "openshell-readiness-failure", + openshellError: lastOpenshellError, + ready: false, + }; } sleep(Math.min(intervalSeconds * 1000, postProbeRemainingMs) / 1000); } - return { failure: "openshell-readiness-failure", ready: false }; + return { + failure: "openshell-readiness-failure", + openshellError: lastOpenshellError, + ready: false, + }; } export function waitForRecreatedSandboxOpenShellReady( @@ -1262,7 +1292,10 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( : ({ failure: "openshell-readiness-failure", ready: false } as const); return readiness.ready ? null - : recreatedSandboxOpenShellReadinessFailureDetail(readiness.failure); + : recreatedSandboxOpenShellReadinessFailureDetail( + readiness.failure, + "openshellError" in readiness ? readiness.openshellError : undefined, + ); })() : null; if (readinessFailureDetail) { From 32b8795c4a6e22f0e880da95f4f334bfcc28dedc Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 24 Jul 2026 15:39:23 -0700 Subject: [PATCH 4/4] test(sandbox): preserve relay recovery diagnostic Signed-off-by: Senthil Ramamoorthy --- src/lib/actions/sandbox/process-recovery.ts | 3 +- ...ocess-recovery-supervisor-relaunch.test.ts | 63 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 29286cf95c8..013cfa5adeb 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -757,7 +757,8 @@ function waitForRecreatedSandboxOpenShellReadyResult( timeout: Math.max(1, Math.min(OPENSHELL_PROBE_TIMEOUT_MS, remainingMs)), }); if (result.status === 0 && !result.error) return { ready: true }; - lastOpenshellError = normalizeOpenshellStructuredError(String(result.stderr ?? "")); + const openshellError = normalizeOpenshellStructuredError(String(result.stderr ?? "")); + if (openshellError) lastOpenshellError = openshellError; // This probe executes only `true`, so an OpenShell process timeout has no // mutation outcome to reconcile. Treat that exact timeout as inconclusive // and retry behind the pinned managed-health guard on the next iteration. diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index af899316aad..cb0fe7ac52e 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -9,6 +9,10 @@ import * as openshellRuntime from "../src/lib/adapters/openshell/runtime.ts"; import * as agentRuntime from "../src/lib/agent/runtime.ts"; import * as registry from "../src/lib/state/registry.ts"; +const OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR = `Error: × status: Unavailable, message: "relay + │ channel dropped", details: [], metadata: MetadataMap { headers: {} } +`; + afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); @@ -351,6 +355,65 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(runOpenshell).not.toHaveBeenCalled(); }); + it("reports the last structured OpenShell error when readiness times out", () => { + mockOpenClawSandbox("relay-dropped-box"); + setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "SUPERVISOR_NOT_RUNNING", + })); + const requestPinnedGatewaySupervisorAction = vi.fn(() => ({ + status: 0, + stdout: "GATEWAY_PID=4242\n", + stderr: "", + })); + const timeoutError = Object.assign(new Error("timed out"), { code: "ETIMEDOUT" }); + const captureOpenshell = vi + .spyOn(openshellRuntime, "captureOpenshell") + .mockReturnValueOnce({ + status: 1, + output: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR.trim(), + stdout: "", + stderr: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR, + }) + .mockReturnValue({ + status: null, + output: "", + stdout: "", + stderr: "", + error: timeoutError, + }); + const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell"); + + const result = checkAndRecoverSandboxProcesses("relay-dropped-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: expect.stringContaining( + 'Last OpenShell readiness error: Error: status: Unavailable, message: "relay channel dropped"', + ), + }); + expect(captureOpenshell).toHaveBeenCalled(); + expect(finalize).toHaveBeenCalledWith(true); + expect(runOpenshell).not.toHaveBeenCalled(); + }); + it("reports a definitive managed health failure separately from OpenShell readiness", () => { mockOpenClawSandbox("managed-failed-box"); setImmediateRecoveryPolling();