fix(sandbox): retry dropped OpenShell relay during recovery

src/lib/actions/sandbox/process-recovery.test.ts

@@ -28,6 +28,9 @@ const OPENSHELL_SUPERVISOR_RELAY_CHANNEL_TIMED_OUT_STDERR = `Error:   × code: '
   │ relay failed: status: DeadlineExceeded, message: \\"relay channel timed
   │ out\\", details: [], metadata: MetadataMap { headers: {} }"
 `;
+const OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR = `Error:   × status: Unavailable, message: "relay
+  │ channel dropped", details: [], metadata: MetadataMap { headers: {} }
+`;
 const OPENSHELL_RELAY_TARGET_NOT_FOUND_STDERR = `Error:   × code: 'The service is currently unavailable', message: "No such file
   │ or directory (os error 2)"
 `;
@@ -139,6 +142,33 @@ describe("recreated sandbox OpenShell readiness", () => {
     expect(sleeps).toEqual([3]);
   });
 
+  it("retries when OpenShell drops the replacement supervisor's reverse relay", () => {
+    const captureOpenshellImpl = vi
+      .fn()
+      .mockReturnValueOnce({
+        status: 1,
+        output: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR.trim(),
+        stdout: "",
+        stderr: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR,
+      })
+      .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" });
+    const beforeProbe = vi.fn(() => true);
+    const sleeps: number[] = [];
+
+    expect(
+      waitForRecreatedSandboxOpenShellReady("recreated-box", {
+        beforeProbe,
+        captureOpenshellImpl,
+        intervalSeconds: 3,
+        sleepImpl: (seconds) => sleeps.push(seconds),
+        timeoutSeconds: 30,
+      }),
+    ).toBe(true);
+    expect(beforeProbe).toHaveBeenCalledTimes(2);
+    expect(captureOpenshellImpl).toHaveBeenCalledTimes(2);
+    expect(sleeps).toEqual([3]);
+  });
+
   it.each([
     OPENSHELL_RELAY_TARGET_NOT_FOUND_STDERR,
     OPENSHELL_RELAY_TARGET_REFUSED_STDERR,

src/lib/actions/sandbox/process-recovery.ts

@@ -587,9 +587,11 @@ function readNonNegativeNumberEnv(name: string, fallback: number): number {
 
 const OPENSHELL_SANDBOX_NOT_READY = `Error: code: 'The system is not in a state required for the operation's execution', message: "sandbox is not ready"`;
 const OPENSHELL_SERVICE_UNAVAILABLE = "code: 'The service is currently unavailable'";
+const OPENSHELL_STATUS_UNAVAILABLE = "status: Unavailable";
 const OPENSHELL_RELAY_OPEN_TIMED_OUT = 'message: "relay open timed out"';
 const OPENSHELL_SUPERVISOR_RELAY_DEADLINE = "supervisor relay failed: status: DeadlineExceeded";
 const OPENSHELL_RELAY_CHANNEL_TIMED_OUT = "relay channel timed out";
+const OPENSHELL_RELAY_CHANNEL_DROPPED = 'message: "relay channel dropped"';
 const OPENSHELL_RELAY_TARGET_NOT_FOUND = 'message: "No such file or directory (os error 2)"';
 const OPENSHELL_RELAY_TARGET_REFUSED = 'message: "Connection refused (os error 111)"';
 
@@ -633,13 +635,18 @@ function isRetryableOpenshellReRegistrationState(
     error.includes(OPENSHELL_SERVICE_UNAVAILABLE) &&
     error.includes(OPENSHELL_SUPERVISOR_RELAY_DEADLINE) &&
     error.includes(OPENSHELL_RELAY_CHANNEL_TIMED_OUT);
+  const relayChannelDropped =
+    (error.includes(OPENSHELL_SERVICE_UNAVAILABLE) ||
+      error.includes(OPENSHELL_STATUS_UNAVAILABLE)) &&
+    error.includes(OPENSHELL_RELAY_CHANNEL_DROPPED);
   const relayTargetUnavailable =
     error.includes(OPENSHELL_SERVICE_UNAVAILABLE) &&
     (error.includes(OPENSHELL_RELAY_TARGET_NOT_FOUND) ||
       error.includes(OPENSHELL_RELAY_TARGET_REFUSED));
   return (
     sessionUnavailable ||
     relayChannelTimedOut ||
+    relayChannelDropped ||
     relayTargetUnavailable ||
     error.includes(OPENSHELL_RELAY_OPEN_TIMED_OUT)
   );
@@ -652,7 +659,11 @@ type RecreatedSandboxOpenShellReadinessFailure =
 
 type RecreatedSandboxOpenShellReadinessResult =
   | { ready: true }
-  | { failure: RecreatedSandboxOpenShellReadinessFailure; ready: false };
+  | {
+      failure: RecreatedSandboxOpenShellReadinessFailure;
+      openshellError?: string;
+      ready: false;
+    };
 
 type RecreatedSandboxOpenShellReadyOptions = {
   captureOpenshellImpl?: typeof captureOpenshell;
@@ -665,15 +676,19 @@ type RecreatedSandboxOpenShellReadyOptions = {
 
 function recreatedSandboxOpenShellReadinessFailureDetail(
   failure: RecreatedSandboxOpenShellReadinessFailure,
+  openshellError?: string,
 ): string {
-  switch (failure) {
-    case "managed-health-definitive-failure":
-      return "the recreated sandbox failed the managed health guard, so the primary dashboard/API host forward was not started";
-    case "managed-health-inconclusive-timeout":
-      return "the recreated sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started";
-    case "openshell-readiness-failure":
-      return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started";
-  }
+  const detail = (() => {
+    switch (failure) {
+      case "managed-health-definitive-failure":
+        return "the recreated sandbox failed the managed health guard, so the primary dashboard/API host forward was not started";
+      case "managed-health-inconclusive-timeout":
+        return "the recreated sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started";
+      case "openshell-readiness-failure":
+        return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started";
+    }
+  })();
+  return openshellError ? `${detail} Last OpenShell readiness error: ${openshellError}` : detail;
 }
 
 /**
@@ -704,6 +719,7 @@ function waitForRecreatedSandboxOpenShellReadyResult(
     intervalSeconds > 0
       ? Math.max(1, Math.floor(timeoutSeconds / intervalSeconds) + 1)
       : Math.max(1, Math.floor(timeoutSeconds) + 1);
+  let lastOpenshellError: string | undefined;
 
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
     const preGuardRemainingMs = deadlineMs - now();
@@ -728,7 +744,11 @@ function waitForRecreatedSandboxOpenShellReadyResult(
     }
     const remainingMs = deadlineMs - now();
     if (attempt > 1 && remainingMs <= 0) {
-      return { failure: "openshell-readiness-failure", ready: false };
+      return {
+        failure: "openshell-readiness-failure",
+        openshellError: lastOpenshellError,
+        ready: false,
+      };
     }
     const result = capture(["sandbox", "exec", "--name", sandboxName, "--", "true"], {
       ignoreError: true,
@@ -737,23 +757,41 @@ function waitForRecreatedSandboxOpenShellReadyResult(
       timeout: Math.max(1, Math.min(OPENSHELL_PROBE_TIMEOUT_MS, remainingMs)),
     });
     if (result.status === 0 && !result.error) return { ready: true };
+    const openshellError = normalizeOpenshellStructuredError(String(result.stderr ?? ""));
+    if (openshellError) lastOpenshellError = openshellError;
     // This probe executes only `true`, so an OpenShell process timeout has no
     // mutation outcome to reconcile. Treat that exact timeout as inconclusive
     // and retry behind the pinned managed-health guard on the next iteration.
     // All other unexpected OpenShell failures remain definitive.
     if (!isRetryableOpenshellReRegistrationState(result) && !isCommandTimeout(result)) {
-      return { failure: "openshell-readiness-failure", ready: false };
+      return {
+        failure: "openshell-readiness-failure",
+        openshellError: lastOpenshellError,
+        ready: false,
+      };
     }
     if (attempt === maxAttempts) {
-      return { failure: "openshell-readiness-failure", ready: false };
+      return {
+        failure: "openshell-readiness-failure",
+        openshellError: lastOpenshellError,
+        ready: false,
+      };
     }
     const postProbeRemainingMs = deadlineMs - now();
     if (postProbeRemainingMs <= 0) {
-      return { failure: "openshell-readiness-failure", ready: false };
+      return {
+        failure: "openshell-readiness-failure",
+        openshellError: lastOpenshellError,
+        ready: false,
+      };
     }
     sleep(Math.min(intervalSeconds * 1000, postProbeRemainingMs) / 1000);
   }
-  return { failure: "openshell-readiness-failure", ready: false };
+  return {
+    failure: "openshell-readiness-failure",
+    openshellError: lastOpenshellError,
+    ready: false,
+  };
 }
 
 export function waitForRecreatedSandboxOpenShellReady(
@@ -1255,7 +1293,10 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
                 : ({ failure: "openshell-readiness-failure", ready: false } as const);
           return readiness.ready
             ? null
-            : recreatedSandboxOpenShellReadinessFailureDetail(readiness.failure);
+            : recreatedSandboxOpenShellReadinessFailureDetail(
+                readiness.failure,
+                "openshellError" in readiness ? readiness.openshellError : undefined,
+              );
         })()
       : null;
     if (readinessFailureDetail) {

test/process-recovery-supervisor-relaunch.test.ts

@@ -9,6 +9,10 @@ import * as openshellRuntime from "../src/lib/adapters/openshell/runtime.ts";
 import * as agentRuntime from "../src/lib/agent/runtime.ts";
 import * as registry from "../src/lib/state/registry.ts";
 
+const OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR = `Error:   × status: Unavailable, message: "relay
+  │ channel dropped", details: [], metadata: MetadataMap { headers: {} }
+`;
+
 afterEach(() => {
   vi.restoreAllMocks();
   vi.unstubAllEnvs();
@@ -351,6 +355,65 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => {
     expect(runOpenshell).not.toHaveBeenCalled();
   });
 
+  it("reports the last structured OpenShell error when readiness times out", () => {
+    mockOpenClawSandbox("relay-dropped-box");
+    setImmediateRecoveryPolling();
+    const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false }));
+    const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({
+      containerId: "replacement-container-id",
+      finalize,
+    }));
+    const requestGatewaySupervisorAction = vi.fn(() => ({
+      status: 1,
+      stdout: "",
+      stderr: "SUPERVISOR_NOT_RUNNING",
+    }));
+    const requestPinnedGatewaySupervisorAction = vi.fn(() => ({
+      status: 0,
+      stdout: "GATEWAY_PID=4242\n",
+      stderr: "",
+    }));
+    const timeoutError = Object.assign(new Error("timed out"), { code: "ETIMEDOUT" });
+    const captureOpenshell = vi
+      .spyOn(openshellRuntime, "captureOpenshell")
+      .mockReturnValueOnce({
+        status: 1,
+        output: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR.trim(),
+        stdout: "",
+        stderr: OPENSHELL_RELAY_CHANNEL_DROPPED_STDERR,
+      })
+      .mockReturnValue({
+        status: null,
+        output: "",
+        stdout: "",
+        stderr: "",
+        error: timeoutError,
+      });
+    const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell");
+
+    const result = checkAndRecoverSandboxProcesses("relay-dropped-box", {
+      quiet: true,
+      isSandboxGatewayRunningImpl: () => false,
+      requestGatewaySupervisorAction,
+      requestPinnedGatewaySupervisorAction,
+      relaunchManagedSupervisorSessionImpl,
+    });
+
+    expect(result).toMatchObject({
+      checked: true,
+      wasRunning: false,
+      recovered: true,
+      forwardRecovered: false,
+      forwardRecoveryFailed: true,
+      forwardRecoveryFailureDetail: expect.stringContaining(
+        'Last OpenShell readiness error: Error: status: Unavailable, message: "relay channel dropped"',
+      ),
+    });
+    expect(captureOpenshell).toHaveBeenCalled();
+    expect(finalize).toHaveBeenCalledWith(true);
+    expect(runOpenshell).not.toHaveBeenCalled();
+  });
+
   it("reports a definitive managed health failure separately from OpenShell readiness", () => {
     mockOpenClawSandbox("managed-failed-box");
     setImmediateRecoveryPolling();