refactor(cli): wire recoverDashboardChain into checkAndRecoverSandboxProcesses

src/nemoclaw.ts

@@ -95,9 +95,13 @@ import {
   persistChannelTokens,
 } from "./lib/sandbox-channels";
 import {
+  OPENSHELL_DOWNLOAD_TIMEOUT_MS,
   OPENSHELL_OPERATION_TIMEOUT_MS,
   OPENSHELL_PROBE_TIMEOUT_MS,
 } from "./lib/openshell-timeouts";
+import { buildChain } from "./lib/dashboard-contract";
+import type { DashboardRecoverDeps } from "./lib/dashboard-recover";
+import { recoverDashboardChain } from "./lib/dashboard-recover";
 const onboardProviders = require("./lib/onboard-providers");
 
 // ── Global commands (derived from command registry) ──────────────
@@ -290,9 +294,10 @@ function isSandboxGatewayRunning(sandboxName: string): boolean | null {
  * Cleans stale lock/temp files, sources proxy config, and launches the gateway
  * in the background. Returns true on success.
  */
-function recoverSandboxProcesses(sandboxName: string): boolean {
+function recoverSandboxProcesses(sandboxName: string, opts: { port?: number } = {}): boolean {
   const agent = agentRuntime.getSessionAgent(sandboxName);
-  const agentScript = agentRuntime.buildRecoveryScript(agent, agent?.forwardPort ?? DASHBOARD_PORT);
+  const port = opts.port ?? agent?.forwardPort ?? DASHBOARD_PORT;
+  const agentScript = agentRuntime.buildRecoveryScript(agent, port);
   const script =
     agentScript ||
     [
@@ -309,7 +314,7 @@ function recoverSandboxProcesses(sandboxName: string): boolean {
       "if [ -r /tmp/nemoclaw-proxy-env.sh ]; then . /tmp/nemoclaw-proxy-env.sh; _PE_MISSING=0; else _PE_MISSING=1; fi;",
       "[ -f ~/.bashrc ] && . ~/.bashrc;",
       'case "${NODE_OPTIONS:-}" in *nemoclaw-sandbox-safety-net*) _GUARDS_MISSING=0 ;; *) _GUARDS_MISSING=1 ;; esac;',
-      `if curl -sf --max-time 3 http://127.0.0.1:${DASHBOARD_PORT}/ > /dev/null 2>&1; then echo ALREADY_RUNNING; exit 0; fi;`,
+      `if curl -sf --max-time 3 http://127.0.0.1:${port}/ > /dev/null 2>&1; then echo ALREADY_RUNNING; exit 0; fi;`,
       "rm -rf /tmp/openclaw-*/gateway.*.lock 2>/dev/null;",
       "rm -f /tmp/gateway.log /tmp/auto-pair.log;",
       "touch /tmp/gateway.log; chmod 600 /tmp/gateway.log;",
@@ -320,7 +325,7 @@ function recoverSandboxProcesses(sandboxName: string): boolean {
       'if [ -z "$OPENCLAW" ]; then echo OPENCLAW_MISSING; exit 1; fi;',
       // Append rather than truncate so [gateway-recovery] WARNING lines
       // written above survive past the launch. (#2478)
-      `nohup "$OPENCLAW" gateway run --port ${DASHBOARD_PORT} >> /tmp/gateway.log 2>&1 &`,
+      `nohup "$OPENCLAW" gateway run --port ${port} >> /tmp/gateway.log 2>&1 &`,
       "GPID=$!; sleep 2;",
       'if kill -0 "$GPID" 2>/dev/null; then echo "GATEWAY_PID=$GPID"; else echo GATEWAY_FAILED; cat /tmp/gateway.log 2>/dev/null | tail -5; fi',
     ].join(" ");
@@ -346,10 +351,68 @@ function ensureSandboxPortForward(sandboxName: string): void {
   });
 }
 
+/** Build bounded DashboardRecoverDeps wired to real openshell calls. */
+function buildDashboardRecoverDeps(): DashboardRecoverDeps {
+  return {
+    executeSandboxCommand: (name: string, script: string) => {
+      const result = executeSandboxCommand(name, script);
+      if (!result) return null;
+      return { status: result.status, stdout: result.stdout };
+    },
+    captureForwardList: () => {
+      const result = captureOpenshell(["forward", "list"], {
+        ignoreError: true,
+        timeout: OPENSHELL_PROBE_TIMEOUT_MS,
+      });
+      return result.status === 0 ? result.output : null;
+    },
+    downloadSandboxConfig: (name: string) => {
+      const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cfg-"));
+      try {
+        const destDir = `${tmpDir}${path.sep}`;
+        const result = runOpenshell(
+          ["sandbox", "download", name, "/sandbox/.openclaw/openclaw.json", destDir],
+          { ignoreError: true, stdio: ["ignore", "ignore", "ignore"], timeout: OPENSHELL_DOWNLOAD_TIMEOUT_MS },
+        );
+        if (result.status !== 0) return null;
+        const files = fs.readdirSync(tmpDir, { recursive: true }) as string[];
+        const jsonFile = files.find((f) => f.endsWith("openclaw.json"));
+        if (!jsonFile) return null;
+        return JSON.parse(fs.readFileSync(path.join(tmpDir, jsonFile), "utf-8"));
+      } catch {
+        return null;
+      } finally {
+        try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
+      }
+    },
+    restartGateway: (name: string, port: number, _agent: unknown) => {
+      const ok = recoverSandboxProcesses(name, { port });
+      if (ok) sleepSeconds(3); // Wait for HTTP listener to bind before re-verify
+      return ok;
+    },
+    stopForward: (port: number) => {
+      runOpenshell(["forward", "stop", String(port)], {
+        ignoreError: true,
+        timeout: OPENSHELL_OPERATION_TIMEOUT_MS,
+      });
+    },
+    startForward: (target: string, name: string) => {
+      runOpenshell(["forward", "start", "--background", target, name], {
+        ignoreError: true,
+        timeout: OPENSHELL_OPERATION_TIMEOUT_MS,
+      });
+    },
+    getSessionAgent: (name: string) => agentRuntime.getSessionAgent(name),
+  };
+}
+
 /**
- * Detect and recover from a sandbox that survived a gateway restart but
- * whose OpenClaw processes are not running. Returns an object describing
- * the outcome: { checked, wasRunning, recovered }.
+ * Detect and recover from a sandbox whose dashboard delivery chain is
+ * unhealthy. Checks all links (gateway process, port forward, CORS) and
+ * repairs whatever is broken — even when the in-sandbox gateway is alive
+ * but the forward or CORS link has drifted (e.g. after a pod restart that
+ * killed only the SSH tunnel). Returns an object describing the outcome:
+ * { checked, wasRunning, recovered }.
  */
 function checkAndRecoverSandboxProcesses(
   sandboxName: string,
@@ -359,23 +422,108 @@ function checkAndRecoverSandboxProcesses(
   if (running === null) {
     return { checked: false, wasRunning: null, recovered: false };
   }
+
+  const agent = agentRuntime.getSessionAgent(sandboxName);
+
+  // Dashboard chain recovery only applies to OpenClaw sandboxes.
+  // Non-OpenClaw agents (Hermes, etc.) use different config paths and don't
+  // expose the OpenClaw control UI — fall back to gateway-only recovery.
+  if (agent !== null) {
+    return checkAndRecoverGatewayOnly(sandboxName, running, agent, { quiet });
+  }
+
+  // OpenClaw sandbox: use the registry's persisted dashboardPort (which
+  // reflects auto-allocated or user-overridden ports) before falling back
+  // to the default. This ensures multi-sandbox setups with custom ports
+  // (e.g. 18790) probe and recover the correct port.
+  const sb = registry.getSandbox(sandboxName);
+  const port = sb?.dashboardPort ?? DASHBOARD_PORT;
+
+  // Fast path: if the gateway is running AND the forward is active, skip
+  // the expensive chain recovery (which includes a 30s downloadSandboxConfig
+  // timeout). Only run full chain verification when something is clearly broken.
+  if (running) {
+    const forwardOutput = captureOpenshell(["forward", "list"], {
+      ignoreError: true,
+      timeout: OPENSHELL_PROBE_TIMEOUT_MS,
+    });
+    const hasForward = forwardOutput.status === 0 &&
+      forwardOutput.output.split("\n").some((line: string) => line.trim().split(/\s+/)[2] === String(port));
+    if (hasForward) {
+      return { checked: true, wasRunning: true, recovered: false };
+    }
+  }
+
+  const chain = buildChain({ port, chatUiUrl: process.env.CHAT_UI_URL });
+  const deps = buildDashboardRecoverDeps();
+  const result = recoverDashboardChain(sandboxName, chain, deps);
+
+  // Chain was already healthy — nothing to do
+  if (!result.attempted) {
+    return { checked: true, wasRunning: running, recovered: false };
+  }
+
+  // Recovery was attempted — report progress
+  if (!quiet && !running) {
+    console.log("");
+    console.log("  OpenClaw gateway is not running inside the sandbox (sandbox likely restarted).");
+    console.log("  Recovering...");
+  }
+
+  if (result.after?.healthy) {
+    if (!quiet) {
+      for (const action of result.actions) {
+        console.log(`  ${G}✓${R} ${action}`);
+      }
+    }
+    return { checked: true, wasRunning: running, recovered: true };
+  }
+
+  // Chain recovery didn't fully succeed — report diagnosis
+  if (!quiet) {
+    if (result.actions.length > 0) {
+      for (const action of result.actions) {
+        console.log(`  • ${action}`);
+      }
+    }
+    if (result.after && !result.after.healthy) {
+      console.error(`  Recovery incomplete: ${result.after.diagnosis || "unknown"}`);
+    }
+    console.error("  Could not fully recover OpenClaw dashboard chain.");
+    console.error("  Connect to the sandbox and run manually:");
+    console.error(`    ${agentRuntime.getGatewayCommand(agent)}`);
+  }
+
+  return { checked: true, wasRunning: running, recovered: false };
+}
+
+/**
+ * Simplified gateway-only recovery for non-OpenClaw agents (Hermes, etc.).
+ * Only checks/restarts the gateway process and re-establishes the port forward.
+ * Does NOT attempt CORS verification (agents use different config paths).
+ */
+function checkAndRecoverGatewayOnly(
+  sandboxName: string,
+  running: boolean,
+  agent: unknown,
+  { quiet = false }: { quiet?: boolean } = {},
+) {
   if (running) {
     return { checked: true, wasRunning: true, recovered: false };
   }
 
-  // Gateway not running — attempt recovery
-  const _recoveryAgent = agentRuntime.getSessionAgent(sandboxName);
   if (!quiet) {
     console.log("");
     console.log(
-      `  ${agentRuntime.getAgentDisplayName(_recoveryAgent)} gateway is not running inside the sandbox (sandbox likely restarted).`,
+      `  ${agentRuntime.getAgentDisplayName(agent)} gateway is not running inside the sandbox (sandbox likely restarted).`,
     );
     console.log("  Recovering...");
   }
 
-  const recovered = recoverSandboxProcesses(sandboxName);
+  const sb = registry.getSandbox(sandboxName);
+  const port = sb?.dashboardPort ?? (agent as { forwardPort?: number })?.forwardPort ?? DASHBOARD_PORT;
+  const recovered = recoverSandboxProcesses(sandboxName, { port });
   if (recovered) {
-    // Wait for gateway to bind its HTTP port before declaring success
     sleepSeconds(3);
     if (isSandboxGatewayRunning(sandboxName) !== true) {
       if (!quiet) {
@@ -387,16 +535,16 @@ function checkAndRecoverSandboxProcesses(
     ensureSandboxPortForward(sandboxName);
     if (!quiet) {
       console.log(
-        `  ${G}✓${R} ${agentRuntime.getAgentDisplayName(_recoveryAgent)} gateway restarted inside sandbox.`,
+        `  ${G}✓${R} ${agentRuntime.getAgentDisplayName(agent)} gateway restarted inside sandbox.`,
       );
-      console.log(`  ${G}✓${R} Dashboard port forward re-established.`);
+      console.log(`  ${G}✓${R} Port forward re-established.`);
     }
   } else if (!quiet) {
     console.error(
-      `  Could not restart ${agentRuntime.getAgentDisplayName(_recoveryAgent)} gateway automatically.`,
+      `  Could not restart ${agentRuntime.getAgentDisplayName(agent)} gateway automatically.`,
     );
     console.error("  Connect to the sandbox and run manually:");
-    console.error(`    ${agentRuntime.getGatewayCommand(_recoveryAgent)}`);
+    console.error(`    ${agentRuntime.getGatewayCommand(agent)}`);
   }
 
   return { checked: true, wasRunning: false, recovered };