Harden gateway recovery after updates

install.sh

@@ -1364,9 +1364,72 @@ step_post_update() {
   # same idempotent setup used by fresh installs so a completed update leaves
   # clawbox-gateway as the active single source of truth.
   step_gateway_setup || echo "  Warning: gateway_setup step failed (non-fatal)"
+  step_gateway_legacy_state_recovery || echo "  Warning: gateway_legacy_state_recovery step failed (non-fatal)"
   step_update_smoke || echo "  Warning: update_smoke reported issues (non-fatal)"
 }
 
+gateway_port_listening() {
+  local gw_port="${GATEWAY_PORT:-18789}"
+  ss -ltn 2>/dev/null | grep -qE "[:.]${gw_port}[[:space:]]"
+}
+
+step_gateway_legacy_state_recovery() {
+  local gw_port="${GATEWAY_PORT:-18789}"
+  if gateway_port_listening; then
+    echo "  Gateway is listening on ${gw_port}, skipping legacy state recovery"
+    return 0
+  fi
+
+  echo "  Gateway is not listening on ${gw_port}; running OpenClaw doctor recovery"
+  as_clawbox "$OPENCLAW_BIN" doctor --fix --yes --non-interactive || true
+  systemctl restart clawbox-gateway.service || true
+  sleep 8
+  if gateway_port_listening; then
+    echo "  Gateway recovered after doctor --fix"
+    return 0
+  fi
+
+  local journal_tail
+  journal_tail=$(journalctl -u clawbox-gateway.service -n 160 --no-pager 2>/dev/null || true)
+  if ! printf '%s\n' "$journal_tail" | grep -Eq 'installs\.json|conflicting plugin install metadata|carl_pir|belongs to agent piper'; then
+    echo "  Gateway still offline, but logs do not match known legacy-state blockers"
+    return 0
+  fi
+
+  local ts qdir moved=0
+  ts=$(date +%Y%m%d-%H%M%S)
+  qdir="$CLAWBOX_HOME/openclaw-legacy-quarantine-$ts"
+  mkdir -p "$qdir"
+
+  echo "  Quarantining known legacy OpenClaw migration blockers in $qdir"
+  systemctl stop clawbox-gateway.service || true
+  for f in \
+    "$CLAWBOX_HOME/.openclaw/plugins/installs.json"* \
+    "$CLAWBOX_HOME/.openclaw/memory/carl_pir.sqlite"* \
+    "$CLAWBOX_HOME/.openclaw/agents/carl_pir/agent/openclaw-agent.sqlite"*
+  do
+    if [ -e "$f" ]; then
+      mv -v "$f" "$qdir/" && moved=1
+    fi
+  done
+
+  if [ "$moved" -eq 0 ]; then
+    echo "  No known legacy migration blocker files found to quarantine"
+  fi
+
+  as_clawbox "$OPENCLAW_BIN" doctor --fix --yes --non-interactive || true
+  systemctl start clawbox-gateway.service || true
+  sleep 12
+
+  if gateway_port_listening; then
+    echo "  Gateway recovered after legacy state quarantine"
+    return 0
+  fi
+
+  echo "  Warning: gateway still not listening on ${gw_port} after legacy state recovery"
+  return 1
+}
+
 step_update_smoke() {
   # Advisory post-update smokes (#151). The rest of post_update only confirms
   # services are *running* — these confirm two flows that can silently break

package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawbox-setup",
-  "version": "3.1.9",
+  "version": "3.1.10",
   "private": true,
   "description": "ClawBox setup wizard and dashboard",
   "scripts": {

src/lib/updater.ts

@@ -4,6 +4,7 @@ import { readFile } from "fs/promises";
 import path from "path";
 import { get, set, setMany } from "./config-store";
 import { findOpenclawBin, restartGateway } from "./openclaw-config";
+import { isPortOpen } from "./port-probe";
 
 const PROJECT_DIR = "/home/clawbox/clawbox";
 const UPDATE_BRANCH_FILE = path.join(PROJECT_DIR, ".update-branch");
@@ -283,6 +284,101 @@ async function updateClawBoxAndReboot(): Promise<void> {
 // 2-3 min; shared across both UPDATE_STEPS and OPENCLAW_UPDATE_STEPS so the
 // two flows can't drift apart.
 const OPENCLAW_INSTALL_TIMEOUT_MS = 300_000;
+const GATEWAY_PORT = Number(process.env.GATEWAY_PORT || "18789");
+const GATEWAY_WAIT_INTERVAL_MS = 1_500;
+const LEGACY_GATEWAY_BLOCKER_RE =
+  /installs\.json|conflicting plugin install metadata|carl_pir|belongs to agent piper/i;
+
+async function delay(ms: number): Promise<void> {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitForGateway(timeoutMs: number): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (await isPortOpen(GATEWAY_PORT, "127.0.0.1", 1_000)) return true;
+    await delay(GATEWAY_WAIT_INTERVAL_MS);
+  }
+  return false;
+}
+
+async function runOpenclawDoctorFix(): Promise<void> {
+  try {
+    await execFile(OPENCLAW_BIN, ["doctor", "--fix", "--yes", "--non-interactive"], {
+      timeout: 90_000,
+      maxBuffer: 2 * 1024 * 1024,
+    });
+  } catch {
+    // Doctor can still repair some state before exiting non-zero. Continue
+    // into a restart + positive gateway probe rather than trusting exit code.
+  }
+}
+
+async function readGatewayJournalTail(): Promise<string> {
+  try {
+    const { stdout } = await execFile(
+      "/usr/bin/journalctl",
+      ["-u", "clawbox-gateway.service", "-n", "160", "--no-pager"],
+      { timeout: 10_000, maxBuffer: 2 * 1024 * 1024 },
+    );
+    return stdout;
+  } catch {
+    return "";
+  }
+}
+
+async function quarantineLegacyOpenclawState(): Promise<void> {
+  const script = `
+set -u
+CLAWBOX_HOME="/home/clawbox"
+TS="$(date +%Y%m%d-%H%M%S)"
+QDIR="$CLAWBOX_HOME/openclaw-legacy-quarantine-$TS"
+mkdir -p "$QDIR"
+/usr/bin/sudo /usr/bin/systemctl stop clawbox-gateway.service || true
+mv -v "$CLAWBOX_HOME/.openclaw/plugins/installs.json"* "$QDIR/" 2>/dev/null || true
+mv -v "$CLAWBOX_HOME/.openclaw/memory/carl_pir.sqlite"* "$QDIR/" 2>/dev/null || true
+mv -v "$CLAWBOX_HOME/.openclaw/agents/carl_pir/agent/openclaw-agent.sqlite"* "$QDIR/" 2>/dev/null || true
+`;
+  await execFile("/bin/bash", ["-lc", script], {
+    timeout: 30_000,
+    maxBuffer: 2 * 1024 * 1024,
+  });
+}
+
+async function ensureGatewayHealthy(options: { restartFirst?: boolean } = {}): Promise<void> {
+  if (options.restartFirst) {
+    await restartGateway();
+  }
+
+  if (await waitForGateway(30_000)) return;
+
+  await runOpenclawDoctorFix();
+  await restartGateway().catch(() => {});
+  if (await waitForGateway(30_000)) return;
+
+  const beforeRecoveryLog = await readGatewayJournalTail();
+  if (!LEGACY_GATEWAY_BLOCKER_RE.test(beforeRecoveryLog)) {
+    const lastLog = getLastLogLine(beforeRecoveryLog);
+    throw new Error(
+      lastLog
+        ? `OpenClaw gateway is not listening on port ${GATEWAY_PORT}: ${lastLog}`
+        : `OpenClaw gateway is not listening on port ${GATEWAY_PORT}`,
+    );
+  }
+
+  await quarantineLegacyOpenclawState();
+  await runOpenclawDoctorFix();
+  await restartGateway();
+  if (await waitForGateway(45_000)) return;
+
+  const afterRecoveryLog = await readGatewayJournalTail();
+  const lastLog = getLastLogLine(afterRecoveryLog);
+  throw new Error(
+    lastLog
+      ? `OpenClaw gateway still offline after legacy state recovery: ${lastLog}`
+      : "OpenClaw gateway still offline after legacy state recovery",
+  );
+}
 
 const UPDATE_STEPS: UpdateStepDef[] = [
   {
@@ -369,6 +465,13 @@ const UPDATE_STEPS: UpdateStepDef[] = [
     requiresRoot: true,
     advisoryOnOverrun: true,
   },
+  {
+    id: "gateway_verify",
+    label: "Verifying gateway health",
+    timeoutMs: 90_000,
+    customRun: () => ensureGatewayHealthy(),
+    failFast: true,
+  },
 ];
 
 /**
@@ -742,7 +845,7 @@ const OPENCLAW_UPDATE_STEPS: UpdateStepDef[] = [
     id: "gateway_restart",
     label: "Restarting OpenClaw gateway",
     timeoutMs: 30_000,
-    customRun: () => restartGateway(),
+    customRun: () => ensureGatewayHealthy({ restartFirst: true }),
   },
 ];
 

src/tests/unit/updater.test.ts

@@ -17,14 +17,20 @@ vi.mock("@/lib/config-store", () => ({
   setMany: vi.fn(),
 }));
 
+vi.mock("@/lib/port-probe", () => ({
+  isPortOpen: vi.fn(),
+}));
+
 import { get, set, setMany } from "@/lib/config-store";
+import { isPortOpen } from "@/lib/port-probe";
 
 const mockGet = vi.mocked(get);
 const mockSet = vi.mocked(set);
 const mockSetMany = vi.mocked(setMany);
 const mockExec = vi.mocked(childProcess.exec);
 const mockExecFile = vi.mocked(childProcess.execFile);
 const mockReadFile = vi.mocked(fs.readFile);
+const mockIsPortOpen = vi.mocked(isPortOpen);
 
 function setupExecMock(results: Record<string, { stdout: string; stderr: string } | Error> = {}) {
   mockExec.mockImplementation(((
@@ -132,6 +138,7 @@ describe("updater", () => {
     mockSet.mockResolvedValue();
     mockSetMany.mockResolvedValue();
     mockReadFile.mockRejectedValue(new Error("ENOENT"));
+    mockIsPortOpen.mockResolvedValue(true);
 
     setupExecMock({
       "ls-remote": { stdout: "abc123\trefs/tags/v1.0.0\ndef456\trefs/tags/v1.1.0\n", stderr: "" },