diff --git a/install.sh b/install.sh index 1d43c33b1..ba698de20 100755 --- a/install.sh +++ b/install.sh @@ -1364,9 +1364,72 @@ step_post_update() { # same idempotent setup used by fresh installs so a completed update leaves # clawbox-gateway as the active single source of truth. step_gateway_setup || echo " Warning: gateway_setup step failed (non-fatal)" + step_gateway_legacy_state_recovery || echo " Warning: gateway_legacy_state_recovery step failed (non-fatal)" step_update_smoke || echo " Warning: update_smoke reported issues (non-fatal)" } +gateway_port_listening() { + local gw_port="${GATEWAY_PORT:-18789}" + ss -ltn 2>/dev/null | grep -qE "[:.]${gw_port}[[:space:]]" +} + +step_gateway_legacy_state_recovery() { + local gw_port="${GATEWAY_PORT:-18789}" + if gateway_port_listening; then + echo " Gateway is listening on ${gw_port}, skipping legacy state recovery" + return 0 + fi + + echo " Gateway is not listening on ${gw_port}; running OpenClaw doctor recovery" + as_clawbox "$OPENCLAW_BIN" doctor --fix --yes --non-interactive || true + systemctl restart clawbox-gateway.service || true + sleep 8 + if gateway_port_listening; then + echo " Gateway recovered after doctor --fix" + return 0 + fi + + local journal_tail + journal_tail=$(journalctl -u clawbox-gateway.service -n 160 --no-pager 2>/dev/null || true) + if ! printf '%s\n' "$journal_tail" | grep -Eq 'installs\.json|conflicting plugin install metadata|carl_pir|belongs to agent piper'; then + echo " Gateway still offline, but logs do not match known legacy-state blockers" + return 0 + fi + + local ts qdir moved=0 + ts=$(date +%Y%m%d-%H%M%S) + qdir="$CLAWBOX_HOME/openclaw-legacy-quarantine-$ts" + mkdir -p "$qdir" + + echo " Quarantining known legacy OpenClaw migration blockers in $qdir" + systemctl stop clawbox-gateway.service || true + for f in \ + "$CLAWBOX_HOME/.openclaw/plugins/installs.json"* \ + "$CLAWBOX_HOME/.openclaw/memory/carl_pir.sqlite"* \ + "$CLAWBOX_HOME/.openclaw/agents/carl_pir/agent/openclaw-agent.sqlite"* + do + if [ -e "$f" ]; then + mv -v "$f" "$qdir/" && moved=1 + fi + done + + if [ "$moved" -eq 0 ]; then + echo " No known legacy migration blocker files found to quarantine" + fi + + as_clawbox "$OPENCLAW_BIN" doctor --fix --yes --non-interactive || true + systemctl start clawbox-gateway.service || true + sleep 12 + + if gateway_port_listening; then + echo " Gateway recovered after legacy state quarantine" + return 0 + fi + + echo " Warning: gateway still not listening on ${gw_port} after legacy state recovery" + return 1 +} + step_update_smoke() { # Advisory post-update smokes (#151). The rest of post_update only confirms # services are *running* — these confirm two flows that can silently break diff --git a/package-lock.json b/package-lock.json index 8a7c08118..a11580e92 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "clawbox-setup", - "version": "3.1.9", + "version": "3.1.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "clawbox-setup", - "version": "3.1.9", + "version": "3.1.10", "dependencies": { "@types/ws": "^8.18.1", "@xterm/addon-fit": "^0.11.0", diff --git a/package.json b/package.json index ee941533b..4076494c1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clawbox-setup", - "version": "3.1.9", + "version": "3.1.10", "private": true, "description": "ClawBox setup wizard and dashboard", "scripts": { diff --git a/src/lib/updater.ts b/src/lib/updater.ts index 7fcc82350..af75b23e1 100644 --- a/src/lib/updater.ts +++ b/src/lib/updater.ts @@ -4,6 +4,7 @@ import { readFile } from "fs/promises"; import path from "path"; import { get, set, setMany } from "./config-store"; import { findOpenclawBin, restartGateway } from "./openclaw-config"; +import { isPortOpen } from "./port-probe"; const PROJECT_DIR = "/home/clawbox/clawbox"; const UPDATE_BRANCH_FILE = path.join(PROJECT_DIR, ".update-branch"); @@ -283,6 +284,101 @@ async function updateClawBoxAndReboot(): Promise { // 2-3 min; shared across both UPDATE_STEPS and OPENCLAW_UPDATE_STEPS so the // two flows can't drift apart. const OPENCLAW_INSTALL_TIMEOUT_MS = 300_000; +const GATEWAY_PORT = Number(process.env.GATEWAY_PORT || "18789"); +const GATEWAY_WAIT_INTERVAL_MS = 1_500; +const LEGACY_GATEWAY_BLOCKER_RE = + /installs\.json|conflicting plugin install metadata|carl_pir|belongs to agent piper/i; + +async function delay(ms: number): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitForGateway(timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await isPortOpen(GATEWAY_PORT, "127.0.0.1", 1_000)) return true; + await delay(GATEWAY_WAIT_INTERVAL_MS); + } + return false; +} + +async function runOpenclawDoctorFix(): Promise { + try { + await execFile(OPENCLAW_BIN, ["doctor", "--fix", "--yes", "--non-interactive"], { + timeout: 90_000, + maxBuffer: 2 * 1024 * 1024, + }); + } catch { + // Doctor can still repair some state before exiting non-zero. Continue + // into a restart + positive gateway probe rather than trusting exit code. + } +} + +async function readGatewayJournalTail(): Promise { + try { + const { stdout } = await execFile( + "/usr/bin/journalctl", + ["-u", "clawbox-gateway.service", "-n", "160", "--no-pager"], + { timeout: 10_000, maxBuffer: 2 * 1024 * 1024 }, + ); + return stdout; + } catch { + return ""; + } +} + +async function quarantineLegacyOpenclawState(): Promise { + const script = ` +set -u +CLAWBOX_HOME="/home/clawbox" +TS="$(date +%Y%m%d-%H%M%S)" +QDIR="$CLAWBOX_HOME/openclaw-legacy-quarantine-$TS" +mkdir -p "$QDIR" +/usr/bin/sudo /usr/bin/systemctl stop clawbox-gateway.service || true +mv -v "$CLAWBOX_HOME/.openclaw/plugins/installs.json"* "$QDIR/" 2>/dev/null || true +mv -v "$CLAWBOX_HOME/.openclaw/memory/carl_pir.sqlite"* "$QDIR/" 2>/dev/null || true +mv -v "$CLAWBOX_HOME/.openclaw/agents/carl_pir/agent/openclaw-agent.sqlite"* "$QDIR/" 2>/dev/null || true +`; + await execFile("/bin/bash", ["-lc", script], { + timeout: 30_000, + maxBuffer: 2 * 1024 * 1024, + }); +} + +async function ensureGatewayHealthy(options: { restartFirst?: boolean } = {}): Promise { + if (options.restartFirst) { + await restartGateway(); + } + + if (await waitForGateway(30_000)) return; + + await runOpenclawDoctorFix(); + await restartGateway().catch(() => {}); + if (await waitForGateway(30_000)) return; + + const beforeRecoveryLog = await readGatewayJournalTail(); + if (!LEGACY_GATEWAY_BLOCKER_RE.test(beforeRecoveryLog)) { + const lastLog = getLastLogLine(beforeRecoveryLog); + throw new Error( + lastLog + ? `OpenClaw gateway is not listening on port ${GATEWAY_PORT}: ${lastLog}` + : `OpenClaw gateway is not listening on port ${GATEWAY_PORT}`, + ); + } + + await quarantineLegacyOpenclawState(); + await runOpenclawDoctorFix(); + await restartGateway(); + if (await waitForGateway(45_000)) return; + + const afterRecoveryLog = await readGatewayJournalTail(); + const lastLog = getLastLogLine(afterRecoveryLog); + throw new Error( + lastLog + ? `OpenClaw gateway still offline after legacy state recovery: ${lastLog}` + : "OpenClaw gateway still offline after legacy state recovery", + ); +} const UPDATE_STEPS: UpdateStepDef[] = [ { @@ -369,6 +465,13 @@ const UPDATE_STEPS: UpdateStepDef[] = [ requiresRoot: true, advisoryOnOverrun: true, }, + { + id: "gateway_verify", + label: "Verifying gateway health", + timeoutMs: 90_000, + customRun: () => ensureGatewayHealthy(), + failFast: true, + }, ]; /** @@ -742,7 +845,7 @@ const OPENCLAW_UPDATE_STEPS: UpdateStepDef[] = [ id: "gateway_restart", label: "Restarting OpenClaw gateway", timeoutMs: 30_000, - customRun: () => restartGateway(), + customRun: () => ensureGatewayHealthy({ restartFirst: true }), }, ]; diff --git a/src/tests/unit/updater.test.ts b/src/tests/unit/updater.test.ts index c97e74b7f..7b2fe7fce 100644 --- a/src/tests/unit/updater.test.ts +++ b/src/tests/unit/updater.test.ts @@ -17,7 +17,12 @@ vi.mock("@/lib/config-store", () => ({ setMany: vi.fn(), })); +vi.mock("@/lib/port-probe", () => ({ + isPortOpen: vi.fn(), +})); + import { get, set, setMany } from "@/lib/config-store"; +import { isPortOpen } from "@/lib/port-probe"; const mockGet = vi.mocked(get); const mockSet = vi.mocked(set); @@ -25,6 +30,7 @@ const mockSetMany = vi.mocked(setMany); const mockExec = vi.mocked(childProcess.exec); const mockExecFile = vi.mocked(childProcess.execFile); const mockReadFile = vi.mocked(fs.readFile); +const mockIsPortOpen = vi.mocked(isPortOpen); function setupExecMock(results: Record = {}) { mockExec.mockImplementation((( @@ -132,6 +138,7 @@ describe("updater", () => { mockSet.mockResolvedValue(); mockSetMany.mockResolvedValue(); mockReadFile.mockRejectedValue(new Error("ENOENT")); + mockIsPortOpen.mockResolvedValue(true); setupExecMock({ "ls-remote": { stdout: "abc123\trefs/tags/v1.0.0\ndef456\trefs/tags/v1.1.0\n", stderr: "" },