-
Notifications
You must be signed in to change notification settings - Fork 2.3k
Gracefully protect live sessions during daemon updates #333
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
4c9dd6a
c1cfff2
1f4a387
47c2962
7a48e04
f2c641e
01a4a6a
20917a7
55c8b52
3a727f7
21f9472
84baecb
da8d772
ffaf388
80fd344
3f7e8ba
0d951d0
f68ebd8
bbbded5
e7d2517
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,7 +9,7 @@ | |
|
|
||
| import { spawn } from "node:child_process"; | ||
| import { existsSync } from "node:fs"; | ||
| import { resolve } from "node:path"; | ||
| import { dirname, resolve } from "node:path"; | ||
| import { appendRotatingLog, expandTildePath, getClientErrorLogPath, VERSION } from "../config.js"; | ||
| import { DaemonClient } from "../modes/daemon/daemon-client.js"; | ||
| import { DAEMON_PROTOCOL_VERSION } from "../modes/daemon/daemon-protocol.js"; | ||
|
|
@@ -35,6 +35,20 @@ function logDaemonLaunch(message: string): void { | |
| appendRotatingLog(getClientErrorLogPath(), `[${new Date().toISOString()}] daemon-launch: ${message}`); | ||
| } | ||
|
|
||
| function reportDaemonLaunchRestoreWarnings(result: DaemonSessionRestoreResult): void { | ||
| if (result.failed.length === 0) { | ||
| return; | ||
| } | ||
| const message = `Warning: restored ${result.restored}/${result.total} daemon session(s), but ${result.failed.length} session(s) failed to reopen after daemon replacement.`; | ||
| logDaemonLaunch(message); | ||
| console.error(message); | ||
| for (const failure of result.failed) { | ||
| const detail = ` ${failure.sessionFile}: ${failure.error}`; | ||
| logDaemonLaunch(detail); | ||
| console.error(detail); | ||
| } | ||
| } | ||
|
|
||
| async function canConnectToDaemon(socketPath: string, timeoutMs: number): Promise<boolean> { | ||
| const client = new DaemonClient(socketPath); | ||
| try { | ||
|
|
@@ -132,12 +146,23 @@ export async function shutdownDaemonAndWait(socketPath: string): Promise<boolean | |
| } | ||
|
|
||
| // activeSessions is undefined when the daemon is reachable but its sessions couldn't | ||
| // be listed — callers must treat that as "possibly busy", not idle. | ||
| // be listed — callers must treat that as "possibly active", not idle. | ||
| export type RunningDaemonProbe = { reachable: false } | { reachable: true; activeSessions?: SessionSummary[] }; | ||
|
|
||
| export function isSessionBusy(summary: SessionSummary): boolean { | ||
| // pendingMessageCount covers queued steering/follow-ups, which live only in | ||
| // memory and would be lost if the daemon were stopped. | ||
| export interface DaemonSessionRestoreFailure { | ||
| sessionFile: string; | ||
| error: string; | ||
| } | ||
|
|
||
| export interface DaemonSessionRestoreResult { | ||
| restored: number; | ||
| total: number; | ||
| failed: DaemonSessionRestoreFailure[]; | ||
| } | ||
|
|
||
| export function hasSessionVolatileWorkForDaemonStop(summary: SessionSummary): boolean { | ||
| // These states live in daemon memory and cannot be reconstructed by reopening | ||
| // the JSONL session file after the daemon restarts. | ||
| return ( | ||
| summary.isStreaming || | ||
| summary.isCompacting || | ||
|
|
@@ -147,6 +172,88 @@ export function isSessionBusy(summary: SessionSummary): boolean { | |
| ); | ||
|
cursor[bot] marked this conversation as resolved.
|
||
| } | ||
|
|
||
| export function isSessionReopenableAfterDaemonStop(summary: SessionSummary): boolean { | ||
| return ( | ||
| summary.activeSessionId !== undefined && summary.runtimeKind !== "subagent" && summary.sessionFile !== undefined | ||
| ); | ||
| } | ||
|
|
||
| export function isSessionRestorableAfterDaemonStop(summary: SessionSummary): boolean { | ||
| return isSessionReopenableAfterDaemonStop(summary) && !hasSessionVolatileWorkForDaemonStop(summary); | ||
| } | ||
|
|
||
| export function isSessionBusy(summary: SessionSummary): boolean { | ||
| return hasSessionVolatileWorkForDaemonStop(summary); | ||
| } | ||
|
|
||
| export function isSessionAtRiskFromDaemonStop(summary: SessionSummary): boolean { | ||
|
macroscopeapp[bot] marked this conversation as resolved.
|
||
| if (summary.activeSessionId === undefined) { | ||
| return hasSessionVolatileWorkForDaemonStop(summary); | ||
| } | ||
| return !isSessionRestorableAfterDaemonStop(summary); | ||
| } | ||
|
|
||
| export function isRunningDaemonProbeAtRiskFromStop(probe: RunningDaemonProbe): boolean { | ||
| if (!probe.reachable) { | ||
| return false; | ||
| } | ||
| return probe.activeSessions === undefined || probe.activeSessions.some(isSessionAtRiskFromDaemonStop); | ||
| } | ||
|
|
||
| function restoreCandidateSessionFiles(sessions: readonly SessionSummary[]): string[] { | ||
| return [ | ||
| ...new Set( | ||
| sessions | ||
| .filter(isSessionRestorableAfterDaemonStop) | ||
| .map((session) => session.sessionFile) | ||
| .filter((sessionFile): sessionFile is string => sessionFile !== undefined) | ||
| .map((sessionFile) => resolve(sessionFile)), | ||
| ), | ||
| ]; | ||
|
cursor[bot] marked this conversation as resolved.
|
||
| } | ||
|
|
||
| export async function restoreDaemonSessionSummaries( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Medium
🚀 Reply "fix it for me" or copy this AI Prompt for your agent: |
||
| socketPath: string, | ||
| sessions: readonly SessionSummary[], | ||
| ): Promise<DaemonSessionRestoreResult> { | ||
| const sessionFiles = restoreCandidateSessionFiles(sessions); | ||
| if (sessionFiles.length === 0) { | ||
| return { restored: 0, total: 0, failed: [] }; | ||
| } | ||
|
|
||
| const client = new DaemonClient(socketPath); | ||
| await client.connect(3000); | ||
| let restored = 0; | ||
| const failed: DaemonSessionRestoreFailure[] = []; | ||
| try { | ||
| for (const sessionFile of sessionFiles) { | ||
| const sourceSummary = sessions.find( | ||
| (session) => | ||
| isSessionRestorableAfterDaemonStop(session) && | ||
| session.sessionFile && | ||
| resolve(session.sessionFile) === sessionFile, | ||
| ); | ||
| const response = await client.request({ | ||
| type: "create", | ||
| ...(sourceSummary?.activeSessionId ? { activeSessionId: sourceSummary.activeSessionId } : {}), | ||
| sessionPath: sessionFile, | ||
| config: { | ||
| sessionDir: dirname(sessionFile), | ||
| ...(sourceSummary?.cwd ? { cwd: sourceSummary.cwd } : {}), | ||
| }, | ||
|
cursor[bot] marked this conversation as resolved.
|
||
| }); | ||
| if (response.success) { | ||
| restored++; | ||
| } else { | ||
| failed.push({ sessionFile, error: response.error }); | ||
| } | ||
| } | ||
| } finally { | ||
| client.close(); | ||
| } | ||
| return { restored, total: sessionFiles.length, failed }; | ||
| } | ||
|
|
||
| export async function probeRunningDaemonSessions(socketPath: string): Promise<RunningDaemonProbe> { | ||
| const client = new DaemonClient(socketPath); | ||
| try { | ||
|
|
@@ -165,23 +272,25 @@ export async function probeRunningDaemonSessions(socketPath: string): Promise<Ru | |
| } | ||
| } | ||
|
|
||
| // Idle-but-loaded sessions reload from disk on the fresh daemon, so only a busy | ||
| // session blocks replacing a stale daemon. | ||
| async function shutdownStaleDaemonIfNotBusy(socketPath: string): Promise<boolean> { | ||
| interface StaleDaemonShutdownResult { | ||
| stopped: boolean; | ||
| restoreSessions: SessionSummary[]; | ||
| } | ||
|
|
||
| // Sessions with volatile in-memory work still block implicit replacement. Idle | ||
| // persisted top-level sessions can be reopened after the fresh daemon starts. | ||
| async function shutdownStaleDaemonForReplacement(socketPath: string): Promise<StaleDaemonShutdownResult> { | ||
| const client = new DaemonClient(socketPath); | ||
| let connected = false; | ||
| let hasBusySessions = false; | ||
| let loadedSessionCount = 0; | ||
| let summaries: SessionSummary[] | undefined; | ||
| try { | ||
| await client.connect(1000); | ||
| connected = true; | ||
| try { | ||
| const summaries = await listActiveDaemonSessionSummaries(client); | ||
| loadedSessionCount = summaries.length; | ||
| hasBusySessions = summaries.some(isSessionBusy); | ||
| summaries = await listActiveDaemonSessionSummaries(client); | ||
| } catch { | ||
| // Couldn't confirm idleness: treat as busy rather than risk interrupting work. | ||
| hasBusySessions = true; | ||
| // Couldn't confirm the session state: treat as active rather than risk interrupting work. | ||
| summaries = undefined; | ||
| } | ||
| } catch { | ||
| // Couldn't reach it to inspect; don't send a blind shutdown, just verify below. | ||
|
|
@@ -190,30 +299,25 @@ async function shutdownStaleDaemonIfNotBusy(socketPath: string): Promise<boolean | |
| } | ||
|
|
||
| if (!connected) { | ||
| return waitForDaemonGone(socketPath); | ||
| return { stopped: await waitForDaemonGone(socketPath), restoreSessions: [] }; | ||
| } | ||
| if (hasBusySessions) { | ||
| logDaemonLaunch(`refusing to replace stale daemon on ${socketPath}: busy session(s) present`); | ||
| return false; | ||
| if (!summaries) { | ||
| logDaemonLaunch(`refusing to replace stale daemon on ${socketPath}: session list unavailable`); | ||
| return { stopped: false, restoreSessions: [] }; | ||
| } | ||
| const atRiskSessions = summaries.filter(isSessionAtRiskFromDaemonStop); | ||
| if (atRiskSessions.length > 0) { | ||
| logDaemonLaunch(`refusing to replace stale daemon on ${socketPath}: unrestorable live session(s) present`); | ||
| return { stopped: false, restoreSessions: [] }; | ||
| } | ||
| const restoreSessions = summaries.filter(isSessionRestorableAfterDaemonStop); | ||
| logDaemonLaunch( | ||
| `replacing stale daemon on ${socketPath} (idle): ${loadedSessionCount} loaded session(s) will reload`, | ||
| `replacing stale daemon on ${socketPath}: ${restoreSessions.length} live session(s) will be reopened`, | ||
| ); | ||
| return shutdownDaemonAndWait(socketPath); | ||
| return { stopped: await shutdownDaemonAndWait(socketPath), restoreSessions }; | ||
| } | ||
|
|
||
| async function ensureDaemonRunning(socketPath: string, spawnCwd?: string): Promise<void> { | ||
| const probe = await probeDaemonVersion(socketPath); | ||
| if (probe === "current") { | ||
| return; | ||
| } | ||
| if (probe === "stale") { | ||
| const stopped = await shutdownStaleDaemonIfNotBusy(socketPath); | ||
| if (!stopped) { | ||
| throw new StaleDaemonError(socketPath); | ||
| } | ||
| } | ||
|
|
||
| async function spawnDaemonAndWait(socketPath: string, spawnCwd?: string): Promise<void> { | ||
| const entrypoint = process.argv[1]; | ||
| if (!entrypoint) { | ||
| throw new Error("Cannot determine current CLI entrypoint for daemon launch"); | ||
|
|
@@ -244,6 +348,56 @@ async function ensureDaemonRunning(socketPath: string, spawnCwd?: string): Promi | |
| throw new Error(`Timed out waiting for daemon to start on ${socketPath}`); | ||
| } | ||
|
|
||
| export async function relaunchDaemonAndRestoreSessions( | ||
| socketPath: string, | ||
| sessions: readonly SessionSummary[], | ||
| spawnCwd?: string, | ||
| options: { allowAtRiskSessions?: boolean; latestProbe?: RunningDaemonProbe } = {}, | ||
| ): Promise<DaemonSessionRestoreResult> { | ||
| const latestProbe = options.latestProbe ?? (await probeRunningDaemonSessions(socketPath)); | ||
| if (latestProbe.reachable && latestProbe.activeSessions === undefined && !options.allowAtRiskSessions) { | ||
| throw new Error(`Cannot stop daemon on ${socketPath}: live session list unavailable`); | ||
|
cursor[bot] marked this conversation as resolved.
|
||
| } | ||
| const sessionsToRestore = latestProbe.reachable ? (latestProbe.activeSessions ?? sessions) : sessions; | ||
|
cursor[bot] marked this conversation as resolved.
|
||
| const atRiskSessions = sessionsToRestore.filter(isSessionAtRiskFromDaemonStop); | ||
| if (atRiskSessions.length > 0 && !options.allowAtRiskSessions) { | ||
| throw new Error(`Cannot stop daemon on ${socketPath}: unrestorable live session(s) present`); | ||
| } | ||
| const stopped = await shutdownDaemonAndWait(socketPath); | ||
| if (!stopped) { | ||
| throw new Error(`Could not stop daemon on ${socketPath}`); | ||
| } | ||
| await spawnDaemonAndWait(socketPath, spawnCwd); | ||
| return restoreDaemonSessionSummaries(socketPath, sessionsToRestore); | ||
| } | ||
|
|
||
| export async function spawnDaemonAndRestoreSessions( | ||
| socketPath: string, | ||
| sessions: readonly SessionSummary[], | ||
| spawnCwd?: string, | ||
| ): Promise<DaemonSessionRestoreResult> { | ||
| await spawnDaemonAndWait(socketPath, spawnCwd); | ||
| return restoreDaemonSessionSummaries(socketPath, sessions); | ||
| } | ||
|
|
||
| async function ensureDaemonRunning(socketPath: string, spawnCwd?: string): Promise<void> { | ||
| const probe = await probeDaemonVersion(socketPath); | ||
| if (probe === "current") { | ||
| return; | ||
| } | ||
| let restoreSessions: SessionSummary[] = []; | ||
| if (probe === "stale") { | ||
| const result = await shutdownStaleDaemonForReplacement(socketPath); | ||
| if (!result.stopped) { | ||
| throw new StaleDaemonError(socketPath); | ||
| } | ||
| restoreSessions = result.restoreSessions; | ||
| } | ||
|
|
||
| const restoreResult = await spawnDaemonAndRestoreSessions(socketPath, restoreSessions, spawnCwd); | ||
| reportDaemonLaunchRestoreWarnings(restoreResult); | ||
| } | ||
|
|
||
| const ensurePromises = new Map<string, Promise<void>>(); | ||
|
|
||
| /** | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.