fix(desktop): recover chat after sleep/wake by revalidating the cached backend

apps/desktop/electron/hardening.cjs

@@ -21,6 +21,81 @@ function resolveTimeoutMs(timeoutMs, fallbackMs = DEFAULT_FETCH_TIMEOUT_MS) {
   return fallback
 }
 
+// ── Sleep/wake reconnect: cached-connection liveness revalidation ────────────
+// After sleep/wake the renderer reconnects through getConnection(), which on the
+// main side hands back a cached backend descriptor. A REMOTE primary spawns no
+// child process, so the 'exit'/'error' handlers that would clear a dead cache
+// never fire — without a liveness check the renderer re-dials the same
+// unreachable remote forever and the composer stays stuck on "Starting Hermes…".
+// These pure helpers back the revalidate-on-reconnect path in main.cjs; they live
+// here so they can be unit-tested without loading Electron.
+
+// Consecutive failed probes required before tearing down and rebuilding a cached
+// remote connection. A single miss is ignored so a transient post-wake blip
+// (captive portal, VPN re-establishing) doesn't trigger a full respawn that the
+// renderer's own backoff would have ridden out.
+const REMOTE_REVALIDATE_FAILURE_THRESHOLD = 2
+// Fast single-shot liveness probe budget. A healthy loopback or remote answers
+// /api/status well under this; long enough to tolerate a momentary stall.
+const REVALIDATE_PROBE_TIMEOUT_MS = 2_500
+// Window within which probe failures count as one reconnect episode. Comfortably
+// larger than the renderer's 15s backoff cap (so two genuinely-consecutive misses
+// always fall inside it) yet short enough that a stale miss from a since-recovered
+// episode expires before the next one.
+const REVALIDATE_STREAK_WINDOW_MS = 60_000
+
+// Probe whether a resolved backend connection descriptor is still reachable by
+// hitting its PUBLIC /api/status endpoint. Deliberately host-liveness only — it
+// does NOT validate auth (an expired OAuth session still returns 200; auth is
+// handled separately by the ws-ticket mint). `fetchJson` is injected (the
+// token-free fetchPublicJson from main.cjs) so this stays pure and testable.
+// Never throws: returns false on any failure.
+async function probeBackendAlive(conn, fetchJson, { timeoutMs = REVALIDATE_PROBE_TIMEOUT_MS } = {}) {
+  if (!conn || typeof conn.baseUrl !== 'string' || !conn.baseUrl || typeof fetchJson !== 'function') {
+    return false
+  }
+
+  const base = conn.baseUrl.replace(/\/+$/, '')
+
+  try {
+    await fetchJson(`${base}/api/status`, { timeoutMs })
+    return true
+  } catch {
+    return false
+  }
+}
+
+// Decide whether a cached connection that just failed a liveness probe should be
+// torn down and rebuilt. Only REMOTE connections are eligible: a local backend
+// self-heals via its child 'exit' handler, so a probe miss there means "the WS
+// hasn't reattached yet", not "the backend is dead" — tearing it down would
+// needlessly SIGTERM a healthy child. Requires a failure streak so a single
+// transient miss can't trigger a respawn.
+function shouldRebuildStaleConnection({
+  alive,
+  mode,
+  failureStreak,
+  threshold = REMOTE_REVALIDATE_FAILURE_THRESHOLD
+}) {
+  if (alive) {
+    return false
+  }
+
+  return mode === 'remote' && Number(failureStreak) >= threshold
+}
+
+// True when a probe failure is far enough from the previous one (or is the first
+// ever) to count as a NEW reconnect episode rather than a continuation — so the
+// caller resets its consecutive-failure streak and a stale miss left over from an
+// earlier, since-recovered episode can't pre-load the "2 consecutive" counter.
+function isFreshRevalidateEpisode(now, lastFailureAt, windowMs = REVALIDATE_STREAK_WINDOW_MS) {
+  if (!lastFailureAt) {
+    return true
+  }
+
+  return Number(now) - Number(lastFailureAt) > windowMs
+}
+
 function encryptDesktopSecret(value, safeStorageApi) {
   const raw = String(value || '')
 
@@ -176,9 +251,15 @@ async function resolveReadableFileForIpc(filePath, options = {}) {
 module.exports = {
   DATA_URL_READ_MAX_BYTES,
   DEFAULT_FETCH_TIMEOUT_MS,
+  REMOTE_REVALIDATE_FAILURE_THRESHOLD,
+  REVALIDATE_PROBE_TIMEOUT_MS,
+  REVALIDATE_STREAK_WINDOW_MS,
   TEXT_PREVIEW_SOURCE_MAX_BYTES,
   encryptDesktopSecret,
+  isFreshRevalidateEpisode,
+  probeBackendAlive,
   resolveReadableFileForIpc,
   resolveTimeoutMs,
-  sensitiveFileBlockReason
+  sensitiveFileBlockReason,
+  shouldRebuildStaleConnection
 }

apps/desktop/electron/hardening.test.cjs

@@ -7,10 +7,16 @@ const { pathToFileURL } = require('node:url')
 
 const {
   DEFAULT_FETCH_TIMEOUT_MS,
+  REMOTE_REVALIDATE_FAILURE_THRESHOLD,
+  REVALIDATE_PROBE_TIMEOUT_MS,
+  REVALIDATE_STREAK_WINDOW_MS,
   encryptDesktopSecret,
+  isFreshRevalidateEpisode,
+  probeBackendAlive,
   resolveReadableFileForIpc,
   resolveTimeoutMs,
-  sensitiveFileBlockReason
+  sensitiveFileBlockReason,
+  shouldRebuildStaleConnection
 } = require('./hardening.cjs')
 
 test('resolveTimeoutMs falls back to defaults and accepts overrides', () => {
@@ -114,3 +120,116 @@ test('resolveReadableFileForIpc validates existence type size and sensitivity',
   })
   assert.equal(envTemplate.resolvedPath, envTemplatePath)
 })
+
+// ── Sleep/wake reconnect: cached-connection liveness revalidation ────────────
+
+test('probeBackendAlive returns true when /api/status responds', async () => {
+  const calls = []
+  const fetchJson = async (url, opts) => {
+    calls.push({ url, opts })
+    return { ok: true }
+  }
+
+  const alive = await probeBackendAlive({ baseUrl: 'https://gw.example.com' }, fetchJson, { timeoutMs: 2500 })
+
+  assert.equal(alive, true)
+  assert.equal(calls.length, 1)
+  // Trailing slash on baseUrl must not double up before /api/status.
+  assert.equal(calls[0].url, 'https://gw.example.com/api/status')
+  assert.equal(calls[0].opts.timeoutMs, 2500)
+})
+
+test('probeBackendAlive normalizes a trailing slash on baseUrl', async () => {
+  let seen = null
+  const fetchJson = async url => {
+    seen = url
+    return null
+  }
+
+  await probeBackendAlive({ baseUrl: 'http://127.0.0.1:8787/' }, fetchJson)
+  assert.equal(seen, 'http://127.0.0.1:8787/api/status')
+})
+
+test('probeBackendAlive returns false (never throws) when the probe rejects', async () => {
+  const fetchJson = async () => {
+    throw new Error('ECONNREFUSED')
+  }
+
+  const alive = await probeBackendAlive({ baseUrl: 'https://gw.example.com' }, fetchJson)
+  assert.equal(alive, false)
+})
+
+test('probeBackendAlive returns false for a missing baseUrl or fetcher', async () => {
+  const ok = async () => ({})
+  assert.equal(await probeBackendAlive(null, ok), false)
+  assert.equal(await probeBackendAlive({ baseUrl: '' }, ok), false)
+  assert.equal(await probeBackendAlive({ baseUrl: 'https://gw.example.com' }, undefined), false)
+})
+
+test('probeBackendAlive defaults the probe timeout to REVALIDATE_PROBE_TIMEOUT_MS', async () => {
+  let seenTimeout
+  const fetchJson = async (_url, opts) => {
+    seenTimeout = opts.timeoutMs
+    return {}
+  }
+
+  await probeBackendAlive({ baseUrl: 'https://gw.example.com' }, fetchJson)
+  assert.equal(seenTimeout, REVALIDATE_PROBE_TIMEOUT_MS)
+})
+
+test('shouldRebuildStaleConnection only tears down a remote past the failure threshold', () => {
+  // A live backend is never rebuilt, regardless of streak.
+  assert.equal(
+    shouldRebuildStaleConnection({ alive: true, mode: 'remote', failureStreak: 99 }),
+    false
+  )
+
+  // A local backend is never rebuilt here — it self-heals via its child 'exit'
+  // handler, so a probe miss means "WS not reattached yet", not "backend dead".
+  assert.equal(
+    shouldRebuildStaleConnection({ alive: false, mode: 'local', failureStreak: 99 }),
+    false
+  )
+
+  // A single remote miss is ignored (rides out transient post-wake blips)…
+  assert.equal(
+    shouldRebuildStaleConnection({ alive: false, mode: 'remote', failureStreak: 1 }),
+    false
+  )
+
+  // …but the second consecutive miss crosses the threshold and rebuilds.
+  assert.equal(
+    shouldRebuildStaleConnection({
+      alive: false,
+      mode: 'remote',
+      failureStreak: REMOTE_REVALIDATE_FAILURE_THRESHOLD
+    }),
+    true
+  )
+
+  // Unknown / undefined mode (e.g. a rejected cache) is not eligible.
+  assert.equal(
+    shouldRebuildStaleConnection({ alive: false, mode: undefined, failureStreak: 5 }),
+    false
+  )
+})
+
+test('REMOTE_REVALIDATE_FAILURE_THRESHOLD requires at least two misses', () => {
+  assert.ok(REMOTE_REVALIDATE_FAILURE_THRESHOLD >= 2)
+})
+
+test('isFreshRevalidateEpisode scopes the failure streak to one reconnect episode', () => {
+  // First-ever failure (no prior timestamp) always starts a fresh episode.
+  assert.equal(isFreshRevalidateEpisode(1_000, 0), true)
+
+  // A miss within the window continues the same episode (streak accumulates).
+  assert.equal(isFreshRevalidateEpisode(10_000, 10_000 - (REVALIDATE_STREAK_WINDOW_MS - 1)), false)
+
+  // A miss past the window is a new episode (streak resets) — this is what stops
+  // a stale miss from an earlier, since-recovered episode pre-loading the counter.
+  assert.equal(isFreshRevalidateEpisode(10_000, 10_000 - (REVALIDATE_STREAK_WINDOW_MS + 1)), true)
+
+  // The window comfortably exceeds the renderer's 15s backoff cap so two
+  // genuinely-consecutive backoff-paced misses always fall inside it.
+  assert.ok(REVALIDATE_STREAK_WINDOW_MS > 15_000)
+})

apps/desktop/electron/main.cjs

@@ -47,8 +47,11 @@ const {
   DEFAULT_FETCH_TIMEOUT_MS,
   TEXT_PREVIEW_SOURCE_MAX_BYTES,
   encryptDesktopSecret: encryptDesktopSecretStrict,
+  isFreshRevalidateEpisode,
+  probeBackendAlive,
   resolveReadableFileForIpc,
-  resolveTimeoutMs
+  resolveTimeoutMs,
+  shouldRebuildStaleConnection
 } = require('./hardening.cjs')
 
 let nodePty = null
@@ -475,6 +478,18 @@ function registerMediaProtocol() {
 let mainWindow = null
 let hermesProcess = null
 let connectionPromise = null
+// Consecutive failed liveness probes of the cached PRIMARY connection during a
+// revalidate-on-reconnect (sleep/wake). Crossing REMOTE_REVALIDATE_FAILURE_THRESHOLD
+// tears the stale remote connection down and rebuilds; reset on any live probe, a
+// fresh build, or when a new reconnect episode starts (see lastRevalidateFailureAt
+// and isFreshRevalidateEpisode). See startHermes().
+let remoteRevalidateFailures = 0
+// Timestamp of the most recent failed liveness probe, used to scope the
+// "consecutive failures" streak to a single reconnect episode: a miss far enough
+// from the previous one resets the streak so a stale failure left over from an
+// earlier, since-recovered episode can't pre-load the counter and force an early
+// rebuild.
+let lastRevalidateFailureAt = 0
 // Additional per-profile backends, keyed by profile name. The PRIMARY backend
 // (the desktop's launch profile) stays managed by hermesProcess +
 // connectionPromise + startHermes(); this pool only holds EXTRA profile
@@ -4132,11 +4147,11 @@ function primaryProfileKey() {
 // profile to startHermes() (the window backend: boot UI, bootstrap, remote
 // mode), and any OTHER profile to a lazily-spawned pool backend. An empty /
 // unknown profile resolves to the primary, so all legacy callers are unchanged.
-async function ensureBackend(profile) {
+async function ensureBackend(profile, options) {
   const key = profile && String(profile).trim() ? String(profile).trim() : primaryProfileKey()
 
   if (key === primaryProfileKey()) {
-    return startHermes()
+    return startHermes(options)
   }
 
   const existing = backendPool.get(key)
@@ -4309,17 +4324,93 @@ function stopAllPoolBackends() {
   }
 }
 
-async function startHermes() {
+async function startHermes(options = {}) {
   // Latched-failure short-circuit: once bootstrap has failed in this
   // process, every subsequent startHermes() call re-throws the same error
   // without re-running install.ps1. This prevents the renderer's
   // ensureGatewayOpen retries (and any other getConnection callers) from
   // restarting a 5-10 minute install loop while the user is still reading
-  // the failure overlay.
+  // the failure overlay. Must stay ABOVE the revalidate block so a latched
+  // failure is never reinterpreted as a dead-cache rebuild.
   if (bootstrapFailure) {
     throw bootstrapFailure
   }
-  if (connectionPromise) return connectionPromise
+
+  if (connectionPromise) {
+    // Steady-state and cold boot reuse the cached connection untouched — no
+    // added latency on the happy paths.
+    if (!options.revalidate) {
+      return connectionPromise
+    }
+
+    // Reconnect-after-wake path (renderer's backoff-paced attemptReconnect):
+    // confirm the cached backend is actually reachable before the renderer
+    // re-dials it. A REMOTE primary has no child process, so the 'exit'/'error'
+    // handlers that would clear a dead connectionPromise never fire — without
+    // this probe the renderer re-dials the same unreachable remote forever and
+    // the composer stays stuck on "Starting Hermes…". Local backends self-heal
+    // via the child 'exit' handler and are deliberately never torn down here.
+    const cached = connectionPromise
+    let conn = null
+
+    try {
+      conn = await cached
+    } catch {
+      // The cached boot rejected (its own catch nulls connectionPromise). Treat
+      // as unreachable; the fall-through below builds a fresh connection.
+      conn = null
+    }
+
+    const alive = conn ? await probeBackendAlive(conn, fetchPublicJson) : false
+
+    if (alive) {
+      remoteRevalidateFailures = 0
+
+      return cached
+    }
+
+    // The cache vanished while we awaited/probed it — a concurrent backend 'exit'
+    // nulled it, or the cached boot itself rejected. There's nothing to
+    // revalidate: hand back a peer's fresh rebuild if one exists, otherwise fall
+    // through and build a new connection below.
+    if (connectionPromise !== cached) {
+      if (connectionPromise) {
+        return connectionPromise
+      }
+    } else {
+      // Cache still present but unreachable. Require consecutive misses within a
+      // single reconnect episode before tearing a REMOTE down (locals self-heal
+      // via their child 'exit' handler, so a probe miss there means "WS not
+      // reattached yet"). The time-window reset stops a stale miss from an
+      // earlier, since-recovered episode from pre-loading this episode's counter.
+      const now = Date.now()
+
+      if (isFreshRevalidateEpisode(now, lastRevalidateFailureAt)) {
+        remoteRevalidateFailures = 0
+      }
+
+      lastRevalidateFailureAt = now
+      remoteRevalidateFailures += 1
+
+      if (!shouldRebuildStaleConnection({ alive, mode: conn?.mode, failureStreak: remoteRevalidateFailures })) {
+        // Not (yet) a confirmed-dead remote: leave the cache in place and let the
+        // renderer's backoff loop retry. The next consecutive miss crosses the
+        // threshold and rebuilds; a local backend never rebuilds here at all.
+        return cached
+      }
+
+      // resetHermesConnection is synchronous (nulls connectionPromise + SIGTERMs
+      // any local child); we re-checked connectionPromise === cached with no
+      // intervening await above, so this can't kill a freshly-rebuilt backend.
+      rememberLog('Cached Hermes backend failed liveness revalidation; rebuilding connection.')
+      resetHermesConnection()
+    }
+  }
+
+  // Any path that reaches a fresh build establishes a new connection, so the
+  // stale-probe streak no longer applies (covers the revalidate teardown above,
+  // a peer's concurrent teardown, and a rebuild from a self-nulled/rejected cache).
+  remoteRevalidateFailures = 0
 
   connectionPromise = (async () => {
     await advanceBootProgress('backend.resolve', 'Resolving Hermes backend', 8)
@@ -4604,7 +4695,7 @@ function createWindow() {
   })
 }
 
-ipcMain.handle('hermes:connection', async (_event, profile) => ensureBackend(profile))
+ipcMain.handle('hermes:connection', async (_event, profile, options) => ensureBackend(profile, options))
 ipcMain.handle('hermes:backend:touch', async (_event, profile) => {
   touchPoolBackend(profile)
   return { ok: true }

apps/desktop/electron/preload.cjs

@@ -1,7 +1,7 @@
 const { contextBridge, ipcRenderer, webUtils } = require('electron')
 
 contextBridge.exposeInMainWorld('hermesDesktop', {
-  getConnection: profile => ipcRenderer.invoke('hermes:connection', profile),
+  getConnection: (profile, options) => ipcRenderer.invoke('hermes:connection', profile, options),
   touchBackend: profile => ipcRenderer.invoke('hermes:backend:touch', profile),
   getGatewayWsUrl: profile => ipcRenderer.invoke('hermes:gateway:ws-url', profile),
   getBootProgress: () => ipcRenderer.invoke('hermes:boot-progress:get'),