Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type { HermesConnection } from '@/global'
import { HermesGateway } from '@/hermes'
import { translateNow } from '@/i18n'
import { desktopDefaultCwd } from '@/lib/desktop-fs'
import { reconnectBackoffDelayMs } from '@/lib/reconnect-backoff'
import {
$desktopBoot,
applyDesktopBootProgress,
Expand Down Expand Up @@ -207,8 +208,11 @@ export function useGatewayBoot({
return
}

// 1s, 2s, 4s … capped at 15s.
const delay = Math.min(15_000, 1_000 * 2 ** Math.min(reconnectAttempt, 4))
// Full-jitter exponential backoff (300ms base, 15s cap) so a gateway
// restart doesn't get redialed by every desktop client in lockstep —
// an immediate-retry reconnect storm can exhaust the gateway's file
// descriptors while it's still coming back up.
const delay = reconnectBackoffDelayMs(reconnectAttempt)
reconnectAttempt += 1
reconnectTimer = setTimeout(() => {
reconnectTimer = null
Expand Down
7 changes: 6 additions & 1 deletion apps/desktop/src/hermes.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { JsonRpcGatewayClient } from '@hermes/shared'

import { reconnectBackoffDelayMs } from '@/lib/reconnect-backoff'
import type {
ActionResponse,
ActionStatusResponse,
Expand Down Expand Up @@ -349,8 +350,12 @@ export function pluginSocket(pluginId: string, path: string, onMessage: (data: u
socket = null

if (!disposed) {
// Full-jitter exponential backoff: same rationale as the gateway
// socket reconnect loops — an immediate-retry loop across many
// desktop clients floods the gateway with connection attempts
// during a restart.
window.setTimeout(() => void connect(), reconnectBackoffDelayMs(attempt, { baseDelayMs: 500, capMs: 30_000 }))
attempt += 1
window.setTimeout(() => void connect(), Math.min(30_000, 1_000 * 2 ** attempt))
}
}
}
Expand Down
92 changes: 92 additions & 0 deletions apps/desktop/src/lib/reconnect-backoff.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
import { describe, expect, it, vi } from 'vitest'

import { reconnectBackoffDelayMs } from './reconnect-backoff'

describe('reconnectBackoffDelayMs', () => {
it('increases the delay ceiling across consecutive failed attempts', () => {
// Pin Math.random so we can read the ceiling directly through the
// returned value instead of statistically sampling it.
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(1)

try {
const delays = [0, 1, 2, 3, 4].map(attempt => reconnectBackoffDelayMs(attempt, { baseDelayMs: 300 }))

expect(delays).toEqual([300, 600, 1200, 2400, 4800])

for (let i = 1; i < delays.length; i++) {
expect(delays[i]).toBeGreaterThan(delays[i - 1])
}
} finally {
randomSpy.mockRestore()
}
})

it('caps the delay ceiling instead of growing unbounded', () => {
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(1)

try {
// Attempt 10 would be 300 * 2**10 = 307_200ms uncapped — must clamp.
expect(reconnectBackoffDelayMs(10, { baseDelayMs: 300, capMs: 15_000 })).toBe(15_000)
expect(reconnectBackoffDelayMs(50, { baseDelayMs: 300, capMs: 15_000 })).toBe(15_000)
} finally {
randomSpy.mockRestore()
}
})

it('applies full jitter: delay is uniformly within [0, ceiling)', () => {
const randomSpy = vi.spyOn(Math, 'random')

try {
randomSpy.mockReturnValue(0)
expect(reconnectBackoffDelayMs(3, { baseDelayMs: 300 })).toBe(0)

randomSpy.mockReturnValue(0.5)
expect(reconnectBackoffDelayMs(3, { baseDelayMs: 300 })).toBe(1200)

randomSpy.mockReturnValue(0.999)
expect(reconnectBackoffDelayMs(3, { baseDelayMs: 300 })).toBeCloseTo(2400 * 0.999, 5)
} finally {
randomSpy.mockRestore()
}
})

it('resets to the attempt-0 ceiling after a successful connection (caller passes attempt back to 0)', () => {
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(1)

try {
// Simulates: fail, fail, fail (attempt climbs), succeed (caller resets
// its counter to 0), fail again — the very next delay must be back at
// the base ceiling, not continuing the climb.
reconnectBackoffDelayMs(0, { baseDelayMs: 300 })
reconnectBackoffDelayMs(1, { baseDelayMs: 300 })
const afterSeveralFailures = reconnectBackoffDelayMs(2, { baseDelayMs: 300 })
const afterReset = reconnectBackoffDelayMs(0, { baseDelayMs: 300 })

expect(afterSeveralFailures).toBe(1200)
expect(afterReset).toBe(300)
} finally {
randomSpy.mockRestore()
}
})

it('treats negative attempt numbers as attempt 0 rather than throwing or returning a negative delay', () => {
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(1)

try {
expect(reconnectBackoffDelayMs(-5, { baseDelayMs: 300 })).toBe(300)
} finally {
randomSpy.mockRestore()
}
})

it('uses sane defaults when no options are passed', () => {
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(1)

try {
expect(reconnectBackoffDelayMs(0)).toBe(300)
expect(reconnectBackoffDelayMs(100)).toBe(15_000)
} finally {
randomSpy.mockRestore()
}
})
})
45 changes: 45 additions & 0 deletions apps/desktop/src/lib/reconnect-backoff.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/**
* Full-jitter exponential backoff for gateway WebSocket reconnects.
*
* A bare exponential backoff still lets every renderer in a fleet retry in
* lockstep — after a gateway restart (e.g. following an update), N desktop
* clients that all disconnected within the same instant all wake up and
* redial at the same instant too, which is a reconnect storm by another
* name. Full jitter (AWS's "Exponential Backoff And Jitter") spreads that
* out: each attempt sleeps a *random* duration between 0 and the exponential
* ceiling, so retries desynchronize instead of pulsing together.
*
* https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
*/

export interface ReconnectBackoffOptions {
/** Ceiling on the exponential delay before jitter is applied, in ms. */
capMs?: number
/** Delay for the first retry (attempt 0) before jitter is applied, in ms. */
baseDelayMs?: number
}

const DEFAULT_BASE_DELAY_MS = 300
const DEFAULT_CAP_MS = 15_000

/**
* Delay before reconnect attempt number `attempt` (0-indexed: the first
* retry after the initial failure is `attempt = 0`). Returns a value in
* `[0, min(capMs, baseDelayMs * 2 ** attempt))` — full jitter, not
* "equal jitter" or "decorrelated jitter", so it can occasionally return a
* very small delay even at a high attempt count. That's intentional: it's
* the variant with the best-documented storm-avoidance behavior and no
* accumulated-delay state to track between calls.
*/
export function reconnectBackoffDelayMs(attempt: number, options: ReconnectBackoffOptions = {}): number {
const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS
const capMs = options.capMs ?? DEFAULT_CAP_MS
const safeAttempt = Math.max(0, attempt)

// 2 ** attempt overflows to Infinity long before it matters (attempt would
// need to be ~1024), and Math.min against a finite cap keeps the ceiling
// sane regardless, so no extra clamping is needed here.
const ceiling = Math.min(capMs, baseDelayMs * 2 ** safeAttempt)

return Math.random() * ceiling
}
6 changes: 4 additions & 2 deletions apps/desktop/src/store/gateway.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { type ConnectionState, type GatewayEvent, resolveGatewayWsUrl } from '@h
import { atom } from 'nanostores'

import { HermesGateway } from '@/hermes'
import { reconnectBackoffDelayMs } from '@/lib/reconnect-backoff'
import { markNativeNotifyBaseline } from '@/store/notify-baseline'
import { setGatewayState } from '@/store/session'

Expand Down Expand Up @@ -184,8 +185,9 @@ function scheduleReconnect(entry: Secondary): void {
return
}

// 1s, 2s, 4s … capped at 15s — same backoff shape as the primary.
const delay = Math.min(15_000, 1_000 * 2 ** Math.min(entry.reconnectAttempt, 4))
// Full-jitter exponential backoff — same shape (and same reason: avoid a
// reconnect storm against a restarting gateway) as the primary's.
const delay = reconnectBackoffDelayMs(entry.reconnectAttempt)
entry.reconnectAttempt += 1
entry.reconnectTimer = setTimeout(() => {
entry.reconnectTimer = null
Expand Down