From e70cbfe5daadbd2dbebbbf6318260a3210d56a50 Mon Sep 17 00:00:00 2001 From: Gille <4317663+helix4u@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:36:23 -0600 Subject: [PATCH] fix(dashboard): retry stalled events feed reconnects --- web/src/components/ChatSidebar.test.tsx | 81 ++++++++++++++++++++++++- web/src/components/ChatSidebar.tsx | 61 +++++++++++++++++-- web/src/lib/events-reconnect.ts | 2 + 3 files changed, 137 insertions(+), 7 deletions(-) diff --git a/web/src/components/ChatSidebar.test.tsx b/web/src/components/ChatSidebar.test.tsx index 523faa093c9b3..74042c55e9ba4 100644 --- a/web/src/components/ChatSidebar.test.tsx +++ b/web/src/components/ChatSidebar.test.tsx @@ -3,6 +3,8 @@ import { act, type ReactNode } from "react"; import { createRoot, type Root } from "react-dom/client"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { EVENTS_CONNECT_TIMEOUT_MS } from "@/lib/events-reconnect"; + const apiMocks = vi.hoisted(() => ({ buildWsUrl: vi.fn(async () => "ws://localhost/api/events?channel=chat-1"), getModelInfo: vi.fn(async () => ({ @@ -113,6 +115,10 @@ async function render(ui: ReactNode) { beforeEach(() => { FakeWebSocket.instances = []; vi.clearAllMocks(); + apiMocks.buildWsUrl.mockReset(); + apiMocks.buildWsUrl.mockResolvedValue( + "ws://localhost/api/events?channel=chat-1", + ); reloadMocks.maybeReloadForLoopbackWsAuthFailure.mockReturnValue(true); vi.stubGlobal("WebSocket", FakeWebSocket); }); @@ -180,6 +186,75 @@ describe("ChatSidebar event socket reconnect", () => { expect(apiMocks.buildWsUrl).toHaveBeenCalledTimes(2); }); + it("keeps retrying when reconnect URL construction fails", async () => { + await renderSidebar(); + apiMocks.buildWsUrl + .mockRejectedValueOnce(new Error("ticket endpoint unavailable")) + .mockResolvedValue("ws://localhost/api/events?channel=chat-1"); + + await act(async () => { + FakeWebSocket.instances[0].emit("close", { code: 1006 }); + }); + + await advance(1_000); + expect(FakeWebSocket.instances).toHaveLength(1); + expect(container.textContent).toContain("reconnecting in 2s"); + + await advance(2_000); + expect(FakeWebSocket.instances).toHaveLength(2); + expect(apiMocks.buildWsUrl).toHaveBeenCalledTimes(3); + }); + + it("times out a stalled URL request and retries", async () => { + let resolveStalledRequest!: (url: string) => void; + await renderSidebar(); + apiMocks.buildWsUrl + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveStalledRequest = resolve; + }), + ) + .mockResolvedValue("ws://localhost/api/events?channel=chat-1"); + + await act(async () => { + FakeWebSocket.instances[0].emit("close", { code: 1006 }); + }); + + await advance(1_000 + EVENTS_CONNECT_TIMEOUT_MS); + expect(FakeWebSocket.instances).toHaveLength(1); + expect(container.textContent).toContain("reconnecting in 2s"); + + // A late ticket response from the timed-out attempt must not create a + // superseded socket alongside the scheduled replacement. + await act(async () => { + resolveStalledRequest("ws://localhost/api/events?channel=stale"); + await Promise.resolve(); + }); + expect(FakeWebSocket.instances).toHaveLength(1); + + await advance(2_000); + expect(FakeWebSocket.instances).toHaveLength(2); + expect(apiMocks.buildWsUrl).toHaveBeenCalledTimes(3); + }); + + it("times out a stalled WebSocket handshake and retries", async () => { + await renderSidebar(); + + await act(async () => { + FakeWebSocket.instances[0].emit("close", { code: 1006 }); + }); + await advance(1_000); + expect(FakeWebSocket.instances).toHaveLength(2); + + await advance(EVENTS_CONNECT_TIMEOUT_MS); + expect(FakeWebSocket.instances[1].closed).toBe(true); + expect(container.textContent).toContain("reconnecting in 2s"); + + await advance(2_000); + expect(FakeWebSocket.instances).toHaveLength(3); + }); + it("backs off exponentially across repeated failures", async () => { await renderSidebar(); @@ -210,7 +285,11 @@ describe("ChatSidebar event socket reconnect", () => { FakeWebSocket.instances[0].emit("close", { code: 1006 }); }); - await advance(30_000); + await advance(1_000); + await act(async () => { + FakeWebSocket.instances[1].emit("open", {}); + }); + await advance(29_000); expect(FakeWebSocket.instances).toHaveLength(2); }); diff --git a/web/src/components/ChatSidebar.tsx b/web/src/components/ChatSidebar.tsx index aa4bd69ea32ed..e7669921129c2 100644 --- a/web/src/components/ChatSidebar.tsx +++ b/web/src/components/ChatSidebar.tsx @@ -36,6 +36,7 @@ import { GatewayClient, type ConnectionState } from "@/lib/gatewayClient"; import { api, buildWsUrl } from "@/lib/api"; import { maybeReloadForLoopbackWsAuthFailure } from "@/lib/dashboard-auth-reload"; import { + EVENTS_CONNECT_TIMEOUT_MS, EVENTS_DISCONNECTED_MESSAGE, EVENTS_MAX_RECONNECT_ATTEMPTS, eventsGaveUpMessage, @@ -256,8 +257,17 @@ export function ChatSidebar({ let unmounting = false; let ws: WebSocket | null = null; let reconnectTimer: ReturnType | null = null; + let connectTimer: ReturnType | null = null; + let connectGeneration = 0; let attempt = 0; + const clearConnectTimer = () => { + if (connectTimer) { + clearTimeout(connectTimer); + connectTimer = null; + } + }; + // The banner is shared with `info.credential_warning` and the JSON-RPC // sidecar, and `error` is those messages' only home — the sidecar does // not re-emit. So the events feed may only write over an empty banner @@ -298,14 +308,49 @@ export function ChatSidebar({ if (unmounting) { return; } - // Re-minted every attempt: tickets are single-use with a short TTL, - // so a reconnect cannot replay the URL from the first connection. - const url = await buildWsUrl("/api/events", { channel }); - if (unmounting) { + + const generation = ++connectGeneration; + let socket: WebSocket | null = null; + + // Cover the whole connection attempt, including gated-mode ticket + // minting. A failed or hanging pre-socket request otherwise emits no + // WebSocket close event and permanently strands the retry loop at its + // last "reconnecting in ..." banner. + clearConnectTimer(); + connectTimer = setTimeout(() => { + connectTimer = null; + if (unmounting || generation !== connectGeneration) { + return; + } + + // Invalidate any late ticket result or socket event from this attempt + // before scheduling its replacement. + connectGeneration += 1; + if (socket && ws === socket) { + ws = null; + socket.close(); + } + scheduleReconnect(); + }, EVENTS_CONNECT_TIMEOUT_MS); + + try { + // Re-minted every attempt: tickets are single-use with a short TTL, + // so a reconnect cannot replay the URL from the first connection. + const url = await buildWsUrl("/api/events", { channel }); + if (unmounting || generation !== connectGeneration) { + return; + } + socket = new WebSocket(url); + ws = socket; + } catch { + if (unmounting || generation !== connectGeneration) { + return; + } + clearConnectTimer(); + connectGeneration += 1; + scheduleReconnect(); return; } - const socket = new WebSocket(url); - ws = socket; // A superseded socket's late close must not schedule a retry on top // of the one that replaced it. @@ -315,6 +360,7 @@ export function ChatSidebar({ if (!isCurrent()) { return; } + clearConnectTimer(); attempt = 0; clearEventsBanner(); }); @@ -332,6 +378,7 @@ export function ChatSidebar({ if (!isCurrent()) { return; } + clearConnectTimer(); if (maybeReloadForLoopbackWsAuthFailure(ev.code)) { return; } @@ -374,6 +421,8 @@ export function ChatSidebar({ return () => { unmounting = true; + connectGeneration += 1; + clearConnectTimer(); if (reconnectTimer) { clearTimeout(reconnectTimer); reconnectTimer = null; diff --git a/web/src/lib/events-reconnect.ts b/web/src/lib/events-reconnect.ts index f533bd924ea53..6babf3f2fd3c3 100644 --- a/web/src/lib/events-reconnect.ts +++ b/web/src/lib/events-reconnect.ts @@ -9,6 +9,8 @@ export const EVENTS_RECONNECT_BASE_MS = 1_000; export const EVENTS_RECONNECT_MAX_MS = 30_000; export const EVENTS_MAX_RECONNECT_ATTEMPTS = 15; +/** Bound ticket minting plus the WebSocket opening handshake. */ +export const EVENTS_CONNECT_TIMEOUT_MS = 15_000; /** Normal closure — the server said goodbye, don't chase it. */ const WS_CLOSE_NORMAL = 1000;