diff --git a/packages/tui/src/context/sdk.tsx b/packages/tui/src/context/sdk.tsx index 93180c6e21da..8edadb4d06a5 100644 --- a/packages/tui/src/context/sdk.tsx +++ b/packages/tui/src/context/sdk.tsx @@ -8,6 +8,70 @@ export type EventSource = { subscribe: (handler: (event: GlobalEvent) => void) => Promise<() => void> } +export type EventStreamLoop = { + readonly signals: readonly [AbortSignal, AbortSignal] + readonly connect: (signal: AbortSignal) => Promise> + readonly event: (event: GlobalEvent) => void + readonly flush: () => void + readonly wait: (delay: number, signal: AbortSignal) => Promise + readonly retry: (entry: { readonly attempt: number; readonly error: unknown }) => void +} + +const retryDelay = 1000 +const maxRetryDelay = 30000 + +export async function runEventStream(input: EventStreamLoop) { + const signal = AbortSignal.any([...input.signals]) + let attempt = 0 + while (!signal.aborted) { + let error: unknown = new Error("event stream completed") + try { + const stream = await input.connect(signal) + for await (const event of stream) { + if (signal.aborted) return + input.event(event) + if (event.payload.type === "server.connected") attempt = 0 + } + } catch (cause) { + error = cause instanceof Error ? cause : new Error("event stream failed", { cause }) + } + try { + input.flush() + } catch (cause) { + error = cause instanceof Error ? cause : new Error("event stream flush failed", { cause }) + } + if (signal.aborted) return + + attempt++ + try { + await input.wait(Math.min(retryDelay * 2 ** (attempt - 1), maxRetryDelay), signal) + } catch (cause) { + error = cause instanceof Error ? cause : new Error("event stream retry wait failed", { cause }) + } + if (signal.aborted) return + try { + input.retry({ attempt, error }) + } catch { + continue + } + } +} + +function waitForRetry(delay: number, signal: AbortSignal) { + if (signal.aborted) return Promise.resolve() + return new Promise((resolve) => { + const abort = () => { + clearTimeout(timer) + resolve() + } + const timer = setTimeout(() => { + signal.removeEventListener("abort", abort) + resolve() + }, delay) + signal.addEventListener("abort", abort, { once: true }) + }) +} + export const { use: useSDK, provider: SDKProvider } = createSimpleContext({ name: "SDK", init: (props: { @@ -48,14 +112,13 @@ export const { use: useSDK, provider: SDKProvider } = createSimpleContext({ let queue: GlobalEvent[] = [] let timer: Timer | undefined let last = 0 - const retryDelay = 1000 - const maxRetryDelay = 30000 const flush = () => { + if (timer) clearTimeout(timer) + timer = undefined if (queue.length === 0) return const events = queue queue = [] - timer = undefined last = Date.now() // Batch all event emissions so all store updates result in a single render batch(() => { @@ -83,13 +146,11 @@ export const { use: useSDK, provider: SDKProvider } = createSimpleContext({ sse?.abort() const ctrl = new AbortController() sse = ctrl - ;(async () => { - let attempt = 0 - while (true) { - if (abort.signal.aborted || ctrl.signal.aborted) break - + const task = runEventStream({ + signals: [abort.signal, ctrl.signal], + connect: async (signal) => { const events = await sdk.global.event({ - signal: ctrl.signal, + signal, sseMaxRetryAttempts: 0, }) @@ -98,37 +159,46 @@ export const { use: useSDK, provider: SDKProvider } = createSimpleContext({ // we've started listening to events await sdk.sync.start().catch(() => {}) } - - for await (const event of events.stream) { - if (ctrl.signal.aborted) break - handleEvent(event) - } - - if (timer) clearTimeout(timer) - if (queue.length > 0) flush() - attempt += 1 - if (abort.signal.aborted || ctrl.signal.aborted) break - - // Exponential backoff - const backoff = Math.min(retryDelay * 2 ** (attempt - 1), maxRetryDelay) - await new Promise((resolve) => setTimeout(resolve, backoff)) - } - })().catch(() => {}) + return events.stream + }, + event: handleEvent, + flush, + wait: waitForRetry, + retry: ({ attempt, error }) => + console.warn("[tui.sdk] event stream disconnected, retrying", { attempt, error }), + }) + void task.catch((error) => { + if (abort.signal.aborted || ctrl.signal.aborted) return + console.error("[tui.sdk] event stream stopped unexpectedly", { error }) + startSSE() + }) } - onMount(async () => { + onMount(() => { if (props.events) { - const unsub = await props.events.subscribe(handleEvent) - onCleanup(unsub) - - if (Flag.OPENCODE_EXPERIMENTAL_WORKSPACES) { - // Start syncing workspaces, it's important to do this after - // we've started listening to events - await sdk.sync.start().catch(() => {}) - } - } else { - startSSE() + let disposed = false + let unsubscribe: (() => void) | undefined + onCleanup(() => { + disposed = true + unsubscribe?.() + }) + void props.events + .subscribe(handleEvent) + .then((cleanup) => { + if (disposed) return cleanup() + unsubscribe = cleanup + if (Flag.OPENCODE_EXPERIMENTAL_WORKSPACES) { + void sdk.sync.start().catch((error) => { + if (!disposed) console.error("[tui.sdk] workspace sync failed", { error }) + }) + } + }) + .catch((error) => { + if (!disposed) console.error("[tui.sdk] injected event source failed", { error }) + }) + return } + startSSE() }) onCleanup(() => { diff --git a/packages/tui/src/context/sync-reconnect.ts b/packages/tui/src/context/sync-reconnect.ts new file mode 100644 index 000000000000..e60ba8993eba --- /dev/null +++ b/packages/tui/src/context/sync-reconnect.ts @@ -0,0 +1,84 @@ +type ReconnectCallbacks = { + readonly bootstrap: () => Promise + readonly targets: () => readonly string[] + readonly exists: (sessionID: string) => boolean + readonly forceSync: (sessionID: string) => Promise + readonly onError: (failure: ReconnectFailure) => void +} + +export type ReconnectFailure = + | { readonly boundary: "bootstrap"; readonly error: unknown } + | { readonly boundary: "session"; readonly sessionID: string; readonly error: unknown } + +export function createReconnectCoordinator(_callbacks: ReconnectCallbacks) { + let epoch = 0 + let disposed = false + let dirty = false + let active: Promise | undefined + const failed = new Set() + + async function pass() { + const targets = [...new Set([..._callbacks.targets(), ...failed])] + let bootstrapSucceeded = true + try { + await _callbacks.bootstrap() + } catch (error) { + if (disposed) return + bootstrapSucceeded = false + _callbacks.onError({ boundary: "bootstrap", error }) + } + if (disposed) return + + const eligible = bootstrapSucceeded ? targets.filter(_callbacks.exists) : targets + if (bootstrapSucceeded) { + for (const sessionID of targets) { + if (!_callbacks.exists(sessionID)) failed.delete(sessionID) + } + } + const results = await Promise.allSettled(eligible.map((sessionID) => _callbacks.forceSync(sessionID))) + if (disposed) return + for (const [index, result] of results.entries()) { + const sessionID = eligible[index] + if (result.status === "fulfilled") { + failed.delete(sessionID) + continue + } + failed.add(sessionID) + _callbacks.onError({ boundary: "session", sessionID, error: result.reason }) + } + } + + function connected() { + if (disposed) return Promise.resolve() + epoch += 1 + if (epoch === 1) return Promise.resolve() + if (active) { + dirty = true + return active + } + + dirty = true + const owner = Promise.withResolvers() + active = owner.promise + void (async () => { + try { + while (dirty) { + if (disposed) return + dirty = false + await pass() + } + } finally { + active = undefined + } + })().then(owner.resolve, owner.reject) + return owner.promise + } + + return { + connected, + dispose() { + disposed = true + dirty = false + }, + } +} diff --git a/packages/tui/src/context/sync.tsx b/packages/tui/src/context/sync.tsx index d0511c5183e4..1c4ae9c5dd3c 100644 --- a/packages/tui/src/context/sync.tsx +++ b/packages/tui/src/context/sync.tsx @@ -28,16 +28,22 @@ import { useTuiStartup } from "./runtime" import { createSimpleContext } from "./helper" import { useExit } from "./exit" import { useArgs } from "./args" -import { batch, onMount } from "solid-js" +import { batch, onCleanup, onMount } from "solid-js" import path from "path" import { useKV } from "./kv" import { usePermission } from "./permission" +import { createReconnectCoordinator } from "./sync-reconnect" const emptyConsoleState: ConsoleState = { consoleManagedProviders: [], switchableOrgCount: 0, } +function responseData(response: { readonly data?: T }) { + if (response.data !== undefined) return response.data + throw new TypeError("Expected response data") +} + function search(items: T[], target: string, key: (item: T) => string) { let left = 0 let right = items.length - 1 @@ -142,8 +148,10 @@ export const { const sdk = useSDK() const fullSyncedSessions = new Set() + const strictRecoverySessions = new Set() const syncingSessions = new Map>() - const hydratingSessions = new Map; parts: Set }>() + const hydratingSessions = new Map; parts: Set; invalidated: boolean }>() + let disposed = false const touchMessage = (sessionID: string, messageID: string) => { hydratingSessions.get(sessionID)?.messages.add(messageID) } @@ -163,12 +171,62 @@ export const { function listSessions() { return sdk.client.session - .list({ start: Date.now() - 30 * 24 * 60 * 60 * 1000, ...sessionListQuery() }) - .then((x) => (x.data ?? []).toSorted((a, b) => a.id.localeCompare(b.id))) + .list({ start: Date.now() - 30 * 24 * 60 * 60 * 1000, ...sessionListQuery() }, { throwOnError: true }) + .then(responseData) + .then((sessions) => sessions.toSorted((a, b) => a.id.localeCompare(b.id))) + } + + function pruneSession(sessionID: string) { + const tracker = hydratingSessions.get(sessionID) + if (tracker) tracker.invalidated = true + fullSyncedSessions.delete(sessionID) + strictRecoverySessions.delete(sessionID) + setStore( + produce((draft) => { + const match = search(draft.session, sessionID, (item) => item.id) + if (match.found) draft.session.splice(match.index, 1) + delete draft.permission[sessionID] + delete draft.question[sessionID] + delete draft.session_status[sessionID] + delete draft.todo[sessionID] + delete draft.session_diff[sessionID] + for (const message of draft.message[sessionID] ?? []) delete draft.part[message.id] + delete draft.message[sessionID] + for (const [messageID, parts] of Object.entries(draft.part)) { + if (parts.some((part) => part.sessionID === sessionID)) delete draft.part[messageID] + } + }), + ) } - event.subscribe((event, { directory, workspace }) => { + function replaceSessions(sessions: Session[]) { + const present = new Set(sessions.map((session) => session.id)) + for (const sessionID of store.session.map((session) => session.id)) { + if (!present.has(sessionID)) pruneSession(sessionID) + } + setStore("session", reconcile(sessions)) + } + + const reconnect = createReconnectCoordinator({ + bootstrap: () => bootstrap({ fatal: false, wait: true, report: false }), + targets: () => [...fullSyncedSessions, ...syncingSessions.keys()], + exists: (sessionID) => store.session.some((session) => session.id === sessionID), + forceSync: (sessionID) => syncSession(sessionID, { force: true }), + onError: (failure) => { + const error = failure.error + console.error("tui reconnect reconciliation failed", { + boundary: failure.boundary, + ...(failure.boundary === "session" ? { sessionID: failure.sessionID } : {}), + error: error instanceof Error ? error.message : String(error), + name: error instanceof Error ? error.name : undefined, + }) + }, + }) + const unsubscribe = event.subscribe((event, { directory, workspace }) => { switch (event.type) { + case "server.connected": + void reconnect.connected() + break case "server.instance.disposed": void bootstrap() break @@ -265,15 +323,7 @@ export const { break case "session.deleted": { - const result = search(store.session, event.properties.info.id, (s) => s.id) - if (result.found) { - setStore( - "session", - produce((draft) => { - draft.splice(result.index, 1) - }), - ) - } + pruneSession(event.properties.info.id) break } case "session.updated": { @@ -438,13 +488,31 @@ export const { } } }) + onCleanup(() => { + disposed = true + reconnect.dispose() + unsubscribe() + }) const exit = useExit() const args = useArgs() - async function bootstrap(input: { fatal?: boolean } = {}) { + async function bootstrap(input: { fatal?: boolean; wait?: boolean; report?: boolean } = {}) { const fatal = input.fatal ?? true + const wait = input.wait ?? false + const report = input.report ?? true const workspace = project.workspace.current() + const fail = async (error: unknown, detached: boolean) => { + if (report || (detached && !fatal)) { + console.error("tui bootstrap failed", { + error: error instanceof Error ? error.message : String(error), + name: error instanceof Error ? error.name : undefined, + stack: error instanceof Error ? error.stack : undefined, + }) + } + if (fatal) return exit(error) + if (!detached) throw error + } const projectPromise = project.sync() const sessionListPromise = projectPromise.then(() => listSessions()) @@ -471,12 +539,12 @@ export const { ...(args.continue ? [sessionListPromise] : []), ]) .then(async () => { - const providersResponse = providersPromise.then((x) => x.data!) - const providerListResponse = providerListPromise.then((x) => x.data!) + const providersResponse = providersPromise.then(responseData) + const providerListResponse = providerListPromise.then(responseData) const capabilitiesResponse = capabilitiesPromise const consoleStateResponse = consoleStatePromise const agentsResponse = agentsPromise.then((x) => x.data ?? []) - const configResponse = configPromise.then((x) => x.data!) + const configResponse = configPromise.then(responseData) const sessionListResponse = args.continue ? sessionListPromise : undefined return Promise.all([ @@ -504,15 +572,15 @@ export const { setStore("console_state", reconcile(consoleState)) setStore("agent", reconcile(agents)) setStore("config", reconcile(config)) - if (sessions !== undefined) setStore("session", reconcile(sessions)) + if (sessions !== undefined) replaceSessions(sessions) }) }) }) .then(() => { if (store.status !== "complete") setStore("status", "partial") // non-blocking - void Promise.all([ - ...(args.continue ? [] : [sessionListPromise.then((sessions) => setStore("session", reconcile(sessions)))]), + const complete = Promise.all([ + ...(args.continue ? [] : [sessionListPromise.then(replaceSessions)]), consoleStatePromise.then((consoleState) => setStore("console_state", reconcile(consoleState))), sdk.client.command.list({ workspace }).then((x) => setStore("command", reconcile(x.data ?? []))), sdk.client.lsp.status({ workspace }).then((x) => setStore("lsp", reconcile(x.data ?? []))), @@ -521,8 +589,15 @@ export const { .list({ workspace }) .then((x) => setStore("mcp_resource", reconcile(x.data ?? {}))), sdk.client.formatter.status({ workspace }).then((x) => setStore("formatter", reconcile(x.data ?? []))), - sdk.client.session.status({ workspace }).then((x) => { - setStore("session_status", reconcile(x.data ?? {})) + Promise.all([ + sessionListPromise, + sdk.client.session.status({ workspace }, { throwOnError: true }).then(responseData), + ]).then(([sessions, statuses]) => { + const present = new Set(sessions.map((session) => session.id)) + setStore( + "session_status", + reconcile(Object.fromEntries(Object.entries(statuses).filter(([sessionID]) => present.has(sessionID)))), + ) }), sdk.client.provider.auth({ workspace }).then((x) => setStore("provider_auth", reconcile(x.data ?? {}))), sdk.client.vcs.get({ workspace }).then((x) => setStore("vcs", reconcile(x.data))), @@ -530,19 +605,103 @@ export const { ]).then(() => { setStore("status", "complete") }) + if (wait) return complete + void complete.catch((error) => fail(error, true)) }) - .catch(async (e) => { - console.error("tui bootstrap failed", { - error: e instanceof Error ? e.message : String(e), - name: e instanceof Error ? e.name : undefined, - stack: e instanceof Error ? e.stack : undefined, - }) - if (fatal) { - exit(e) - } else { - throw e - } + .catch((error) => fail(error, false)) + } + + async function syncSession(sessionID: string, input: { force: boolean }) { + if (disposed) return + if (!input.force && fullSyncedSessions.has(sessionID)) return + const older = syncingSessions.get(sessionID) + if (older) { + if (!input.force) return older + await Promise.allSettled([older]) + if (disposed) return + } + if (input.force) fullSyncedSessions.delete(sessionID) + const strict = input.force || strictRecoverySessions.has(sessionID) + const tracker = { messages: new Set(), parts: new Set(), invalidated: false } + hydratingSessions.set(sessionID, tracker) + const task = (async () => { + const [session, messages, todo, diff] = await Promise.all([ + sdk.client.session.get({ sessionID }, { throwOnError: true }), + sdk.client.session.messages({ sessionID, limit: 100 }, { throwOnError: strict }), + sdk.client.session.todo({ sessionID }, { throwOnError: strict }), + sdk.client.session.diff({ sessionID }, { throwOnError: strict }), + ]) + if (disposed || tracker.invalidated) return + const info = session.data + if (!info) return + setStore( + produce((draft) => { + const match = search(draft.session, sessionID, (s) => s.id) + if (match.found) draft.session[match.index] = info + if (!match.found) draft.session.splice(match.index, 0, info) + draft.todo[sessionID] = todo.data ?? [] + const currentMessages = draft.message[sessionID] ?? [] + const infos = (messages.data ?? []).flatMap((message) => { + if (!tracker.messages.has(message.info.id)) return [message.info] + const current = currentMessages.find((item) => item.id === message.info.id) + return current ? [current] : [] + }) + infos.push( + ...currentMessages.filter( + (message) => tracker.messages.has(message.id) && !infos.some((item) => item.id === message.id), + ), + ) + const removed = infos.slice(0, -100) + const visible = infos.slice(-100) + const visibleIDs = new Set(visible.map((message) => message.id)) + for (const message of currentMessages) { + if (!visibleIDs.has(message.id)) delete draft.part[message.id] + } + for (const message of messages.data ?? []) { + if (!visibleIDs.has(message.info.id)) { + delete draft.part[message.info.id] + continue + } + const currentParts = draft.part[message.info.id] ?? [] + const parts = message.parts.flatMap((part) => { + const current = currentParts.find((item) => item.id === part.id) + if (tracker.parts.has(part.id)) return current ? [current] : [] + if ( + current && + (part.type === "text" || part.type === "reasoning") && + (current.type === "text" || current.type === "reasoning") && + part.text.length === 0 && + current.text.length > 0 + ) { + return [current] + } + return [part] + }) + parts.push( + ...currentParts.filter( + (part) => tracker.parts.has(part.id) && !parts.some((item) => item.id === part.id), + ), + ) + draft.part[message.info.id] = parts + } + for (const message of removed) delete draft.part[message.id] + draft.message[sessionID] = visible + draft.session_diff[sessionID] = diff.data ?? [] + }), + ) + fullSyncedSessions.add(sessionID) + strictRecoverySessions.delete(sessionID) + })() + .catch((error) => { + if (strict && !disposed && !tracker.invalidated) strictRecoverySessions.add(sessionID) + throw error + }) + .finally(() => { + syncingSessions.delete(sessionID) + hydratingSessions.delete(sessionID) }) + syncingSessions.set(sessionID, task) + return task } onMount(() => { @@ -573,7 +732,7 @@ export const { }, async refresh() { const list = await listSessions() - setStore("session", reconcile(list)) + replaceSessions(list) }, status(sessionID: string) { const session = result.session.get(sessionID) @@ -586,77 +745,7 @@ export const { return last.time.completed ? "idle" : "working" }, async sync(sessionID: string) { - if (fullSyncedSessions.has(sessionID)) return - const syncing = syncingSessions.get(sessionID) - if (syncing) return syncing - const tracker = { messages: new Set(), parts: new Set() } - hydratingSessions.set(sessionID, tracker) - const task = (async () => { - const [session, messages, todo, diff] = await Promise.all([ - sdk.client.session.get({ sessionID }, { throwOnError: true }), - sdk.client.session.messages({ sessionID, limit: 100 }), - sdk.client.session.todo({ sessionID }), - sdk.client.session.diff({ sessionID }), - ]) - setStore( - produce((draft) => { - const match = search(draft.session, sessionID, (s) => s.id) - if (match.found) draft.session[match.index] = session.data! - if (!match.found) draft.session.splice(match.index, 0, session.data!) - draft.todo[sessionID] = todo.data ?? [] - const currentMessages = draft.message[sessionID] ?? [] - const infos = (messages.data ?? []).flatMap((message) => { - if (!tracker.messages.has(message.info.id)) return [message.info] - const current = currentMessages.find((item) => item.id === message.info.id) - return current ? [current] : [] - }) - infos.push( - ...currentMessages.filter( - (message) => tracker.messages.has(message.id) && !infos.some((item) => item.id === message.id), - ), - ) - const removed = infos.slice(0, -100) - const visible = infos.slice(-100) - const visibleIDs = new Set(visible.map((message) => message.id)) - for (const message of messages.data ?? []) { - if (!visibleIDs.has(message.info.id)) { - delete draft.part[message.info.id] - continue - } - const currentParts = draft.part[message.info.id] ?? [] - const parts = message.parts.flatMap((part) => { - const current = currentParts.find((item) => item.id === part.id) - if (tracker.parts.has(part.id)) return current ? [current] : [] - if ( - current && - (part.type === "text" || part.type === "reasoning") && - (current.type === "text" || current.type === "reasoning") && - part.text.length === 0 && - current.text.length > 0 - ) { - return [current] - } - return [part] - }) - parts.push( - ...currentParts.filter( - (part) => tracker.parts.has(part.id) && !parts.some((item) => item.id === part.id), - ), - ) - draft.part[message.info.id] = parts - } - for (const message of removed) delete draft.part[message.id] - draft.message[sessionID] = visible - draft.session_diff[sessionID] = diff.data ?? [] - }), - ) - fullSyncedSessions.add(sessionID) - })().finally(() => { - syncingSessions.delete(sessionID) - hydratingSessions.delete(sessionID) - }) - syncingSessions.set(sessionID, task) - return task + return syncSession(sessionID, { force: false }) }, }, bootstrap, diff --git a/packages/tui/test/cli/cmd/tui/sync-reconnect-hydration.test.tsx b/packages/tui/test/cli/cmd/tui/sync-reconnect-hydration.test.tsx new file mode 100644 index 000000000000..871d7cf5a7db --- /dev/null +++ b/packages/tui/test/cli/cmd/tui/sync-reconnect-hydration.test.tsx @@ -0,0 +1,721 @@ +/** @jsxImportSource @opentui/solid */ +import { expect, spyOn, test } from "bun:test" +import type { GlobalEvent } from "@opencode-ai/sdk/v2" +import { createEffect, createRoot } from "solid-js" +import { tmpdir } from "../../../fixture/fixture" +import type { FetchHandler } from "../../../fixture/tui-sdk" +import { directory, json, mount } from "./sync-fixture" + +function deferred() { + let resolvePromise: ((value: T) => void) | undefined + const promise = new Promise((resolve) => (resolvePromise = resolve)) + return { + promise, + resolve(value: T) { + if (!resolvePromise) throw new Error("deferred promise is not initialized") + resolvePromise(value) + }, + } +} + +function observe(check: () => boolean) { + return new Promise((resolve) => + createRoot((dispose) => + createEffect(() => { + if (!check()) return + dispose() + resolve() + }), + ), + ) +} + +function global(payload: GlobalEvent["payload"]): GlobalEvent { + return { directory, project: "proj_test", payload } +} + +function connected(id: string) { + return global({ id, type: "server.connected", properties: {} }) +} + +function instanceDisposed(id: string) { + return global({ id, type: "server.instance.disposed", properties: { directory } }) +} + +function session(id: string, title: string, parentID?: string) { + return { + id, + slug: id, + projectID: "proj_test", + title, + parentID, + time: { created: 0, updated: 0 }, + version: "1.18.3", + directory, + } +} + +function message(sessionID: string, id: string, completed = true) { + return { + id, + sessionID, + role: "assistant" as const, + agent: "build", + modelID: "model", + providerID: "test", + mode: "build", + parentID: "msg_user", + path: { cwd: directory, root: directory }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 1, ...(completed ? { completed: 2 } : {}) }, + } +} + +function payload(sessionID: string, id: string, text: string) { + return [{ info: message(sessionID, id), parts: [{ id: `prt_${id}`, sessionID, messageID: id, type: "text", text }] }] +} + +async function scenario( + override: FetchHandler | undefined, + run: (app: Awaited>) => Promise, +) { + await using tmp = await tmpdir() + await Bun.write(`${tmp.path}/kv.json`, "{}") + const app = await mount(override, tmp.path) + try { + await run(app) + } finally { + app.app.renderer.destroy() + } +} + +test("later connection epochs repair all loaded domains without changing initial counts", async () => { + const parentID = "ses_parent" + const childID = "ses_child" + const counts = new Map() + const listed = deferred() + const forced = new Map([parentID, childID].map((id) => [id, deferred()])) + let current: "initial" | "reconnected" = "initial" + const sessions = () => [session(parentID, `${current} parent`), session(childID, `${current} child`, parentID)] + await scenario( + (url) => { + const count = (counts.get(url.pathname) ?? 0) + 1 + counts.set(url.pathname, count) + if (url.pathname === "/session") { + if (count === 2) listed.resolve() + return json(sessions()) + } + if (url.pathname === "/session/status") + return json({ [parentID]: { type: current === "initial" ? "idle" : "busy" }, [childID]: { type: "idle" } }) + for (const id of [parentID, childID]) { + if (url.pathname === `/session/${id}`) return json(sessions().find((item) => item.id === id)) + if (url.pathname === `/session/${id}/message`) { + if (count === 2) forced.get(id)?.resolve() + if (current === "initial") return json(payload(id, `msg_${id}_old`, "old")) + return json( + Array.from( + { length: id === parentID ? 101 : 1 }, + (_, index) => payload(id, `msg_${id}_${String(index).padStart(3, "0")}`, `new ${index}`)[0], + ), + ) + } + if (url.pathname === `/session/${id}/todo`) + return json([{ content: current, status: "pending", priority: "high" }]) + if (url.pathname === `/session/${id}/diff`) + return json([{ file: `${current}.ts`, patch: current, additions: 1, deletions: 0, status: "modified" }]) + } + }, + async ({ emit, sync }) => { + await Promise.all([sync.session.sync(parentID), sync.session.sync(childID)]) + const initial = new Map(counts) + emit(connected("initial")) + await Promise.resolve() + expect(counts).toEqual(initial) + current = "reconnected" + emit(connected("reconnect")) + await Promise.all([listed.promise, ...[...forced.values()].map((item) => item.promise)]) + await Promise.all([sync.session.sync(parentID), sync.session.sync(childID)]) + expect(counts.get("/session")).toBe(2) + for (const id of [parentID, childID]) + expect(["", "/message", "/todo", "/diff"].map((suffix) => counts.get(`/session/${id}${suffix}`))).toEqual([ + 2, 2, 2, 2, + ]) + expect([sync.session.get(parentID)?.title, sync.session.get(childID)?.parentID]).toEqual([ + "reconnected parent", + parentID, + ]) + expect(sync.data.session_status[parentID]).toEqual({ type: "busy" }) + expect(sync.data.todo[parentID]).toEqual([{ content: "reconnected", status: "pending", priority: "high" }]) + expect(sync.data.session_diff[parentID]).toEqual([ + { file: "reconnected.ts", patch: "reconnected", additions: 1, deletions: 0, status: "modified" }, + ]) + expect(sync.data.message[parentID]).toHaveLength(100) + expect(sync.data.message[parentID][0]?.id).toBe(`msg_${parentID}_001`) + expect(sync.data.part[`msg_${parentID}_old`]).toBeUndefined() + expect(sync.data.part[`msg_${parentID}_000`]).toBeUndefined() + expect(sync.data.part[`msg_${parentID}_100`]?.[0]).toMatchObject({ text: "new 100" }) + expect(sync.data.part[`msg_${childID}_000`]?.[0]).toMatchObject({ text: "new 0" }) + }, + ) +}) + +test("reconnect waits for old hydration and live updates win over stale responses", async () => { + const id = "ses_inflight" + const messageID = "msg_inflight" + const old = deferred() + const forced = deferred() + const listed = deferred() + const forceRequested = deferred() + let lists = 0 + let messages = 0 + await scenario( + (url) => { + if (url.pathname === "/session") { + if (++lists === 2) listed.resolve() + return json([session(id, "current")]) + } + if (url.pathname === `/session/${id}`) return json(session(id, "current")) + if (url.pathname === `/session/${id}/message`) + return ++messages === 1 ? old.promise : (forceRequested.resolve(), forced.promise) + if (url.pathname === `/session/${id}/todo` || url.pathname === `/session/${id}/diff`) return json([]) + }, + async ({ emit, sync }) => { + const oldHydration = sync.session.sync(id) + emit(connected("initial")) + emit(connected("reconnect")) + await listed.promise + expect(messages).toBe(1) + const livePart = { + id: `prt_${messageID}`, + sessionID: id, + messageID, + type: "text" as const, + text: "live before force", + } + const live = observe(() => sync.data.part[messageID]?.[0]?.type === "text") + emit( + global({ + id: "message", + type: "message.updated", + properties: { sessionID: id, info: { ...message(id, messageID, false), time: { created: 10 } } }, + }), + ) + emit( + global({ id: "part", type: "message.part.updated", properties: { sessionID: id, time: 10, part: livePart } }), + ) + await live + old.resolve(json(payload(id, messageID, "stale old"))) + await Promise.all([oldHydration, forceRequested.promise]) + const forceLive = observe( + () => + sync.data.part[messageID]?.[0]?.type === "text" && sync.data.part[messageID][0].text === "live during force", + ) + emit( + global({ + id: "force-part", + type: "message.part.updated", + properties: { sessionID: id, time: 11, part: { ...livePart, text: "live during force" } }, + }), + ) + await forceLive + forced.resolve(json(payload(id, messageID, "stale force"))) + await sync.session.sync(id) + expect(messages).toBe(2) + expect(sync.data.message[id]).toHaveLength(1) + expect(sync.data.part[messageID]?.[0]).toMatchObject({ text: "live during force" }) + }, + ) +}) + +test("bootstrap and session failures isolate and remain route and epoch retryable", async () => { + const failedID = "ses_failed" + const healthyID = "ses_healthy" + const gets = new Map() + const requested = [deferred(), deferred(), deferred(), deferred()] + let reconnecting = false + let recover = false + let lists = 0 + await scenario( + (url) => { + if (url.pathname === "/session") { + if (++lists === 2) requested[0].resolve() + return json([session(failedID, recover ? "recovered" : "failed"), session(healthyID, "healthy")]) + } + if (url.pathname === "/config/providers" && reconnecting && !recover) return json({}, { status: 500 }) + for (const id of [failedID, healthyID]) { + if (url.pathname === `/session/${id}`) { + const count = (gets.get(id) ?? 0) + 1 + gets.set(id, count) + if (count === 2) requested[id === failedID ? 1 : 2].resolve() + if (id === failedID && count === 4) requested[3].resolve() + if (id === failedID && reconnecting && !recover) return json({}, { status: 500 }) + return json(session(id, recover ? "recovered" : id)) + } + if (url.pathname === `/session/${id}/message`) return json(payload(id, `msg_${id}`, id)) + if (url.pathname === `/session/${id}/todo` || url.pathname === `/session/${id}/diff`) return json([]) + } + }, + async ({ emit, sync }) => { + await Promise.all([sync.session.sync(failedID), sync.session.sync(healthyID)]) + emit(connected("initial")) + reconnecting = true + emit(connected("reconnect")) + await Promise.all(requested.slice(0, 3).map((item) => item.promise)) + await expect(sync.session.sync(failedID)).rejects.toBeDefined() + await expect(sync.session.sync(healthyID)).resolves.toBeUndefined() + await expect(sync.session.sync(failedID)).rejects.toBeDefined() + expect(gets.get(failedID)).toBe(3) + recover = true + emit(connected("retry")) + await requested[3].promise + await sync.session.sync(failedID) + expect(gets.get(failedID)).toBe(4) + expect(sync.session.get(failedID)?.title).toBe("recovered") + expect(sync.data.message[healthyID]?.[0]?.id).toBe(`msg_${healthyID}`) + }, + ) +}) + +test("default detached bootstrap owns strict failures and reconnect reports once", async () => { + const id = "ses_bootstrap_policy" + const statusFailure = Promise.withResolvers() + const statusRequested = Promise.withResolvers() + const reconnectRequested = Promise.withResolvers() + let phase: "initial" | "default-failed" | "reconnect-failed" = "initial" + await scenario( + (url) => { + if (url.pathname === "/session") { + if (phase === "reconnect-failed") { + reconnectRequested.resolve() + return json({}, { status: 500 }) + } + return json([session(id, "present")]) + } + if (url.pathname === "/session/status") { + if (phase === "default-failed") { + statusRequested.resolve() + return statusFailure.promise + } + return json({ [id]: { type: "idle" } }) + } + }, + async ({ emit }) => { + emit(connected("initial")) + const reports: string[] = [] + const reported = Promise.withResolvers() + const reconnectReported = Promise.withResolvers() + const error = spyOn(console, "error").mockImplementation((message) => { + if (typeof message !== "string") return + reports.push(message) + if (message === "tui bootstrap failed") reported.resolve() + if (message === "tui reconnect reconciliation failed") reconnectReported.resolve() + }) + const unhandled: unknown[] = [] + const unhandledObserved = Promise.withResolvers() + const onUnhandled = (reason: unknown) => { + unhandled.push(reason) + unhandledObserved.resolve() + } + process.on("unhandledRejection", onUnhandled) + try { + phase = "default-failed" + emit(instanceDisposed("default-failed")) + await statusRequested.promise + statusFailure.resolve(json({}, { status: 500 })) + await Promise.race([reported.promise, unhandledObserved.promise]) + await new Promise((resolve) => setImmediate(resolve)) + expect(reports).toEqual(["tui bootstrap failed"]) + expect(unhandled).toEqual([]) + + phase = "reconnect-failed" + emit(connected("reconnect-failed")) + await reconnectRequested.promise + await reconnectReported.promise + expect(reports).toEqual(["tui bootstrap failed", "tui reconnect reconciliation failed"]) + expect(unhandled).toEqual([]) + } finally { + process.off("unhandledRejection", onUnhandled) + error.mockRestore() + } + }, + ) +}) + +test("session list snapshot governs status in either settlement order", async () => { + const id = "ses_status_order" + const firstList = Promise.withResolvers() + const firstStatusRequested = Promise.withResolvers() + const secondListRequested = Promise.withResolvers() + const secondStatus = Promise.withResolvers() + let phase: "initial" | "status-first" | "list-first" = "initial" + await scenario( + (url) => { + if (url.pathname === "/session") { + if (phase === "status-first") return firstList.promise + if (phase === "list-first") { + secondListRequested.resolve() + return json([]) + } + return json([session(id, "present")]) + } + if (url.pathname === "/session/status") { + if (phase === "status-first") { + firstStatusRequested.resolve() + return json({ [id]: { type: "busy" } }) + } + if (phase === "list-first") return secondStatus.promise + return json({ [id]: { type: "idle" } }) + } + }, + async ({ sync }) => { + phase = "status-first" + const first = sync.bootstrap({ wait: true, fatal: false, report: false }) + await firstStatusRequested.promise + firstList.resolve(json([session(id, "present")])) + await first + expect(sync.data.session_status[id]).toEqual({ type: "busy" }) + + phase = "list-first" + const second = sync.bootstrap({ wait: true, fatal: false, report: false }) + await secondListRequested.promise + await observe(() => sync.session.get(id) === undefined) + secondStatus.resolve(json({ [id]: { type: "busy" } })) + await second + expect(sync.data.session_status[id]).toBeUndefined() + }, + ) +}) + +for (const domain of ["message", "todo", "diff"] as const) { + test(`forced ${domain} failure preserves every session domain`, async () => { + const id = `ses_${domain}_failure` + const failure = Promise.withResolvers() + const requested = Promise.withResolvers() + let failing = false + await scenario( + (url) => { + if (url.pathname === "/session") return json([session(id, "initial")]) + if (url.pathname === "/session/status") return json({ [id]: { type: "idle" } }) + if (url.pathname === `/session/${id}`) return json(session(id, failing ? "failed" : "initial")) + if (url.pathname === `/session/${id}/${domain}` && failing) { + requested.resolve() + return failure.promise + } + const value = failing ? "failed" : "initial" + if (url.pathname === `/session/${id}/message`) return json(payload(id, `msg_${value}`, value)) + if (url.pathname === `/session/${id}/todo`) + return json([{ content: value, status: "pending", priority: "high" }]) + if (url.pathname === `/session/${id}/diff`) + return json([{ file: `${value}.ts`, patch: value, additions: 1, deletions: 0, status: "modified" }]) + }, + async ({ emit, sync }) => { + await sync.session.sync(id) + emit(connected("initial")) + failing = true + emit(connected(`failed-${domain}`)) + await requested.promise + const joined = sync.session.sync(id) + failure.resolve(json({}, { status: 500 })) + const outcome = await joined.then( + () => "resolved" as const, + () => "rejected" as const, + ) + expect({ + outcome, + title: sync.session.get(id)?.title, + messageID: sync.data.message[id]?.[0]?.id, + part: sync.data.part.msg_initial?.[0]?.type === "text" ? sync.data.part.msg_initial[0].text : undefined, + todo: sync.data.todo[id]?.[0]?.content, + diff: sync.data.session_diff[id]?.[0]?.file, + }).toEqual({ + outcome: "rejected", + title: "initial", + messageID: "msg_initial", + part: "initial", + todo: "initial", + diff: "initial.ts", + }) + }, + ) + }) +} + +test("HTTP failures preserve every reconnect domain and remain retryable", async () => { + const id = "ses_http_failure" + const bootstrapRequests = new Set() + const failedRequests = new Set() + const domainCounts = new Map() + const bootstrapFailed = Promise.withResolvers() + const domainsFailed = Promise.withResolvers() + const bootstrapPaths = new Set(["/session", "/session/status"]) + const domainPaths = new Set([`/session/${id}/message`, `/session/${id}/todo`, `/session/${id}/diff`]) + const forceFailures = new Map([...domainPaths].map((pathname) => [pathname, Promise.withResolvers()])) + let phase: "initial" | "bootstrap-failed" | "domains-failed" | "recovered" = "initial" + await scenario( + (url) => { + if (domainPaths.has(url.pathname)) domainCounts.set(url.pathname, (domainCounts.get(url.pathname) ?? 0) + 1) + if (phase === "bootstrap-failed" && bootstrapPaths.has(url.pathname)) { + bootstrapRequests.add(url.pathname) + if (bootstrapRequests.size === bootstrapPaths.size) bootstrapFailed.resolve() + return json({}, { status: 500 }) + } + if (phase === "domains-failed" && domainPaths.has(url.pathname)) { + failedRequests.add(url.pathname) + if (failedRequests.size === domainPaths.size) domainsFailed.resolve() + const deferredFailure = forceFailures.get(url.pathname) + if (domainCounts.get(url.pathname) === 1 && deferredFailure) return deferredFailure.promise + return json({}, { status: 500 }) + } + const stable = phase === "recovered" ? "recovered" : "initial" + const hydration = phase === "domains-failed" ? phase : stable + if (url.pathname === "/session") return json([session(id, stable)]) + if (url.pathname === "/session/status") return json({ [id]: { type: phase === "recovered" ? "busy" : "idle" } }) + if (url.pathname === `/session/${id}`) return json(session(id, hydration)) + if (url.pathname === `/session/${id}/message`) return json(payload(id, `msg_${hydration}`, hydration)) + if (url.pathname === `/session/${id}/todo`) + return json([{ content: hydration, status: "pending", priority: "high" }]) + if (url.pathname === `/session/${id}/diff`) + return json([{ file: `${hydration}.ts`, patch: hydration, additions: 1, deletions: 0, status: "modified" }]) + }, + async ({ emit, sync }) => { + await sync.session.sync(id) + emit(connected("initial")) + phase = "bootstrap-failed" + emit(connected("bootstrap-failed")) + await bootstrapFailed.promise + await Bun.sleep(0) + + expect(sync.session.get(id)?.title).toBe("initial") + expect(sync.data.session_status[id]).toEqual({ type: "idle" }) + + domainCounts.clear() + phase = "domains-failed" + emit(connected("domains-failed")) + await domainsFailed.promise + const joined = sync.session.sync(id) + for (const failure of forceFailures.values()) failure.resolve(json({}, { status: 500 })) + const joinedOutcome = await joined.then( + () => "resolved" as const, + () => "rejected" as const, + ) + expect({ + outcome: joinedOutcome, + requests: [...domainPaths].map((pathname) => domainCounts.get(pathname)), + }).toEqual({ + outcome: "rejected", + requests: [1, 1, 1], + }) + + expect(sync.session.get(id)?.title).toBe("initial") + expect(sync.data.session_status[id]).toEqual({ type: "idle" }) + expect(sync.data.message[id]?.[0]?.id).toBe("msg_initial") + expect(sync.data.part.msg_initial?.[0]).toMatchObject({ text: "initial" }) + expect(sync.data.todo[id]).toEqual([{ content: "initial", status: "pending", priority: "high" }]) + expect(sync.data.session_diff[id]).toEqual([ + { file: "initial.ts", patch: "initial", additions: 1, deletions: 0, status: "modified" }, + ]) + + const routeOutcome = await sync.session.sync(id).then( + () => "resolved" as const, + () => "rejected" as const, + ) + expect({ + outcome: routeOutcome, + requests: [...domainPaths].map((pathname) => domainCounts.get(pathname)), + title: sync.session.get(id)?.title, + messageID: sync.data.message[id]?.[0]?.id, + partText: sync.data.part.msg_initial?.[0]?.type === "text" ? sync.data.part.msg_initial[0].text : undefined, + todo: sync.data.todo[id]?.[0]?.content, + diff: sync.data.session_diff[id]?.[0]?.file, + }).toEqual({ + outcome: "rejected", + requests: [2, 2, 2], + title: "initial", + messageID: "msg_initial", + partText: "initial", + todo: "initial", + diff: "initial.ts", + }) + + phase = "recovered" + await sync.session.sync(id) + expect({ + requests: [...domainPaths].map((pathname) => domainCounts.get(pathname)), + title: sync.session.get(id)?.title, + messageID: sync.data.message[id]?.[0]?.id, + todo: sync.data.todo[id]?.[0]?.content, + diff: sync.data.session_diff[id]?.[0]?.file, + }).toEqual({ + requests: [3, 3, 3], + title: "recovered", + messageID: "msg_recovered", + todo: "recovered", + diff: "recovered.ts", + }) + await sync.session.sync(id) + expect([...domainPaths].map((pathname) => domainCounts.get(pathname))).toEqual([3, 3, 3]) + }, + ) +}) + +test("successful reconnect bootstrap skips a loaded session deleted during the outage", async () => { + const id = "ses_deleted" + let deleted = false + let gets = 0 + await scenario( + (url) => { + if (url.pathname === "/session") return json(deleted ? [] : [session(id, "present")]) + if (url.pathname === "/session/status") return json({ [id]: { type: "busy" } }) + if (url.pathname === `/session/${id}`) return ((gets += 1), json(session(id, "present"))) + if (url.pathname === `/session/${id}/message`) return json(payload(id, "msg_deleted", "present")) + if (url.pathname === `/session/${id}/todo`) + return json([{ content: "present", status: "pending", priority: "high" }]) + if (url.pathname === `/session/${id}/diff`) + return json([{ file: "present.ts", patch: "present", additions: 1, deletions: 0, status: "modified" }]) + }, + async ({ emit, sync }) => { + await sync.session.sync(id) + sync.set("permission", id, []) + sync.set("question", id, []) + sync.set("part", "msg_deleted", []) + sync.set("part", "msg_orphan", [ + { id: "prt_orphan", sessionID: id, messageID: "msg_orphan", type: "text" as const, text: "orphan" }, + ]) + emit(connected("initial")) + deleted = true + const removed = observe(() => sync.data.session.length === 0) + emit(connected("reconnect")) + await removed + await Promise.resolve() + expect(gets).toBe(1) + expect(sync.session.get(id)).toBeUndefined() + expect(sync.data.session_status[id]).toBeUndefined() + expect(sync.data.message[id]).toBeUndefined() + expect(sync.data.part.msg_deleted).toBeUndefined() + expect(sync.data.part.msg_orphan).toBeUndefined() + expect(sync.data.todo[id]).toBeUndefined() + expect(sync.data.session_diff[id]).toBeUndefined() + expect(sync.data.permission[id]).toBeUndefined() + expect(sync.data.question[id]).toBeUndefined() + + deleted = false + emit( + global({ + id: "restored", + type: "session.updated", + properties: { sessionID: id, info: session(id, "restored") }, + }), + ) + await observe(() => sync.session.get(id)?.title === "restored") + await sync.session.sync(id) + expect(gets).toBe(2) + }, + ) +}) + +test("reconnect prunes every adjacent deleted session", async () => { + const ids = ["ses_deleted_a", "ses_deleted_b"] + let deleted = false + await scenario( + (url) => { + if (url.pathname === "/session") return json(deleted ? [] : ids.map((id) => session(id, "present"))) + if (url.pathname === "/session/status") return json(Object.fromEntries(ids.map((id) => [id, { type: "busy" }]))) + for (const id of ids) { + if (url.pathname === `/session/${id}`) return json(session(id, "present")) + if (url.pathname === `/session/${id}/message`) return json(payload(id, `msg_${id}`, "present")) + if (url.pathname === `/session/${id}/todo`) + return json([{ content: "present", status: "pending", priority: "high" }]) + if (url.pathname === `/session/${id}/diff`) + return json([{ file: `${id}.ts`, patch: "present", additions: 1, deletions: 0, status: "modified" }]) + } + }, + async ({ emit, sync }) => { + await Promise.all(ids.map((id) => sync.session.sync(id))) + emit(connected("initial")) + deleted = true + emit(connected("reconnect")) + await observe(() => sync.data.session.length === 0) + + for (const id of ids) { + expect(sync.data.session_status[id]).toBeUndefined() + expect(sync.data.message[id]).toBeUndefined() + expect(sync.data.part[`msg_${id}`]).toBeUndefined() + expect(sync.data.todo[id]).toBeUndefined() + expect(sync.data.session_diff[id]).toBeUndefined() + } + }, + ) +}) + +test("deleting a session invalidates in-flight hydration", async () => { + const id = "ses_deleted_inflight" + const response = Promise.withResolvers() + const requested = Promise.withResolvers() + await scenario( + (url) => { + if (url.pathname === "/session") return json([session(id, "present")]) + if (url.pathname === "/session/status") return json({ [id]: { type: "busy" } }) + if (url.pathname === `/session/${id}`) return json(session(id, "stale")) + if (url.pathname === `/session/${id}/message`) { + requested.resolve() + return response.promise + } + if (url.pathname === `/session/${id}/todo`) + return json([{ content: "stale", status: "pending", priority: "high" }]) + if (url.pathname === `/session/${id}/diff`) + return json([{ file: "stale.ts", patch: "stale", additions: 1, deletions: 0, status: "modified" }]) + }, + async ({ emit, sync }) => { + const hydration = sync.session.sync(id) + await requested.promise + emit( + global({ id: "deleted", type: "session.deleted", properties: { sessionID: id, info: session(id, "deleted") } }), + ) + await observe(() => sync.session.get(id) === undefined) + response.resolve(json(payload(id, "msg_stale", "stale"))) + await hydration + + expect(sync.session.get(id)).toBeUndefined() + expect(sync.data.session_status[id]).toBeUndefined() + expect(sync.data.message[id]).toBeUndefined() + expect(sync.data.part.msg_stale).toBeUndefined() + expect(sync.data.todo[id]).toBeUndefined() + expect(sync.data.session_diff[id]).toBeUndefined() + }, + ) +}) + +test("disposal prevents old work and remount storms produce one trailing pass", async () => { + let firstRequests = 0 + await scenario( + (url) => { + if (url.pathname === "/session") firstRequests += 1 + return undefined + }, + async ({ app, emit }) => { + app.renderer.destroy() + expect(() => emit(connected("after-dispose"))).toThrow("event source not ready") + await Promise.resolve() + expect(firstRequests).toBe(1) + }, + ) + const id = "ses_remount" + let lists = 0 + let messages = 0 + await scenario( + (url) => { + if (url.pathname === "/session") return ((lists += 1), json([session(id, `pass ${lists}`)])) + if (url.pathname === `/session/${id}`) return json(session(id, `pass ${lists}`)) + if (url.pathname === `/session/${id}/message`) + return ((messages += 1), json(payload(id, `msg_pass_${messages}`, `pass ${messages}`))) + if (url.pathname === `/session/${id}/todo` || url.pathname === `/session/${id}/diff`) return json([]) + }, + async ({ emit, sync }) => { + await sync.session.sync(id) + for (const epoch of ["initial", "two", "three", "four"]) emit(connected(epoch)) + await observe(() => sync.data.message[id]?.[0]?.id === "msg_pass_3") + expect([lists, messages]).toEqual([3, 3]) + }, + ) +}) diff --git a/packages/tui/test/cli/tui/sdk-reconnect.test.tsx b/packages/tui/test/cli/tui/sdk-reconnect.test.tsx new file mode 100644 index 000000000000..17c58c2b33f6 --- /dev/null +++ b/packages/tui/test/cli/tui/sdk-reconnect.test.tsx @@ -0,0 +1,343 @@ +/** @jsxImportSource @opentui/solid */ +import { expect, test } from "bun:test" +import { testRender } from "@opentui/solid" +import type { GlobalEvent } from "@opencode-ai/sdk/v2" +import { onMount } from "solid-js" +import { runEventStream, SDKProvider, useSDK, type EventSource } from "../../../src/context/sdk" + +const connected = { + directory: "global", + payload: { + id: "evt_connected", + type: "server.connected", + properties: {}, + }, +} satisfies GlobalEvent + +const queued = { + directory: "global", + payload: { + id: "evt_disposed", + type: "global.disposed", + properties: {}, + }, +} satisfies GlobalEvent + +type Connect = (signal: AbortSignal) => Promise> +type CallbackBoundary = "flush" | "wait" | "retry" + +function createHarness(connect: Connect, fault?: CallbackBoundary) { + const lifecycle = new AbortController() + const subscription = new AbortController() + const waits: Array<{ readonly delay: number; readonly release: () => void }> = [] + const queue: GlobalEvent[] = [] + const seen: GlobalEvent[] = [] + const retries: Array<{ readonly attempt: number; readonly error: unknown }> = [] + let flushes = 0 + let faulted = false + const fail = (boundary: CallbackBoundary) => { + if (fault !== boundary || faulted) return + faulted = true + throw new Error(`${boundary} failed`) + } + + return { + lifecycle, + subscription, + waits, + seen, + retries, + get flushes() { + return flushes + }, + start: () => + runEventStream({ + signals: [lifecycle.signal, subscription.signal], + connect, + event: (event) => queue.push(event), + flush: () => { + fail("flush") + flushes++ + seen.push(...queue.splice(0)) + }, + wait: (delay, signal) => { + fail("wait") + const deferred = Promise.withResolvers() + const release = () => { + signal.removeEventListener("abort", release) + deferred.resolve() + } + signal.addEventListener("abort", release, { once: true }) + waits.push({ delay, release }) + return deferred.promise + }, + retry: (entry) => { + fail("retry") + retries.push(entry) + }, + }), + } +} + +async function settleUntil(condition: () => boolean) { + for (let count = 0; count < 100; count++) { + if (condition()) return + await Promise.resolve() + } + throw new Error("condition did not settle") +} + +async function releaseRetry(harness: ReturnType, index = 0) { + await settleUntil(() => harness.waits.length > index) + const call = harness.waits[index] + if (!call) throw new Error("retry wait not found") + call.release() +} + +function untilAbort(signal: AbortSignal): Promise { + return new Promise((_, reject) => { + const abort = () => reject(new DOMException("Aborted", "AbortError")) + if (signal.aborted) return abort() + signal.addEventListener("abort", abort, { once: true }) + }) +} + +async function* emptyStream(): AsyncIterable {} + +test("fans out an injected event to subscribers once", async () => { + const sourceReady = Promise.withResolvers() + const listenerReady = Promise.withResolvers() + const received = Promise.withResolvers() + let handler: ((event: GlobalEvent) => void) | undefined + const source: EventSource = { + subscribe: async (next) => { + handler = next + sourceReady.resolve() + return () => { + handler = undefined + } + }, + } + + function Probe() { + const sdk = useSDK() + onMount(() => { + const unsubscribe = sdk.event.on("event", received.resolve) + listenerReady.resolve() + return unsubscribe + }) + return + } + + const app = await testRender(() => ( + + + + )) + + try { + await Promise.all([sourceReady.promise, listenerReady.promise]) + if (!handler) throw new Error("event source not ready") + handler(connected) + + expect(await received.promise).toEqual(connected) + } finally { + app.renderer.destroy() + } +}) + +test("unsubscribes an injected source disposed before subscription resolves", async () => { + const subscribed = Promise.withResolvers() + const release = Promise.withResolvers() + let unsubscribed = 0 + const source: EventSource = { + subscribe: async () => { + subscribed.resolve() + await release.promise + return () => unsubscribed++ + }, + } + const app = await testRender(() => ( + + + + )) + + await subscribed.promise + app.renderer.destroy() + release.resolve() + await settleUntil(() => unsubscribed === 1) + + expect(unsubscribed).toBe(1) +}) + +test("reconnects after connection establishment rejects", async () => { + const failure = new Error("connect failed") + let connects = 0 + const harness = createHarness(async (signal) => { + connects++ + if (connects === 1) throw failure + return untilAbort(signal) + }) + + const task = harness.start() + await releaseRetry(harness) + await settleUntil(() => connects === 2) + harness.subscription.abort() + await task + + expect(connects).toBe(2) + expect(harness.waits.map((call) => call.delay)).toEqual([1000]) + expect(harness.retries).toEqual([{ attempt: 1, error: failure }]) +}) + +test("reconnects after a stream completes normally", async () => { + let connects = 0 + const harness = createHarness(async (signal) => { + connects++ + if (connects === 1) return emptyStream() + return untilAbort(signal) + }) + + const task = harness.start() + await releaseRetry(harness) + await settleUntil(() => connects === 2) + harness.subscription.abort() + await task + + expect(connects).toBe(2) + expect(harness.retries).toHaveLength(1) + expect(harness.retries[0]?.attempt).toBe(1) +}) + +test("flushes a failed stream and delivers the recovered first event once", async () => { + const failure = new Error("iterator failed") + const recovered = Promise.withResolvers() + let connects = 0 + let active = 0 + let maxActiveStreams = 0 + const harness = createHarness(async (signal) => { + connects++ + return (async function* () { + active++ + maxActiveStreams = Math.max(maxActiveStreams, active) + try { + if (connects === 1) { + yield queued + throw failure + } + yield connected + recovered.resolve() + await untilAbort(signal) + } finally { + active-- + } + })() + }) + + const task = harness.start() + await releaseRetry(harness) + await recovered.promise + harness.lifecycle.abort() + await task + + expect(connects).toBe(2) + expect(harness.seen).toEqual([queued, connected]) + expect(harness.retries).toEqual([{ attempt: 1, error: failure }]) + expect(harness.flushes).toBe(2) + expect(maxActiveStreams).toBe(1) + expect(active).toBe(0) +}) + +test("caps exponential delays and resets them after server.connected", async () => { + let connects = 0 + const harness = createHarness(async () => { + connects++ + if (connects === 8) + return (async function* () { + yield connected + })() + return emptyStream() + }) + + const task = harness.start() + for (let index = 0; index < 7; index++) await releaseRetry(harness, index) + await settleUntil(() => harness.waits.length === 8) + harness.subscription.abort() + await task + + expect(harness.waits.map((call) => call.delay)).toEqual([1000, 2000, 4000, 8000, 16000, 30000, 30000, 1000]) + expect(harness.retries).toHaveLength(7) +}) + +test.each(["flush", "wait", "retry"] as const)("%s callback failure remains owned and reconnects", async (boundary) => { + let connects = 0 + const harness = createHarness(async (signal) => { + connects++ + if (connects === 1) return emptyStream() + return untilAbort(signal) + }, boundary) + + const task = harness.start() + const outcome = task.then( + () => "resolved" as const, + () => "rejected" as const, + ) + if (boundary !== "wait") await releaseRetry(harness) + const state = await Promise.race([outcome, settleUntil(() => connects === 2).then(() => "running" as const)]) + expect(state).toBe("running") + harness.subscription.abort() + + expect(await outcome).toBe("resolved") + expect(connects).toBe(2) +}) + +test.each(["lifecycle", "subscription"] as const)("%s abort during connect exits without retry", async (owner) => { + const started = Promise.withResolvers() + const harness = createHarness(async (signal) => { + started.resolve() + return untilAbort(signal) + }) + + const task = harness.start() + await started.promise + harness[owner].abort() + await task + + expect(harness.waits).toEqual([]) + expect(harness.retries).toEqual([]) +}) + +test.each(["lifecycle", "subscription"] as const)("%s abort during iteration exits without retry", async (owner) => { + const iterating = Promise.withResolvers() + const harness = createHarness(async (signal) => + (async function* () { + iterating.resolve() + await untilAbort(signal) + })(), + ) + + const task = harness.start() + await iterating.promise + harness[owner].abort() + await task + + expect(harness.waits).toEqual([]) + expect(harness.retries).toEqual([]) +}) + +test.each(["lifecycle", "subscription"] as const)("%s abort during backoff exits without retry", async (owner) => { + let connects = 0 + const harness = createHarness(async () => { + connects++ + return emptyStream() + }) + + const task = harness.start() + await settleUntil(() => harness.waits.length === 1) + harness[owner].abort() + await task + + expect(connects).toBe(1) + expect(harness.retries).toEqual([]) +}) diff --git a/packages/tui/test/context/sync-reconnect.test.ts b/packages/tui/test/context/sync-reconnect.test.ts new file mode 100644 index 000000000000..e159e9a2a4d4 --- /dev/null +++ b/packages/tui/test/context/sync-reconnect.test.ts @@ -0,0 +1,216 @@ +import { expect, test } from "bun:test" +import { createReconnectCoordinator, type ReconnectFailure } from "../../src/context/sync-reconnect" + +function deferred() { + let resolvePromise: ((value: T) => void) | undefined + const promise = new Promise((resolve) => { + resolvePromise = resolve + }) + return { + promise, + resolve(value: T) { + if (!resolvePromise) throw new Error("deferred promise is not initialized") + resolvePromise(value) + }, + } +} + +test("ignores epoch one, snapshots before bootstrap, and coalesces a storm into one trailing pass", async () => { + const firstBootstrap = deferred() + const bootstraps: number[] = [] + const snapshots: string[][] = [] + const forced: string[] = [] + const failures: ReconnectFailure[] = [] + const sessionError = new Error("inflight failed") + let targets = ["ses_full", "ses_inflight"] + const coordinator = createReconnectCoordinator({ + bootstrap: async () => { + bootstraps.push(bootstraps.length + 1) + if (bootstraps.length === 1) await firstBootstrap.promise + }, + targets: () => { + snapshots.push([...targets]) + return targets + }, + exists: () => true, + forceSync: async (sessionID) => { + forced.push(sessionID) + if (sessionID === "ses_inflight" && forced.filter((item) => item === sessionID).length === 1) throw sessionError + }, + onError: (failure) => failures.push(failure), + }) + + await coordinator.connected() + expect(bootstraps).toEqual([]) + + const active = coordinator.connected() + await Promise.resolve() + expect(snapshots).toEqual([["ses_full", "ses_inflight"]]) + + targets = ["ses_new"] + const trailing = coordinator.connected() + void coordinator.connected() + firstBootstrap.resolve() + await active + await trailing + + expect(bootstraps).toEqual([1, 2]) + expect(snapshots).toEqual([["ses_full", "ses_inflight"], ["ses_new"]]) + expect(forced).toEqual(["ses_full", "ses_inflight", "ses_new", "ses_inflight"]) + expect(failures).toEqual([{ boundary: "session", sessionID: "ses_inflight", error: sessionError }]) +}) + +test("filters deleted targets only after bootstrap succeeds and keeps targets when bootstrap fails", async () => { + const forced: string[] = [] + const failures: ReconnectFailure[] = [] + const bootstrapError = new Error("bootstrap failed") + let bootstrapFails = false + const coordinator = createReconnectCoordinator({ + bootstrap: async () => { + if (bootstrapFails) throw bootstrapError + }, + targets: () => ["ses_present", "ses_deleted"], + exists: (sessionID) => sessionID === "ses_present", + forceSync: async (sessionID) => { + forced.push(sessionID) + }, + onError: (failure) => failures.push(failure), + }) + + await coordinator.connected() + await coordinator.connected() + bootstrapFails = true + await coordinator.connected() + + expect(forced).toEqual(["ses_present", "ses_present", "ses_deleted"]) + expect(failures).toEqual([{ boundary: "bootstrap", error: bootstrapError }]) +}) + +test("disposal during an active pass cancels queued and future work", async () => { + const blocked = deferred() + const forced: string[] = [] + const failures: ReconnectFailure[] = [] + const coordinator = createReconnectCoordinator({ + bootstrap: () => blocked.promise, + targets: () => ["ses_loaded"], + exists: () => true, + forceSync: async (sessionID) => { + forced.push(sessionID) + }, + onError: (failure) => failures.push(failure), + }) + + await coordinator.connected() + const active = coordinator.connected() + await Promise.resolve() + void coordinator.connected() + coordinator.dispose() + blocked.resolve() + await active + await coordinator.connected() + + expect(forced).toEqual([]) + expect(failures).toEqual([]) +}) + +test("an epoch arriving during the trailing pass schedules another pass", async () => { + const releases = [deferred(), deferred()] + const entered = [deferred(), deferred(), deferred()] + let bootstraps = 0 + const coordinator = createReconnectCoordinator({ + bootstrap: async () => { + const index = bootstraps++ + entered[index].resolve() + await releases[index]?.promise + }, + targets: () => [], + exists: () => true, + forceSync: async () => {}, + onError: () => {}, + }) + + await coordinator.connected() + const active = coordinator.connected() + await entered[0].promise + void coordinator.connected() + releases[0].resolve() + await entered[1].promise + void coordinator.connected() + releases[1].resolve() + await active + + expect(bootstraps).toBe(3) +}) + +test("an epoch arriving during ownership release waits for its reconciliation pass", async () => { + const handoffReady = deferred() + let bootstraps = 0 + let running = 0 + let maxRunning = 0 + let handoff: Promise | undefined + let coordinator: ReturnType + coordinator = createReconnectCoordinator({ + bootstrap: async () => { + bootstraps += 1 + running += 1 + maxRunning = Math.max(maxRunning, running) + if (bootstraps === 1) { + queueMicrotask(() => + queueMicrotask(() => + queueMicrotask(() => + queueMicrotask(() => { + handoff = coordinator.connected() + handoffReady.resolve() + }), + ), + ), + ) + } + running -= 1 + }, + targets: () => [], + exists: () => true, + forceSync: async () => {}, + onError: () => {}, + }) + + await coordinator.connected() + const active = coordinator.connected() + await handoffReady.promise + if (!handoff) throw new Error("ownership handoff promise is not initialized") + await handoff + const afterHandoff = bootstraps + await coordinator.connected() + await active + + expect({ afterHandoff, afterNextEpoch: bootstraps, maxRunning }).toEqual({ + afterHandoff: 2, + afterNextEpoch: 3, + maxRunning: 1, + }) +}) + +test("disposal suppresses failures settled after a session pass", async () => { + const forced = Promise.withResolvers() + const entered = Promise.withResolvers() + const failures: ReconnectFailure[] = [] + const coordinator = createReconnectCoordinator({ + bootstrap: async () => {}, + targets: () => ["ses_loaded"], + exists: () => true, + forceSync: () => { + entered.resolve() + return forced.promise + }, + onError: (failure) => failures.push(failure), + }) + + await coordinator.connected() + const active = coordinator.connected() + await entered.promise + coordinator.dispose() + forced.reject(new Error("settled after disposal")) + await active + + expect(failures).toEqual([]) +})