diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index 8c7b34f9e..3e158e69f 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -28,6 +28,7 @@ import { SessionBlocker } from "./blocker" const log = Log.create({ service: "llm" }) export const OUTPUT_TOKEN_MAX = ProviderTransform.OUTPUT_TOKEN_MAX export const SILENT_STREAM_TIMEOUT_MS = Duration.toMillis(Duration.minutes(10)) +export const CONNECT_STREAM_TIMEOUT_MS = Duration.toMillis(Duration.seconds(30)) type Result = Awaited> export type StreamInput = { @@ -42,6 +43,7 @@ export type StreamInput = { small?: boolean tools: Record retries?: number + connectTimeoutMs?: number streamTimeoutMs?: number toolChoice?: "auto" | "required" | "none" trace?: Pick @@ -427,18 +429,51 @@ const live: Layer.Layer< Stream.scoped( Stream.unwrap( Effect.gen(function* () { - const timeoutMsInput = input.streamTimeoutMs - const timeoutMs = - typeof timeoutMsInput === "number" && Number.isFinite(timeoutMsInput) && timeoutMsInput > 0 - ? timeoutMsInput + const connectTimeoutMsInput = input.connectTimeoutMs + const connectTimeoutMs = + typeof connectTimeoutMsInput === "number" && + Number.isFinite(connectTimeoutMsInput) && + connectTimeoutMsInput > 0 + ? connectTimeoutMsInput + : CONNECT_STREAM_TIMEOUT_MS + const streamTimeoutMsInput = input.streamTimeoutMs + const streamTimeoutMs = + typeof streamTimeoutMsInput === "number" && + Number.isFinite(streamTimeoutMsInput) && + streamTimeoutMsInput > 0 + ? streamTimeoutMsInput : SILENT_STREAM_TIMEOUT_MS const ctx = yield* Effect.context() const request = yield* Effect.acquireRelease( Effect.sync(() => { const ctrl = new AbortController() let disposed = false + let providerProgressed = false let sequence = 0 let timeout: Timer | undefined + let timeoutError: Error | undefined + let rejectTimeout: ((error: Error) => void) | undefined + const timeoutFailure = new Promise((_, reject) => { + rejectTimeout = reject + }) + // Keep the timeout rejection observed even if no iterator.next() is racing yet. + void timeoutFailure.catch(() => {}) + const currentTimeoutMs = () => (providerProgressed ? streamTimeoutMs : connectTimeoutMs) + const failConnectTimeout = () => { + if (timeoutError) return + timeoutError = new Error( + `LLM stream connection timed out after ${connectTimeoutMs}ms without provider progress`, + ) + rejectTimeout?.(timeoutError) + ctrl.abort() + } + const timeoutStream = () => { + if (providerProgressed) { + ctrl.abort() + return + } + failConnectTimeout() + } const arm = () => { const current = ++sequence timeout = setTimeout(() => { @@ -451,17 +486,23 @@ const live: Layer.Layer< arm() return } - ctrl.abort() + timeoutStream() }) .catch(() => { - if (!disposed && current === sequence) ctrl.abort() + if (!disposed && current === sequence) timeoutStream() }) - }, timeoutMs) + }, currentTimeoutMs()) } arm() return { ctrl, - resetTimeout() { + timeoutFailure, + timeoutError() { + return timeoutError + }, + resetTimeout(event: Event) { + if (!providerProgressed && !isProviderProgressEvent(event)) return + providerProgressed = true if (timeout) clearTimeout(timeout) arm() }, @@ -479,9 +520,9 @@ const live: Layer.Layer< // This is a silent-stream timeout: it limits how long we wait for // the next provider event, not the total model runtime. - return Stream.fromAsyncIterable(result.fullStream, (e) => (e instanceof Error ? e : new Error(String(e)))).pipe( - Stream.tap(() => Effect.sync(() => request.resetTimeout())), - ) + return Stream.fromAsyncIterable(failOnTimeout(result.fullStream, request), (e) => + e instanceof Error ? e : new Error(String(e)), + ).pipe(Stream.tap((event) => Effect.sync(() => request.resetTimeout(event)))) }), ), ) @@ -490,6 +531,59 @@ const live: Layer.Layer< }), ) +function isProviderProgressEvent(event: Event) { + switch (event.type) { + case "text-start": + case "text-delta": + case "reasoning-start": + case "reasoning-delta": + case "tool-input-start": + case "tool-input-delta": + case "tool-call": + case "tool-result": + case "tool-error": + return true + default: + return false + } +} + +function failOnTimeout( + iterable: AsyncIterable, + request: { + timeoutFailure: Promise + timeoutError: () => Error | undefined + }, +): AsyncIterable { + return { + [Symbol.asyncIterator]() { + const iterator = iterable[Symbol.asyncIterator]() + return { + async next() { + const timeoutError = request.timeoutError() + if (timeoutError) throw timeoutError + const nextPromise = iterator.next() + void nextPromise.catch(() => {}) + const next = await Promise.race([nextPromise, request.timeoutFailure]) + const nextTimeoutError = request.timeoutError() + if (nextTimeoutError) throw nextTimeoutError + return next + }, + async return(value?: unknown) { + // The abort signal is the cleanup path; return() is protocol cleanup and + // may never resolve for a hung provider iterator. + void iterator.return?.().catch(() => {}) + return { done: true, value: value as T } + }, + async throw(error?: unknown) { + if (iterator.throw) return iterator.throw(error) + throw error + }, + } + }, + } +} + export const layer = live.pipe(Layer.provide(Permission.defaultLayer)) export const defaultLayer: Layer.Layer = Layer.suspend(() => diff --git a/packages/opencode/test/session/llm.test.ts b/packages/opencode/test/session/llm.test.ts index 508e1f99c..567420ce0 100644 --- a/packages/opencode/test/session/llm.test.ts +++ b/packages/opencode/test/session/llm.test.ts @@ -237,6 +237,107 @@ function waitSilentStreamingRequest(pathname: string) { } } +function waitStartOnlyStreamingRequest(pathname: string) { + const request = deferred() + const requestAborted = deferred() + const responseCanceled = deferred() + const encoder = new TextEncoder() + let interval: Timer | undefined + + state.queue.push({ + path: pathname, + resolve: request.resolve, + response(req: Request) { + req.signal.addEventListener("abort", () => requestAborted.resolve(), { once: true }) + + return new Response( + new ReadableStream({ + start(controller) { + interval = setInterval(() => { + controller.enqueue( + encoder.encode( + [ + `data: ${JSON.stringify({ + id: "chatcmpl-start-only", + object: "chat.completion.chunk", + choices: [{ delta: { role: "assistant" } }], + })}`, + ].join("\n\n") + "\n\n", + ), + ) + }, 5) + }, + cancel() { + if (interval) clearInterval(interval) + responseCanceled.resolve() + }, + }), + { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + }, + ) + }, + }) + + return { + request: request.promise, + requestAborted: requestAborted.promise, + responseCanceled: responseCanceled.promise, + } +} + +function waitProgressThenSilentStreamingRequest(pathname: string) { + const request = deferred() + const requestAborted = deferred() + const responseCanceled = deferred() + const encoder = new TextEncoder() + + state.queue.push({ + path: pathname, + resolve: request.resolve, + response(req: Request) { + req.signal.addEventListener("abort", () => requestAborted.resolve(), { once: true }) + + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + [ + `data: ${JSON.stringify({ + id: "chatcmpl-progress-then-silent", + object: "chat.completion.chunk", + choices: [{ delta: { role: "assistant" } }], + })}`, + `data: ${JSON.stringify({ + id: "chatcmpl-progress-then-silent", + object: "chat.completion.chunk", + choices: [{ delta: { content: "Hello" } }], + })}`, + ].join("\n\n") + "\n\n", + ), + ) + }, + cancel() { + responseCanceled.resolve() + }, + }), + { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + }, + ) + }, + }) + + return { + request: request.promise, + requestAborted: requestAborted.promise, + responseCanceled: responseCanceled.promise, + } +} + beforeAll(() => { state.server = Bun.serve({ port: 0, @@ -521,7 +622,7 @@ describe("session.llm.stream", () => { const modelID = "qwen-plus" const fixture = await loadFixture(providerID, modelID) const model = fixture.model - const pending = waitSilentStreamingRequest("/chat/completions") + const pending = waitProgressThenSilentStreamingRequest("/chat/completions") await using tmp = await tmpdir({ init: async (dir) => { @@ -573,6 +674,7 @@ describe("session.llm.stream", () => { system: ["You are a helpful assistant."], messages: [{ role: "user", content: "Hello" }], tools: {}, + connectTimeoutMs: 1_000, streamTimeoutMs: 20, }) .pipe(Stream.runDrain), @@ -586,7 +688,158 @@ describe("session.llm.stream", () => { }) }) - test("does not count awaiting question blockers as silent provider timeout", async () => { + test("connect timeout produces stream failure not success drain", async () => { + const server = state.server + if (!server) throw new Error("Server not initialized") + + const providerID = "alibaba" + const modelID = "qwen-plus" + const fixture = await loadFixture(providerID, modelID) + const model = fixture.model + const pending = waitSilentStreamingRequest("/chat/completions") + + await using tmp = await tmpdir({ + init: async (dir) => { + await Bun.write( + path.join(dir, "opencode.json"), + JSON.stringify({ + $schema: "https://opencode.ai/config.json", + enabled_providers: [providerID], + provider: { + [providerID]: { + options: { + apiKey: "test-key", + baseURL: `${server.url.origin}/v1`, + }, + }, + }, + }), + ) + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const resolved = await getModel(ProviderID.make(providerID), ModelID.make(model.id)) + const sessionID = SessionID.make("session-test-connect-timeout-failure") + const agent = { + name: "test", + mode: "primary", + options: {}, + permission: [{ permission: "*", pattern: "*", action: "allow" }], + } satisfies Agent.Info + const user = { + id: MessageID.make("user-connect-timeout-failure"), + sessionID, + role: "user", + time: { created: Date.now() }, + agent: agent.name, + model: { providerID: ProviderID.make(providerID), modelID: resolved.id }, + } satisfies MessageV2.User + + const exit = await llm.runPromiseExit((svc) => + svc + .stream({ + user, + sessionID, + model: resolved, + agent, + system: ["You are a helpful assistant."], + messages: [{ role: "user", content: "Hello" }], + tools: {}, + connectTimeoutMs: 20, + streamTimeoutMs: 1_000, + }) + .pipe(Stream.runDrain), + ) + + await Promise.race([pending.responseCanceled, timeout(500)]) + await Promise.race([pending.requestAborted, timeout(500)]).catch(() => undefined) + await pending.request + expect(Exit.isFailure(exit)).toBe(true) + }, + }) + }) + + test("start events do not reset connect timer and timeout produces stream failure", async () => { + const server = state.server + if (!server) throw new Error("Server not initialized") + + const providerID = "alibaba" + const modelID = "qwen-plus" + const fixture = await loadFixture(providerID, modelID) + const model = fixture.model + const pending = waitStartOnlyStreamingRequest("/chat/completions") + + await using tmp = await tmpdir({ + init: async (dir) => { + await Bun.write( + path.join(dir, "opencode.json"), + JSON.stringify({ + $schema: "https://opencode.ai/config.json", + enabled_providers: [providerID], + provider: { + [providerID]: { + options: { + apiKey: "test-key", + baseURL: `${server.url.origin}/v1`, + }, + }, + }, + }), + ) + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const resolved = await getModel(ProviderID.make(providerID), ModelID.make(model.id)) + const sessionID = SessionID.make("session-test-start-only-connect-timeout") + const agent = { + name: "test", + mode: "primary", + options: {}, + permission: [{ permission: "*", pattern: "*", action: "allow" }], + } satisfies Agent.Info + const user = { + id: MessageID.make("user-start-only-connect-timeout"), + sessionID, + role: "user", + time: { created: Date.now() }, + agent: agent.name, + model: { providerID: ProviderID.make(providerID), modelID: resolved.id }, + } satisfies MessageV2.User + + const exit = await Promise.race([ + llm.runPromiseExit((svc) => + svc + .stream({ + user, + sessionID, + model: resolved, + agent, + system: ["You are a helpful assistant."], + messages: [{ role: "user", content: "Hello" }], + tools: {}, + connectTimeoutMs: 20, + streamTimeoutMs: 1_000, + }) + .pipe(Stream.runDrain), + ), + timeout(500), + ]) + + await Promise.race([pending.responseCanceled, timeout(500)]) + await Promise.race([pending.requestAborted, timeout(500)]).catch(() => undefined) + await pending.request + expect(Exit.isFailure(exit)).toBe(true) + }, + }) + }) + + test("connect timeout pauses during awaiting question and fires failure after answer", async () => { const server = state.server if (!server) throw new Error("Server not initialized") @@ -668,7 +921,8 @@ describe("session.llm.stream", () => { system: ["You are a helpful assistant."], messages: [{ role: "user", content: "Hello" }], tools: {}, - streamTimeoutMs: 20, + connectTimeoutMs: 20, + streamTimeoutMs: 1_000, }) .pipe(Stream.runDrain), ) @@ -694,7 +948,7 @@ describe("session.llm.stream", () => { await Promise.race([pending.requestAborted, timeout(500)]) const exit = await Promise.race([exitPromise, timeout(500)]) - expect(Exit.isFailure(exit)).toBe(false) + expect(Exit.isFailure(exit)).toBe(true) }, }) }) diff --git a/packages/opencode/test/session/processor-effect.test.ts b/packages/opencode/test/session/processor-effect.test.ts index 24668d84d..3d42add10 100644 --- a/packages/opencode/test/session/processor-effect.test.ts +++ b/packages/opencode/test/session/processor-effect.test.ts @@ -1033,6 +1033,71 @@ it.live("session.processor effect tests record aborted errors and idle state", ( ), ) +it.live("connect timeout writes assistant info.error and flips session_status idle", () => + provideTmpdirServer( + ({ dir, llm }) => + Effect.gen(function* () { + const seen = defer() + const { processors, session, provider } = yield* boot() + const bus = yield* Bus.Service + const sts = yield* SessionStatus.Service + + yield* llm.hang + + const chat = yield* session.create({}) + const parent = yield* user(chat.id, "bad model") + const msg = yield* assistant(chat.id, parent.id, path.resolve(dir)) + const mdl = yield* provider.getModel(ref.providerID, ref.modelID) + const errs: string[] = [] + const off = yield* bus.subscribeCallback(Session.Event.Error, (evt) => { + if (evt.properties.sessionID !== chat.id) return + if (!evt.properties.error) return + errs.push(evt.properties.error.name) + seen.resolve() + }) + const handle = yield* processors.create({ + assistantMessage: msg, + sessionID: chat.id, + model: mdl, + }) + + const result = yield* handle.process({ + user: { + id: parent.id, + sessionID: chat.id, + role: "user", + time: parent.time, + agent: parent.agent, + model: { providerID: ref.providerID, modelID: ref.modelID }, + } satisfies MessageV2.User, + sessionID: chat.id, + model: mdl, + agent: agent(), + system: [], + messages: [{ role: "user", content: "bad model" }], + tools: {}, + connectTimeoutMs: 20, + streamTimeoutMs: 1_000, + }) + + yield* Effect.promise(() => seen.promise) + const stored = MessageV2.get({ sessionID: chat.id, messageID: msg.id }) + const state = yield* sts.get(chat.id) + off() + + expect(result).toBe("stop") + expect(handle.message.error).toBeTruthy() + expect(stored.info.role).toBe("assistant") + if (stored.info.role === "assistant") { + expect(stored.info.error).toBeTruthy() + } + expect(state).toMatchObject({ type: "idle" }) + expect(errs.length).toBeGreaterThan(0) + }), + { git: true, config: (url) => providerCfg(url) }, + ), +) + it.live("session.processor effect tests mark interruptions aborted without manual abort", () => provideTmpdirServer( ({ dir, llm }) =>