diff --git a/packages/desktop-electron/src/main/index-sidecar-source.test.ts b/packages/desktop-electron/src/main/index-sidecar-source.test.ts index 0f32530f7..5d8079665 100644 --- a/packages/desktop-electron/src/main/index-sidecar-source.test.ts +++ b/packages/desktop-electron/src/main/index-sidecar-source.test.ts @@ -9,6 +9,7 @@ describe("desktop sidecar source guard", () => { expect(source).toContain('app.setPath("logs", join(app.getPath("userData"), "logs"))') expect(source).toContain('logger.log("server ready", { url: res.url })') expect(source).toContain('logger.log("init done")') + expect(source).toContain("server.stop(true)") expect(source).not.toContain("sqliteFileExists") expect(source).not.toContain('username: "opencode"') }) diff --git a/packages/desktop-electron/src/main/index.ts b/packages/desktop-electron/src/main/index.ts index a358f63e6..038cb0fd0 100644 --- a/packages/desktop-electron/src/main/index.ts +++ b/packages/desktop-electron/src/main/index.ts @@ -615,7 +615,7 @@ registerAboutIpc() function killSidecar() { if (!server) return - server.stop() + server.stop(true) server = null } diff --git a/packages/opencode/src/pty/index.ts b/packages/opencode/src/pty/index.ts index 8a1af375b..79d120f72 100644 --- a/packages/opencode/src/pty/index.ts +++ b/packages/opencode/src/pty/index.ts @@ -10,6 +10,7 @@ import { lazy } from "@opencode-ai/util/lazy" import { Shell } from "@/shell/shell" import { Plugin } from "@/plugin" import { envValueCaseInsensitive, withoutInternalServerAuthEnv } from "@/util/env" +import { Process } from "@/util/process" import { PtyID } from "./schema" import { Effect, Layer, Context } from "effect" import * as EffectLogger from "@opencode-ai/core/effect/logger" @@ -19,6 +20,7 @@ export namespace Pty { const BUFFER_LIMIT = 1024 * 1024 * 2 const BUFFER_CHUNK = 64 * 1024 + const EXIT_WAIT_MS = 1000 const encoder = new TextEncoder() type Socket = { @@ -33,6 +35,7 @@ export namespace Pty { type Active = { info: Info process: Proc + exitCode?: number buffer: string bufferCursor: number cursor: number @@ -42,6 +45,7 @@ export namespace Pty { type State = { dir: string sessions: Map + cleanupTasks: Set> } // WebSocket control frame: 0x00 + UTF-8 JSON. @@ -121,10 +125,8 @@ export namespace Pty { Effect.gen(function* () { const bus = yield* Bus.Service const plugin = yield* Plugin.Service - function teardown(session: Active) { - try { - session.process.kill() - } catch {} + + function closeSubscribers(session: Active) { for (const [sub, ws] of session.subscribers.entries()) { try { if (sock(ws) === sub) ws.close() @@ -133,17 +135,71 @@ export namespace Pty { session.subscribers.clear() } + function waitForExit(session: Active) { + if (session.info.status === "exited") return Promise.resolve(session.exitCode) + return new Promise((resolve) => { + let timer: ReturnType | undefined + const sub = session.process.onExit(({ exitCode }) => { + if (timer) clearTimeout(timer) + sub.dispose() + session.exitCode = exitCode + resolve(exitCode) + }) + timer = setTimeout(() => { + sub.dispose() + resolve(undefined) + }, EXIT_WAIT_MS) + }) + } + + const hasExited = (session: Active) => session.info.status === "exited" || !Process.exists(session.process.pid) + + function signalProcess(session: Active, signal: string) { + if (session.info.status === "exited") return + try { + session.process.kill(signal) + } catch {} + } + + const terminate = Effect.fn("Pty.terminate")(function* (session: Active) { + if (hasExited(session)) return session.exitCode + const exited = waitForExit(session) + if (process.platform === "win32") { + signalProcess(session, "SIGTERM") + return yield* Effect.promise(() => exited) + } + + yield* Effect.promise(() => + Process.terminateTree({ + pid: session.process.pid, + signalRoot: (signal) => session.process.kill(signal), + waitForExit: exited, + }), + ) + if (hasExited(session)) return session.exitCode + return yield* Effect.promise(() => exited) + }) + + const teardown = Effect.fn("Pty.teardown")(function* (session: Active) { + closeSubscribers(session) + return yield* terminate(session) + }) + const state = yield* InstanceState.make( Effect.fn("Pty.state")(function* (ctx) { const state = { dir: ctx.directory, sessions: new Map(), + cleanupTasks: new Set>(), } yield* Effect.addFinalizer(() => - Effect.sync(() => { + Effect.gen(function* () { + if (state.cleanupTasks.size) { + yield* Effect.promise(() => Promise.allSettled(state.cleanupTasks)).pipe(Effect.asVoid) + } for (const session of state.sessions.values()) { - teardown(session) + yield* teardown(session) } state.sessions.clear() }), @@ -153,13 +209,30 @@ export namespace Pty { }), ) + const trackCleanup = (s: State, id: PtyID, effect: Effect.Effect) => { + const task = Effect.runPromise(effect.pipe(Effect.provide(EffectLogger.layer))) + .catch((error) => { + log.error("cleanup task failed", { id, error }) + }) + .finally(() => { + s.cleanupTasks.delete(task) + }) + s.cleanupTasks.add(task) + } + const remove = Effect.fn("Pty.remove")(function* (id: PtyID) { const s = yield* InstanceState.get(state) const session = s.sessions.get(id) if (!session) return s.sessions.delete(id) log.info("removing session", { id }) - teardown(session) + const alreadyExited = session.info.status === "exited" + const exitCode = yield* teardown(session) + if (!alreadyExited && session.info.status !== "exited") { + session.info.status = "exited" + session.exitCode = exitCode ?? -1 + yield* bus.publish(Event.Exited, { id: session.info.id, exitCode: session.exitCode }) + } yield* bus.publish(Event.Deleted, { id: session.info.id }) }) @@ -261,11 +334,18 @@ export namespace Pty { ) proc.onExit( Instance.bind(({ exitCode }) => { - if (session.info.status === "exited") return + if (session.info.status === "exited" || !s.sessions.has(id)) return log.info("session exited", { id, exitCode }) session.info.status = "exited" - Effect.runFork(bus.publish(Event.Exited, { id, exitCode }).pipe(Effect.provide(EffectLogger.layer))) - Effect.runFork(remove(id).pipe(Effect.provide(EffectLogger.layer))) + session.exitCode = exitCode + trackCleanup( + s, + id, + Effect.gen(function* () { + yield* bus.publish(Event.Exited, { id, exitCode }) + yield* remove(id) + }), + ) }), ) yield* bus.publish(Event.Created, { info }) diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index 8e9059e5c..f77fcc5b5 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -1,6 +1,6 @@ import { Provider } from "@/provider/provider" import { Log } from "@opencode-ai/core/util/log" -import { Context, Effect, Layer, Record } from "effect" +import { Context, Duration, Effect, Layer, Record } from "effect" import * as Stream from "effect/Stream" import { streamText, wrapLanguageModel, type ModelMessage, type Tool, tool, jsonSchema } from "ai" import { mergeDeep, pipe } from "remeda" @@ -25,6 +25,7 @@ import * as Option from "effect/Option" const log = Log.create({ service: "llm" }) export const OUTPUT_TOKEN_MAX = ProviderTransform.OUTPUT_TOKEN_MAX +export const SILENT_STREAM_TIMEOUT_MS = Duration.toMillis(Duration.minutes(10)) type Result = Awaited> export type StreamInput = { @@ -39,6 +40,7 @@ export type StreamInput = { small?: boolean tools: Record retries?: number + streamTimeoutMs?: number toolChoice?: "auto" | "required" | "none" } @@ -399,14 +401,38 @@ const live: Layer.Layer< Stream.scoped( Stream.unwrap( Effect.gen(function* () { - const ctrl = yield* Effect.acquireRelease( - Effect.sync(() => new AbortController()), - (ctrl) => Effect.sync(() => ctrl.abort()), + const timeoutMsInput = input.streamTimeoutMs + const timeoutMs = + typeof timeoutMsInput === "number" && Number.isFinite(timeoutMsInput) && timeoutMsInput > 0 + ? timeoutMsInput + : SILENT_STREAM_TIMEOUT_MS + const request = yield* Effect.acquireRelease( + Effect.sync(() => { + const ctrl = new AbortController() + let timeout = setTimeout(() => ctrl.abort(), timeoutMs) + return { + ctrl, + resetTimeout() { + clearTimeout(timeout) + timeout = setTimeout(() => ctrl.abort(), timeoutMs) + }, + cleanup() { + clearTimeout(timeout) + ctrl.abort() + }, + } + }), + (request) => Effect.sync(() => request.cleanup()), ) - const result = yield* run({ ...input, abort: ctrl.signal }) + const result = yield* run({ ...input, abort: request.ctrl.signal }) - return Stream.fromAsyncIterable(result.fullStream, (e) => (e instanceof Error ? e : new Error(String(e)))) + // This is a silent-stream timeout: it limits how long we wait for + // the next provider event, not the total model runtime. + return Stream.fromAsyncIterable(result.fullStream, (e) => (e instanceof Error ? e : new Error(String(e)))).pipe( + Stream.tap(() => Effect.sync(() => request.resetTimeout())), + Stream.timeout(Duration.millis(timeoutMs)), + ) }), ), ) diff --git a/packages/opencode/src/session/processor.ts b/packages/opencode/src/session/processor.ts index 4066ccefd..e1b638b40 100644 --- a/packages/opencode/src/session/processor.ts +++ b/packages/opencode/src/session/processor.ts @@ -23,6 +23,7 @@ import { isRecord } from "@/util/record" import { TurnChange } from "./turn-change" const log = Log.create({ service: "session.processor" }) +const TOOL_CLEANUP_TIMEOUT_MS = 1_000 export type Result = "compact" | "stop" | "continue" @@ -99,8 +100,24 @@ type ToolCall = { done: Deferred.Deferred } +type PendingLoopAction = { + loopAction: "block" | "stop" + tool: string + sigKey: string + kind: SessionDiagnostics.SignatureKind + outcome: SessionDiagnostics.LoopOutcome + completedCount: number + completedFailures?: number + nextOccurrenceCount: number + attemptedInput?: unknown + errorMessage: string + renderedText?: string +} + interface ProcessorContext extends Input { toolcalls: Record + pendingLoopActions: Record + pendingToolUpdates: Record MessageV2.ToolPart>> shouldBreak: boolean snapshot: string | undefined blocked: boolean @@ -150,6 +167,8 @@ export const layer: Layer.Layer< sessionID: input.sessionID, model: input.model, toolcalls: {}, + pendingLoopActions: {}, + pendingToolUpdates: {}, shouldBreak: false, snapshot: initialSnapshot, blocked: false, @@ -192,7 +211,10 @@ export const layer: Layer.Layer< update: (part: MessageV2.ToolPart) => MessageV2.ToolPart, ) { const match = yield* readToolCall(toolCallID) - if (!match) return + if (!match) { + ;(ctx.pendingToolUpdates[toolCallID] ??= []).push(update) + return + } const part = yield* session.updatePart(update(match.part)) ctx.toolcalls[toolCallID] = { ...match.call, @@ -203,6 +225,22 @@ export const layer: Layer.Layer< return part }) + const applyPendingToolUpdates = Effect.fn("SessionProcessor.applyPendingToolUpdates")(function* (toolCallID: string) { + const pending = ctx.pendingToolUpdates[toolCallID] + if (!pending?.length) return + const match = yield* readToolCall(toolCallID) + if (!match) return + const next = pending.reduce((part, update) => update(part), match.part) + const part = yield* session.updatePart(next) + delete ctx.pendingToolUpdates[toolCallID] + ctx.toolcalls[toolCallID] = { + ...match.call, + partID: part.id, + messageID: part.messageID, + sessionID: part.sessionID, + } + }) + const toolStateMetadata = (part: MessageV2.ToolPart) => "metadata" in part.state && isRecord(part.state.metadata) ? part.state.metadata : undefined @@ -440,6 +478,50 @@ export const layer: Layer.Layer< const failToolCall = Effect.fn("SessionProcessor.failToolCall")(function* (toolCallID: string, error: unknown) { const match = yield* readToolCall(toolCallID) if (!match) return false + const pending = ctx.pendingLoopActions[toolCallID] + if (pending) { + delete ctx.pendingLoopActions[toolCallID] + const existingMeta = toolStateMetadata(match.part) + const metadata = SessionDiagnostics.mergeMetadata(existingMeta, { + diagnostics: { + loop: { + loopAction: pending.loopAction, + loopType: pending.kind, + loopSigKey: pending.sigKey, + outcome: pending.outcome, + loopCompletedCount: pending.completedCount, + loopCompletedFailures: pending.completedFailures, + loopOccurrenceCount: pending.nextOccurrenceCount, + attemptedInput: pending.attemptedInput, + }, + }, + }) + const end = Date.now() + const start = "time" in match.part.state ? match.part.state.time.start : end + yield* session.updatePart({ + ...match.part, + state: { + status: "error", + input: match.part.state.input, + error: pending.errorMessage, + metadata, + time: { start, end }, + }, + }) + if (pending.loopAction === "stop" && pending.renderedText) { + yield* session.updatePart({ + id: PartID.ascending(), + sessionID: ctx.sessionID, + messageID: ctx.assistantMessage.id, + type: "text", + text: pending.renderedText, + synthetic: true, + }) + ctx.blocked = true + } + yield* settleToolCall(toolCallID) + return true + } if (match.part.state.status !== "running") { yield* settleToolCall(toolCallID) return false @@ -539,6 +621,7 @@ export const layer: Layer.Layer< messageID: part.messageID, sessionID: part.sessionID, } + yield* applyPendingToolUpdates(value.id) return case "tool-input-delta": @@ -551,7 +634,7 @@ export const layer: Layer.Layer< if (ctx.assistantMessage.summary) { throw new Error(`Tool call not allowed while generating summary: ${value.toolName}`) } - const running = yield* updateToolCall(value.toolCallId, (match) => ({ + let running = yield* updateToolCall(value.toolCallId, (match) => ({ ...match, tool: value.toolName, state: { @@ -564,7 +647,10 @@ export const layer: Layer.Layer< ? { ...value.providerMetadata, providerExecuted: true } : value.providerMetadata, })) - if (!running || !ctx.assistantMessage.parentID) return + yield* applyPendingToolUpdates(value.toolCallId) + const refreshed = yield* readToolCall(value.toolCallId) + if (refreshed) running = refreshed.part + if (!ctx.assistantMessage.parentID) return const info = yield* session.get(ctx.sessionID) const observed = SessionDiagnostics.observeToolCall({ records: loopRecords(ctx.assistantMessage.parentID), @@ -577,13 +663,15 @@ export const layer: Layer.Layer< modelID: ctx.model.id, providerID: ctx.model.providerID, }) - yield* session.updatePart({ - ...running, + const withDiagnostics = (part: MessageV2.ToolPart) => ({ + ...part, state: { - ...running.state, - metadata: SessionDiagnostics.mergeMetadata(toolStateMetadata(running), observed.record.metadata), + ...part.state, + metadata: SessionDiagnostics.mergeMetadata(toolStateMetadata(part), observed.record.metadata), }, }) + if (running) yield* session.updatePart(withDiagnostics(running)) + else yield* updateToolCall(value.toolCallId, withDiagnostics) return } @@ -751,7 +839,7 @@ export const layer: Layer.Layer< yield* Effect.forEach( Object.values(ctx.toolcalls), - (call) => Deferred.await(call.done).pipe(Effect.timeout("250 millis"), Effect.ignore), + (call) => Deferred.await(call.done).pipe(Effect.timeout(`${TOOL_CLEANUP_TIMEOUT_MS} millis`), Effect.ignore), { concurrency: "unbounded" }, ) @@ -860,8 +948,21 @@ export const layer: Layer.Layer< attemptedInput?: unknown errorMessage: string }) { + ctx.pendingLoopActions[input.toolCallId] = { + loopAction: "block", + tool: input.tool, + sigKey: input.sigKey, + kind: input.kind, + outcome: input.outcome, + completedCount: input.completedCount, + completedFailures: input.completedFailures, + nextOccurrenceCount: input.nextOccurrenceCount, + attemptedInput: input.attemptedInput, + errorMessage: input.errorMessage, + } const match = yield* readToolCall(input.toolCallId) if (!match) return + delete ctx.pendingLoopActions[input.toolCallId] // Idempotence guard: if the model emits multiple parallel tool calls of the same // sigKey within one assistant step, applyLoopGate can decide block for several of // them before any has persisted. Re-check existing block sigKeys here so we record @@ -935,8 +1036,22 @@ export const layer: Layer.Layer< renderedText: string toolErrorMessage: string }) { + ctx.pendingLoopActions[input.toolCallId] = { + loopAction: "stop", + tool: input.tool, + sigKey: input.sigKey, + kind: input.kind, + outcome: input.outcome, + completedCount: input.completedCount, + completedFailures: input.completedFailures, + nextOccurrenceCount: input.nextOccurrenceCount, + attemptedInput: input.attemptedInput, + errorMessage: input.toolErrorMessage, + renderedText: input.renderedText, + } const match = yield* readToolCall(input.toolCallId) if (!match) return + delete ctx.pendingLoopActions[input.toolCallId] // Idempotence guard (see recordSyntheticBlock for the full rationale): re-check // hasStopped here. The duplicate-stop case writes two Chinese summaries which is // visible UX noise; this guard closes the most common parallel-call window. diff --git a/packages/opencode/src/session/retry.ts b/packages/opencode/src/session/retry.ts index 17dc659ee..91c4bb2c8 100644 --- a/packages/opencode/src/session/retry.ts +++ b/packages/opencode/src/session/retry.ts @@ -13,6 +13,7 @@ export const RETRY_INITIAL_DELAY = 2000 export const RETRY_BACKOFF_FACTOR = 2 export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds export const RETRY_MAX_DELAY = 2_147_483_647 // max 32-bit signed integer for setTimeout +export const RETRY_MAX_ATTEMPTS = 10 function cap(ms: number) { return Math.min(ms, RETRY_MAX_DELAY) @@ -112,6 +113,7 @@ export function policy(opts: { const error = opts.parse(meta.input) const message = retryable(error) if (!message) return Cause.done(meta.attempt) + if (meta.attempt >= RETRY_MAX_ATTEMPTS) return Cause.done(meta.attempt) return Effect.gen(function* () { const wait = delay(meta.attempt, MessageV2.APIError.isInstance(error) ? error : undefined) const now = yield* Clock.currentTimeMillis diff --git a/packages/opencode/src/tool/bash.ts b/packages/opencode/src/tool/bash.ts index cc176b2da..6c579cbc7 100644 --- a/packages/opencode/src/tool/bash.ts +++ b/packages/opencode/src/tool/bash.ts @@ -13,6 +13,7 @@ import { AppFileSystem } from "@opencode-ai/core/filesystem" import { fileURLToPath } from "url" import { Flag } from "@opencode-ai/core/flag/flag" import { Shell } from "@/shell/shell" +import { Process } from "@/util/process" import { BashArity } from "@/permission/arity" import * as Truncate from "./truncate" @@ -513,11 +514,15 @@ export const BashTool = Tool.define( if (exit.kind === "abort") { aborted = true - yield* handle.kill({ forceKillAfter: "3 seconds" }).pipe(Effect.orDie) + yield* Effect.promise(() => + Process.terminateTree({ pid: handle.pid, waitForExit: Effect.runPromise(handle.exitCode) }), + ).pipe(Effect.orDie) } if (exit.kind === "timeout") { expired = true - yield* handle.kill({ forceKillAfter: "3 seconds" }).pipe(Effect.orDie) + yield* Effect.promise(() => + Process.terminateTree({ pid: handle.pid, waitForExit: Effect.runPromise(handle.exitCode) }), + ).pipe(Effect.orDie) } return exit.kind === "exit" ? exit.code : null diff --git a/packages/opencode/src/util/process.ts b/packages/opencode/src/util/process.ts index e45ceb471..4f57353cb 100644 --- a/packages/opencode/src/util/process.ts +++ b/packages/opencode/src/util/process.ts @@ -1,9 +1,15 @@ import { type ChildProcess } from "child_process" import launch from "cross-spawn" import { buffer } from "node:stream/consumers" +import { setTimeout as sleep } from "node:timers/promises" import { errorMessage } from "./error" +import { Log } from "@opencode-ai/core/util/log" + +const log = Log.create({ service: "util.process" }) export namespace Process { + export const TERMINATION_GRACE_MS = 500 + export type Stdio = "inherit" | "pipe" | "ignore" export type Shell = boolean | string @@ -69,24 +75,37 @@ export namespace Process { }) let closed = false - let timer: ReturnType | undefined + let exited: Promise const abort = () => { if (closed) return if (proc.exitCode !== null || proc.signalCode !== null) return closed = true - proc.kill(opts.kill ?? "SIGTERM") - const ms = opts.timeout ?? 5_000 - if (ms <= 0) return - timer = setTimeout(() => proc.kill("SIGKILL"), ms) + if (ms <= 0) { + proc.kill(opts.kill ?? "SIGTERM") + return + } + if (!proc.pid) { + proc.kill(opts.kill ?? "SIGTERM") + return + } + + void terminateTree({ + pid: proc.pid, + graceMs: ms, + signalRoot: (signal) => proc.kill(signal), + waitForExit: exited, + }).catch((error) => { + log.debug("failed to terminate aborted process tree", { pid: proc.pid, error: errorMessage(error) }) + proc.kill("SIGKILL") + }) } - const exited = new Promise((resolve, reject) => { + exited = new Promise((resolve, reject) => { const done = () => { opts.abort?.removeEventListener("abort", abort) - if (timer) clearTimeout(timer) } proc.once("exit", (code, signal) => { @@ -144,22 +163,136 @@ export namespace Process { throw new RunFailedError(cmd, out.code, out.stdout, out.stderr) } - // Duplicated in `packages/sdk/js/src/process.ts` because the SDK cannot import - // `opencode` without creating a cycle. Keep both copies in sync. + // The SDK keeps a sync stop variant because it cannot import opencode without + // creating a cycle. Keep platform behavior aligned when changing this path. export async function stop(proc: ChildProcess) { if (proc.exitCode !== null || proc.signalCode !== null) return - if (process.platform !== "win32" || !proc.pid) { + if (!proc.pid) { proc.kill() return } - const out = await run(["taskkill", "/pid", String(proc.pid), "/T", "/F"], { - nothrow: true, + const waitForExit = new Promise((resolve) => { + const done = () => { + proc.off("exit", done) + proc.off("error", done) + resolve() + } + proc.once("exit", done) + proc.once("error", done) + }) + + await terminateTree({ + pid: proc.pid, + signalRoot: (signal) => proc.kill(signal), + waitForExit, + }) + } + + export function exists(pid: number) { + try { + process.kill(pid, 0) + return true + } catch { + return false + } + } + + export async function descendants(pid: number): Promise { + if (process.platform === "win32") return [] + const seen = new Set() + const pending = [pid] + while (pending.length) { + const parent = pending.pop()! + const out = await Bun.$`pgrep -P ${parent}` + .quiet() + .nothrow() + .text() + .catch((error) => { + log.debug("failed to enumerate child processes", { pid: parent, error: errorMessage(error) }) + return "" + }) + for (const line of out.split(/\s+/)) { + const child = Number(line) + if (!Number.isInteger(child) || child <= 0 || seen.has(child)) continue + seen.add(child) + pending.push(child) + } + } + return Array.from(seen) + } + + function signalPid(pid: number, signal: NodeJS.Signals) { + try { + if (exists(pid)) process.kill(pid, signal) + return true + } catch { + return false + } + } + + function signalGroup(pid: number, signal: NodeJS.Signals) { + try { + process.kill(-pid, signal) + return true + } catch { + return false + } + } + + export async function terminateTree(input: { + pid: number + graceMs?: number + signalRoot?: (signal: NodeJS.Signals) => void + waitForExit?: Promise + findDescendants?: (pid: number) => Promise + }) { + const graceMs = input.graceMs ?? TERMINATION_GRACE_MS + if (process.platform === "win32") { + await Bun.$`taskkill /pid ${input.pid} /f /t`.quiet().nothrow() + return + } + + // Descendants are a best-effort snapshot for normal child processes. A + // daemonized double-fork can intentionally leave this tree before cleanup. + const children = await (input.findDescendants ?? descendants)(input.pid).catch((error) => { + log.debug("failed to enumerate process tree", { pid: input.pid, error: errorMessage(error) }) + return [] }) + const signalRoot = (signal: NodeJS.Signals) => { + if (input.signalRoot && exists(input.pid)) { + try { + input.signalRoot(signal) + return + } catch {} + } + signalPid(input.pid, signal) + } - if (out.code === 0) return - proc.kill() + const groupSignaled = signalGroup(input.pid, "SIGTERM") + log.debug("sent process tree terminate signal", { pid: input.pid, groupSignaled, descendantCount: children.length }) + if (!groupSignaled) { + signalRoot("SIGTERM") + for (const child of children) signalPid(child, "SIGTERM") + } + + // With waitForExit, worst case is one grace period before SIGKILL and one + // bounded wait after SIGKILL so callers can observe the final exit. + const rootExited = await (input.waitForExit + ? Promise.race([input.waitForExit.then(() => true, () => true), sleep(graceMs).then(() => false)]) + : sleep(graceMs).then(() => false)) + + if (!exists(input.pid) && children.every((child) => !exists(child))) return + if (groupSignaled && !rootExited && exists(input.pid)) { + signalGroup(input.pid, "SIGKILL") + log.debug("sent process group kill signal", { pid: input.pid }) + } else { + signalRoot("SIGKILL") + } + for (const child of children) signalPid(child, "SIGKILL") + log.debug("sent process tree kill signals", { pid: input.pid, groupSignaled, descendantCount: children.length }) + if (input.waitForExit) await Promise.race([input.waitForExit.catch(() => undefined), sleep(graceMs)]) } export async function text(cmd: string[], opts: RunOptions = {}): Promise { diff --git a/packages/opencode/test/pty/pty-session.test.ts b/packages/opencode/test/pty/pty-session.test.ts index 1835ba213..3475b5401 100644 --- a/packages/opencode/test/pty/pty-session.test.ts +++ b/packages/opencode/test/pty/pty-session.test.ts @@ -5,6 +5,7 @@ import { Pty } from "../../src/pty" import type { PtyID } from "../../src/pty/schema" import { tmpdir } from "../fixture/fixture" import { setTimeout as sleep } from "node:timers/promises" +import { existsSync } from "node:fs" const wait = async (fn: () => boolean, ms = 5000) => { const end = Date.now() + ms @@ -44,11 +45,9 @@ describe("pty", () => { }) id = info.id - await wait(() => pick(log, id!).includes("exited")) - - await Pty.remove(id) await wait(() => pick(log, id!).length >= 3) expect(pick(log, id!)).toEqual(["created", "exited", "deleted"]) + expect(await Pty.get(id)).toBeUndefined() } finally { off.forEach((x) => x()) if (id) await Pty.remove(id) @@ -74,7 +73,11 @@ describe("pty", () => { let id: PtyID | undefined try { - const info = await Pty.create({ command: "/bin/sh", title: "sh" }) + const info = await Pty.create({ + command: "/bin/sh", + args: ["-c", "trap 'exit 0' TERM; while :; do sleep 1; done"], + title: "sh", + }) id = info.id await sleep(100) @@ -90,6 +93,45 @@ describe("pty", () => { }) }) + test("remove terminates child processes started inside the pty", async () => { + if (process.platform === "win32") return + + await using dir = await tmpdir({ git: true }) + + await Instance.provide({ + directory: dir.path, + fn: async () => { + const pidFile = `${dir.path}/pty-child.pid` + let id: PtyID | undefined + try { + const child = `trap '' HUP TERM; echo $$ > ${JSON.stringify(pidFile)}; while :; do sleep 1; done` + const info = await Pty.create({ + command: "/bin/sh", + args: ["-c", `trap 'exit 0' TERM; /bin/sh -c ${JSON.stringify(child)} & wait`], + title: "child-tree", + }) + id = info.id + + await wait(() => existsSync(pidFile)) + const pid = Number((await Bun.file(pidFile).text()).trim()) + expect(pid).toBeGreaterThan(0) + + await Pty.remove(id) + await wait(() => { + try { + process.kill(pid, 0) + return false + } catch { + return true + } + }) + } finally { + if (id) await Pty.remove(id) + } + }, + }) + }) + test("does not expose internal server auth env to terminal sessions", async () => { if (process.platform === "win32") return @@ -109,7 +151,12 @@ describe("pty", () => { let id: PtyID | undefined try { const info = await Pty.create({ - command: "/bin/sh", + command: "/usr/bin/env", + args: [ + "sh", + "-c", + 'printf "username=%s\\n" "${OPENCODE_SERVER_USERNAME}" && printf "password=%s\\n" "${OPENCODE_SERVER_PASSWORD}" && printf "custom=%s\\n" "${PAWWORK_E2E_CUSTOM_ENV-unset}"', + ], title: "env", }) id = info.id @@ -121,10 +168,6 @@ describe("pty", () => { close: () => undefined, } as any) - await Pty.write( - info.id, - 'printf "username=%s\\n" "${OPENCODE_SERVER_USERNAME}" && printf "password=%s\\n" "${OPENCODE_SERVER_PASSWORD}" && printf "custom=%s\\n" "${PAWWORK_E2E_CUSTOM_ENV-unset}"\nexit\n', - ) await wait(() => output.join("").includes("custom=")) const text = output.join("") @@ -165,7 +208,12 @@ describe("pty", () => { let id: PtyID | undefined try { const info = await Pty.create({ - command: "/bin/sh", + command: "/usr/bin/env", + args: [ + "sh", + "-c", + 'printf "username=%s\\n" "${OPENCODE_SERVER_USERNAME}" && printf "password=%s\\n" "${OPENCODE_SERVER_PASSWORD}"', + ], title: "explicit-env", env: { OPENCODE_SERVER_USERNAME: "explicit-user", @@ -181,10 +229,6 @@ describe("pty", () => { close: () => undefined, } as any) - await Pty.write( - info.id, - 'printf "username=%s\\n" "${OPENCODE_SERVER_USERNAME}" && printf "password=%s\\n" "${OPENCODE_SERVER_PASSWORD}"\nexit\n', - ) await wait(() => output.join("").includes("password=")) const text = output.join("") diff --git a/packages/opencode/test/session/llm.test.ts b/packages/opencode/test/session/llm.test.ts index d5e4afb72..8c9ee02c6 100644 --- a/packages/opencode/test/session/llm.test.ts +++ b/packages/opencode/test/session/llm.test.ts @@ -200,6 +200,38 @@ function waitStreamingRequest(pathname: string) { } } +function waitSilentStreamingRequest(pathname: string) { + const request = deferred() + const requestAborted = deferred() + const responseCanceled = deferred() + + state.queue.push({ + path: pathname, + resolve: request.resolve, + response(req: Request) { + req.signal.addEventListener("abort", () => requestAborted.resolve(), { once: true }) + + return new Response( + new ReadableStream({ + cancel() { + responseCanceled.resolve() + }, + }), + { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + }, + ) + }, + }) + + return { + request: request.promise, + requestAborted: requestAborted.promise, + responseCanceled: responseCanceled.promise, + } +} + beforeAll(() => { state.server = Bun.serve({ port: 0, @@ -476,6 +508,79 @@ describe("session.llm.stream", () => { }) }) + test("silent stream timeout cancels provider response body promptly", async () => { + const server = state.server + if (!server) throw new Error("Server not initialized") + + const providerID = "alibaba" + const modelID = "qwen-plus" + const fixture = await loadFixture(providerID, modelID) + const model = fixture.model + const pending = waitSilentStreamingRequest("/chat/completions") + + await using tmp = await tmpdir({ + init: async (dir) => { + await Bun.write( + path.join(dir, "opencode.json"), + JSON.stringify({ + $schema: "https://opencode.ai/config.json", + enabled_providers: [providerID], + provider: { + [providerID]: { + options: { + apiKey: "test-key", + baseURL: `${server.url.origin}/v1`, + }, + }, + }, + }), + ) + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const resolved = await getModel(ProviderID.make(providerID), ModelID.make(model.id)) + const sessionID = SessionID.make("session-test-silent-timeout") + const agent = { + name: "test", + mode: "primary", + options: {}, + permission: [{ permission: "*", pattern: "*", action: "allow" }], + } satisfies Agent.Info + const user = { + id: MessageID.make("user-silent-timeout"), + sessionID, + role: "user", + time: { created: Date.now() }, + agent: agent.name, + model: { providerID: ProviderID.make(providerID), modelID: resolved.id }, + } satisfies MessageV2.User + + const exit = await llm.runPromiseExit((svc) => + svc + .stream({ + user, + sessionID, + model: resolved, + agent, + system: ["You are a helpful assistant."], + messages: [{ role: "user", content: "Hello" }], + tools: {}, + streamTimeoutMs: 20, + }) + .pipe(Stream.runDrain), + ) + + await Promise.race([pending.responseCanceled, timeout(500)]) + await Promise.race([pending.requestAborted, timeout(500)]).catch(() => undefined) + await pending.request + expect(Exit.isFailure(exit)).toBe(false) + }, + }) + }) + test("keeps tools enabled by prompt permissions", async () => { const server = state.server if (!server) { diff --git a/packages/opencode/test/session/retry.test.ts b/packages/opencode/test/session/retry.test.ts index 46dc48f67..93b0c4153 100644 --- a/packages/opencode/test/session/retry.test.ts +++ b/packages/opencode/test/session/retry.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from "bun:test" import type { NamedError } from "@opencode-ai/util/error" import { APICallError } from "ai" import { setTimeout as sleep } from "node:timers/promises" -import { Effect, Schedule } from "effect" +import { Effect, Exit, Schedule } from "effect" import { SessionRetry } from "../../src/session/retry" import { MessageV2 } from "../../src/session/message-v2" import { ProviderID } from "../../src/provider/schema" @@ -121,6 +121,33 @@ describe("session.retry.delay", () => { }, }) }) + + test("policy stops retrying after the configured max attempts", async () => { + const attempts: number[] = [] + let runs = 0 + const error = apiError({ "retry-after-ms": "0" }) + + const exit = await Effect.runPromiseExit( + Effect.try({ + try: () => { + runs++ + throw error + }, + catch: (err) => err as MessageV2.APIError, + }).pipe( + Effect.retry( + SessionRetry.policy({ + parse: (err) => err as MessageV2.APIError, + set: (info) => Effect.sync(() => attempts.push(info.attempt)), + }), + ), + ), + ) + + expect(Exit.isFailure(exit)).toBe(true) + expect(runs).toBe(SessionRetry.RETRY_MAX_ATTEMPTS) + expect(attempts).toEqual(Array.from({ length: SessionRetry.RETRY_MAX_ATTEMPTS - 1 }, (_, index) => index + 1)) + }) }) describe("session.retry.retryable", () => { diff --git a/packages/opencode/test/tool/bash.test.ts b/packages/opencode/test/tool/bash.test.ts index 4411652fc..9b6ec18ef 100644 --- a/packages/opencode/test/tool/bash.test.ts +++ b/packages/opencode/test/tool/bash.test.ts @@ -2,6 +2,8 @@ import { describe, expect, test } from "bun:test" import { Effect, Layer, ManagedRuntime } from "effect" import os from "os" import path from "path" +import { existsSync } from "node:fs" +import { setTimeout as sleep } from "node:timers/promises" import { Shell } from "../../src/shell/shell" import { BashTool } from "../../src/tool/bash" import { Instance } from "../../src/project/instance" @@ -90,6 +92,15 @@ const forms = (dir: string) => { return Array.from(new Set([full, slash, root, root.toLowerCase()])) } +const wait = async (fn: () => boolean, ms = 5000) => { + const end = Date.now() + ms + while (Date.now() < end) { + if (fn()) return + await sleep(25) + } + throw new Error("timeout waiting for condition") +} + const withShell = (item: { label: string; shell: string }, fn: () => Promise) => async () => { const prev = process.env.SHELL process.env.SHELL = item.shell @@ -1067,6 +1078,52 @@ describe("tool.bash abort", () => { }) }, 15_000) + test("terminates child processes that ignore TERM on abort", async () => { + if (process.platform === "win32") return + + await using dir = await tmpdir() + + await Instance.provide({ + directory: dir.path, + fn: async () => { + const bash = await initBash() + const controller = new AbortController() + const pidFile = path.join(dir.path, "bash-child.pid") + const child = `trap '' HUP TERM; echo $$ > ${JSON.stringify(pidFile)}; while :; do sleep 1; done` + const res = await Effect.runPromise( + bash.execute( + { + command: `/bin/sh -c ${JSON.stringify(child)} & echo before; wait`, + description: "Abort child tree", + }, + { + ...ctx, + abort: controller.signal, + metadata: (input) => + Effect.sync(() => { + const output = (input.metadata as { output?: string })?.output + if (output?.includes("before") && !controller.signal.aborted) controller.abort() + }), + }, + ), + ) + + expect(res.output).toContain("before") + expect(res.output).toContain("User aborted the command") + await wait(() => existsSync(pidFile)) + const pid = Number((await Bun.file(pidFile).text()).trim()) + await wait(() => { + try { + process.kill(pid, 0) + return false + } catch { + return true + } + }) + }, + }) + }, 15_000) + test("terminates command on timeout", async () => { await Instance.provide({ directory: projectRoot, diff --git a/packages/opencode/test/util/process.test.ts b/packages/opencode/test/util/process.test.ts index 1d08cba6b..29d26649f 100644 --- a/packages/opencode/test/util/process.test.ts +++ b/packages/opencode/test/util/process.test.ts @@ -8,6 +8,15 @@ function node(script: string) { return [process.execPath, "-e", script] } +async function waitForFile(file: string) { + for (let i = 0; i < 50; i++) { + const text = await fs.readFile(file, "utf8").catch(() => undefined) + if (text) return text + await new Promise((resolve) => setTimeout(resolve, 20)) + } + throw new Error(`timed out waiting for ${file}`) +} + describe("util.process", () => { test("captures stdout and stderr", async () => { const out = await Process.run(node('process.stdout.write("out");process.stderr.write("err")')) @@ -43,6 +52,24 @@ describe("util.process", () => { expect(Date.now() - started).toBeLessThan(1000) }, 3000) + test("aborts child processes started by a spawned process", async () => { + if (process.platform === "win32") return + + await using tmp = await tmpdir() + const pidFile = path.join(tmp.path, "child.pid") + const child = `trap '' HUP TERM; echo $$ > ${JSON.stringify(pidFile)}; while :; do sleep 1; done` + const abort = new AbortController() + const proc = Process.spawn(["/bin/sh", "-c", `/bin/sh -c ${JSON.stringify(child)} & wait`], { + abort: abort.signal, + timeout: 50, + }) + + const childPid = Number((await waitForFile(pidFile)).trim()) + abort.abort() + expect(await proc.exited).not.toBe(0) + expect(Process.exists(childPid)).toBe(false) + }, 3000) + test("kills after timeout when process ignores terminate signal", async () => { if (process.platform === "win32") return @@ -60,6 +87,56 @@ describe("util.process", () => { expect(Date.now() - started).toBeLessThan(1000) }, 3000) + test("terminateTree uses the platform process-tree cleanup path on Windows", async () => { + if (process.platform !== "win32") return + + const proc = Process.spawn(node("setInterval(() => {}, 1000)")) + await Process.terminateTree({ pid: proc.pid! }) + const exit = await Promise.race([ + proc.exited, + new Promise((_, reject) => setTimeout(() => reject(new Error("timeout waiting for process exit")), 3000)), + ]) + + expect(exit).not.toBe(0) + }, 5000) + + test("terminateTree honors grace period without waitForExit", async () => { + if (process.platform === "win32") return + + const proc = Process.spawn(node('process.on("SIGTERM", () => {}); setInterval(() => {}, 1000)')) + const started = Date.now() + await Process.terminateTree({ pid: proc.pid!, graceMs: 120 }) + const elapsed = Date.now() - started + + expect(elapsed).toBeGreaterThanOrEqual(100) + expect(await proc.exited).not.toBe(0) + }, 3000) + + test("terminateTree falls back to root process when descendant enumeration fails", async () => { + if (process.platform === "win32") return + + const proc = Process.spawn(node('process.on("SIGTERM", () => {}); setInterval(() => {}, 1000)')) + await Process.terminateTree({ + pid: proc.pid!, + graceMs: 25, + waitForExit: proc.exited, + findDescendants: async () => { + throw new Error("pgrep unavailable") + }, + }) + + expect(await proc.exited).not.toBe(0) + }, 3000) + + test("stop uses the shared process tree termination path", async () => { + if (process.platform === "win32") return + + const proc = Process.spawn(node('process.on("SIGTERM", () => {}); setInterval(() => {}, 1000)')) + await Process.stop(proc) + + expect(await proc.exited).not.toBe(0) + }, 3000) + test("uses cwd when spawning commands", async () => { await using tmp = await tmpdir() const out = await Process.run(node("process.stdout.write(process.cwd())"), { diff --git a/packages/sdk/js/src/process.ts b/packages/sdk/js/src/process.ts index 3111b424a..926eb41f6 100644 --- a/packages/sdk/js/src/process.ts +++ b/packages/sdk/js/src/process.ts @@ -1,14 +1,59 @@ import { type ChildProcess, spawnSync } from "node:child_process" -// Duplicated from `packages/opencode/src/util/process.ts` because the SDK cannot -// import `opencode` without creating a cycle (`opencode` depends on `@opencode-ai/sdk`). +function descendants(pid: number) { + if (process.platform === "win32") return [] + const seen = new Set() + const pending = [pid] + while (pending.length) { + const parent = pending.pop()! + const out = spawnSync("pgrep", ["-P", String(parent)], { encoding: "utf8" }) + if (out.error || out.status !== 0) continue + for (const line of out.stdout.split(/\s+/)) { + const child = Number(line) + if (!Number.isInteger(child) || child <= 0 || seen.has(child)) continue + seen.add(child) + pending.push(child) + } + } + return Array.from(seen) +} + +function signal(pid: number, signal: NodeJS.Signals) { + try { + process.kill(pid, signal) + } catch {} +} + +// Kept in sync with `packages/opencode/src/util/process.ts` without importing +// opencode, which depends on this SDK package. export function stop(proc: ChildProcess) { if (proc.exitCode !== null || proc.signalCode !== null) return if (process.platform === "win32" && proc.pid) { const out = spawnSync("taskkill", ["/pid", String(proc.pid), "/T", "/F"], { windowsHide: true }) if (!out.error && out.status === 0) return } - proc.kill() + if (!proc.pid) { + proc.kill() + return + } + const children = descendants(proc.pid) + const clearKillTimer = () => { + clearTimeout(killTimer) + proc.off("exit", clearKillTimer) + proc.off("close", clearKillTimer) + proc.off("error", clearKillTimer) + } + signal(proc.pid, "SIGTERM") + for (const child of children) signal(child, "SIGTERM") + const killTimer = setTimeout(() => { + clearKillTimer() + signal(proc.pid!, "SIGKILL") + for (const child of children) signal(child, "SIGKILL") + }, 500) + proc.once("exit", clearKillTimer) + proc.once("close", clearKillTimer) + proc.once("error", clearKillTimer) + killTimer.unref?.() } export function bindAbort(proc: ChildProcess, signal?: AbortSignal, onAbort?: () => void) {