diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md index 6e32cfd5e7..bfb02f8928 100644 --- a/docs/advanced-setup.md +++ b/docs/advanced-setup.md @@ -481,6 +481,8 @@ host. Without this variable the behavior is unchanged. | `OPENCLAUDE_MAX_TURNS` | No | Per-prompt **local** interactive REPL turn cap for the in-process query loop. Defaults to `50`. Set a larger positive integer for long autonomous local interactive sessions (for example models that take many small tool steps). CLI `--max-turns 0` explicitly disables this cap and prints a cautionary warning. Precedence for a valid override: CLI `--max-turns` → this env var → legacy `CLAUDE_CODE_MAX_TURNS` (only when this var is unset/empty) → `/config` → Max turns (interactive) → `50`. If this env var is set but invalid (zero, negative, non-integer), the default `50` is used and lower layers are not consulted — same pattern as `OPENCLAUDE_MAX_RETRIES`. Does not apply to remote-backed interactive sessions (`connect` / `ssh` / `--remote`). | | `OPENCLAUDE_RETRY_DELAY_MS` | No | Base retry delay in milliseconds for APIs that do not send `Retry-After`; exponential backoff starts from this value, capped at 60000 (default: 500) | | `OPENCLAUDE_QUERY_HARD_MAX_MS` | No | Foreground query hard maximum in milliseconds. Defaults to 1800000 (30 minutes). Use a larger positive integer for long autonomous sessions; invalid, zero, negative, fractional, or timer-overflow values are ignored with a warning. | +| `OPENCLAUDE_INTERRUPT_TRACE` | No | Set to `1` or `true` to retain a bounded, privacy-safe interruption lifecycle trace in memory. Disabled by default. The trace contains only allowlisted lifecycle metadata—never prompts, responses, tool arguments, credentials, or raw error messages. | +| `OPENCLAUDE_INTERRUPT_TRACE_FILE` | No | Optional absolute JSONL output path used only when `OPENCLAUDE_INTERRUPT_TRACE` is enabled. On Linux, missing parent directories are created privately and every parent is opened through `/proc/self/fd` without following symbolic links before the final regular file is appended. If the file already exists, its mode is reset to `0600` on every append, so do not configure a shared file. Other platforms retain the bounded trace in memory but do not write this file because Node does not expose an equivalent safe descriptor-relative traversal API there. Writes are best-effort and never change request behavior. Use a separate path per OpenClaude process and keep the resulting diagnostic file private. | | `OPENCLAUDE_DISABLE_CO_AUTHORED_BY` | No | Suppress the default `Co-Authored-By` trailer in generated git commits | | `OPENCLAUDE_LOG_TOKEN_USAGE` | No | When truthy (e.g. `verbose`), emits one JSON line on stderr per API request with input/output/cache tokens and the resolved provider. **User-facing debug output** — complements the REPL display controlled by `/config showCacheStats`. Distinct from `CLAUDE_CODE_ENABLE_TOKEN_USAGE_ATTACHMENT`, which is **model-facing** (injects context usage info into the prompt itself). Both can run together. | diff --git a/src/QueryEngine.interruptionTrace.test.ts b/src/QueryEngine.interruptionTrace.test.ts new file mode 100644 index 0000000000..5e4a231f96 --- /dev/null +++ b/src/QueryEngine.interruptionTrace.test.ts @@ -0,0 +1,183 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from './test/sharedMutationLock.js' +import { QueryEngine } from './QueryEngine.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, + registerInterruptionController, +} from './utils/interruptionTrace.js' + +const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + +beforeEach(async () => { + await acquireSharedMutationLock('QueryEngine.interruptionTrace.test.ts') +}) + +afterEach(async () => { + try { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } finally { + releaseSharedMutationLock() + } +}) + +describe('QueryEngine interruption tracing', () => { + test('does not record lifecycle entries while tracing is disabled', async () => { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + const engine = Object.create(QueryEngine.prototype) as QueryEngine + const controller = new AbortController() + ;(engine as unknown as { abortController: AbortController }).abortController = + controller + ;(engine as unknown as { + submitMessageImpl(): AsyncGenerator + }).submitMessageImpl = async function* () {} + + for await (const _message of engine.submitMessage('hello')) { + // The stub deliberately yields nothing. + } + + expect(__getInterruptionTraceSnapshotForTests()).toEqual([]) + }) + + test('records a programmatic query-root interruption before aborting', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const controller = new AbortController() + const engine = Object.create(QueryEngine.prototype) as QueryEngine + ;(engine as unknown as { + abortController: AbortController + }).abortController = controller + + engine.interrupt('sdk_interrupt') + + const requested = __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'abort.requested', + ) + expect(controller.signal.aborted).toBe(true) + expect(requested).toMatchObject({ + source: 'sdk_interrupt', + subsystem: 'query_engine', + controllerRole: 'query-root', + }) + }) + + test('records start and terminal lifecycle for successful SDK turns', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const engine = Object.create(QueryEngine.prototype) as QueryEngine + const controller = new AbortController() + ;(engine as unknown as { abortController: AbortController }).abortController = + controller + ;(engine as unknown as { + submitMessageImpl(): AsyncGenerator + }).submitMessageImpl = async function* () {} + + for await (const _message of engine.submitMessage('hello')) { + // The stub deliberately yields nothing. + } + + const trace = __getInterruptionTraceSnapshotForTests() + const started = trace.find(entry => entry.event === 'query.started') + const terminal = trace.find(entry => entry.event === 'query.terminal') + expect(started).toMatchObject({ + subsystem: 'query_engine', + querySource: 'sdk', + controllerRole: 'query-root', + }) + expect(terminal).toMatchObject({ + subsystem: 'query_engine', + queryId: started?.queryId, + outcome: 'completed', + }) + expect(typeof started?.eventId).toBe('string') + expect(typeof terminal?.causalEventId).toBe('string') + expect(terminal!.causalEventId).toBe(started!.eventId) + }) + + test('records aborted and failed SDK turn terminals', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + + for (const scenario of ['aborted', 'failed'] as const) { + __resetInterruptionTraceForTests() + const engine = Object.create(QueryEngine.prototype) as QueryEngine + const controller = new AbortController() + ;(engine as unknown as { abortController: AbortController }).abortController = + controller + ;(engine as unknown as { + submitMessageImpl(): AsyncGenerator + }).submitMessageImpl = async function* () { + if (scenario === 'aborted') { + controller.abort('interrupt') + return + } + throw new Error('turn failed') + } + + const drain = async () => { + for await (const _message of engine.submitMessage('hello')) { + // The stub deliberately yields nothing. + } + } + if (scenario === 'failed') await expect(drain()).rejects.toThrow('turn failed') + else await drain() + + const trace = __getInterruptionTraceSnapshotForTests() + const started = trace.find(entry => entry.event === 'query.started') + const terminal = trace.find(entry => entry.event === 'query.terminal') + expect(terminal?.outcome).toBe(scenario) + expect(typeof started?.eventId).toBe('string') + expect(typeof terminal?.eventId).toBe('string') + if (scenario === 'aborted') { + const observed = trace.find( + entry => entry.event === 'signal.observed', + ) + expect(typeof observed?.eventId).toBe('string') + expect(terminal?.causalEventId).toBe(observed!.eventId) + } else { + expect(terminal?.causalEventId).toBe(started!.eventId) + } + } + }) + + test('registers the query root when tracing is enabled at the turn boundary', async () => { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + const engine = Object.create(QueryEngine.prototype) as QueryEngine + const controller = new AbortController() + ;(engine as unknown as { abortController: AbortController }).abortController = + controller + registerInterruptionController(controller, { + subsystem: 'query_engine', + controllerRole: 'query-root', + }) + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + ;(engine as unknown as { + submitMessageImpl(): AsyncGenerator + }).submitMessageImpl = async function* () { + controller.abort() + } + + for await (const _message of engine.submitMessage('hello')) { + // The stub deliberately yields nothing. + } + + const trace = __getInterruptionTraceSnapshotForTests() + const registered = trace.find( + entry => + entry.event === 'controller.registered' && + entry.controllerRole === 'query-root', + ) + const observed = trace.find(entry => entry.event === 'signal.observed') + const terminal = trace.find(entry => entry.event === 'query.terminal') + expect(registered).toBeDefined() + expect(typeof observed?.eventId).toBe('string') + expect(terminal).toMatchObject({ + outcome: 'aborted', + causalEventId: observed!.eventId, + }) + }) +}) diff --git a/src/QueryEngine.ts b/src/QueryEngine.ts index f8e6edba4b..fac90015e1 100644 --- a/src/QueryEngine.ts +++ b/src/QueryEngine.ts @@ -45,6 +45,14 @@ import { SYNTHETIC_OUTPUT_TOOL_NAME } from './tools/SyntheticOutputTool/Syntheti import type { Message } from './types/message.js' import type { OrphanedPermission } from './types/textInputTypes.js' import { createAbortController } from './utils/abortController.js' +import { + flushInterruptionTrace, + getInterruptionSignalAbortEventId, + isInterruptionTraceEnabled, + registerInterruptionController, + requestAbort, + traceInterruptionEvent, +} from './utils/interruptionTrace.js' import { validateArrayOf, assertNonEmptyString, assertObject, assertFunction } from './utils/validation.js' import { invalidateRemovedToolSchemas } from './utils/toolSchemaCache.js' import type { AttributionState } from './utils/commitAttribution.js' @@ -205,6 +213,10 @@ export class QueryEngine { this.config = config this.mutableMessages = config.initialMessages ?? [] this.abortController = config.abortController ?? createAbortController() + registerInterruptionController(this.abortController, { + subsystem: 'query_engine', + controllerRole: 'query-root', + }) this.permissionDenials = [] this.readFileState = config.readFileCache this.totalUsage = EMPTY_USAGE @@ -213,6 +225,56 @@ export class QueryEngine { async *submitMessage( prompt: string | ContentBlockParam[], options?: { uuid?: string; isMeta?: boolean }, + ): AsyncGenerator { + const queryId = isInterruptionTraceEnabled() ? randomUUID() : undefined + registerInterruptionController(this.abortController, { + subsystem: 'query_engine', + controllerRole: 'query-root', + queryId, + querySource: 'sdk', + }, { refreshQueryContext: true }) + const startedEventId = traceInterruptionEvent('query.started', { + subsystem: 'query_engine', + phase: 'running', + queryId, + querySource: 'sdk', + controllerRole: 'query-root', + }) + let outcome = 'consumer_closed' + let terminalError: unknown + try { + yield* this.submitMessageImpl(prompt, options) + outcome = this.abortController.signal.aborted ? 'aborted' : 'completed' + } catch (error) { + terminalError = error + outcome = this.abortController.signal.aborted ? 'aborted' : 'failed' + throw error + } finally { + const terminalOutcome = this.abortController.signal.aborted + ? 'aborted' + : outcome + traceInterruptionEvent('query.terminal', { + subsystem: 'query_engine', + phase: terminalOutcome, + queryId, + querySource: 'sdk', + controllerRole: 'query-root', + outcome: terminalOutcome, + reason: this.abortController.signal.reason, + error: terminalError, + causalEventId: terminalOutcome === 'aborted' + ? getInterruptionSignalAbortEventId(this.abortController.signal) + : startedEventId, + }) + if (this.abortController.signal.aborted) { + flushInterruptionTrace('query_terminal') + } + } + } + + private async *submitMessageImpl( + prompt: string | ContentBlockParam[], + options?: { uuid?: string; isMeta?: boolean }, ): AsyncGenerator { const { cwd, @@ -1212,8 +1274,12 @@ export class QueryEngine { } } - interrupt(): void { - this.abortController.abort() + interrupt(source = 'programmatic_interrupt'): void { + requestAbort(this.abortController, undefined, { + source, + subsystem: 'query_engine', + controllerRole: 'query-root', + }) } getMessages(): readonly Message[] { diff --git a/src/cli/print.interruptionTrace.test.ts b/src/cli/print.interruptionTrace.test.ts new file mode 100644 index 0000000000..d4d51f1619 --- /dev/null +++ b/src/cli/print.interruptionTrace.test.ts @@ -0,0 +1,108 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../test/sharedMutationLock.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, +} from '../utils/interruptionTrace.js' +import { + abortPrintModeControlRequest, + type PrintModeControlAbortSource, +} from './printInterruption.js' + +const originalInterruptionTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE +let hasSharedMutationLock = false + +beforeEach(async () => { + await acquireSharedMutationLock('cli/print.interruptionTrace.test.ts') + hasSharedMutationLock = true +}) + +afterEach(async () => { + try { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalInterruptionTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalInterruptionTrace + } + } finally { + if (hasSharedMutationLock) { + releaseSharedMutationLock() + hasSharedMutationLock = false + } + } +}) + +describe('print-mode interruption tracing', () => { + test.each([ + ['sdk_control_interrupt', 'interrupt'], + ['sdk_end_session', undefined], + ] as const)( + 'links %s input to the query and speculation aborts', + (source: PrintModeControlAbortSource, queryReason: unknown) => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const queryController = new AbortController() + const suggestionController = new AbortController() + + const causalEventId = abortPrintModeControlRequest( + queryController, + suggestionController, + source, + queryReason, + ) + + expect(queryController.signal.aborted).toBe(true) + expect(suggestionController.signal.aborted).toBe(true) + const trace = __getInterruptionTraceSnapshotForTests() + expect(trace.find(entry => entry.eventId === causalEventId)).toMatchObject({ + event: `input.${source}`, + source, + subsystem: 'print_mode', + }) + expect( + trace.find( + entry => + entry.event === 'abort.requested' && + entry.controllerRole === 'query-root', + ), + ).toMatchObject({ source, causalEventId, subsystem: 'print_mode' }) + expect( + trace.find( + entry => + entry.event === 'abort.requested' && + entry.controllerRole === 'speculation', + ), + ).toMatchObject({ + source, + causalEventId, + subsystem: 'prompt_suggestion', + }) + }, + ) + + test('preserves native abort behavior when tracing is disabled', () => { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + __resetInterruptionTraceForTests() + const queryController = new AbortController() + const suggestionController = new AbortController() + + const causalEventId = abortPrintModeControlRequest( + queryController, + suggestionController, + 'sdk_control_interrupt', + 'interrupt', + ) + + expect(causalEventId).toBeUndefined() + expect(queryController.signal.reason).toBe('interrupt') + expect(suggestionController.signal.reason).toBeInstanceOf(DOMException) + expect(suggestionController.signal.reason.name).toBe('AbortError') + expect(__getInterruptionTraceSnapshotForTests()).toEqual([]) + }) +}) diff --git a/src/cli/print.ts b/src/cli/print.ts index a049ef0f32..fbd3b13c6d 100644 --- a/src/cli/print.ts +++ b/src/cli/print.ts @@ -148,6 +148,10 @@ import { permissionPromptToolResultToPermissionDecision, } from 'src/utils/permissions/PermissionPromptToolResultSchema.js' import { createAbortController } from 'src/utils/abortController.js' +import { + registerInterruptionController, + requestAbort, +} from 'src/utils/interruptionTrace.js' import { createCombinedAbortSignal } from 'src/utils/combinedAbortSignal.js' import { generateSessionTitle, @@ -348,6 +352,7 @@ import { type HeadlessHeartbeatEvent, type HeadlessHeartbeatState, } from './headlessHeartbeat.js' +import { abortPrintModeControlRequest } from './printInterruption.js' import { isTeamLead, hasActiveInProcessTeammates, @@ -1204,6 +1209,14 @@ function runHeadlessStreaming( let shutdownPromptInjected = false let heldBackResult: StdoutMessage | null = null let abortController: AbortController | undefined + const abortActiveQuery = (source: string, reason: unknown): void => { + if (!abortController || abortController.signal.aborted) return + requestAbort(abortController, reason, { + source, + subsystem: 'print_mode', + controllerRole: 'query-root', + }) + } // Same queue sendRequest() enqueues to — one FIFO for everything. const output = structuredIO.outbound @@ -1213,9 +1226,7 @@ function runHeadlessStreaming( const sigintHandler = () => { logForDiagnosticsNoPII('info', 'shutdown_signal', { signal: 'SIGINT' }) options.heartbeat?.setPhase('shutting_down') - if (abortController && !abortController.signal.aborted) { - abortController.abort('interrupt') - } + abortActiveQuery('print_sigint', 'interrupt') void gracefulShutdown(0) } process.on('SIGINT', sigintHandler) @@ -2044,7 +2055,7 @@ function runHeadlessStreaming( // Abort the current operation when a 'now' priority message arrives. subscribeToCommandQueue(() => { if (abortController && getCommandsByMaxPriority('now').length > 0) { - abortController.abort('interrupt') + abortActiveQuery('priority_now', 'interrupt') } }) @@ -2318,6 +2329,10 @@ function runHeadlessStreaming( } abortController = createAbortController() + registerInterruptionController(abortController, { + subsystem: 'print_mode', + controllerRole: 'query-root', + }) const turnStartTime = feature('FILE_PERSISTENCE') ? Date.now() : undefined @@ -3029,10 +3044,12 @@ function runHeadlessStreaming( }, })) } - if (abortController) { - abortController.abort('interrupt') - } - suggestionState.abortController?.abort() + abortPrintModeControlRequest( + abortController, + suggestionState.abortController, + 'sdk_control_interrupt', + 'interrupt', + ) suggestionState.abortController = null suggestionState.lastEmitted = null suggestionState.pendingSuggestion = null @@ -3041,10 +3058,12 @@ function runHeadlessStreaming( logForDebugging( `[print.ts] end_session received, reason=${message.request.reason ?? 'unspecified'}`, ) - if (abortController) { - abortController.abort() - } - suggestionState.abortController?.abort() + abortPrintModeControlRequest( + abortController, + suggestionState.abortController, + 'sdk_end_session', + undefined, + ) suggestionState.abortController = null suggestionState.lastEmitted = null suggestionState.pendingSuggestion = null @@ -4127,7 +4146,7 @@ function runHeadlessStreaming( structuredIO.injectControlResponse(response) }, onInterrupt() { - abortController?.abort('interrupt') + abortActiveQuery('bridge_interrupt', 'interrupt') }, onSetModel(model) { const resolved = diff --git a/src/cli/printInterruption.ts b/src/cli/printInterruption.ts new file mode 100644 index 0000000000..f0bc76aac7 --- /dev/null +++ b/src/cli/printInterruption.ts @@ -0,0 +1,37 @@ +import { + requestAbort, + traceInterruptionEvent, +} from '../utils/interruptionTrace.js' + +export type PrintModeControlAbortSource = + | 'sdk_control_interrupt' + | 'sdk_end_session' + +export function abortPrintModeControlRequest( + queryController: AbortController | undefined, + suggestionController: AbortController | null, + source: PrintModeControlAbortSource, + queryReason: unknown, +): string | undefined { + const causalEventId = traceInterruptionEvent(`input.${source}`, { + source, + subsystem: 'print_mode', + }) + if (queryController && !queryController.signal.aborted) { + requestAbort(queryController, queryReason, { + source, + causalEventId, + subsystem: 'print_mode', + controllerRole: 'query-root', + }) + } + if (suggestionController && !suggestionController.signal.aborted) { + requestAbort(suggestionController, undefined, { + source, + causalEventId, + subsystem: 'prompt_suggestion', + controllerRole: 'speculation', + }) + } + return causalEventId +} diff --git a/src/components/permissions/MonitorPermissionRequest/MonitorPermissionRequest.test.tsx b/src/components/permissions/MonitorPermissionRequest/MonitorPermissionRequest.test.tsx index 698d2cd3cd..8fd3e2e2a7 100644 --- a/src/components/permissions/MonitorPermissionRequest/MonitorPermissionRequest.test.tsx +++ b/src/components/permissions/MonitorPermissionRequest/MonitorPermissionRequest.test.tsx @@ -34,6 +34,11 @@ import type { } from '../../../utils/permissions/PermissionResult.js' import type { PermissionUpdate } from '../../../utils/permissions/PermissionUpdateSchema.js' import type { ToolUseConfirm } from '../PermissionRequest.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, +} from '../../../utils/interruptionTrace.js' function createTestStreams(): { stdout: PassThrough @@ -344,6 +349,8 @@ async function renderMonitorPermission( } } +const originalInterruptionTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + beforeEach(async () => { await acquireSharedMutationLock( 'components/permissions/MonitorPermissionRequest.test.tsx', @@ -353,9 +360,16 @@ beforeEach(async () => { })) }) -afterEach(() => { +afterEach(async () => { try { mock.restore() + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalInterruptionTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalInterruptionTrace + } } finally { releaseSharedMutationLock() } @@ -486,6 +500,8 @@ describe('MonitorPermissionRequest', () => { }) test('escape cancels the pending permission request and closes the dialog', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() const onDone = mock(() => {}) const onReject = mock(() => {}) const { toolUseConfirm, toolUseContext, decisionPromise } = @@ -504,6 +520,16 @@ describe('MonitorPermissionRequest', () => { expect(decision.behavior).toBe('ask') expect(toolUseContext.abortController.signal.aborted).toBe(true) expect(onReject).toHaveBeenCalledTimes(1) + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'abort.requested' && + entry.source === 'permission_abort', + ), + ).toMatchObject({ + subsystem: 'tool_permission', + controllerRole: 'query-root', + }) } finally { mounted.cleanup() } diff --git a/src/components/permissions/PermissionRequest.tsx b/src/components/permissions/PermissionRequest.tsx index b15e5ce55e..5b64570482 100644 --- a/src/components/permissions/PermissionRequest.tsx +++ b/src/components/permissions/PermissionRequest.tsx @@ -120,7 +120,7 @@ export type ToolUseConfirm = { classifierMatchedRule?: string; workerBadge?: WorkerBadgeProps; onUserInteraction(): void; - onAbort(): void; + onAbort(source?: string, causalEventId?: string): void; onDismissCheckmark?(): void; onAllow(updatedInput: z.infer, permissionUpdates: PermissionUpdate[], feedback?: string, contentBlocks?: ContentBlockParam[]): void; onReject(feedback?: string, contentBlocks?: ContentBlockParam[]): void; diff --git a/src/entrypoints/sdk/interruption.ts b/src/entrypoints/sdk/interruption.ts new file mode 100644 index 0000000000..e83eeea908 --- /dev/null +++ b/src/entrypoints/sdk/interruption.ts @@ -0,0 +1,14 @@ +import { requestAbort } from '../../utils/interruptionTrace.js' + +export function requestSdkRootAbort( + controller: AbortController, + source: string, + subsystem: 'sdk_query' | 'sdk_session', +): void { + if (controller.signal.aborted) return + requestAbort(controller, undefined, { + source, + subsystem, + controllerRole: 'query-root', + }) +} diff --git a/src/entrypoints/sdk/query.ts b/src/entrypoints/sdk/query.ts index 0d5d894ed5..8250501849 100644 --- a/src/entrypoints/sdk/query.ts +++ b/src/entrypoints/sdk/query.ts @@ -20,6 +20,7 @@ import { } from '../../Tool.js' import { assembleToolPool, getTools } from '../../tools.js' import { createFileStateCacheWithSizeLimit } from '../../utils/fileStateCache.js' +import { requestSdkRootAbort } from './interruption.js' import { init } from '../init.js' import { resolveSessionFilePath, @@ -756,8 +757,9 @@ class QueryImpl implements Query { } close(): void { - this.interrupt() - this.abortController.abort() + const wrapperController = this.abortController + this.interruptWithSource('sdk_close') + requestSdkRootAbort(wrapperController, 'sdk_close', 'sdk_query') // Disconnect MCP clients to prevent resource leaks const mcpClients = this._engine?.getMcpClients?.() ?? [] for (const client of mcpClients) { @@ -773,8 +775,14 @@ class QueryImpl implements Query { } interrupt(): void { + this.interruptWithSource('sdk_interrupt') + } + + private interruptWithSource(source: string): void { if (this._engine) { - this._engine.interrupt() + this._engine.interrupt(source) + } else { + requestSdkRootAbort(this.abortController, source, 'sdk_query') } // Deny all pending permission prompts before clearing for (const [toolUseId, pending] of this.pendingPermissionPrompts) { diff --git a/src/entrypoints/sdk/v2.ts b/src/entrypoints/sdk/v2.ts index 621e693e79..3af4052e7d 100644 --- a/src/entrypoints/sdk/v2.ts +++ b/src/entrypoints/sdk/v2.ts @@ -19,6 +19,7 @@ import { } from '../../Tool.js' import { assembleToolPool, getTools } from '../../tools.js' import { createFileStateCacheWithSizeLimit } from '../../utils/fileStateCache.js' +import { requestSdkRootAbort } from './interruption.js' import { init } from '../init.js' import { resolveSessionFilePath, @@ -387,8 +388,14 @@ class SDKSessionImpl implements SDKSession { } interrupt(): void { + this.interruptWithSource('sdk_interrupt') + } + + private interruptWithSource(source: string): void { if (this._engine) { - this._engine.interrupt() + this._engine.interrupt(source) + } else if (this._abortController) { + requestSdkRootAbort(this._abortController, source, 'sdk_session') } // Deny all pending permission prompts before clearing for (const [toolUseId, pending] of this.pendingPermissionPrompts) { @@ -403,10 +410,11 @@ class SDKSessionImpl implements SDKSession { } close(): void { - this.interrupt() - // Abort the AbortController to cancel any in-flight HTTP requests or - // async operations tied to the signal. Mirrors QueryImpl.close(). - this._abortController?.abort() + const wrapperController = this._abortController + this.interruptWithSource('sdk_close') + if (wrapperController) { + requestSdkRootAbort(wrapperController, 'sdk_close', 'sdk_session') + } this._abortController = null // Disconnect MCP clients to prevent resource leaks const mcpClients = this._engine?.getMcpClients?.() ?? [] diff --git a/src/grpc/server.interruptionTrace.test.ts b/src/grpc/server.interruptionTrace.test.ts new file mode 100644 index 0000000000..a010105bd8 --- /dev/null +++ b/src/grpc/server.interruptionTrace.test.ts @@ -0,0 +1,89 @@ +import { EventEmitter } from 'node:events' +import { expect, spyOn, test } from 'bun:test' +import { QueryEngine } from '../QueryEngine.js' +import { GrpcServer } from './server.js' + +class FakeCall extends EventEmitter { + writes: unknown[] = [] + ended = false + + write(value: unknown): void { + this.writes.push(value) + } + + end(): void { + this.ended = true + } +} + +async function exerciseInterruption( + event: 'cancel' | 'end', +): Promise { + let releaseSubmit!: () => void + const submitBlocked = new Promise(resolve => { + releaseSubmit = resolve + }) + let submitEntered = false + const submitMessage = spyOn( + QueryEngine.prototype, + 'submitMessage', + ).mockImplementation( + async function* () { + submitEntered = true + await submitBlocked + }, + ) + const interrupt = spyOn(QueryEngine.prototype, 'interrupt') + try { + const server = new GrpcServer() + const call = new FakeCall() + ;(server as unknown as { + handleChat(call: FakeCall): void + }).handleChat(call) + call.emit('data', { + request: { + message: 'test', + working_directory: process.cwd(), + model: 'sonnet', + }, + }) + let interruptionEmitted = false + for ( + let attempts = 0; + attempts < 100 && interrupt.mock.calls.length === 0; + attempts++ + ) { + if (event === 'cancel' && submitEntered && !interruptionEmitted) { + call.emit('data', { cancel: {} }) + interruptionEmitted = true + } + if (event === 'end' && submitEntered && !interruptionEmitted) { + call.emit('end') + interruptionEmitted = true + } + await Bun.sleep(10) + } + if (!submitEntered) { + throw new Error('submitMessage was never entered') + } + if (interrupt.mock.calls.length === 0) { + throw new Error( + `Timed out waiting for QueryEngine.interrupt after the '${event}' event`, + ) + } + return interrupt.mock.calls[0]?.[0] + } finally { + releaseSubmit() + await Bun.sleep(10) + interrupt.mockRestore() + submitMessage.mockRestore() + } +} + +test('labels an explicit gRPC cancellation', async () => { + expect(await exerciseInterruption('cancel')).toBe('grpc_cancel') +}) + +test('labels a gRPC stream ending while a query is active', async () => { + expect(await exerciseInterruption('end')).toBe('grpc_stream_end') +}) diff --git a/src/grpc/server.ts b/src/grpc/server.ts index d367ba7788..6d8f3fae47 100644 --- a/src/grpc/server.ts +++ b/src/grpc/server.ts @@ -289,7 +289,7 @@ export class GrpcServer { } else if (clientMessage.cancel) { interrupted = true if (engine) { - engine.interrupt() + engine.interrupt('grpc_cancel') } call.end() } @@ -312,7 +312,7 @@ export class GrpcServer { resolve('no') } if (engine) { - engine.interrupt() + engine.interrupt('grpc_stream_end') } engine = null pendingRequests.clear() diff --git a/src/hooks/toolPermission/PermissionContext.ts b/src/hooks/toolPermission/PermissionContext.ts index 7548388135..0834d588cd 100644 --- a/src/hooks/toolPermission/PermissionContext.ts +++ b/src/hooks/toolPermission/PermissionContext.ts @@ -22,6 +22,7 @@ import type { } from '../../types/permissions.js' import { setClassifierApproval } from '../../utils/classifierApprovals.js' import { logForDebugging } from '../../utils/debug.js' +import { requestAbort } from '../../utils/interruptionTrace.js' import { executePermissionRequestHooks } from '../../utils/hooks.js' import { REJECT_MESSAGE, @@ -210,6 +211,7 @@ function createPermissionContext( feedback?: string, isAbort?: boolean, contentBlocks?: ContentBlockParam[], + abortTrace?: { source: string; causalEventId?: string }, ): PermissionDecision { const sub = !!toolUseContext.agentId const baseMessage = feedback @@ -222,7 +224,12 @@ function createPermissionContext( logForDebugging( `Aborting: tool=${tool.name} isAbort=${isAbort} hasFeedback=${!!feedback} isSubagent=${sub}`, ) - toolUseContext.abortController.abort('interrupt') + requestAbort(toolUseContext.abortController, 'interrupt', { + source: abortTrace?.source ?? 'permission_abort', + causalEventId: abortTrace?.causalEventId, + subsystem: 'tool_permission', + controllerRole: 'query-root', + }) } return { behavior: 'ask', message, contentBlocks } }, @@ -367,7 +374,11 @@ function createPermissionContext( logForDebugging( `Hook interrupt: tool=${tool.name} hookMessage=${decision.message}`, ) - toolUseContext.abortController.abort('interrupt') + requestAbort(toolUseContext.abortController, 'interrupt', { + source: 'permission_hook', + subsystem: 'tool_permission', + controllerRole: 'query-root', + }) } return this.buildDeny( decision.message || 'Permission denied by hook', diff --git a/src/hooks/toolPermission/handlers/interactiveHandler.test.ts b/src/hooks/toolPermission/handlers/interactiveHandler.test.ts index 908eaf4f23..fc8afbf8a0 100644 --- a/src/hooks/toolPermission/handlers/interactiveHandler.test.ts +++ b/src/hooks/toolPermission/handlers/interactiveHandler.test.ts @@ -1,4 +1,10 @@ import { describe, expect, test, vi } from 'vitest' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, + requestAbort, +} from '../../../utils/interruptionTrace.js' import { handleInteractivePermission, type InteractivePermissionParams, @@ -9,7 +15,7 @@ import { // bypasses resolveOnce fails here instead of silently stranding the watchdog. type QueueItem = { - onAbort: () => void + onAbort: (source?: string, causalEventId?: string) => void onAllow: ( updatedInput: Record, permissionUpdates: unknown[], @@ -118,10 +124,19 @@ describe('handleInteractivePermission watchdog suspension', () => { }) test('resumes exactly once on abort', () => { - const { getQueueItem, resume, resolve } = setup() - getQueueItem().onAbort() + const { ctx, getQueueItem, resume, resolve } = setup() + getQueueItem().onAbort('cancel_keybinding', 'input-event-1') expect(resume).toHaveBeenCalledTimes(1) expect(resolve).toHaveBeenCalledTimes(1) + expect(ctx.cancelAndAbort).toHaveBeenCalledWith( + undefined, + true, + undefined, + { + source: 'cancel_keybinding', + causalEventId: 'input-event-1', + }, + ) }) test('resumes only once when two resolution paths race', async () => { @@ -166,6 +181,55 @@ describe('handleInteractivePermission watchdog suspension', () => { expect(ctx.removeFromQueue).toHaveBeenCalledTimes(1) }) + test('preserves the originating trace when an external abort closes the dialog', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const { ctx, abortController } = setup() + + try { + requestAbort(abortController, undefined, { + source: 'cancel_keybinding', + subsystem: 'query_engine', + controllerRole: 'query-root', + }) + + const trace = __getInterruptionTraceSnapshotForTests() + const originatingAbort = trace.find( + entry => + entry.event === 'abort.requested' && + entry.source === 'cancel_keybinding', + ) + const permissionResolution = trace.find( + entry => entry.event === 'permission.abort_resolved', + ) + expect(typeof originatingAbort?.eventId).toBe('string') + expect(permissionResolution).toMatchObject({ + source: 'cancel_keybinding', + subsystem: 'tool_permission', + causalEventId: originatingAbort!.eventId, + outcome: 'denied', + }) + expect(ctx.cancelAndAbort).toHaveBeenCalledWith( + undefined, + true, + undefined, + { + source: 'cancel_keybinding', + causalEventId: originatingAbort!.eventId, + }, + ) + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + } + }) + test('resolves and resumes immediately if already aborted when shown', () => { const { ctx, resume, resolve } = setup({ preAbort: true }) expect(resume).toHaveBeenCalledTimes(1) diff --git a/src/hooks/toolPermission/handlers/interactiveHandler.ts b/src/hooks/toolPermission/handlers/interactiveHandler.ts index 9ec8a4f358..54d250c67b 100644 --- a/src/hooks/toolPermission/handlers/interactiveHandler.ts +++ b/src/hooks/toolPermission/handlers/interactiveHandler.ts @@ -27,6 +27,10 @@ import { errorMessage } from '../../../utils/errors.js' import type { PermissionDecision } from '../../../utils/permissions/PermissionResult.js' import type { PermissionUpdate } from '../../../utils/permissions/PermissionUpdateSchema.js' import { hasPermissionsToUseTool } from '../../../utils/permissions/permissions.js' +import { + getInterruptionSignalAbortTrace, + tracePermissionAbortResolution, +} from '../../../utils/interruptionTrace.js' import type { PermissionContext } from '../PermissionContext.js' import { createResolveOnce } from '../PermissionContext.js' @@ -127,12 +131,23 @@ function handleInteractivePermission( const abortSignal = ctx.toolUseContext.abortController.signal const onExternalAbort = () => { if (!claim()) return + const abortTrace = getInterruptionSignalAbortTrace(abortSignal) + tracePermissionAbortResolution( + abortTrace.source, + abortTrace.causalEventId, + 'tool_permission', + ) if (bridgeCallbacks && bridgeRequestId) { bridgeCallbacks.cancelRequest(bridgeRequestId) } channelUnsubscribe?.() ctx.removeFromQueue() - resolveOnce(ctx.cancelAndAbort(undefined, true)) + resolveOnce( + ctx.cancelAndAbort(undefined, true, undefined, { + source: abortTrace.source ?? 'permission_abort', + causalEventId: abortTrace.causalEventId, + }), + ) } if (abortSignal.aborted) { // Already aborted: cancel and stop setup so we never enqueue a stale prompt. @@ -190,7 +205,7 @@ function handleInteractivePermission( ctx.removeFromQueue() } }, - onAbort() { + onAbort(source, causalEventId) { if (!claim()) return if (bridgeCallbacks && bridgeRequestId) { bridgeCallbacks.sendResponse(bridgeRequestId, { @@ -205,7 +220,12 @@ function handleInteractivePermission( { decision: 'reject', source: { type: 'user_abort' } }, { permissionPromptStartTimeMs }, ) - resolveOnce(ctx.cancelAndAbort(undefined, true)) + resolveOnce( + ctx.cancelAndAbort(undefined, true, undefined, { + source: source ?? 'permission_dialog', + causalEventId, + }), + ) }, async onAllow( updatedInput, diff --git a/src/hooks/useBackgroundTaskNavigation.interruptionTrace.test.tsx b/src/hooks/useBackgroundTaskNavigation.interruptionTrace.test.tsx new file mode 100644 index 0000000000..46b1a66212 --- /dev/null +++ b/src/hooks/useBackgroundTaskNavigation.interruptionTrace.test.tsx @@ -0,0 +1,223 @@ +import { PassThrough } from 'node:stream' +import { afterEach, beforeEach, expect, test } from 'bun:test' +import React, { useEffect } from 'react' +import { createRoot } from '../ink.js' +import { KeyboardEvent } from '../ink/events/keyboard-event.js' +import { AppStateProvider, type AppState } from '../state/AppState.js' +import { getDefaultAppState } from '../state/AppStateStore.js' +import type { InProcessTeammateTaskState } from '../tasks/InProcessTeammateTask/types.js' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../test/sharedMutationLock.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, +} from '../utils/interruptionTrace.js' +import { useBackgroundTaskNavigation } from './useBackgroundTaskNavigation.js' + +const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + +beforeEach(async () => { + await acquireSharedMutationLock( + 'hooks/useBackgroundTaskNavigation.interruptionTrace.test.tsx', + ) +}) + +afterEach(() => { + try { + __resetInterruptionTraceForTests() + if (originalTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + } finally { + releaseSharedMutationLock() + } +}) + +function Harness({ + onReady, +}: { + onReady: (handler: (event: KeyboardEvent) => void) => void +}): React.ReactNode { + const { handleKeyDown } = useBackgroundTaskNavigation() + useEffect(() => onReady(handleKeyDown), [handleKeyDown, onReady]) + return null +} + +function createTeammateTask(): { + task: InProcessTeammateTaskState + currentWorkAbortController: AbortController + lifecycleAbortController: AbortController +} { + const currentWorkAbortController = new AbortController() + const lifecycleAbortController = new AbortController() + return { + currentWorkAbortController, + lifecycleAbortController, + task: { + id: 'teammate-task-1', + type: 'in_process_teammate', + status: 'running', + description: 'test teammate', + startTime: Date.now(), + outputFile: '/tmp/test-teammate-output', + outputOffset: 0, + notified: false, + identity: { + agentId: 'researcher@test-team', + agentName: 'researcher', + teamName: '', + planModeRequired: false, + parentSessionId: 'parent-session', + }, + prompt: 'test', + abortController: lifecycleAbortController, + currentWorkAbortController, + awaitingPlanApproval: false, + permissionMode: 'default', + isIdle: false, + shutdownRequested: false, + pendingUserMessages: [], + lastReportedToolCount: 0, + lastReportedTokenCount: 0, + }, + } +} + +async function renderNavigation(initialState: AppState): Promise<{ + invoke: (key: 'escape' | 'k') => void + cleanup: () => Promise +}> { + const stdout = new PassThrough() + const stdin = new PassThrough() as PassThrough & { + isTTY: boolean + setRawMode: (mode: boolean) => void + ref: () => void + unref: () => void + } + stdin.isTTY = true + stdin.setRawMode = () => {} + stdin.ref = () => {} + stdin.unref = () => {} + ;(stdout as unknown as { columns: number }).columns = 120 + const root = await createRoot({ + stdout: stdout as unknown as NodeJS.WriteStream, + stdin: stdin as unknown as NodeJS.ReadStream, + patchConsole: false, + }) + let handler: ((event: KeyboardEvent) => void) | undefined + try { + root.render( + + { + handler = value + }} + /> + , + ) + for (let attempts = 0; attempts < 100 && !handler; attempts++) { + await Bun.sleep(10) + } + expect(handler).toBeDefined() + return { + invoke(key) { + handler!( + new KeyboardEvent({ + kind: 'key', + name: key, + sequence: key === 'escape' ? '\u001b' : key, + raw: key === 'escape' ? '\u001b' : key, + ctrl: false, + shift: false, + meta: false, + option: false, + super: false, + fn: false, + isPasted: false, + }), + ) + }, + async cleanup() { + root.unmount() + await Bun.sleep(30) + stdin.end() + stdout.end() + }, + } + } catch (error) { + root.unmount() + await Bun.sleep(30) + stdin.end() + stdout.end() + throw error + } +} + +test('records Escape causality before aborting the current teammate turn', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const { task, currentWorkAbortController } = createTeammateTask() + const rendered = await renderNavigation({ + ...getDefaultAppState(), + tasks: { [task.id]: task }, + viewingAgentTaskId: task.id, + viewSelectionMode: 'viewing-agent', + }) + + try { + rendered.invoke('escape') + const entries = __getInterruptionTraceSnapshotForTests() + const input = entries.find(entry => entry.event === 'input.teammate_escape') + const requested = entries.find(entry => entry.event === 'abort.requested') + expect(currentWorkAbortController.signal.aborted).toBe(true) + expect(currentWorkAbortController.signal.reason).toBeInstanceOf( + DOMException, + ) + expect( + (currentWorkAbortController.signal.reason as DOMException).name, + ).toBe('AbortError') + expect(input).toBeDefined() + expect(requested).toMatchObject({ + source: 'teammate_escape', + subsystem: 'in_process_teammate', + controllerRole: 'subagent-turn', + subagentId: 'researcher@test-team', + causalEventId: input?.eventId, + }) + } finally { + await rendered.cleanup() + } +}) + +test('records kill-key causality before aborting the teammate lifecycle', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const { task, lifecycleAbortController } = createTeammateTask() + const rendered = await renderNavigation({ + ...getDefaultAppState(), + tasks: { [task.id]: task }, + expandedView: 'teammates', + selectedIPAgentIndex: 0, + viewSelectionMode: 'selecting-agent', + }) + + try { + rendered.invoke('k') + const entries = __getInterruptionTraceSnapshotForTests() + const input = entries.find(entry => entry.event === 'input.teammate_kill') + const requested = entries.find(entry => entry.event === 'abort.requested') + expect(lifecycleAbortController.signal.aborted).toBe(true) + expect(input).toBeDefined() + expect(requested).toMatchObject({ + source: 'teammate_kill', + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: 'researcher@test-team', + causalEventId: input?.eventId, + }) + } finally { + await rendered.cleanup() + } +}) diff --git a/src/hooks/useBackgroundTaskNavigation.ts b/src/hooks/useBackgroundTaskNavigation.ts index c1a9b14fd7..be6f504492 100644 --- a/src/hooks/useBackgroundTaskNavigation.ts +++ b/src/hooks/useBackgroundTaskNavigation.ts @@ -11,15 +11,17 @@ import { enterTeammateView, exitTeammateView, } from '../state/teammateViewHelpers.js' -import { - getRunningTeammatesSorted, - InProcessTeammateTask, -} from '../tasks/InProcessTeammateTask/InProcessTeammateTask.js' +import { getRunningTeammatesSorted } from '../tasks/InProcessTeammateTask/InProcessTeammateTask.js' import { type InProcessTeammateTaskState, isInProcessTeammateTask, } from '../tasks/InProcessTeammateTask/types.js' import { isBackgroundTask } from '../tasks/types.js' +import { + requestAbort, + traceInterruptionEvent, +} from '../utils/interruptionTrace.js' +import { killInProcessTeammate } from '../utils/swarm/spawnInProcess.js' // Step teammate selection by delta, wrapping across leader(-1)..teammates(0..n-1)..hide(n). // First step from a collapsed tree expands it and parks on leader. @@ -155,7 +157,23 @@ export function useBackgroundTaskNavigation(options?: { const task = tasks[taskId] if (isInProcessTeammateTask(task) && task.status === 'running') { // Abort currentWorkAbortController (stops current turn) NOT abortController (kills teammate) - task.currentWorkAbortController?.abort() + const causalEventId = traceInterruptionEvent( + 'input.teammate_escape', + { + source: 'teammate_escape', + subsystem: 'in_process_teammate', + subagentId: task.identity.agentId, + }, + ) + if (task.currentWorkAbortController) { + requestAbort(task.currentWorkAbortController, undefined, { + source: 'teammate_escape', + subsystem: 'in_process_teammate', + controllerRole: 'subagent-turn', + subagentId: task.identity.agentId, + causalEventId, + }) + } return } } @@ -233,7 +251,15 @@ export function useBackgroundTaskNavigation(options?: { e.preventDefault() const selected = getSelectedTeammate() if (selected && selected.task.status === 'running') { - void InProcessTeammateTask.kill(selected.taskId, setAppState) + const causalEventId = traceInterruptionEvent('input.teammate_kill', { + source: 'teammate_kill', + subsystem: 'in_process_teammate', + subagentId: selected.task.identity.agentId, + }) + killInProcessTeammate(selected.taskId, setAppState, { + source: 'teammate_kill', + causalEventId, + }) } return } diff --git a/src/hooks/useCancelRequest.test.tsx b/src/hooks/useCancelRequest.test.tsx new file mode 100644 index 0000000000..753fba9dba --- /dev/null +++ b/src/hooks/useCancelRequest.test.tsx @@ -0,0 +1,304 @@ +import { PassThrough } from 'node:stream' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + afterEach, + beforeEach, + describe, + expect, + mock, + spyOn, + test, +} from 'bun:test' +import React from 'react' +import * as analytics from '../services/analytics/index.js' +import { createRoot } from '../ink.js' +import { KeybindingProvider } from '../keybindings/KeybindingContext.js' +import type { + KeybindingContextName, + ParsedKeystroke, +} from '../keybindings/types.js' +import { AppStateProvider } from '../state/AppState.js' +import { + getDefaultAppState, + type AppState, +} from '../state/AppStateStore.js' +import { CancelRequestHandler } from './useCancelRequest.js' +import type { LocalAgentTaskState } from '../tasks/LocalAgentTask/LocalAgentTask.js' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../test/sharedMutationLock.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, +} from '../utils/interruptionTrace.js' + +type HandlerRegistration = { + action: string + context: KeybindingContextName + handler: () => void +} + +function TestKeybindingProvider({ + children, + registry, +}: { + children: React.ReactNode + registry: React.RefObject>> +}): React.ReactNode { + const pendingChordRef = React.useRef(null) + const [pendingChord, setPendingChord] = React.useState< + ParsedKeystroke[] | null + >(null) + const [activeContexts] = React.useState( + () => new Set(['Chat', 'Global']), + ) + return ( + activeContexts.add(context)} + unregisterActiveContext={context => activeContexts.delete(context)} + handlerRegistryRef={registry} + > + {children} + + ) +} + +function createTestStreams() { + const stdout = new PassThrough() + const stdin = new PassThrough() as PassThrough & { + isTTY: boolean + setRawMode: (mode: boolean) => void + ref: () => void + unref: () => void + } + stdin.isTTY = true + stdin.setRawMode = () => {} + stdin.ref = () => {} + stdin.unref = () => {} + ;(stdout as unknown as { columns: number }).columns = 120 + return { stdout, stdin } +} + +async function waitFor( + predicate: () => boolean, + label: string, + timeoutMs = 2_000, +): Promise { + const startedAt = Date.now() + while (Date.now() - startedAt < timeoutMs) { + if (predicate()) return + await Bun.sleep(10) + } + throw new Error(`Timed out waiting for ${label}`) +} + +async function renderCancelHandler( + initialState: AppState, + options: { failAfterRender?: boolean; onCleanup?: () => void } = {}, +) { + const { stdout, stdin } = createTestStreams() + const root = await createRoot({ + stdout: stdout as unknown as NodeJS.WriteStream, + stdin: stdin as unknown as NodeJS.ReadStream, + patchConsole: false, + }) + const registry = { + current: new Map>(), + } + const onCancel = mock((_source: string, _causalEventId?: string) => {}) + let latestState = initialState + const cleanup = async () => { + root.unmount() + await Bun.sleep(30) + stdin.end() + stdout.end() + options.onCleanup?.() + } + try { + root.render( + { + latestState = newState + }} + > + + {}} + onCancel={onCancel} + onAgentsKilled={() => {}} + isMessageSelectorVisible={false} + screen="prompt" + abortSignal={new AbortController().signal} + /> + + , + ) + if (options.failAfterRender) throw new Error('synthetic setup failure') + await waitFor( + () => + registry.current.has('app:interrupt') && + (initialState.viewSelectionMode === 'viewing-agent' || + registry.current.has('chat:cancel')), + 'cancel keybinding registration', + ) + } catch (error) { + await cleanup() + throw error + } + return { + onCancel, + invoke(action: 'chat:cancel' | 'app:interrupt') { + const registration = registry.current.get(action)?.values().next().value + if (!registration) throw new Error(`Missing ${action} handler`) + registration.handler() + }, + getState: () => latestState, + cleanup, + } +} + +const originalInterruptionTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE +let hasSharedMutationLock = false + +beforeEach(async () => { + await acquireSharedMutationLock('hooks/useCancelRequest.test.tsx') + hasSharedMutationLock = true +}) + +afterEach(async () => { + try { + mock.restore() + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalInterruptionTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalInterruptionTrace + } + } finally { + if (hasSharedMutationLock) { + releaseSharedMutationLock() + hasSharedMutationLock = false + } + } +}) + +describe('CancelRequestHandler interruption sources', () => { + test('cleans up the Ink root and streams when setup fails', async () => { + let cleanupCount = 0 + + await expect( + renderCancelHandler(getDefaultAppState(), { + failAfterRender: true, + onCleanup: () => { + cleanupCount++ + }, + }), + ).rejects.toThrow('synthetic setup failure') + expect(cleanupCount).toBe(1) + }) + + test('chat:cancel preserves escape analytics and passes the precise cancel source', async () => { + const logEvent = spyOn(analytics, 'logEvent').mockImplementation(() => {}) + const rendered = await renderCancelHandler(getDefaultAppState()) + try { + rendered.invoke('chat:cancel') + + expect(rendered.onCancel).toHaveBeenCalledWith( + 'cancel_keybinding', + undefined, + ) + expect(logEvent).toHaveBeenCalledWith( + 'tengu_cancel', + expect.objectContaining({ source: 'escape' }), + ) + } finally { + await rendered.cleanup() + } + }) + + test('app:interrupt preserves escape analytics while passing ctrl_c', async () => { + const logEvent = spyOn(analytics, 'logEvent').mockImplementation(() => {}) + const rendered = await renderCancelHandler({ + ...getDefaultAppState(), + viewSelectionMode: 'viewing-agent', + }) + try { + rendered.invoke('app:interrupt') + + expect(rendered.onCancel).toHaveBeenCalledWith('ctrl_c', undefined) + expect(rendered.getState().viewSelectionMode).toBe('none') + expect(logEvent).toHaveBeenCalledWith( + 'tengu_cancel', + expect.objectContaining({ source: 'escape' }), + ) + } finally { + await rendered.cleanup() + } + }) + + test('app:interrupt links background-agent aborts to the Ctrl-C input', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const agentAbortController = new AbortController() + const agentTask: LocalAgentTaskState = { + id: 'agent-1', + type: 'local_agent', + status: 'running', + description: 'trace test agent', + startTime: Date.now(), + outputFile: join(tmpdir(), 'openclaude-trace-test-agent.output'), + outputOffset: 0, + notified: false, + agentId: 'agent-1', + prompt: 'test', + agentType: 'general-purpose', + abortController: agentAbortController, + retrieved: false, + lastReportedToolCount: 0, + lastReportedTokenCount: 0, + isBackgrounded: true, + pendingMessages: [], + retain: false, + diskLoaded: false, + } + const rendered = await renderCancelHandler({ + ...getDefaultAppState(), + viewSelectionMode: 'viewing-agent', + viewingAgentTaskId: 'agent-1', + tasks: { 'agent-1': agentTask }, + }) + try { + rendered.invoke('app:interrupt') + await waitFor( + () => agentAbortController.signal.aborted, + 'background-agent abort', + ) + + const trace = __getInterruptionTraceSnapshotForTests() + const input = trace.find(entry => entry.event === 'input.ctrl_c') + const agentAbort = trace.find( + entry => + entry.event === 'abort.requested' && + entry.controllerRole === 'background-agent', + ) + expect(input).toBeDefined() + expect(agentAbort).toMatchObject({ + source: 'ctrl_c', + subagentId: 'agent-1', + causalEventId: input!.eventId, + }) + } finally { + await rendered.cleanup() + } + }) +}) diff --git a/src/hooks/useCancelRequest.ts b/src/hooks/useCancelRequest.ts index 4382e27623..18058313dd 100644 --- a/src/hooks/useCancelRequest.ts +++ b/src/hooks/useCancelRequest.ts @@ -33,15 +33,18 @@ import { hasCommandsInQueue, } from '../utils/messageQueueManager.js' import { emitTaskTerminatedSdk } from '../utils/sdkEventQueue.js' +import { traceInterruptionEvent } from '../utils/interruptionTrace.js' /** Time window in ms during which a second press kills all background agents. */ const KILL_AGENTS_CONFIRM_WINDOW_MS = 3000 +export type CancelRequestSource = 'cancel_keybinding' | 'ctrl_c' + type CancelRequestHandlerProps = { setToolUseConfirmQueue: ( f: (toolUseConfirmQueue: ToolUseConfirm[]) => ToolUseConfirm[], ) => void - onCancel: () => void + onCancel: (source: CancelRequestSource, causalEventId?: string) => void onAgentsKilled: () => void isMessageSelectorVisible: boolean screen: Screen @@ -84,7 +87,10 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null { const lastKillAgentsPressRef = useRef(0) const viewSelectionMode = useAppState(s => s.viewSelectionMode) - const handleCancel = useCallback(() => { + const handleCancel = useCallback(( + source: CancelRequestSource, + causalEventId?: string, + ) => { const cancelProps = { source: 'escape' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, @@ -97,7 +103,7 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null { if (abortSignal !== undefined && !abortSignal.aborted) { logEvent('tengu_cancel', cancelProps) setToolUseConfirmQueue(() => []) - onCancel() + onCancel(source, causalEventId) return } @@ -112,7 +118,7 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null { // Fallback: nothing to cancel or pop (shouldn't reach here if isActive is correct) logEvent('tengu_cancel', cancelProps) setToolUseConfirmQueue(() => []) - onCancel() + onCancel(source, causalEventId) }, [ abortSignal, popCommandFromQueue, @@ -161,7 +167,18 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null { isContextActive && (canCancelRunningTask || hasQueuedCommands || isViewingTeammate) - useKeybinding('chat:cancel', handleCancel, { + const handleCancelKeybinding = useCallback( + () => { + const causalEventId = traceInterruptionEvent('input.cancel_keybinding', { + source: 'cancel_keybinding', + subsystem: 'cancel_request', + }) + handleCancel('cancel_keybinding', causalEventId) + }, + [handleCancel], + ) + + useKeybinding('chat:cancel', handleCancelKeybinding, { context: 'Chat', isActive: isEscapeActive, }) @@ -169,13 +186,14 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null { // Shared kill path: stop all agents, suppress per-agent notifications, // emit SDK events, enqueue a single aggregate model-facing notification. // Returns true if anything was killed. - const killAllAgentsAndNotify = useCallback((): boolean => { + const killAllAgentsAndNotify = useCallback( + (source: string, causalEventId?: string): boolean => { const tasks = store.getState().tasks const running = Object.entries(tasks).filter( ([, t]) => t.type === 'local_agent' && t.status === 'running', ) if (running.length === 0) return false - killAllRunningAgentTasks(tasks, setAppState) + killAllRunningAgentTasks(tasks, setAppState, { source, causalEventId }) const descriptions: string[] = [] for (const [taskId, task] of running) { markAgentsNotified(taskId, setAppState) @@ -191,19 +209,26 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null { : `${descriptions.length} background agents were stopped by the user: ${descriptions.map(d => `"${d}"`).join(', ')}.` enqueuePendingNotification({ value: summary, mode: 'task-notification' }) onAgentsKilled() - return true - }, [store, setAppState, onAgentsKilled]) + return true + }, + [store, setAppState, onAgentsKilled], + ) // Ctrl+C (app:interrupt). Scoped to teammate-view: killing agents from the // main prompt stays a deliberate gesture (chat:killAgents), not a // side-effect of cancelling a turn. const handleInterrupt = useCallback(() => { + const causalEventId = traceInterruptionEvent('input.ctrl_c', { + source: 'ctrl_c', + subsystem: 'cancel_request', + phase: isViewingTeammate ? 'teammate_view' : 'main_view', + }) if (isViewingTeammate) { - killAllAgentsAndNotify() + killAllAgentsAndNotify('ctrl_c', causalEventId) exitTeammateView(setAppState) } if (canCancelRunningTask || hasQueuedCommands) { - handleCancel() + handleCancel('ctrl_c', causalEventId) } }, [ isViewingTeammate, @@ -247,7 +272,11 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null { 'kill_agents' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, }) clearCommandQueue() - killAllAgentsAndNotify() + const causalEventId = traceInterruptionEvent('input.kill_agents', { + source: 'kill_agents', + subsystem: 'cancel_request', + }) + killAllAgentsAndNotify('kill_agents', causalEventId) return } // First press -- show confirmation hint in status bar diff --git a/src/hooks/useInboxPoller.ts b/src/hooks/useInboxPoller.ts index ffb0b793b0..12119c1975 100644 --- a/src/hooks/useInboxPoller.ts +++ b/src/hooks/useInboxPoller.ts @@ -21,6 +21,7 @@ import { handlePlanApprovalResponse, } from '../utils/inProcessTeammateHelpers.js' import { createAssistantMessage } from '../utils/messages.js' +import { tracePermissionAbortResolution } from '../utils/interruptionTrace.js' import { requestPermissionModeChange } from '../utils/permissions/permissionModeChange.js' import { applyPermissionModeChange, @@ -313,7 +314,12 @@ export function useInboxPoller({ onUserInteraction() { // No-op for tmux workers (no classifier auto-approval) }, - onAbort() { + onAbort(source, causalEventId) { + tracePermissionAbortResolution( + source, + causalEventId, + 'tmux_permission_bridge', + ) void sendPermissionResponseViaMailbox( parsed.agent_id, { decision: 'rejected', resolvedBy: 'leader' }, diff --git a/src/hooks/useReplBridge.tsx b/src/hooks/useReplBridge.tsx index 5e799c306a..826e20b886 100644 --- a/src/hooks/useReplBridge.tsx +++ b/src/hooks/useReplBridge.tsx @@ -20,6 +20,7 @@ import type { Message } from '../types/message.js'; import { getCwd } from '../utils/cwd.js'; import { logForDebugging } from '../utils/debug.js'; import { errorMessage } from '../utils/errors.js'; +import { requestBridgeInterrupt } from '../utils/replInterruption.js'; import { enqueue } from '../utils/messageQueueManager.js'; import { buildSystemInitMessage } from '../utils/messages/systemInit.js'; import { createBridgeStatusMessage, createSystemMessage } from '../utils/messages.js'; @@ -393,7 +394,7 @@ export function useReplBridge(messages: Message[], setMessages: (action: React.S onInboundMessage: handleInboundMessage, onPermissionResponse: handlePermissionResponse, onInterrupt() { - abortControllerRef.current?.abort('interrupt'); + requestBridgeInterrupt(abortControllerRef); }, onSetModel(model) { const resolved = model === 'default' ? null : model ?? null; diff --git a/src/hooks/useSSHSession.ts b/src/hooks/useSSHSession.ts index 35b3a067dc..901825ae8e 100644 --- a/src/hooks/useSSHSession.ts +++ b/src/hooks/useSSHSession.ts @@ -28,6 +28,7 @@ import type { Message as MessageType } from '../types/message.js' import type { PermissionAskDecision } from '../types/permissions.js' import { logForDebugging } from '../utils/debug.js' import { gracefulShutdown } from '../utils/gracefulShutdown.js' +import { tracePermissionAbortResolution } from '../utils/interruptionTrace.js' import type { RemoteMessageContent } from '../utils/teleport/api.js' type UseSSHSessionResult = { @@ -121,7 +122,12 @@ export function useSSHSession({ permissionResult, permissionPromptStartTimeMs: Date.now(), onUserInteraction() {}, - onAbort() { + onAbort(source, causalEventId) { + tracePermissionAbortResolution( + source, + causalEventId, + 'ssh_permission_bridge', + ) manager.respondToPermissionRequest(requestId, { behavior: 'deny', message: 'User aborted', diff --git a/src/query.abortClassification.test.ts b/src/query.abortClassification.test.ts index f1b4baf62a..d2971bff3a 100644 --- a/src/query.abortClassification.test.ts +++ b/src/query.abortClassification.test.ts @@ -11,6 +11,16 @@ import { INTERRUPT_MESSAGE, } from './utils/messages.js' import { asSystemPrompt } from './utils/systemPromptType.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, + requestAbort, +} from './utils/interruptionTrace.js' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from './test/sharedMutationLock.js' const DEFAULT_ABORT = Symbol('default abort') @@ -55,11 +65,10 @@ function makeToolUseContext(tools: Tools = []): QueryParams['toolUseContext'] { } function abort(controller: AbortController, reason: AbortInput): void { - if (reason === DEFAULT_ABORT) { - controller.abort() - return - } - controller.abort(reason) + requestAbort(controller, reason === DEFAULT_ABORT ? undefined : reason, { + source: 'query_abort_test', + controllerRole: 'query-root', + }) } function makeBaseParams( @@ -170,6 +179,33 @@ function interruptionMessages(yielded: any[]) { } describe('query abort classification', () => { + test.serial('links abort classification to the winning root abort', async () => { + await acquireSharedMutationLock('query abort classification trace') + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + try { + await drainWithReturn(makeParams('query-timeout')) + + const trace = __getInterruptionTraceSnapshotForTests() + const rootAbort = trace.find(entry => entry.event === 'abort.requested') + const classified = trace.find( + entry => entry.event === 'query.abort_classified', + ) + expect(rootAbort).toBeDefined() + expect(classified).toBeDefined() + expect(typeof rootAbort!.eventId).toBe('string') + expect(typeof classified!.causalEventId).toBe('string') + expect(classified!.causalEventId).toBe(rootAbort!.eventId) + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + releaseSharedMutationLock() + } + }) + test('query timeout aborts produce timeout transcript text without user interruption', async () => { const { yielded, returned } = await drainWithReturn( makeParams('query-timeout'), diff --git a/src/query.ts b/src/query.ts index 1a99f9da43..b7d76c9155 100644 --- a/src/query.ts +++ b/src/query.ts @@ -5,6 +5,7 @@ import type { } from '@anthropic-ai/sdk/resources/index.mjs' import type { CanUseToolFn } from './hooks/useCanUseTool.js' import { FallbackTriggeredError } from './services/api/withRetry.js' +import { isMainThreadGoalSource } from './services/goal/controller.js' import { calculateTokenWarningState, getAutoCompactThreshold, @@ -55,6 +56,11 @@ import { normalizeAbortReason, shouldCreateUserInterruptionMessage, } from './utils/abortReasons.js' +import { + flushInterruptionTrace, + getInterruptionSignalAbortEventId, + traceInterruptionEvent, +} from './utils/interruptionTrace.js' import { createAssistantMessage, createUserMessage, @@ -185,6 +191,27 @@ async function cleanupComputerUseAtTerminal( } } +function traceAbortMessageSelection( + signal: AbortSignal, + phase: 'streaming' | 'tools' | 'post-tools', +): void { + const abortReason = signal.reason + const createsUserInterruption = shouldCreateUserInterruptionMessage(abortReason) + const createsSystemWarning = getQueryAbortSystemMessage(abortReason) !== null + traceInterruptionEvent('query.abort_classified', { + subsystem: 'query', + phase, + reason: abortReason, + causalEventId: getInterruptionSignalAbortEventId(signal), + outcome: createsUserInterruption + ? 'user_interruption' + : createsSystemWarning + ? 'system_warning' + : 'silent', + }) + flushInterruptionTrace('query_abort_classified') +} + async function* emitAbortedStreaming( signal: AbortSignal, toolUseContext: ToolUseContext, @@ -194,6 +221,7 @@ async function* emitAbortedStreaming( > { await cleanupComputerUseAtTerminal(toolUseContext) const abortReason = signal.reason + traceAbortMessageSelection(signal, 'streaming') const abortSystemMessage = getQueryAbortSystemMessage(abortReason) if (abortSystemMessage) { yield createSystemMessage(abortSystemMessage, 'warning') @@ -211,6 +239,7 @@ function* emitAbortedToolsAfterCleanup( hasSharedTurnBudget: boolean, ): Generator> { const abortReason = signal.reason + traceAbortMessageSelection(signal, 'tools') const abortSystemMessage = getQueryAbortSystemMessage(abortReason) if (abortSystemMessage) { yield createSystemMessage(abortSystemMessage, 'warning') @@ -794,6 +823,19 @@ async function* queryLoop( state.toolUseContext, ) + const activeGoal = state.toolUseContext.getAppState().goal + if ( + activeGoal?.status === 'active' && + isMainThreadGoalSource(querySource, state.toolUseContext) + ) { + traceInterruptionEvent('goal.main_turn_started', { + subsystem: 'goal', + phase: 'main_query', + querySource, + attemptId: activeGoal.id, + }) + } + // eslint-disable-next-line no-constant-condition while (true) { // Destructure state at the top of each iteration. toolUseContext alone @@ -2007,6 +2049,7 @@ async function* queryLoop( // Without this, tool_use blocks would lack matching tool_result blocks. if (toolUseContext.abortController.signal.aborted) { const abortReason = toolUseContext.abortController.signal.reason + traceAbortMessageSelection(toolUseContext.abortController.signal, 'post-tools') if (streamingToolExecutor) { // Consume remaining results - executor generates synthetic tool_results for // aborted tools since it checks the abort signal in executeTool() diff --git a/src/remote/remotePermissionBridge.ts b/src/remote/remotePermissionBridge.ts index dab2968a97..d3a463594e 100644 --- a/src/remote/remotePermissionBridge.ts +++ b/src/remote/remotePermissionBridge.ts @@ -5,6 +5,7 @@ import type { AssistantMessage } from '../types/message.js' import type { PermissionAskDecision } from '../types/permissions.js' import { jsonStringify } from '../utils/slowOperations.js' import type { ToolUseConfirm } from '../components/permissions/PermissionRequest.js' +import { tracePermissionAbortResolution } from '../utils/interruptionTrace.js' /** * Create a synthetic AssistantMessage for remote permission requests. @@ -123,7 +124,12 @@ export function createRemotePermissionQueueItem({ onUserInteraction() { // No-op for remote permission prompts. }, - onAbort() { + onAbort(source, causalEventId) { + tracePermissionAbortResolution( + source, + causalEventId, + 'remote_permission_bridge', + ) respond({ behavior: 'deny', message: 'User aborted', diff --git a/src/screens/REPL.tsx b/src/screens/REPL.tsx index 93f30714b0..7fb1e1f8c3 100644 --- a/src/screens/REPL.tsx +++ b/src/screens/REPL.tsx @@ -38,7 +38,7 @@ import { asSessionId, asAgentId } from '../types/ids.js'; import { logForDebugging } from '../utils/debug.js'; import { QueryGuard } from '../utils/QueryGuard.js'; import { getQueryGuardOptionsFromEnv } from '../utils/queryGuardConfig.js'; -import { QueryLifecycleOperationTracker, formatQueryLifecycleAbortSignalReason, formatQueryLifecycleLogMessage, getQueryTerminalReason, type QueryActiveOperationSnapshot, type QueryGuardTimeoutInfo, type QueryLifecycleContext, type QueryTerminalReason } from '../utils/queryLifecycle.js'; +import { QueryLifecycleOperationTracker, formatQueryLifecycleAbortSignalReason, formatQueryLifecycleLogMessage, getQueryTerminalOutcome, getQueryTerminalReason, type QueryActiveOperationSnapshot, type QueryGuardTimeoutInfo, type QueryLifecycleContext, type QueryTerminalReason } from '../utils/queryLifecycle.js'; import { claimBackgroundTurnBudget, canRestoreDeferredMaxTurnsCap, computeDeferredMaxTurnsCapForBackgroundHandoff, createForegroundTurnBudgetHandoff, getReplMaxTurnsWarning, releaseForegroundTurnBudget, resolveReplMaxTurnsForSession, shouldShowReplMaxTurnsUnlimitedWarning, shouldContinueBackgroundAfterForegroundQuery, waitForForegroundTurnBudgetSettlement, type ForegroundTurnBudgetHandoff } from './replMaxTurns.js'; import { createCombinedAbortSignal } from '../utils/combinedAbortSignal.js'; import { isEnvTruthy } from '../utils/envUtils.js'; @@ -92,7 +92,10 @@ import { CommandKeybindingHandlers } from '../hooks/useCommandKeybindings.js'; import { KeybindingSetup } from '../keybindings/KeybindingProviderSetup.js'; import { useShortcutDisplay } from '../keybindings/useShortcutDisplay.js'; import { getShortcutDisplay } from '../keybindings/shortcutFormat.js'; -import { CancelRequestHandler } from '../hooks/useCancelRequest.js'; +import { + CancelRequestHandler, + type CancelRequestSource, +} from '../hooks/useCancelRequest.js'; import { useBackgroundTaskNavigation } from '../hooks/useBackgroundTaskNavigation.js'; import { useSwarmInitialization } from '../hooks/useSwarmInitialization.js'; import { useTeammateViewAutoExit } from '../hooks/useTeammateViewAutoExit.js'; @@ -224,6 +227,18 @@ import { RemoteCallout } from '../components/RemoteCallout.js'; import { getAPIProvider } from '../utils/model/providers.js'; import { activityManager } from '../utils/activityManager.js'; import { createAbortController } from '../utils/abortController.js'; +import { + flushInterruptionTrace, + getInterruptionSignalAbortEventId, + registerInterruptionController, + requestAbort, + traceInterruptionEvent, +} from '../utils/interruptionTrace.js'; +import { driveQueryEvents } from '../utils/queryEventDriver.js'; +import { + requestBackgroundHandoffAbort, + requestPriorityNowAbort, +} from '../utils/replInterruption.js'; import { MCPConnectionManager } from 'src/services/mcp/MCPConnectionManager.js'; import { useFeedbackSurvey } from 'src/components/FeedbackSurvey/useFeedbackSurvey.js'; import { useMemorySurvey } from 'src/components/FeedbackSurvey/useMemorySurvey.js'; @@ -1887,7 +1902,20 @@ export function REPL({ const activeAbortController = abortControllerRef.current; if (activeAbortController && !activeAbortController.signal.aborted) { logQueryLifecycle('abort_requested', timeout.context, formatQueryLifecycleAbortSignalReason(timeoutAbortReason)); - activeAbortController.abort(timeoutAbortReason); + requestAbort(activeAbortController, timeoutAbortReason, { + source: 'query_guard', + causalEventId: timeout.causalEventId, + subsystem: 'repl', + phase: 'timeout', + queryId: timeout.context.queryId, + queryGeneration: timeout.generation, + querySource: timeout.context.querySource, + controllerRole: 'query-root', + elapsedQueryMs: timeout.elapsedMs, + activeApiCallCount: timeout.activeOperations.apiCalls.length, + activeToolUseCount: timeout.activeOperations.toolUses.length, + }); + flushInterruptionTrace('query_guard_timeout'); } if (timeout.activeOperations.apiCalls.length > 0) { logForDebugging(`api.call.active_on_abort queryId=${timeout.context.queryId} generation=${timeout.generation} ${timeoutOperations}`); @@ -2382,7 +2410,11 @@ export function REPL({ }, [focusedInputDialog, repinScroll]); // Omitted means a programmatic edit/restore cancellation, which must not arm // correction context because those flows rewind the conversation themselves. - function onCancel(isUserInitiated = false) { + function onCancel( + isUserInitiated = false, + cancelSource: CancelRequestSource | 'programmatic' = 'programmatic', + causalEventId?: string, + ) { if (focusedInputDialog === 'elicitation') { // Elicitation dialog handles its own Escape, and closing it shouldn't affect any loading state. return; @@ -2436,7 +2468,7 @@ export function REPL({ } if (focusedInputDialog === 'tool-permission') { // Tool use confirm handles the abort signal itself - toolUseConfirmQueue[0]?.onAbort(); + toolUseConfirmQueue[0]?.onAbort(cancelSource, causalEventId); setToolUseConfirmQueue([]); } else if (focusedInputDialog === 'prompt') { // Reject all pending prompts and clear the queue @@ -2444,12 +2476,26 @@ export function REPL({ item.reject(new Error('Prompt cancelled by user')); } setPromptQueue([]); - abortController?.abort('user-cancel'); + if (abortController) { + requestAbort(abortController, 'user-cancel', { + source: cancelSource, + causalEventId, + subsystem: 'repl', + controllerRole: 'query-root', + }); + } } else if (activeRemote.isRemoteMode) { // Remote mode: send interrupt signal to CCR activeRemote.cancelRequest(); } else { - abortController?.abort('user-cancel'); + if (abortController) { + requestAbort(abortController, 'user-cancel', { + source: cancelSource, + causalEventId, + subsystem: 'repl', + controllerRole: 'query-root', + }); + } } if (cancelContext) { logQueryLifecycle('abort_acknowledged', cancelContext, formatQueryLifecycleAbortSignalReason('user-cancel')); @@ -2496,7 +2542,7 @@ export function REPL({ // CancelRequestHandler props - rendered inside KeybindingSetup const cancelRequestProps = { setToolUseConfirmQueue, - onCancel: () => onCancel(true), + onCancel: (source, causalEventId) => onCancel(true, source, causalEventId), onAgentsKilled: () => setMessages(prev => [...prev, createAgentsKilledMessage()]), isMessageSelectorVisible: isMessageSelectorVisible || !!showBashesDialog, screen, @@ -3018,7 +3064,7 @@ export function REPL({ // but its controller must be reachable during preparation so Escape can // cancel the handoff before it dispatches a provider request. setAbortController(backgroundSession.abortController); - abortController?.abort('background'); + requestBackgroundHandoffAbort(abortController); }, [abortController, mainLoopModel, toolPermissionContext, mainThreadAgentDefinition, getToolUseContext, customSystemPrompt, appendSystemPrompt, canUseTool, setAppState, getAutoCompactTrackingForSession, setAutoCompactTrackingForSession, fallbackModel, setAbortController, addNotification, terminalTitle]); const { handleBackgroundSession @@ -3266,23 +3312,11 @@ export function REPL({ } } }); - let queryTerminal: QueryTerminal; - let generatorDone = false; - try { - while (true) { - const next = await queryGenerator.next(); - if (next.done) { - generatorDone = true; - queryTerminal = next.value; - break; - } - const event = next.value; - queryGuard.registerActivity(`query_event:${event.type}`, queryGeneration); - onQueryEvent(event); - } - } finally { - if (!generatorDone) await queryGenerator.return(undefined as never); - } + const queryTerminal = await driveQueryEvents( + queryGenerator, + reason => queryGuard.registerActivity(reason, queryGeneration), + onQueryEvent, + ); if (isBuddyEnabled()) { void fireCompanionObserver(messagesRef.current, reaction => setAppState(prev => prev.companionReaction === reaction ? prev : { ...prev, @@ -3359,6 +3393,20 @@ export function REPL({ const turnBudget = turnBudgetHandoff.budget; foregroundTurnBudgetRef.current = turnBudgetHandoff; const queryContext = startResult.context; + registerInterruptionController(abortController, { + subsystem: 'repl', + controllerRole: 'query-root', + queryId: queryContext.queryId, + queryGeneration: thisGeneration, + querySource: queryContext.querySource, + }); + traceInterruptionEvent('query.started', { + subsystem: 'repl', + phase: 'running', + queryId: queryContext.queryId, + queryGeneration: thisGeneration, + querySource: queryContext.querySource, + }); logQueryLifecycle('start', queryContext); logQueryLifecycle('guard_start', queryContext); let didThrow = false; @@ -3456,8 +3504,24 @@ export function REPL({ // then clear it only when this query reaches its terminal cleanup. interruptionCorrectionTracker.finishModelTurn(queryContext.queryId); const terminalReason = getQueryTerminalReason(abortController.signal, didThrow); + const terminalOutcome = getQueryTerminalOutcome(abortController.signal, didThrow); const abortReason = getAbortReasonLabel(abortController.signal.reason); const activeOperations = lifecycleTracker.snapshot(); + traceInterruptionEvent('query.terminal', { + subsystem: 'repl', + phase: terminalReason, + outcome: terminalOutcome, + queryId: queryContext.queryId, + queryGeneration: thisGeneration, + querySource: queryContext.querySource, + reason: abortController.signal.reason, + causalEventId: getInterruptionSignalAbortEventId( + abortController.signal, + ), + activeApiCallCount: activeOperations.apiCalls.length, + activeToolUseCount: activeOperations.toolUses.length, + }); + if (abortController.signal.aborted) flushInterruptionTrace('query_terminal'); const completedContext = { ...queryContext, terminalReason, @@ -4660,7 +4724,7 @@ export function REPL({ // (e.g. from a chat UI client via UDS). useEffect(() => { if (queuedCommands.some(cmd => cmd.priority === 'now')) { - abortControllerRef.current?.abort('interrupt'); + requestPriorityNowAbort(abortControllerRef); } }, [queuedCommands]); diff --git a/src/services/PromptSuggestion/speculation.test.ts b/src/services/PromptSuggestion/speculation.test.ts index 09c1600811..c6d93a8fbf 100644 --- a/src/services/PromptSuggestion/speculation.test.ts +++ b/src/services/PromptSuggestion/speculation.test.ts @@ -3,6 +3,10 @@ import type { AppState } from '../../state/AppStateStore.js' import { IDLE_SPECULATION_STATE } from '../../state/AppStateStore.js' import type { ToolUseContext } from '../../Tool.js' import type { REPLHookContext } from '../../utils/hooks/postSamplingHooks.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, +} from '../../utils/interruptionTrace.js' import { acquireSharedMutationLock, releaseSharedMutationLock, @@ -59,6 +63,9 @@ afterAll(() => { describe('startSpeculation', () => { test('stops speculative writes in plan mode even when bypass is available', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() let appState = { speculation: IDLE_SPECULATION_STATE, toolPermissionContext: { @@ -103,8 +110,35 @@ describe('startSpeculation', () => { toolName: 'Write', }, }) + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'abort.requested', + ), + ).toMatchObject({ + source: 'speculation_edit_boundary', + subsystem: 'prompt_suggestion', + controllerRole: 'speculation', + }) + abortSpeculation(setAppState) + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'abort.repeated', + ), + ).toMatchObject({ + source: 'speculation_cancelled', + subsystem: 'prompt_suggestion', + controllerRole: 'speculation', + outcome: 'ignored_first_abort_wins', + repeatedCount: 1, + }) } finally { abortSpeculation(setAppState) + __resetInterruptionTraceForTests() + if (originalTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } } }) }) diff --git a/src/services/PromptSuggestion/speculation.ts b/src/services/PromptSuggestion/speculation.ts index e761aadc12..b374251149 100644 --- a/src/services/PromptSuggestion/speculation.ts +++ b/src/services/PromptSuggestion/speculation.ts @@ -36,6 +36,7 @@ import { import { formatDuration, formatNumber } from '../../utils/format.js' import type { REPLHookContext } from '../../utils/hooks/postSamplingHooks.js' import { logError } from '../../utils/log.js' +import { requestAbort } from '../../utils/interruptionTrace.js' import type { SetAppState } from '../../utils/messageQueueManager.js' import { createSystemMessage, @@ -61,6 +62,17 @@ import { const MAX_SPECULATION_TURNS = 20 const MAX_SPECULATION_MESSAGES = 100 +function requestSpeculationAbort( + controller: AbortController, + source: string, +): void { + requestAbort(controller, undefined, { + source, + subsystem: 'prompt_suggestion', + controllerRole: 'speculation', + }) +} + const WRITE_TOOLS = new Set(['Edit', 'Write', 'NotebookEdit']) const SAFE_READ_ONLY_TOOLS = new Set([ 'Read', @@ -371,6 +383,11 @@ async function generatePipelinedSuggestion( const pipelineAbortController = createChildAbortController( parentAbortController, + undefined, + { + subsystem: 'prompt_suggestion', + controllerRole: 'suggestion-pipeline', + }, ) if (pipelineAbortController.signal.aborted) return @@ -418,6 +435,11 @@ export async function startSpeculation( const abortController = createChildAbortController( context.toolUseContext.abortController, + undefined, + { + subsystem: 'prompt_suggestion', + controllerRole: 'speculation', + }, ) if (abortController.signal.aborted) return @@ -442,7 +464,7 @@ export async function startSpeculation( speculation: { status: 'active', id, - abort: () => abortController.abort(), + abort: () => requestSpeculationAbort(abortController, 'speculation_cancelled'), startTime, messagesRef, writtenPathsRef, @@ -495,7 +517,10 @@ export async function startSpeculation( completedAt: Date.now(), }, })) - abortController.abort() + requestSpeculationAbort( + abortController, + 'speculation_edit_boundary', + ) return denySpeculation( 'Speculation paused: file edit requires permission', 'speculation_edit_boundary', @@ -599,7 +624,10 @@ export async function startSpeculation( updateActiveSpeculationState(setAppState, () => ({ boundary: { type: 'bash', command, completedAt: Date.now() }, })) - abortController.abort() + requestSpeculationAbort( + abortController, + 'speculation_bash_boundary', + ) return denySpeculation( 'Speculation paused: bash boundary', 'speculation_bash_boundary', @@ -633,7 +661,7 @@ export async function startSpeculation( completedAt: Date.now(), }, })) - abortController.abort() + requestSpeculationAbort(abortController, 'speculation_unknown_tool') return denySpeculation( `Tool ${tool.name} not allowed during speculation`, 'speculation_unknown_tool', @@ -647,7 +675,10 @@ export async function startSpeculation( if (msg.type === 'assistant' || msg.type === 'user') { messagesRef.current.push(msg) if (messagesRef.current.length >= MAX_SPECULATION_MESSAGES) { - abortController.abort() + requestSpeculationAbort( + abortController, + 'speculation_message_limit', + ) } if (isUserMessageWithArrayContent(msg)) { const newTools = count( @@ -687,7 +718,7 @@ export async function startSpeculation( abortController, ) } catch (error) { - abortController.abort() + requestSpeculationAbort(abortController, 'speculation_failure') if (error instanceof Error && error.name === 'AbortError') { safeRemoveOverlay(overlayPath) diff --git a/src/services/api/claude.abortClassification.test.ts b/src/services/api/claude.abortClassification.test.ts index 35439bbf32..e48f86f364 100644 --- a/src/services/api/claude.abortClassification.test.ts +++ b/src/services/api/claude.abortClassification.test.ts @@ -83,6 +83,19 @@ describe('Claude stream abort classification wiring', () => { ) }) + test('does not infer user intent from an external AbortError while root is active', () => { + const activeRoot = new AbortController() + const externalAbort = new APIUserAbortError() + + expect(activeRoot.signal.aborted).toBe(false) + expect(shouldCreateUserInterruptionMessage(activeRoot.signal.reason)).toBe( + false, + ) + expect( + getClaudeStreamingAbortLogMessage(activeRoot.signal, externalAbort), + ).toBe('Streaming aborted by parent signal: Request was aborted.') + }) + test('labels expected side-task retry aborts without reclassifying user aborts', () => { const expectedAbort = new AbortController() expectedAbort.abort('agent-summary-superseded') diff --git a/src/services/api/claude.lifecycle.test.ts b/src/services/api/claude.lifecycle.test.ts index 7832961950..9e24b00470 100644 --- a/src/services/api/claude.lifecycle.test.ts +++ b/src/services/api/claude.lifecycle.test.ts @@ -14,6 +14,10 @@ import { resetGrowthBook } from '../analytics/growthbook.js' import { getEmptyToolPermissionContext } from '../../Tool.js' import type { Message } from '../../types/message.js' import { QueryLifecycleOperationTracker } from '../../utils/queryLifecycle.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, +} from '../../utils/interruptionTrace.js' import { asSystemPrompt } from '../../utils/systemPromptType.js' import { executeNonStreamingRequest, @@ -39,6 +43,7 @@ const envKeys = [ 'CLAUDE_CODE_USE_OPENAI', 'CLAUDE_CODE_USE_VERTEX', 'CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK', + 'CLAUDE_DISABLE_STREAM_WATCHDOG', 'CLAUDE_FEATURE_FLAGS_FILE', 'CLAUDE_STREAM_IDLE_TIMEOUT_MS', 'GEMINI_API_KEY', @@ -46,6 +51,7 @@ const envKeys = [ 'OPENAI_API_KEY', 'OPENAI_BASE_URL', 'OPENAI_MODEL', + 'OPENCLAUDE_INTERRUPT_TRACE', 'OPENCLAUDE_MAX_RETRIES', 'VCR_RECORD', ] as const @@ -490,6 +496,8 @@ describe('Claude API lifecycle tracking', () => { test('preserves provider override and query source during 404 non-streaming fallback', async () => { setClientTestEnv() process.env.OPENCLAUDE_MAX_RETRIES = '0' + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() const queryLifecycle = new QueryLifecycleOperationTracker() const providerBaseURL = 'https://provider.example/v1' const requests: { @@ -541,6 +549,13 @@ describe('Claude API lifecycle tracking', () => { for await (const message of generator) { messages.push(message) + if ( + typeof message === 'object' && + message !== null && + (message as { type?: unknown }).type === 'assistant' + ) { + break + } } const streamingRequest = requests.find(request => request.stream === true) @@ -562,6 +577,27 @@ describe('Claude API lifecycle tracking', () => { querySource: 'sdk', }) expect(queryLifecycle.snapshot().apiCalls).toEqual([]) + const trace = __getInterruptionTraceSnapshotForTests() + const creationError = trace.find( + entry => + entry.event === 'claude_stream.error' && + entry.phase === 'stream_creation', + ) + const fallbackStarted = trace.find( + entry => entry.event === 'claude_stream.fallback_started', + ) + const fallbackSettled = trace.find( + entry => entry.event === 'claude_stream.fallback_settled', + ) + expect(creationError).toBeDefined() + expect(fallbackStarted).toMatchObject({ + trigger: '404_stream_creation', + causalEventId: creationError?.eventId, + }) + expect(fallbackSettled).toMatchObject({ + outcome: 'completed', + causalEventId: fallbackStarted?.eventId, + }) }) test('parent abort during OpenAI-compatible stream does not start non-streaming fallback', async () => { @@ -658,9 +694,12 @@ describe('Claude API lifecycle tracking', () => { test('stream idle timeout respects disabled non-streaming fallback guard', async () => { setClientTestEnv() + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() process.env.OPENCLAUDE_MAX_RETRIES = '0' process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = String(TEST_STREAM_IDLE_TIMEOUT_MS) process.env.CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK = '1' + process.env.CLAUDE_DISABLE_STREAM_WATCHDOG = '1' const queryLifecycle = new QueryLifecycleOperationTracker() const parent = new AbortController() let fallbackRequests = 0 @@ -726,6 +765,17 @@ describe('Claude API lifecycle tracking', () => { JSON.stringify((message as { message?: { content?: unknown } }).message?.content).includes('Stream idle timeout'), ), ).toBe(true) + const trace = __getInterruptionTraceSnapshotForTests() + const providerIdle = trace.find( + entry => + entry.event === 'provider_stream.idle_timeout' && + entry.transport === 'openai_chat_completions', + ) + const claudeError = trace.find( + entry => entry.event === 'claude_stream.error', + ) + expect(providerIdle).toBeDefined() + expect(claudeError?.causalEventId).toBe(providerIdle?.eventId) }) test('tracks each non-streaming fallback request and clears it on success', async () => { diff --git a/src/services/api/claude.streamWatchdog.test.ts b/src/services/api/claude.streamWatchdog.test.ts index 3ccbbbb38d..38ecf5e50c 100644 --- a/src/services/api/claude.streamWatchdog.test.ts +++ b/src/services/api/claude.streamWatchdog.test.ts @@ -11,7 +11,7 @@ import type { BetaRawMessageStreamEvent, } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs' import type { Stream } from '@anthropic-ai/sdk/streaming.mjs' -import { mkdtempSync, rmSync } from 'fs' +import { mkdtempSync, readFileSync, rmSync } from 'fs' import { tmpdir } from 'os' import { join } from 'path' import { @@ -22,6 +22,13 @@ import { getEmptyToolPermissionContext } from '../../Tool.js' import type { Message } from '../../types/message.js' import { asSystemPrompt } from '../../utils/systemPromptType.js' import { QueryLifecycleOperationTracker } from '../../utils/queryLifecycle.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, + registerInterruptionController, + requestAbort, +} from '../../utils/interruptionTrace.js' import { EMPTY_USAGE } from './emptyUsage.js' import type { Options } from './claude.js' @@ -39,6 +46,8 @@ const envKeys = [ 'CLAUDE_ENABLE_STREAM_WATCHDOG', 'CLAUDE_STREAM_IDLE_TIMEOUT_MS', 'OPENCLAUDE_MAX_RETRIES', + 'OPENCLAUDE_INTERRUPT_TRACE', + 'OPENCLAUDE_INTERRUPT_TRACE_FILE', 'VCR_RECORD', ] as const @@ -295,6 +304,8 @@ beforeEach(async () => { await acquireSharedMutationLock('claude.streamWatchdog.test.ts') installClientSpy() setTestMacro() + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() for (const key of envKeys) { delete process.env[key] } @@ -306,11 +317,13 @@ beforeEach(async () => { process.env.VCR_RECORD = '1' }) -afterEach(() => { +afterEach(async () => { try { restoreClientSpy?.() restoreClientSpy = undefined createHandler = undefined + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() for (const key of envKeys) { const envKey: string = key if ( @@ -343,6 +356,9 @@ afterEach(() => { describe('Claude stream watchdog', () => { test('falls back when the top-level stream iterator never settles', async () => { + const traceFile = join(fixturesRoot!, 'interruption-trace.jsonl') + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = traceFile const wedged = makeWedgedStream() const streamModes: unknown[] = [] createHandler = params => { @@ -369,6 +385,64 @@ describe('Claude stream watchdog', () => { expect(streamModes).toEqual([true, undefined]) expect(wedged.abortSignal.aborted).toBe(true) expect(wedged.returnCalled()).toBe(true) + await __waitForInterruptionTraceFlushForTests() + const trace = readFileSync(traceFile, 'utf8') + .trim() + .split('\n') + .map(line => JSON.parse(line) as { + event: string + eventId?: string + causalEventId?: string + source?: string + outcome?: string + }) + const events = trace.map(entry => entry.event) + expect(events.indexOf('claude_stream.idle_timeout')).toBeGreaterThanOrEqual(0) + expect(events.indexOf('claude_stream.loop_settled')).toBeGreaterThan( + events.indexOf('claude_stream.idle_timeout'), + ) + expect(events.indexOf('claude_stream.fallback_started')).toBeGreaterThan( + events.indexOf('claude_stream.loop_settled'), + ) + expect(trace).toContainEqual( + expect.objectContaining({ + event: 'abort.requested', + source: 'claude_stream_watchdog', + }), + ) + const idleTimeout = trace.find( + entry => entry.event === 'claude_stream.idle_timeout', + ) + const providerAbort = trace.find( + entry => + entry.event === 'abort.requested' && + entry.source === 'claude_stream_watchdog', + ) + const loopSettled = trace.find( + entry => entry.event === 'claude_stream.loop_settled', + ) + const fallbackStarted = trace.find( + entry => entry.event === 'claude_stream.fallback_started', + ) + const fallbackSettled = trace.find( + entry => entry.event === 'claude_stream.fallback_settled', + ) + expect(idleTimeout).toBeDefined() + expect(providerAbort).toBeDefined() + expect(loopSettled).toBeDefined() + expect(fallbackStarted).toBeDefined() + expect(fallbackSettled).toBeDefined() + expect(typeof idleTimeout!.eventId).toBe('string') + expect(typeof providerAbort!.causalEventId).toBe('string') + expect(typeof loopSettled!.causalEventId).toBe('string') + expect(typeof fallbackStarted!.causalEventId).toBe('string') + expect(providerAbort!.causalEventId).toBe(idleTimeout!.eventId) + expect(loopSettled!.causalEventId).toBe(idleTimeout!.eventId) + expect(fallbackStarted!.causalEventId).toBe(idleTimeout!.eventId) + expect(fallbackSettled).toMatchObject({ + causalEventId: fallbackStarted!.eventId, + outcome: 'completed', + }) expect( (result as unknown[]).some( message => @@ -385,8 +459,10 @@ describe('Claude stream watchdog', () => { test('does not attempt fallback when the parent signal aborts first', async () => { process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = '250' + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' const wedged = makeWedgedStream() const controller = new AbortController() + registerInterruptionController(controller, { controllerRole: 'query-root' }) const streamModes: unknown[] = [] createHandler = params => { streamModes.push(params.stream) @@ -406,7 +482,10 @@ describe('Claude stream watchdog', () => { error instanceof Error ? error.name : String(error), ) await wedged.nextStarted - controller.abort() + requestAbort(controller, undefined, { + source: 'cancel_keybinding', + controllerRole: 'query-root', + }) try { const result = await Promise.race([request, delay(150)]) @@ -414,6 +493,97 @@ describe('Claude stream watchdog', () => { expect(streamModes).toEqual([true]) expect(wedged.abortSignal.aborted).toBe(true) expect(wedged.returnCalled()).toBe(true) + const trace = __getInterruptionTraceSnapshotForTests() + const rootAbort = trace.find( + entry => + entry.event === 'abort.requested' && + entry.controllerRole === 'query-root', + ) + const parentAbort = trace.find( + entry => entry.event === 'claude_stream.parent_abort', + ) + const providerAbort = trace.find( + entry => + entry.event === 'abort.requested' && + entry.source === 'claude_stream_parent', + ) + expect(rootAbort).toBeDefined() + expect(parentAbort).toBeDefined() + expect(providerAbort).toBeDefined() + expect(typeof rootAbort!.eventId).toBe('string') + expect(typeof parentAbort!.eventId).toBe('string') + expect(typeof parentAbort!.causalEventId).toBe('string') + expect(typeof providerAbort!.causalEventId).toBe('string') + expect(parentAbort!.causalEventId).toBe(rootAbort!.eventId) + expect(providerAbort!.causalEventId).toBe(parentAbort!.eventId) + } finally { + wedged.rejectPendingNext(new Error('test cleanup')) + await settleForCleanup(request) + } + }) + + test('records a terminal failed outcome when non-streaming fallback rejects', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const wedged = makeWedgedStream() + createHandler = params => { + if (params.stream === true) return makeWithResponse(wedged.stream) + return Promise.reject(new Error('synthetic fallback failure')) + } + + const request = collectStreamingMessages( + new AbortController().signal, + makeOptions(), + ) + await wedged.nextStarted + try { + await Promise.race([request, delay(250)]) + const trace = __getInterruptionTraceSnapshotForTests() + const started = trace.find( + entry => entry.event === 'claude_stream.fallback_started', + ) + const settled = trace.find( + entry => entry.event === 'claude_stream.fallback_settled', + ) + expect(started).toBeDefined() + expect(settled).toMatchObject({ + outcome: 'failed', + causalEventId: started!.eventId, + }) + } finally { + wedged.rejectPendingNext(new Error('test cleanup')) + await settleForCleanup(request) + } + }) + + test('records fallback failure when the returned message cannot be normalized', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const wedged = makeWedgedStream() + createHandler = params => { + if (params.stream === true) return makeWithResponse(wedged.stream) + return Promise.resolve( + makeBetaMessage('msg-invalid-fallback', [null] as never), + ) + } + + const request = collectStreamingMessages( + new AbortController().signal, + makeOptions(), + ) + await wedged.nextStarted + try { + const result = await Promise.race([ + request.then( + () => 'resolved', + error => error, + ), + delay(250), + ]) + expect(result).not.toBe('timeout') + const settlements = __getInterruptionTraceSnapshotForTests().filter( + entry => entry.event === 'claude_stream.fallback_settled', + ) + expect(settlements).toHaveLength(1) + expect(settlements[0]?.outcome).toBe('failed') } finally { wedged.rejectPendingNext(new Error('test cleanup')) await settleForCleanup(request) diff --git a/src/services/api/claude.ts b/src/services/api/claude.ts index 909f526916..a577b13111 100644 --- a/src/services/api/claude.ts +++ b/src/services/api/claude.ts @@ -175,6 +175,13 @@ import { CHROME_TOOL_SEARCH_INSTRUCTIONS } from 'src/utils/claudeInChrome/prompt import { COMPACT_MAX_OUTPUT_TOKENS, getContextWindowForModel, getMaxThinkingTokensForModel } from 'src/utils/context.js' import { logForDebugging } from 'src/utils/debug.js' import { logForDiagnosticsNoPII } from 'src/utils/diagLogs.js' +import { + flushInterruptionTrace, + getInterruptionErrorCausalEventId, + getInterruptionSignalAbortEventId, + requestAbort, + traceInterruptionEvent, +} from 'src/utils/interruptionTrace.js' import { type EffortValue, modelSupportsEffort } from 'src/utils/effort.js' import type { QueryLifecycleOperationTracker } from 'src/utils/queryLifecycle.js' import { @@ -1157,6 +1164,21 @@ async function* queryModel( void > { const providerRequestModel = options.requestModel ?? options.model + function traceFallbackSettlement( + outcome: 'superseded' | 'aborted' | 'failed' | 'completed', + causalEventId: string | undefined, + error?: unknown, + ): void { + traceInterruptionEvent('claude_stream.fallback_settled', { + subsystem: 'claude_stream', + transport: 'anthropic_messages', + model: options.model, + outcome, + causalEventId, + ...(error === undefined ? {} : { error }), + }) + flushInterruptionTrace('claude_stream_fallback_settled') + } // Check cheap conditions first — the off-switch await blocks on GrowthBook // init (~10ms). For non-Opus models (haiku, sonnet) this skips the await // entirely. Subscribers don't hit this path at all. @@ -2101,6 +2123,7 @@ async function* queryModel( const STREAM_IDLE_TIMEOUT_MS = getStreamIdleTimeoutMs() const STREAM_IDLE_WARNING_MS = STREAM_IDLE_TIMEOUT_MS / 2 let streamIdleAborted = false + let streamSettlementCausalEventId: string | undefined // performance.now() snapshot when watchdog fires, for measuring abort propagation delay let streamWatchdogFiredAt: number | null = null let streamIdleWarningTimer: ReturnType | null = null @@ -2123,19 +2146,36 @@ async function* queryModel( { level: 'warn' }, ) logForDiagnosticsNoPII('warn', 'cli_streaming_idle_warning') + traceInterruptionEvent('claude_stream.idle_warning', { + subsystem: 'claude_stream', + transport: 'anthropic_messages', + model: options.model, + sinceLastYieldMs: warnMs, + }) } function closeStreamIterator( iterator: AsyncIterator, reason: Error, + source: 'claude_stream_watchdog' | 'claude_stream_parent', + causalEventId?: string, ): void { const activeStream = stream - releaseStreamResources() try { - activeStream?.controller?.abort(reason) + if (activeStream?.controller) { + requestAbort(activeStream.controller, reason, { + source, + causalEventId, + subsystem: 'claude_stream', + transport: 'anthropic_messages', + model: options.model, + controllerRole: 'provider-stream', + }) + } } catch { // Ignore - the stream may already be closed by the SDK. } + releaseStreamResources() try { const returned = iterator.return?.() @@ -2158,6 +2198,14 @@ async function* queryModel( { level: 'error' }, ) logForDiagnosticsNoPII('error', 'cli_streaming_idle_timeout') + const causalEventId = traceInterruptionEvent('claude_stream.idle_timeout', { + subsystem: 'claude_stream', + transport: 'anthropic_messages', + model: options.model, + sinceLastYieldMs: STREAM_IDLE_TIMEOUT_MS, + }) + streamSettlementCausalEventId = causalEventId + flushInterruptionTrace('claude_stream_idle_timeout') logEvent('tengu_streaming_idle_timeout', { model: options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, @@ -2166,7 +2214,12 @@ async function* queryModel( timeout_ms: STREAM_IDLE_TIMEOUT_MS, }) - closeStreamIterator(iterator, timeoutError) + closeStreamIterator( + iterator, + timeoutError, + 'claude_stream_watchdog', + causalEventId, + ) } function readNextStreamPart( @@ -2174,7 +2227,14 @@ async function* queryModel( ): Promise> { if (signal.aborted) { const abortError = new APIUserAbortError() - closeStreamIterator(iterator, abortError) + streamSettlementCausalEventId = + getInterruptionSignalAbortEventId(signal) + closeStreamIterator( + iterator, + abortError, + 'claude_stream_parent', + getInterruptionSignalAbortEventId(signal), + ) return Promise.reject(abortError) } @@ -2200,7 +2260,22 @@ async function* queryModel( } const onAbort = () => { const abortError = new APIUserAbortError() - closeStreamIterator(iterator, abortError) + const parentCausalEventId = getInterruptionSignalAbortEventId(signal) + const causalEventId = traceInterruptionEvent('claude_stream.parent_abort', { + subsystem: 'claude_stream', + transport: 'anthropic_messages', + model: options.model, + reason: signal.reason, + causalEventId: parentCausalEventId, + }) + streamSettlementCausalEventId = + causalEventId ?? parentCausalEventId + closeStreamIterator( + iterator, + abortError, + 'claude_stream_parent', + causalEventId ?? parentCausalEventId, + ) settleReject(abortError) } @@ -2630,6 +2705,14 @@ async function* queryModel( streamWatchdogFiredAt !== null ? Math.round(performance.now() - streamWatchdogFiredAt) : -1 + traceInterruptionEvent('claude_stream.loop_settled', { + subsystem: 'claude_stream', + transport: 'anthropic_messages', + model: options.model, + outcome: 'clean', + causalEventId: streamSettlementCausalEventId, + sinceLastYieldMs: exitDelayMs, + }) logForDiagnosticsNoPII( 'info', 'cli_stream_loop_exited_after_watchdog_clean', @@ -2719,6 +2802,20 @@ async function* queryModel( } catch (streamingError) { // Clear the idle timeout watchdog on error path too clearStreamIdleTimers() + streamSettlementCausalEventId = + getInterruptionErrorCausalEventId(streamingError) ?? + streamSettlementCausalEventId + traceInterruptionEvent('claude_stream.error', { + subsystem: 'claude_stream', + transport: 'anthropic_messages', + model: options.model, + outcome: signal.aborted ? 'root_aborted' : 'external_error', + reason: signal.reason, + causalEventId: + getInterruptionSignalAbortEventId(signal) ?? + streamSettlementCausalEventId, + error: streamingError, + }) // Instrumentation: if the watchdog had already fired and the for-await // threw (rather than exiting cleanly), record that the loop DID exit and @@ -2727,6 +2824,15 @@ async function* queryModel( const exitDelayMs = Math.round( performance.now() - streamWatchdogFiredAt, ) + traceInterruptionEvent('claude_stream.loop_settled', { + subsystem: 'claude_stream', + transport: 'anthropic_messages', + model: options.model, + outcome: 'error', + causalEventId: streamSettlementCausalEventId, + error: streamingError, + sinceLastYieldMs: exitDelayMs, + }) logForDiagnosticsNoPII( 'info', 'cli_stream_loop_exited_after_watchdog_error', @@ -2862,6 +2968,16 @@ async function* queryModel( // Instrumentation: proves executeNonStreamingRequest was entered (vs. the // fallback event firing but the call itself hanging at dispatch). logForDiagnosticsNoPII('info', 'cli_nonstreaming_fallback_started') + const fallbackStartedEventId = traceInterruptionEvent( + 'claude_stream.fallback_started', { + subsystem: 'claude_stream', + transport: 'anthropic_messages', + model: options.model, + trigger: streamIdleAborted ? 'watchdog' : 'other', + causalEventId: streamSettlementCausalEventId, + }, + ) + flushInterruptionTrace('claude_stream_fallback_started') logEvent('tengu_nonstreaming_fallback_started', { request_id: (streamRequestId ?? 'unknown') as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, @@ -2872,54 +2988,69 @@ async function* queryModel( : 'other') as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, }) endActiveApiCall() - const result = yield* executeNonStreamingRequest( - { model: providerRequestModel, source: options.querySource, providerOverride: options.providerOverride, effortValue: effort }, - { - model: providerRequestModel, - fallbackModel: options.fallbackModel, - thinkingConfig, - ...(isFastModeEnabled() && { fastMode: isFastMode }), - signal, - initialConsecutive529Errors: is529Error(streamingError) ? 1 : 0, - querySource: options.querySource, - }, - paramsFromContext, - (attempt, _startTime, tokens) => { - attemptNumber = attempt - maxOutputTokens = tokens - }, - params => captureAPIRequest(params, options.querySource), - streamRequestId, - options.queryLifecycle, - options.onProviderRequestStart, - ) + let result: BetaMessage | null + let fallbackResultMessage: AssistantMessage | undefined + try { + result = yield* executeNonStreamingRequest( + { model: providerRequestModel, source: options.querySource, providerOverride: options.providerOverride, effortValue: effort }, + { + model: providerRequestModel, + fallbackModel: options.fallbackModel, + thinkingConfig, + ...(isFastModeEnabled() && { fastMode: isFastMode }), + signal, + initialConsecutive529Errors: is529Error(streamingError) ? 1 : 0, + querySource: options.querySource, + }, + paramsFromContext, + (attempt, _startTime, tokens) => { + attemptNumber = attempt + maxOutputTokens = tokens + }, + params => captureAPIRequest(params, options.querySource), + streamRequestId, + options.queryLifecycle, + options.onProviderRequestStart, + ) - if (result === null) return + if (result === null) { + traceFallbackSettlement('superseded', fallbackStartedEventId) + return + } - const m: AssistantMessage = { - message: { - ...result, - content: normalizeContentFromAPI( - result.content, - tools, - options.agentId, - ), - }, - requestId: streamRequestId ?? undefined, - type: 'assistant', - uuid: randomUUID(), - timestamp: new Date().toISOString(), - ...(process.env.USER_TYPE === 'ant' && - research !== undefined && { - research, + fallbackResultMessage = { + message: { + ...result, + content: normalizeContentFromAPI( + result.content, + tools, + options.agentId, + ), + }, + requestId: streamRequestId ?? undefined, + type: 'assistant', + uuid: randomUUID(), + timestamp: new Date().toISOString(), + ...(process.env.USER_TYPE === 'ant' && + research !== undefined && { + research, + }), + ...(advisorModel && { + advisorModel, }), - ...(advisorModel && { - advisorModel, - }), + } + newMessages.push(fallbackResultMessage) + fallbackMessage = fallbackResultMessage + } catch (error) { + traceFallbackSettlement( + signal.aborted ? 'aborted' : 'failed', + fallbackStartedEventId, + error, + ) + throw error } - newMessages.push(m) - fallbackMessage = m - yield m + traceFallbackSettlement('completed', fallbackStartedEventId) + yield fallbackResultMessage } finally { clearStreamIdleTimers() } @@ -2959,6 +3090,16 @@ async function* queryModel( errorFromRetry.originalError.status === 404 if (is404StreamCreationError) { + const streamCreationErrorEventId = traceInterruptionEvent( + 'claude_stream.error', + { + subsystem: 'claude_stream', + phase: 'stream_creation', + transport: 'anthropic_messages', + model: options.model, + error: errorFromRetry, + }, + ) // 404 is thrown at .withResponse() before streamRequestId is assigned, // and CannotRetryError means every retry failed — so grab the failed // request's ID from the error header instead. @@ -2988,6 +3129,18 @@ async function* queryModel( '404_stream_creation' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, }) + const fallbackStartedEventId = traceInterruptionEvent( + 'claude_stream.fallback_started', + { + subsystem: 'claude_stream', + transport: 'anthropic_messages', + model: options.model, + trigger: '404_stream_creation', + causalEventId: streamCreationErrorEventId, + }, + ) + flushInterruptionTrace('claude_stream_fallback_started') + try { // Fall back to non-streaming mode endActiveApiCall() @@ -3017,7 +3170,10 @@ async function* queryModel( options.onProviderRequestStart, ) - if (result === null) return + if (result === null) { + traceFallbackSettlement('superseded', fallbackStartedEventId) + return + } const m: AssistantMessage = { message: { @@ -3038,10 +3194,16 @@ async function* queryModel( } newMessages.push(m) fallbackMessage = m + traceFallbackSettlement('completed', fallbackStartedEventId) yield m // Continue to success logging below } catch (fallbackError) { + traceFallbackSettlement( + signal.aborted ? 'aborted' : 'failed', + fallbackStartedEventId, + fallbackError, + ) // Propagate model-fallback signal to query.ts (see comment above). if (fallbackError instanceof FallbackTriggeredError) { throw fallbackError @@ -3265,7 +3427,11 @@ export function cleanupStream( try { // Abort the stream via its controller if not already aborted if (!stream.controller.signal.aborted) { - stream.controller.abort() + requestAbort(stream.controller, undefined, { + source: 'claude_stream_cleanup', + subsystem: 'claude_stream', + controllerRole: 'provider-stream', + }) } } catch { // Ignore - stream may already be closed diff --git a/src/services/api/codexShim.interruption.test.ts b/src/services/api/codexShim.interruption.test.ts new file mode 100644 index 0000000000..cb12e0126a --- /dev/null +++ b/src/services/api/codexShim.interruption.test.ts @@ -0,0 +1,841 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../../test/sharedMutationLock.js' +import type { AnthropicStreamEvent } from './codexShim.js' +import { + __codexStreamToAnthropicForTests, + codexStreamToAnthropic, +} from './codexShim.js' +import { QueryGuard, type QueryGuardTimeoutReason } from '../../utils/QueryGuard.js' +import type { QueryGuardTimeoutInfo } from '../../utils/queryLifecycle.js' +import { driveQueryEvents } from '../../utils/queryEventDriver.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, + requestAbort, +} from '../../utils/interruptionTrace.js' + +const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + +beforeEach(async () => { + await acquireSharedMutationLock('codexShim.interruption.test.ts') +}) + +afterEach(async () => { + try { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } finally { + releaseSharedMutationLock() + } +}) + +async function bounded(promise: Promise, timeoutMs = 2_000): Promise { + let timer: ReturnType | undefined + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error('issue-1830 test did not settle')), + timeoutMs, + ) + }), + ]) + } finally { + if (timer !== undefined) clearTimeout(timer) + } +} + +function responseFromText(text: string): Response { + const encoder = new TextEncoder() + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(text)) + controller.close() + }, + }), + ) +} + +async function collectCodex(response: Response): Promise { + const events: AnthropicStreamEvent[] = [] + for await (const event of codexStreamToAnthropic(response, 'gpt-test')) { + events.push(event) + } + return events +} + +function makeTimedStream( + makeFrame: (index: number) => string, + intervalMs: number, +): { + response: Response + cancelReasons: unknown[] + getEmissionCount: () => number + stop: () => void +} { + const cancelReasons: unknown[] = [] + const encoder = new TextEncoder() + let index = 0 + let timer: ReturnType | undefined + let stopped = false + let streamController: ReadableStreamDefaultController | undefined + const stream = new ReadableStream({ + start(controller) { + streamController = controller + timer = setInterval(() => { + if (stopped) return + try { + controller.enqueue(encoder.encode(makeFrame(index++))) + } catch { + stopped = true + } + }, intervalMs) + }, + cancel(reason) { + stopped = true + if (timer !== undefined) clearInterval(timer) + cancelReasons.push(reason) + }, + }) + return { + response: new Response(stream), + cancelReasons, + getEmissionCount: () => index, + stop: () => { + stopped = true + if (timer !== undefined) clearInterval(timer) + try { + streamController?.close() + } catch { + // The production reader may already have cancelled the stream. + } + }, + } +} + +async function driveWithGuard( + response: Response, + options: { + idleTimeoutMs: number + hardMaxQueryMs: number + /** Raw-reader deadline, intentionally distinct from QueryGuard activity. */ + readerIdleTimeoutMs: number + }, +): Promise<{ + events: AnthropicStreamEvent[] + timeout: QueryGuardTimeoutInfo + result: { status: 'resolved' } | { status: 'rejected'; error: unknown } + signalReason: unknown +}> { + const controller = new AbortController() + const guard = new QueryGuard(options) + const start = guard.tryStart({ + queryId: 'issue-1830-query', + querySource: 'issue-1830-test', + }) + if (!start) throw new Error('QueryGuard did not start') + let timeout: QueryGuardTimeoutInfo | undefined + guard.setTimeoutHandler(info => { + timeout = info + requestAbort(controller, info.context.terminalReason, { + source: 'query_guard', + subsystem: 'issue_1830_test', + controllerRole: 'query-root', + causalEventId: info.causalEventId, + }) + }) + const events: AnthropicStreamEvent[] = [] + const stream = __codexStreamToAnthropicForTests( + response, + 'gpt-test', + controller.signal, + { idleTimeoutMs: options.readerIdleTimeoutMs }, + ) + try { + const result = await bounded( + driveQueryEvents( + stream, + reason => guard.registerActivity(reason, start.generation), + event => events.push(event), + ).then( + () => ({ status: 'resolved' as const }), + error => ({ status: 'rejected' as const, error }), + ), + ) + if (!timeout) throw new Error('QueryGuard did not own the terminal decision') + return { events, timeout, result, signalReason: controller.signal.reason } + } finally { + if (!controller.signal.aborted) { + requestAbort(controller, 'test-cleanup', { + source: 'issue_1830_test_cleanup', + subsystem: 'issue_1830_test', + controllerRole: 'query-root', + }) + } + guard.forceEnd('unknown', 'test-cleanup') + await bounded(stream.return(undefined), 100).catch(() => {}) + } +} + +describe('issue #1830 Codex interruption ownership', () => { + test('raw transport silence is owned by the Codex reader deadline', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const cancelReasons: unknown[] = [] + const response = new Response( + new ReadableStream({ + cancel(reason) { + cancelReasons.push(reason) + }, + }), + ) + const iterator = __codexStreamToAnthropicForTests( + response, + 'gpt-test', + undefined, + { idleTimeoutMs: 25 }, + )[Symbol.asyncIterator]() + + try { + expect((await bounded(iterator.next())).value?.type).toBe('message_start') + const result = await bounded( + iterator.next().then( + value => ({ status: 'resolved' as const, value }), + error => ({ status: 'rejected' as const, error }), + ), + ) + + expect(result.status).toBe('rejected') + if (result.status === 'rejected') { + expect((result.error as Error).message).toContain( + 'Codex SSE stream idle', + ) + } + expect(cancelReasons).toHaveLength(1) + const trace = __getInterruptionTraceSnapshotForTests() + const idleTimeout = trace.find( + entry => entry.event === 'codex_stream.idle_timeout', + ) + const readerError = trace.find( + entry => entry.event === 'codex_stream.error', + ) + const readerCancelled = trace.find( + entry => entry.event === 'codex_stream.cancelled', + ) + const converterClosed = trace.find( + entry => entry.event === 'codex_stream.converter_closed', + ) + expect(idleTimeout).toBeDefined() + expect(readerError).toBeDefined() + expect(readerCancelled).toBeDefined() + expect(converterClosed).toBeDefined() + expect(typeof idleTimeout!.eventId).toBe('string') + expect(typeof readerError!.causalEventId).toBe('string') + expect(typeof readerCancelled!.causalEventId).toBe('string') + expect(typeof converterClosed!.causalEventId).toBe('string') + expect(readerError!.causalEventId).toBe(idleTimeout!.eventId) + expect(readerCancelled!.causalEventId).toBe(idleTimeout!.eventId) + expect(converterClosed!.causalEventId).toBe(idleTimeout!.eventId) + } finally { + const returned = iterator.return?.(undefined) + if (returned) await bounded(Promise.resolve(returned), 100).catch(() => {}) + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + } + }) + + test('parent abort settles the pending read before its idle deadline', async () => { + const cancelReasons: unknown[] = [] + const response = new Response( + new ReadableStream({ + cancel(reason) { + cancelReasons.push(reason) + }, + }), + ) + const controller = new AbortController() + const iterator = __codexStreamToAnthropicForTests( + response, + 'gpt-test', + controller.signal, + { idleTimeoutMs: 1_000 }, + )[Symbol.asyncIterator]() + try { + expect((await bounded(iterator.next())).value?.type).toBe('message_start') + + const pending = iterator.next().then( + value => ({ status: 'resolved' as const, value }), + error => ({ status: 'rejected' as const, error }), + ) + controller.abort('query-timeout') + const result = await bounded(pending) + + expect(controller.signal.reason).toBe('query-timeout') + expect(result.status).toBe('rejected') + if (result.status === 'rejected') { + expect((result.error as { name?: unknown }).name).toBe('AbortError') + } + expect(cancelReasons).toHaveLength(1) + } finally { + if (!controller.signal.aborted) controller.abort('test-cleanup') + const returned = iterator.return?.(undefined) + if (returned) await bounded(Promise.resolve(returned), 100).catch(() => {}) + } + }) + + test('a done-only Codex stream completes without waiting for transport EOF', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const cancelReasons: unknown[] = [] + const encoder = new TextEncoder() + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('data: [DONE]\n\n')) + }, + cancel(reason) { + cancelReasons.push(reason) + }, + }), + ) + + try { + const events: AnthropicStreamEvent[] = [] + for await (const event of __codexStreamToAnthropicForTests( + response, + 'gpt-test', + undefined, + { idleTimeoutMs: 25 }, + )) { + events.push(event) + } + expect(events.at(-1)?.type).toBe('message_stop') + expect( + __getInterruptionTraceSnapshotForTests().some( + entry => entry.event === 'codex_stream.idle_timeout', + ), + ).toBe(false) + expect(cancelReasons).toHaveLength(1) + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + }) + + test('normal completed and incomplete frames after a done marker close without interruption traces', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + try { + for (const terminalEvent of [ + 'response.completed', + 'response.incomplete', + ]) { + __resetInterruptionTraceForTests() + const cancelReasons: unknown[] = [] + const encoder = new TextEncoder() + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array()) + controller.enqueue(encoder.encode([ + ': keepalive', + '', + 'data: [DONE]', + '', + `event: ${terminalEvent}`, + `data: {"type":"${terminalEvent}","response":{"status":"${terminalEvent.slice('response.'.length)}","output":[]}}`, + '', + '', + ].join('\n'))) + }, + cancel(reason) { + cancelReasons.push(reason) + }, + }), + ) + const events: AnthropicStreamEvent[] = [] + await bounded((async () => { + for await (const event of codexStreamToAnthropic(response, 'gpt-test')) { + events.push(event) + } + })()) + + expect(events.at(-1)?.type).toBe('message_stop') + const trace = __getInterruptionTraceSnapshotForTests() + const firstRawBytes = trace.filter( + entry => entry.event === 'codex_stream.first_raw_byte', + ) + const terminal = trace.find( + entry => + entry.event === 'codex_stream.protocol_terminal' && + entry.phase === terminalEvent, + ) + expect(firstRawBytes).toHaveLength(1) + expect(firstRawBytes[0]!.rawByteCount).toBeGreaterThan(0) + expect(terminal).toBeDefined() + expect(terminal!.controlFrameCount).toBe(2) + expect(terminal!.ignoredFrameCount).toBe(0) + expect( + trace.some(entry => entry.event === 'codex_stream.cancelled'), + ).toBe(false) + expect(cancelReasons).toHaveLength(1) + } + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + } + }) + + test('reports non-negative idle evidence and forwards its causal event id', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const causalEventIds: string[] = [] + const encoder = new TextEncoder() + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(': keepalive\n\n')) + }, + }), + ) + const iterator = __codexStreamToAnthropicForTests( + response, + 'gpt-test', + undefined, + { + idleTimeoutMs: 25, + onCausalEventId: eventId => causalEventIds.push(eventId), + }, + )[Symbol.asyncIterator]() + + try { + expect((await bounded(iterator.next())).value?.type).toBe('message_start') + const pending = iterator.next().catch(error => error) + await bounded(pending) + + const idleTimeout = __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'codex_stream.idle_timeout', + ) + expect(idleTimeout).toBeDefined() + expect(idleTimeout?.sinceLastRawByteMs).toBeGreaterThanOrEqual(0) + expect(idleTimeout?.sinceLastParsedFrameMs).toBeGreaterThanOrEqual(0) + expect(causalEventIds).toEqual([idleTimeout!.eventId]) + } finally { + await bounded(iterator.return?.(undefined) ?? Promise.resolve()).catch( + () => {}, + ) + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + }) + + test('marks the converter complete before yielding message_stop', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const cancelReasons: unknown[] = [] + const encoder = new TextEncoder() + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode([ + 'event: response.completed', + 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}', + '', + '', + ].join('\n'))) + }, + cancel(reason) { + cancelReasons.push(reason) + }, + }), + ) + const iterator = codexStreamToAnthropic(response, 'gpt-test') + + try { + let next: IteratorResult + do { + next = await bounded(iterator.next()) + } while (!next.done && next.value.type !== 'message_stop') + expect(next.done).toBe(false) + + await bounded(iterator.return(undefined)) + + const converterClosed = __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'codex_stream.converter_closed', + ) + expect(converterClosed).toBeDefined() + expect(converterClosed!.outcome).toBe('complete') + expect(cancelReasons).toHaveLength(1) + } finally { + await bounded(iterator.return(undefined)).catch(() => {}) + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + } + }) + + test('reports root_aborted when cancellation follows terminal evidence', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const controller = new AbortController() + const response = responseFromText([ + 'event: response.completed', + 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}', + '', + '', + ].join('\n')) + const iterator = codexStreamToAnthropic( + response, + 'gpt-test', + controller.signal, + ) + + let next: IteratorResult + do { + next = await bounded(iterator.next()) + } while (!next.done && next.value.type !== 'message_stop') + expect(next.done).toBe(false) + + requestAbort(controller, undefined, { + source: 'test_root_abort', + subsystem: 'query_engine', + controllerRole: 'query-root', + }) + await bounded(iterator.return(undefined)) + + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'codex_stream.converter_closed', + )?.outcome, + ).toBe('root_aborted') + }) + + test('keeps terminal ownership when the consumer returns after message_delta', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const response = responseFromText([ + 'event: response.completed', + 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}', + '', + '', + ].join('\n')) + const iterator = codexStreamToAnthropic(response, 'gpt-test') + + try { + let next: IteratorResult + do { + next = await bounded(iterator.next()) + } while (!next.done && next.value.type !== 'message_delta') + expect(next.done).toBe(false) + await bounded(iterator.return(undefined)) + + const converterClosed = __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'codex_stream.converter_closed', + ) + expect(converterClosed?.outcome).toBe('complete') + } finally { + await bounded(iterator.return(undefined)).catch(() => {}) + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + }) + + test('classifies each Codex reader and converter frame exactly once', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const text = [ + ': keepalive', + '', + 'event: response.created', + 'data: not-json', + '', + 'event: response.created', + 'data: []', + '', + 'event: response.created', + 'data: {"type":"response.created","sequence_number":1}', + '', + 'data: [DONE]', + '', + 'event: response.completed', + 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}', + '', + '', + ].join('\n') + + try { + await bounded((async () => { + for await (const _event of codexStreamToAnthropic( + responseFromText(text), + 'gpt-test', + )) { + // Drain the converter to terminal diagnostics. + } + })()) + + const trace = __getInterruptionTraceSnapshotForTests() + const terminal = trace.find( + entry => entry.event === 'codex_stream.protocol_terminal', + ) + const converterClosed = trace.find( + entry => entry.event === 'codex_stream.converter_closed', + ) + expect(terminal).toMatchObject({ + rawByteCount: new TextEncoder().encode(text).byteLength, + parsedFrameCount: 2, + controlFrameCount: 2, + ignoredFrameCount: 2, + }) + expect(converterClosed?.ignoredParsedFrameCount).toBe(1) + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + }) + + test('classifies event-only and typed data-only frames without inflating ignored input', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const text = [ + 'event: ping', + '', + 'data: {"type":"response.created","sequence_number":1}', + '', + 'event: response.completed', + 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}', + '', + '', + ].join('\n') + + try { + await collectCodex(responseFromText(text)) + const terminal = __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'codex_stream.protocol_terminal', + ) + expect(terminal).toMatchObject({ + parsedFrameCount: 2, + controlFrameCount: 1, + ignoredFrameCount: 0, + }) + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + }) + + test('reports parsed-but-unhandled converter events with a distinct counter', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const encoder = new TextEncoder() + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode([ + 'event: response.created', + 'data: {"type":"response.created","sequence_number":1}', + '', + 'event: response.completed', + 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}', + '', + '', + ].join('\n'))) + }, + }), + ) + + try { + await bounded((async () => { + for await (const _event of codexStreamToAnthropic(response, 'gpt-test')) { + // Drain the converter to its terminal diagnostics. + } + })()) + + const converterClosed = __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'codex_stream.converter_closed', + ) + expect(converterClosed?.ignoredParsedFrameCount).toBe(1) + expect(converterClosed?.ignoredFrameCount).toBeUndefined() + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + }) + + test('keepalives and parsed-but-ignored frames cannot reset QueryGuard', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const timed = makeTimedStream( + index => { + if (index % 4 === 0) return ': keepalive\n\n' + if (index % 4 === 1) { + return `event: response.created\ndata: {"type":"response.created","sequence_number":${index}}\n\n` + } + if (index % 4 === 2) { + return 'event: response.created\ndata: []\n\n' + } + return `event: response.reasoning_summary_text.delta\ndata: {"type":"response.reasoning_summary_text.delta","delta":"r","sequence_number":${index}}\n\n` + }, + 25, + ) + try { + const outcome = await driveWithGuard(timed.response, { + idleTimeoutMs: 250, + hardMaxQueryMs: 1_500, + readerIdleTimeoutMs: 500, + }) + + expect(outcome.timeout.reason satisfies QueryGuardTimeoutReason).toBe( + 'idle', + ) + expect(outcome.signalReason).toBe('query-timeout') + expect(outcome.result.status).toBe('rejected') + expect(outcome.events.map(event => event.type)).toEqual([ + 'message_start', + ]) + const readerClosed = __getInterruptionTraceSnapshotForTests() + .filter(entry => entry.event === 'codex_stream.cancelled') + .at(-1) + const rootAbort = __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'abort.requested' && entry.source === 'query_guard', + ) + const traceEvents = __getInterruptionTraceSnapshotForTests().map( + entry => entry.event, + ) + expect(timed.getEmissionCount()).toBeGreaterThan(3) + expect(readerClosed?.rawByteCount).toBeGreaterThan(0) + expect(readerClosed?.parsedFrameCount).toBeGreaterThan(0) + expect(readerClosed?.controlFrameCount).toBeGreaterThan(0) + expect(readerClosed?.ignoredFrameCount).toBeGreaterThan(0) + const converterClosed = __getInterruptionTraceSnapshotForTests() + .filter(entry => entry.event === 'codex_stream.converter_closed') + .at(-1) + expect(converterClosed?.ignoredParsedFrameCount).toBeGreaterThan(0) + expect(converterClosed?.ignoredFrameCount).toBeUndefined() + expect(rootAbort).toBeDefined() + expect(readerClosed).toBeDefined() + expect(typeof rootAbort!.eventId).toBe('string') + expect(typeof readerClosed!.causalEventId).toBe('string') + expect(readerClosed!.causalEventId).toBe(rootAbort!.eventId) + expect(traceEvents.indexOf('codex_stream.converter_closed')).toBeGreaterThan( + traceEvents.indexOf('codex_stream.cancelled'), + ) + expect(timed.cancelReasons).toHaveLength(1) + } finally { + timed.stop() + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + } + }) + + test('valid deltas extend idle activity but cannot bypass the hard maximum', async () => { + const timed = makeTimedStream( + index => + `event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"x","sequence_number":${index}}\n\n`, + 25, + ) + try { + const outcome = await driveWithGuard(timed.response, { + idleTimeoutMs: 250, + hardMaxQueryMs: 600, + readerIdleTimeoutMs: 500, + }) + const countAtAbort = outcome.events.length + await Bun.sleep(30) + + expect(outcome.timeout.reason satisfies QueryGuardTimeoutReason).toBe( + 'hard_max', + ) + expect(outcome.signalReason).toBe('hard-max-query-timeout') + expect(outcome.result.status).toBe('rejected') + expect(countAtAbort).toBeGreaterThan(3) + expect(outcome.events).toHaveLength(countAtAbort) + expect(timed.cancelReasons).toHaveLength(1) + } finally { + timed.stop() + } + }) + + test('repeated pending-read abort cycles cancel exactly once', async () => { + for (let cycle = 0; cycle < 40; cycle++) { + const cancelReasons: unknown[] = [] + const response = new Response( + new ReadableStream({ + cancel(reason) { + cancelReasons.push(reason) + }, + }), + ) + const controller = new AbortController() + const iterator = __codexStreamToAnthropicForTests( + response, + 'gpt-test', + controller.signal, + { idleTimeoutMs: 1_000 }, + )[Symbol.asyncIterator]() + try { + expect((await bounded(iterator.next())).value?.type).toBe( + 'message_start', + ) + const pending = iterator.next().catch(error => error as unknown) + controller.abort('user-cancel') + await bounded(pending) + expect(cancelReasons).toHaveLength(1) + } finally { + if (!controller.signal.aborted) controller.abort('test-cleanup') + const returned = iterator.return?.(undefined) + if (returned) { + await bounded(Promise.resolve(returned), 100).catch(() => {}) + } + } + } + }) +}) diff --git a/src/services/api/codexShim.ts b/src/services/api/codexShim.ts index da9b1a15a2..2b1e9e2aa6 100644 --- a/src/services/api/codexShim.ts +++ b/src/services/api/codexShim.ts @@ -12,6 +12,18 @@ import { createThinkTagFilter, stripThinkTags, } from './thinkTagSanitizer.js' +import { + createReaderCanceller, + createStreamAbortError, + readWithIdleTimeout, + throwIfStreamAborted, +} from './openaiShim/streamControl.js' +import { + flushInterruptionTrace, + getInterruptionSignalAbortEventId, + setInterruptionErrorCausalEventId, + traceInterruptionEvent, +} from '../../utils/interruptionTrace.js' export interface AnthropicUsage { input_tokens: number @@ -81,42 +93,6 @@ type CodexSseEvent = { data: Record } -function createStreamAbortError(): DOMException { - return new DOMException('Aborted', 'AbortError') -} - -function throwIfStreamAborted(signal?: AbortSignal): void { - if (signal?.aborted) { - throw createStreamAbortError() - } -} - -function createReaderCanceller( - reader: ReadableStreamDefaultReader, - signal?: AbortSignal, -): { - cancel: (error?: unknown) => void - cleanup: () => void - } { - let cancelled = false - const cancel = (error: unknown = createStreamAbortError()) => { - if (cancelled) return - cancelled = true - void reader.cancel(error).catch(() => {}) - } - const onAbort = () => cancel(createStreamAbortError()) - - signal?.addEventListener('abort', onAbort, { once: true }) - if (signal?.aborted) { - onAbort() - } - - return { - cancel, - cleanup: () => signal?.removeEventListener('abort', onAbort), - } -} - function makeUsage(usage?: Record): AnthropicUsage { // Single source of truth for raw → Anthropic shape. Lives in // cacheMetrics.ts alongside the raw-shape extractor so any new @@ -708,83 +684,108 @@ export async function performCodexRequest(options: { return response } -async function* readSseEvents(response: Response, signal?: AbortSignal): AsyncGenerator { +type CodexStreamReadOptions = { + /** Internal deterministic-test seam; production keeps the 120-second owner. */ + idleTimeoutMs?: number + onCausalEventId?: (eventId: string) => void +} + +const STREAM_IDLE_TIMEOUT_MS = 120_000 + +async function* readSseEvents( + response: Response, + signal?: AbortSignal, + options: CodexStreamReadOptions = {}, +): AsyncGenerator { const reader = response.body?.getReader() if (!reader) return const readerCanceller = createReaderCanceller(reader, signal) const decoder = new TextDecoder() let buffer = '' - const STREAM_IDLE_TIMEOUT_MS = 120_000 // 2 minutes without data - let lastDataTime = Date.now() - let streamComplete = false + const streamIdleTimeoutMs = options.idleTimeoutMs ?? STREAM_IDLE_TIMEOUT_MS + let lastDataTime = performance.now() + let lastParsedFrameTime = lastDataTime + let transportComplete = false + let protocolComplete = false + let doneMarkerObserved = false + let rawByteCount = 0 + let sawFirstRawByte = false + let parsedFrameCount = 0 + let controlFrameCount = 0 + let ignoredFrameCount = 0 + let streamCausalEventId: string | undefined + + traceInterruptionEvent('codex_stream.read_started', { + subsystem: 'codex_stream', + transport: 'codex_responses', + }) /** * Read from the stream with an idle timeout. Respects the caller's * AbortSignal — clears the idle timer on abort so the AbortError * surfaces cleanly instead of a spurious idle timeout. */ - async function readWithTimeout(): Promise>> { - return new Promise((resolve, reject) => { - let settled = false - let timeoutId: ReturnType | undefined = setTimeout(() => { - const elapsed = Math.round((Date.now() - lastDataTime) / 1000) - cancelAndReject(new Error( - `Codex SSE stream idle for ${elapsed}s (limit: ${STREAM_IDLE_TIMEOUT_MS / 1000}s). Connection likely dropped.`, - )) - }, STREAM_IDLE_TIMEOUT_MS) - - const cleanup = () => { - if (timeoutId !== undefined) { - clearTimeout(timeoutId) - timeoutId = undefined + async function readWithTimeout() { + return readWithIdleTimeout(reader!, streamIdleTimeoutMs, { + signal, + cancelReader: readerCanceller.cancel, + createTimeoutError: () => { + const elapsed = Math.round((performance.now() - lastDataTime) / 1000) + return new Error( + `Codex SSE stream idle for ${elapsed}s (limit: ${streamIdleTimeoutMs / 1000}s). Connection likely dropped.`, + ) + }, + onTimeout: error => { + const now = performance.now() + streamCausalEventId = traceInterruptionEvent('codex_stream.idle_timeout', { + subsystem: 'codex_stream', + transport: 'codex_responses', + sinceLastRawByteMs: now - lastDataTime, + sinceLastParsedFrameMs: now - lastParsedFrameTime, + rawByteCount, + parsedFrameCount, + controlFrameCount, + ignoredFrameCount, + }) + if (streamCausalEventId) { + options.onCausalEventId?.(streamCausalEventId) } - signal?.removeEventListener('abort', onAbort) - } - const finishResolve = ( - value: Bun.ReadableStreamDefaultReadResult>, - ) => { - if (settled) return - settled = true - cleanup() - if (value.value) lastDataTime = Date.now() - resolve(value) - } - const finishReject = (error: unknown) => { - if (settled) return - settled = true - cleanup() - reject(error) - } - const cancelAndReject = (error: unknown) => { - readerCanceller.cancel(error) - finishReject(error) - } - const onAbort = () => cancelAndReject(createStreamAbortError()) - - signal?.addEventListener('abort', onAbort, { once: true }) - if (signal?.aborted) { - onAbort() - return - } - - // reader is guarded non-null above; hoisted function escapes TS narrowing. - reader!.read().then( - result => finishResolve(result), - err => finishReject(err), - ) + setInterruptionErrorCausalEventId(error, streamCausalEventId) + flushInterruptionTrace('codex_stream_idle_timeout') + }, }) } try { - while (true) { + streamLoop: while (true) { const { done, value } = await readWithTimeout() if (done) { - streamComplete = true + transportComplete = true + traceInterruptionEvent('codex_stream.eof', { + subsystem: 'codex_stream', + transport: 'codex_responses', + rawByteCount, + parsedFrameCount, + controlFrameCount, + ignoredFrameCount, + }) break } throwIfStreamAborted(signal) + if (value && value.byteLength > 0) { + lastDataTime = performance.now() + rawByteCount += value.byteLength + if (!sawFirstRawByte) { + sawFirstRawByte = true + traceInterruptionEvent('codex_stream.first_raw_byte', { + subsystem: 'codex_stream', + transport: 'codex_responses', + rawByteCount, + }) + } + } buffer += decoder.decode(value, { stream: true }) const chunks = buffer.split('\n\n') buffer = chunks.pop() ?? '' @@ -797,33 +798,132 @@ async function* readSseEvents(response: Response, signal?: AbortSignal): AsyncGe .filter(Boolean) if (lines.length === 0) continue + if (lines.every(line => line.startsWith(':'))) { + controlFrameCount++ + continue + } + const eventLine = lines.find(line => line.startsWith('event: ')) const dataLines = lines.filter(line => line.startsWith('data: ')) - if (!eventLine || dataLines.length === 0) continue + if (dataLines.length === 0) { + controlFrameCount++ + continue + } - const event = eventLine.slice(7).trim() const rawData = dataLines.map(line => line.slice(6)).join('\n') - if (rawData === '[DONE]') continue + if (rawData === '[DONE]') { + doneMarkerObserved = true + controlFrameCount++ + traceInterruptionEvent('codex_stream.control_frame', { + subsystem: 'codex_stream', + transport: 'codex_responses', + phase: 'done_marker', + rawByteCount, + parsedFrameCount, + controlFrameCount, + ignoredFrameCount, + }) + continue + } let data: Record try { const parsed = JSON.parse(rawData) - if (!parsed || typeof parsed !== 'object') continue + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + ignoredFrameCount++ + continue + } data = parsed as Record } catch { + ignoredFrameCount++ + continue + } + + const event = eventLine?.slice(7).trim() ?? + (typeof data.type === 'string' ? data.type : undefined) + if (!event) { + controlFrameCount++ continue } throwIfStreamAborted(signal) + parsedFrameCount++ + lastParsedFrameTime = performance.now() + if (parsedFrameCount === 1) { + traceInterruptionEvent('codex_stream.first_parsed_frame', { + subsystem: 'codex_stream', + transport: 'codex_responses', + rawByteCount, + parsedFrameCount, + controlFrameCount, + ignoredFrameCount, + }) + } + const isTerminalEvent = + event === 'response.completed' || + event === 'response.incomplete' || + event === 'response.failed' + if (isTerminalEvent) { + protocolComplete = true + traceInterruptionEvent('codex_stream.protocol_terminal', { + subsystem: 'codex_stream', + transport: 'codex_responses', + phase: event, + rawByteCount, + parsedFrameCount, + controlFrameCount, + ignoredFrameCount, + }) + } yield { event, data } + if (isTerminalEvent) break streamLoop + } + if (doneMarkerObserved) { + protocolComplete = true + break } } + } catch (error) { + traceInterruptionEvent('codex_stream.error', { + subsystem: 'codex_stream', + transport: 'codex_responses', + outcome: signal?.aborted ? 'root_aborted' : 'external_error', + reason: signal?.reason, + causalEventId: + (signal && getInterruptionSignalAbortEventId(signal)) ?? + streamCausalEventId, + error, + rawByteCount, + parsedFrameCount, + controlFrameCount, + ignoredFrameCount, + }) + flushInterruptionTrace('codex_stream_error') + throw error } finally { - if (!streamComplete || signal?.aborted) { - readerCanceller.cancel(createStreamAbortError()) + const readerWasInterrupted = + (!transportComplete && !protocolComplete) || signal?.aborted + if (readerWasInterrupted) { + const causalEventId = + (signal && getInterruptionSignalAbortEventId(signal)) ?? + streamCausalEventId + traceInterruptionEvent('codex_stream.cancelled', { + subsystem: 'codex_stream', + transport: 'codex_responses', + reason: signal?.reason, + causalEventId, + rawByteCount, + parsedFrameCount, + controlFrameCount, + ignoredFrameCount, + }) } + if (!transportComplete) readerCanceller.cancel(createStreamAbortError()) readerCanceller.cleanup() reader.releaseLock() + if (readerWasInterrupted) { + flushInterruptionTrace('codex_stream_reader_closed') + } } } @@ -882,10 +982,11 @@ export async function collectCodexCompletedResponse( return completedResponse } -export async function* codexStreamToAnthropic( +async function* codexStreamToAnthropicWithReadOptions( response: Response, model: string, signal?: AbortSignal, + readOptions: CodexStreamReadOptions = {}, ): AsyncGenerator { const messageId = makeMessageId() const toolBlocksByItemId = new Map< @@ -898,6 +999,10 @@ export async function* codexStreamToAnthropic( let sawToolUse = false let finalResponse: Record | undefined let streamComplete = false + let providerTerminalObserved = false + let converterFailed = false + let streamCausalEventId: string | undefined + let ignoredParsedFrameCount = 0 const cancelResponseBody = () => { void response.body?.cancel(createStreamAbortError()).catch(() => {}) } @@ -953,7 +1058,13 @@ export async function* codexStreamToAnthropic( }, } - for await (const event of readSseEvents(response, signal)) { + for await (const event of readSseEvents(response, signal, { + ...readOptions, + onCausalEventId: eventId => { + streamCausalEventId = eventId + readOptions.onCausalEventId?.(eventId) + }, + })) { throwIfStreamAborted(signal) const payload = event.data @@ -1113,15 +1224,26 @@ export async function* codexStreamToAnthropic( event.event === 'response.completed' || event.event === 'response.incomplete' ) { + providerTerminalObserved = true finalResponse = payload.response break } if (event.event === 'response.failed') { + providerTerminalObserved = true const msg = payload?.response?.error?.message ?? payload?.error?.message ?? 'Codex response failed' throw APIError.generate(500, undefined, msg, new Headers()) } + + ignoredParsedFrameCount++ + if (ignoredParsedFrameCount === 1) { + traceInterruptionEvent('codex_stream.first_parsed_frame_ignored', { + subsystem: 'codex_stream', + transport: 'codex_responses', + ignoredParsedFrameCount, + }) + } } throwIfStreamAborted(signal) @@ -1151,16 +1273,60 @@ export async function* codexStreamToAnthropic( ), } throwIfStreamAborted(signal) - yield { type: 'message_stop' } streamComplete = true + yield { type: 'message_stop' } + } catch (error) { + converterFailed = true + throw error } finally { - if (!streamComplete || signal?.aborted) { + const terminalComplete = providerTerminalObserved && !converterFailed + traceInterruptionEvent('codex_stream.converter_closed', { + subsystem: 'codex_stream', + transport: 'codex_responses', + outcome: signal?.aborted + ? 'root_aborted' + : converterFailed + ? 'failed' + : streamComplete || terminalComplete + ? 'complete' + : 'incomplete', + reason: signal?.reason, + causalEventId: + (signal && getInterruptionSignalAbortEventId(signal)) ?? + streamCausalEventId, + ignoredParsedFrameCount, + }) + if ((!streamComplete && !terminalComplete) || signal?.aborted) { cancelResponseBody() + flushInterruptionTrace('codex_stream_converter_closed') } signal?.removeEventListener('abort', cancelResponseBody) } } +export function codexStreamToAnthropic( + response: Response, + model: string, + signal?: AbortSignal, +): AsyncGenerator { + return codexStreamToAnthropicWithReadOptions(response, model, signal) +} + +/** Deterministic reader-deadline seam for interruption regressions only. */ +export function __codexStreamToAnthropicForTests( + response: Response, + model: string, + signal: AbortSignal | undefined, + readOptions: CodexStreamReadOptions, +): AsyncGenerator { + return codexStreamToAnthropicWithReadOptions( + response, + model, + signal, + readOptions, + ) +} + export function convertCodexResponseToAnthropicMessage( data: Record, model: string, diff --git a/src/services/api/openaiShim.test.ts b/src/services/api/openaiShim.test.ts index 256f425016..cc54441ec4 100644 --- a/src/services/api/openaiShim.test.ts +++ b/src/services/api/openaiShim.test.ts @@ -6,6 +6,10 @@ import { asMockFetch } from '../../test/typedMocks.js' import { _clearRegistryForTesting, ensureIntegrationsLoaded, registerGateway } from '../../integrations/index.ts' import { applyProviderFlag } from '../../utils/providerFlag.ts' import { applyProviderProfileToProcessEnv } from '../../utils/providerProfiles.ts' +import { + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, +} from '../../utils/interruptionTrace.js' import { getAssistantMessageFromError, OPENCODE_GO_FREE_LIMIT_ERROR_MESSAGE, @@ -4751,6 +4755,49 @@ test('caller abort winning the timeout catch race prevents a retry', async () => expect(fetchCalls).toBe(1) }) +test('interruption tracing preserves the native AbortSignal.any request path', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + const originalAbortSignalAny = Object.getOwnPropertyDescriptor( + AbortSignal, + 'any', + ) + const nativeAny = AbortSignal.any.bind(AbortSignal) + let nativeAnyCalls = 0 + Object.defineProperty(AbortSignal, 'any', { + configurable: true, + value: (signals: AbortSignal[]) => { + nativeAnyCalls++ + return nativeAny(signals) + }, + }) + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + globalThis.fetch = asMockFetch( + mock(async () => makeChatCompletionResponse('gpt-4o-mini')), + ) + + try { + const client = createOpenAIShimClient({}) as OpenAIShimClient + await client.beta.messages.create( + { + model: 'gpt-4o-mini', + messages: [{ role: 'user', content: 'hello' }], + max_tokens: 64, + stream: false, + }, + { signal: new AbortController().signal }, + ) + expect(nativeAnyCalls).toBe(1) + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + if (originalAbortSignalAny) { + Object.defineProperty(AbortSignal, 'any', originalAbortSignalAny) + } + } +}) + test('manual signal fallback preserves caller cancellation after headers arrive', async () => { process.env.API_TIMEOUT_MS = '200' const fetchSignals: AbortSignal[] = [] diff --git a/src/services/api/openaiShim/clientDispatch.test.ts b/src/services/api/openaiShim/clientDispatch.test.ts index 4c5f335437..d744c2d2ef 100644 --- a/src/services/api/openaiShim/clientDispatch.test.ts +++ b/src/services/api/openaiShim/clientDispatch.test.ts @@ -1,4 +1,13 @@ import { expect, test } from 'bun:test' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../../../test/sharedMutationLock.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, +} from '../../../utils/interruptionTrace.js' import type { AnthropicStreamEvent, ShimCreateParams } from '../codexShim.js' import { createShimRequest, @@ -107,6 +116,47 @@ test('OpenAIShimStream combines parent and controller cancellation', async () => expect(receivedSignal?.aborted).toBe(true) }) +test('OpenAIShimStream records its parent signal relationship', async () => { + await acquireSharedMutationLock('openaiShim-clientDispatch-parent-trace') + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const parent = new AbortController() + let stream: OpenAIShimStream | undefined + try { + stream = new OpenAIShimStream(async function* () { + yield { type: 'unused' } + }, parent.signal) + const trace = __getInterruptionTraceSnapshotForTests() + const parentRegistration = trace.find( + entry => + entry.event === 'signal.registered' && + entry.controllerRole === 'combined-parent', + ) + const streamRegistration = trace.find( + entry => + entry.event === 'controller.registered' && + entry.controllerRole === 'stream-controller', + ) + expect(parentRegistration).toBeDefined() + expect(streamRegistration).toBeDefined() + if (!parentRegistration?.controllerId || !streamRegistration) { + throw new Error('missing interruption controller registration') + } + expect(streamRegistration.parentControllerIds).toEqual([ + parentRegistration.controllerId, + ]) + } finally { + stream?.controller.abort('test-cleanup') + parent.abort('test-cleanup') + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + releaseSharedMutationLock() + } +}) + test('OpenAIShimStream cancels the response before iteration starts', () => { let cancellations = 0 const stream = new OpenAIShimStream( @@ -123,12 +173,61 @@ test('OpenAIShimStream cancels the response before iteration starts', () => { }) test('OpenAIShimStream aborts its controller when a consumer returns early', async () => { + await acquireSharedMutationLock('openaiShim-clientDispatch-closure-trace') + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() const stream = new OpenAIShimStream(async function* () { yield { type: 'first' } yield { type: 'second' } }) - for await (const _event of stream) break - expect(stream.controller.signal.aborted).toBe(true) + try { + for await (const _event of stream) break + expect(stream.controller.signal.aborted).toBe(true) + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'abort.requested', + ), + ).toMatchObject({ source: 'iterator_closed' }) + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + releaseSharedMutationLock() + } +}) + +test('OpenAIShimStream does not relabel a provider exception as consumer closure', async () => { + await acquireSharedMutationLock('openaiShim-clientDispatch-failure-trace') + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const stream = new OpenAIShimStream(async function* () { + yield { type: 'message_start' } + throw new Error('provider generator failed') + }) + + try { + const iterator = stream[Symbol.asyncIterator]() + expect((await iterator.next()).done).toBe(false) + await expect(iterator.next()).rejects.toThrow('provider generator failed') + expect(stream.controller.signal.aborted).toBe(false) + expect( + __getInterruptionTraceSnapshotForTests().some( + entry => + entry.event === 'abort.requested' && + entry.source === 'iterator_closed', + ), + ).toBe(false) + } finally { + stream.controller.abort('test-cleanup') + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + releaseSharedMutationLock() + } }) test.each([ diff --git a/src/services/api/openaiShim/clientDispatch.ts b/src/services/api/openaiShim/clientDispatch.ts index 014d9fc73d..539698da73 100644 --- a/src/services/api/openaiShim/clientDispatch.ts +++ b/src/services/api/openaiShim/clientDispatch.ts @@ -1,5 +1,10 @@ import { APIError } from '@anthropic-ai/sdk' import { createCombinedAbortSignal } from '../../../utils/combinedAbortSignal.js' +import { + registerInterruptionController, + registerInterruptionSignal, + requestAbort, +} from '../../../utils/interruptionTrace.js' import type { AnthropicStreamEvent, ShimCreateParams } from '../codexShim.js' import { isLikelyOllamaEndpoint, @@ -90,6 +95,17 @@ export class OpenAIShimStream implements AsyncIterable { ) { this.makeGenerator = makeGenerator this.parentSignal = parentSignal + const parentId = parentSignal + ? registerInterruptionSignal(parentSignal, { + subsystem: 'openai_shim_dispatch', + controllerRole: 'combined-parent', + }) + : undefined + registerInterruptionController(this.controller, { + subsystem: 'openai_shim_dispatch', + controllerRole: 'stream-controller', + ...(parentId && { parentControllerIds: [parentId] }), + }) if (cancelBeforeIteration) { let cleaned = false @@ -129,6 +145,10 @@ export class OpenAIShimStream implements AsyncIterable { const combined = createCombinedAbortSignal(this.parentSignal, { signalB: this.controller.signal, + trace: { + subsystem: 'openai_shim_dispatch', + controllerRole: 'stream-combined', + }, }) this.cleanupCombinedSignal = combined.cleanup this.generator = this.makeGenerator(combined.signal) @@ -138,12 +158,20 @@ export class OpenAIShimStream implements AsyncIterable { async *[Symbol.asyncIterator](): AsyncGenerator { const generator = this.getGenerator() let completed = false + let failed = false try { yield* generator completed = true + } catch (error) { + failed = true + throw error } finally { - if (!completed && !this.controller.signal.aborted) { - this.controller.abort() + if (!completed && !failed && !this.controller.signal.aborted) { + requestAbort(this.controller, undefined, { + source: 'iterator_closed', + subsystem: 'openai_shim_dispatch', + controllerRole: 'stream-controller', + }) } this.cleanupCombinedSignal?.() this.cleanupCombinedSignal = undefined diff --git a/src/services/api/openaiShim/geminiStreamConversion.test.ts b/src/services/api/openaiShim/geminiStreamConversion.test.ts index 78b56d94dc..fcd4ee05bc 100644 --- a/src/services/api/openaiShim/geminiStreamConversion.test.ts +++ b/src/services/api/openaiShim/geminiStreamConversion.test.ts @@ -1,5 +1,6 @@ import { expect, test } from 'bun:test' import { + createProviderStreamTrace, createReaderCanceller, createStreamAbortError, readWithIdleTimeout, @@ -8,6 +9,7 @@ import { import { geminiSseToAnthropic } from './geminiStreamConversion.js' const dependencies = { + createProviderStreamTrace, createReaderCanceller, createStreamAbortError, getStreamIdleTimeoutMs: () => 1_000, diff --git a/src/services/api/openaiShim/geminiStreamConversion.ts b/src/services/api/openaiShim/geminiStreamConversion.ts index b245433612..fb70fccce1 100644 --- a/src/services/api/openaiShim/geminiStreamConversion.ts +++ b/src/services/api/openaiShim/geminiStreamConversion.ts @@ -1,6 +1,7 @@ import { buildAnthropicUsageFromRawUsage } from '../cacheMetrics.js' import type { AnthropicStreamEvent, AnthropicUsage } from '../codexShim.js' import { logForDebugging } from '../../../utils/debug.js' +import type { createProviderStreamTrace } from './streamControl.js' type ReaderCanceller = { cancel(error?: unknown): void @@ -16,6 +17,7 @@ export type GeminiStreamDependencies = { reader: ReadableStreamDefaultReader, signal?: AbortSignal, ): ReaderCanceller + createProviderStreamTrace: typeof createProviderStreamTrace createStreamAbortError(): DOMException getStreamIdleTimeoutMs(): number makeMessageId(): string @@ -25,7 +27,7 @@ export type GeminiStreamDependencies = { options?: { signal?: AbortSignal cancelReader?: (error?: unknown) => void - onTimeout?: () => void + onTimeout?: (error: unknown) => void }, ): Promise throwIfStreamAborted(signal?: AbortSignal): void @@ -39,6 +41,7 @@ export async function* geminiSseToAnthropic( ): AsyncGenerator { const { createReaderCanceller, + createProviderStreamTrace, createStreamAbortError, getStreamIdleTimeoutMs, makeMessageId, @@ -59,8 +62,11 @@ export async function* geminiSseToAnthropic( let usage: Partial | undefined let finishReason: string | undefined const streamIdleTimeoutMs = getStreamIdleTimeoutMs() - let lastDataTime = Date.now() + let lastDataTime = performance.now() let streamComplete = false + let protocolComplete = false + let readerFailed = false + const streamTrace = createProviderStreamTrace('gemini_sse') const emitMessageStart = async function* () { if (hasEmittedStart) return @@ -98,20 +104,23 @@ export async function* geminiSseToAnthropic( { signal, cancelReader: readerCanceller.cancel, - onTimeout: () => { - const elapsed = Math.round((Date.now() - lastDataTime) / 1000) + onTimeout: error => { + const elapsed = Math.round((performance.now() - lastDataTime) / 1000) logForDebugging( `Gemini SSE stream idle for ${elapsed}s (limit: ${streamIdleTimeoutMs / 1000}s). Connection likely dropped.`, { level: 'error' }, ) + streamTrace.recordIdleTimeout(error) }, }, ) if (done) { streamComplete = true + streamTrace.recordEof() break } - if (value) lastDataTime = Date.now() + if (value) lastDataTime = performance.now() + streamTrace.recordRaw(value) throwIfStreamAborted(signal) buffer += decoder.decode(value, { stream: true }) @@ -122,10 +131,15 @@ export async function* geminiSseToAnthropic( throwIfStreamAborted(signal) const lines = chunk.split('\n').map(line => line.trim()).filter(Boolean) const dataLines = lines.filter(line => line.startsWith('data: ')) - if (dataLines.length === 0) continue + if (dataLines.length === 0) { + streamTrace.recordControl() + continue + } const rawData = dataLines.map(line => line.slice(6)).join('\n') if (rawData === '[DONE]') { + streamTrace.recordControl() + protocolComplete = true yield* emitMessageStart() if (hasEmittedTextStart || hasEmittedCurrentTool) { throwIfStreamAborted(signal) @@ -149,8 +163,15 @@ export async function* geminiSseToAnthropic( let parsed: Record try { - parsed = JSON.parse(rawData) as Record + const value: unknown = JSON.parse(rawData) + if (!value || typeof value !== 'object' || Array.isArray(value)) { + streamTrace.recordIgnored() + } else { + streamTrace.recordParsed() + } + parsed = value as Record } catch { + streamTrace.recordIgnored() continue } @@ -173,6 +194,7 @@ export async function* geminiSseToAnthropic( const candidate = candidates[0] if (typeof candidate.finishReason === 'string') { finishReason = candidate.finishReason + protocolComplete = true } const content = candidate.content as @@ -267,8 +289,22 @@ export async function* geminiSseToAnthropic( } throwIfStreamAborted(signal) yield { type: 'message_stop' } - streamComplete = true + } catch (error) { + readerFailed = true + throw error } finally { + streamTrace.recordClosed( + signal?.aborted + ? 'root_aborted' + : readerFailed + ? 'failed' + : protocolComplete + ? 'complete' + : streamComplete + ? 'eof_without_terminal' + : 'incomplete', + signal, + ) if (!streamComplete || signal?.aborted) { readerCanceller.cancel(createStreamAbortError()) } diff --git a/src/services/api/openaiShim/providerStreamInterruptionTrace.test.ts b/src/services/api/openaiShim/providerStreamInterruptionTrace.test.ts new file mode 100644 index 0000000000..2eb967da43 --- /dev/null +++ b/src/services/api/openaiShim/providerStreamInterruptionTrace.test.ts @@ -0,0 +1,366 @@ +import { afterAll, afterEach, beforeAll, expect, test } from 'bun:test' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../../../test/sharedMutationLock.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, +} from '../../../utils/interruptionTrace.js' +import type { AnthropicStreamEvent } from '../codexShim.js' +import { + anthropicSsePassthrough, + geminiSseToAnthropic, + openaiStreamToAnthropic, +} from './responseAdapters.js' + +const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE +const originalIdleTimeout = process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS + +beforeAll(async () => { + await acquireSharedMutationLock('providerStreamInterruptionTrace.test.ts') + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' +}) + +afterEach(() => { + if (originalIdleTimeout === undefined) { + delete process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS + } else { + process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = originalIdleTimeout + } +}) + +afterAll(() => { + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + if (originalIdleTimeout === undefined) { + delete process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS + } else { + process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = originalIdleTimeout + } + releaseSharedMutationLock() +}) + +function responseFromText(text: string): Response { + const encoder = new TextEncoder() + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(text)) + controller.close() + }, + }), + { headers: { 'content-type': 'text/event-stream' } }, + ) +} + +function openResponseFromText( + text: string, + onCancel?: () => void, +): Response { + const encoder = new TextEncoder() + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(text)) + }, + cancel() { + onCancel?.() + }, + }), + { headers: { 'content-type': 'text/event-stream' } }, + ) +} + +async function collect( + stream: AsyncIterable, +): Promise { + const events: AnthropicStreamEvent[] = [] + for await (const event of stream) events.push(event) + return events +} + +test('all non-Codex readers distinguish raw, parsed, control, and ignored frames', async () => { + const cases = [ + { + transport: 'openai_chat_completions', + text: [ + ': keepalive', + 'data: not-json', + 'data: []', + 'data: {"choices":[{"delta":{"content":"hello"},"finish_reason":"stop"}]}', + 'data: [DONE]', + '', + ].join('\n'), + expected: { parsed: 1, control: 2, ignored: 2 }, + stream: (text: string) => + openaiStreamToAnthropic(responseFromText(text), 'test-model'), + }, + { + transport: 'gemini_sse', + text: [ + ': keepalive', + '', + 'data: not-json', + '', + 'data: []', + '', + 'data: {"candidates":[{"content":{"parts":[{"text":"hello"}]},"finishReason":"STOP"}]}', + '', + 'data: [DONE]', + '', + '', + ].join('\n'), + expected: { parsed: 1, control: 2, ignored: 2 }, + stream: (text: string) => + geminiSseToAnthropic(responseFromText(text), 'gemini-test'), + }, + { + transport: 'anthropic_messages', + text: [ + ': keepalive', + '', + 'data: not-json', + '', + 'data: {}', + '', + 'data: {"type":"message_stop"}', + '', + 'data: [DONE]', + '', + '', + ].join('\n'), + expected: { parsed: 1, control: 2, ignored: 2 }, + stream: (text: string) => + anthropicSsePassthrough(responseFromText(text), 'claude-test'), + }, + ] + + for (const scenario of cases) { + __resetInterruptionTraceForTests() + await collect(scenario.stream(scenario.text)) + + const closed = __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'provider_stream.reader_closed' && + entry.transport === scenario.transport, + ) + expect(closed).toMatchObject({ + outcome: 'complete', + rawByteCount: new TextEncoder().encode(scenario.text).byteLength, + parsedFrameCount: scenario.expected.parsed, + controlFrameCount: scenario.expected.control, + ignoredFrameCount: scenario.expected.ignored, + }) + } +}) + +for (const traceEnabled of [false, true]) { + test(`OpenAI-compatible and Gemini null payloads fail with tracing ${traceEnabled ? 'enabled' : 'disabled'}`, async () => { + const previousTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + try { + if (traceEnabled) process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + else delete process.env.OPENCLAUDE_INTERRUPT_TRACE + + const cases = [ + { + transport: 'openai_chat_completions', + stream: () => + openaiStreamToAnthropic( + responseFromText('data: null\n\ndata: [DONE]\n\n'), + 'test-model', + ), + }, + { + transport: 'gemini_sse', + stream: () => + geminiSseToAnthropic( + responseFromText('data: null\n\ndata: [DONE]\n\n'), + 'gemini-test', + ), + }, + ] + + for (const scenario of cases) { + __resetInterruptionTraceForTests() + await expect(collect(scenario.stream())).rejects.toBeInstanceOf(TypeError) + const snapshot = __getInterruptionTraceSnapshotForTests() + if (!traceEnabled) { + expect(snapshot).toEqual([]) + continue + } + expect( + snapshot.find( + entry => + entry.event === 'provider_stream.reader_closed' && + entry.transport === scenario.transport, + ), + ).toMatchObject({ + outcome: 'failed', + ignoredFrameCount: 1, + }) + } + } finally { + __resetInterruptionTraceForTests() + if (previousTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = previousTrace + } + }) +} + +test('Anthropic terminal consumer closure cancels an open response body', async () => { + __resetInterruptionTraceForTests() + let cancellations = 0 + const iterator = anthropicSsePassthrough( + openResponseFromText( + 'data: {"type":"message_stop"}\n\n', + () => { + cancellations++ + }, + ), + 'claude-test', + )[Symbol.asyncIterator]() + + expect((await iterator.next()).value?.type).toBe('message_stop') + await iterator.return?.(undefined) + expect(cancellations).toBe(1) +}) + +test('all non-Codex readers report a failure after terminal evidence then timeout', async () => { + process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = '20' + const cases = [ + { + transport: 'openai_chat_completions', + stream: () => + openaiStreamToAnthropic( + openResponseFromText( + 'data: {"choices":[{"delta":{"content":"hello"},"finish_reason":"stop"}]}\n', + ), + 'test-model', + ), + }, + { + transport: 'gemini_sse', + stream: () => + geminiSseToAnthropic( + openResponseFromText( + 'data: {"candidates":[{"content":{"parts":[{"text":"hello"}]},"finishReason":"STOP"}]}\n\n', + ), + 'gemini-test', + ), + }, + { + transport: 'anthropic_messages', + stream: () => + anthropicSsePassthrough( + openResponseFromText('data: {"type":"message_stop"}\n\n'), + 'claude-test', + ), + }, + ] + + for (const scenario of cases) { + __resetInterruptionTraceForTests() + await expect(collect(scenario.stream())).rejects.toBeDefined() + const closed = __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'provider_stream.reader_closed' && + entry.transport === scenario.transport, + ) + expect(closed?.outcome).toBe('failed') + } +}) + +test('all non-Codex readers distinguish transport EOF from protocol completion', async () => { + const cases = [ + { + transport: 'openai_chat_completions', + stream: () => + openaiStreamToAnthropic( + responseFromText( + 'data: {"choices":[{"delta":{"content":"hello"},"finish_reason":null}]}\n', + ), + 'test-model', + ), + }, + { + transport: 'gemini_sse', + stream: () => + geminiSseToAnthropic( + responseFromText( + 'data: {"candidates":[{"content":{"parts":[{"text":"hello"}]}}]}\n\n', + ), + 'gemini-test', + ), + }, + { + transport: 'anthropic_messages', + stream: () => + anthropicSsePassthrough( + responseFromText( + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}\n\n', + ), + 'claude-test', + ), + }, + ] + + for (const scenario of cases) { + __resetInterruptionTraceForTests() + await collect(scenario.stream()) + + const closed = __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'provider_stream.reader_closed' && + entry.transport === scenario.transport, + ) + expect(closed?.outcome).toBe('eof_without_terminal') + } +}) + +test('all non-Codex reader idle errors carry the provider timeout causal event', async () => { + process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = '20' + const cases = [ + { + transport: 'openai_chat_completions', + stream: (response: Response) => + openaiStreamToAnthropic(response, 'test-model'), + }, + { + transport: 'gemini_sse', + stream: (response: Response) => + geminiSseToAnthropic(response, 'gemini-test'), + }, + { + transport: 'anthropic_messages', + stream: (response: Response) => + anthropicSsePassthrough(response, 'claude-test'), + }, + ] + + for (const scenario of cases) { + __resetInterruptionTraceForTests() + const response = new Response(new ReadableStream()) + let caught: unknown + try { + await collect(scenario.stream(response)) + } catch (error) { + caught = error + } + + const idleTimeout = __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'provider_stream.idle_timeout' && + entry.transport === scenario.transport, + ) + const readerClosed = __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'provider_stream.reader_closed' && + entry.transport === scenario.transport, + ) + expect(caught).toBeDefined() + expect(idleTimeout).toBeDefined() + expect(readerClosed?.causalEventId).toBe(idleTimeout?.eventId) + } +}) diff --git a/src/services/api/openaiShim/responseAdapters.ts b/src/services/api/openaiShim/responseAdapters.ts index fc626183f2..de8850acf0 100644 --- a/src/services/api/openaiShim/responseAdapters.ts +++ b/src/services/api/openaiShim/responseAdapters.ts @@ -25,6 +25,7 @@ import { openaiStreamToAnthropic as convertOpenAIStream } from './streamConversi import { geminiSseToAnthropic as convertGeminiStream } from './geminiStreamConversion.js' import { anthropicSsePassthrough as parseAnthropicSsePassthrough, + createProviderStreamTrace, createReaderCanceller, createStreamAbortError, getStreamIdleTimeoutMs, @@ -93,6 +94,7 @@ export async function* geminiSseToAnthropic( signal?: AbortSignal, ): AsyncGenerator { yield* convertGeminiStream(response, model, signal, { + createProviderStreamTrace, createReaderCanceller, createStreamAbortError, getStreamIdleTimeoutMs, @@ -135,6 +137,7 @@ export async function* openaiStreamToAnthropic( streamModel, ), couldBeRawToolCallsRequestedPrefix, + createProviderStreamTrace, createReaderCanceller, createStreamAbortError, findXmlToolCallOpener, diff --git a/src/services/api/openaiShim/streamControl.test.ts b/src/services/api/openaiShim/streamControl.test.ts index 02c8fdb4ad..4069d4e939 100644 --- a/src/services/api/openaiShim/streamControl.test.ts +++ b/src/services/api/openaiShim/streamControl.test.ts @@ -142,6 +142,28 @@ test('readWithIdleTimeout settles when a custom canceller throws synchronously', reader.releaseLock() }) +test('readWithIdleTimeout contains a throwing custom error factory', async () => { + const cancelReasons: unknown[] = [] + const reader = new ReadableStream({ + cancel(reason) { + cancelReasons.push(reason) + }, + }).getReader() + + await expect( + withDeadline( + readWithIdleTimeout(reader, 20, { + createTimeoutError: () => { + throw new Error('factory failed') + }, + }), + 'idle timeout did not reject within 500ms', + ), + ).rejects.toBeInstanceOf(StreamIdleTimeoutError) + expect(cancelReasons).toHaveLength(1) + expect(cancelReasons[0]).toBeInstanceOf(StreamIdleTimeoutError) +}) + test('stream idle timeout parser validates and bounds overrides', () => { expect(getStreamIdleTimeoutMs()).toBe(90_000) process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = '25' diff --git a/src/services/api/openaiShim/streamControl.ts b/src/services/api/openaiShim/streamControl.ts index 755588e56e..20ec21cf22 100644 --- a/src/services/api/openaiShim/streamControl.ts +++ b/src/services/api/openaiShim/streamControl.ts @@ -1,3 +1,10 @@ +import { + flushInterruptionTrace, + getInterruptionSignalAbortEventId, + setInterruptionErrorCausalEventId, + traceInterruptionEvent, +} from '../../../utils/interruptionTrace.js' + const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 90_000 const MAX_STREAM_IDLE_TIMEOUT_MS = 2_147_483_647 @@ -57,13 +64,79 @@ export function getStreamIdleTimeoutMs(): number { : DEFAULT_STREAM_IDLE_TIMEOUT_MS } +export function createProviderStreamTrace(transport: string) { + let lastRawByteTime = performance.now() + let lastParsedFrameTime = lastRawByteTime + let rawByteCount = 0 + let parsedFrameCount = 0 + let controlFrameCount = 0 + let ignoredFrameCount = 0 + let causalEventId: string | undefined + + const fields = () => ({ + subsystem: 'provider_stream', + transport, + rawByteCount, + parsedFrameCount, + controlFrameCount, + ignoredFrameCount, + }) + + traceInterruptionEvent('provider_stream.read_started', fields()) + + return { + recordRaw(value: Uint8Array | undefined): void { + if (!value || value.byteLength === 0) return + lastRawByteTime = performance.now() + rawByteCount += value.byteLength + if (rawByteCount === value.byteLength) { + traceInterruptionEvent('provider_stream.first_raw_byte', fields()) + } + }, + recordParsed(): void { + lastParsedFrameTime = performance.now() + parsedFrameCount++ + }, + recordControl(): void { + controlFrameCount++ + }, + recordIgnored(): void { + ignoredFrameCount++ + }, + recordIdleTimeout(error: unknown): void { + const now = performance.now() + causalEventId = traceInterruptionEvent('provider_stream.idle_timeout', { + ...fields(), + sinceLastRawByteMs: now - lastRawByteTime, + sinceLastParsedFrameMs: now - lastParsedFrameTime, + }) + setInterruptionErrorCausalEventId(error, causalEventId) + flushInterruptionTrace('provider_stream_idle_timeout') + }, + recordEof(): void { + traceInterruptionEvent('provider_stream.eof', fields()) + }, + recordClosed(outcome: string, signal?: AbortSignal): void { + traceInterruptionEvent('provider_stream.reader_closed', { + ...fields(), + outcome, + reason: signal?.reason, + causalEventId: + (signal && getInterruptionSignalAbortEventId(signal)) ?? + causalEventId, + }) + }, + } +} + export async function readWithIdleTimeout( reader: ReadableStreamDefaultReader, timeoutMs: number, options: { signal?: AbortSignal cancelReader?: (error?: unknown) => void - onTimeout?: () => void + onTimeout?: (error: unknown) => void + createTimeoutError?: () => unknown } = {}, ): Promise { const signal = options.signal @@ -111,9 +184,15 @@ export async function readWithIdleTimeout( } timeoutId = setTimeout(() => { - const error = new StreamIdleTimeoutError(timeoutMs) + let error: unknown + try { + error = options.createTimeoutError?.() + } catch { + // Fall back to the standard timeout error. + } + error ??= new StreamIdleTimeoutError(timeoutMs) try { - options.onTimeout?.() + options.onTimeout?.(error) } catch { // Ignore diagnostic callback failures. } @@ -140,27 +219,33 @@ export async function* anthropicSsePassthrough( const decoder = new TextDecoder() let buffer = '' const streamIdleTimeoutMs = getStreamIdleTimeoutMs() - let lastDataTime = Date.now() + let lastDataTime = performance.now() let streamComplete = false + let protocolComplete = false + let readerFailed = false + const streamTrace = createProviderStreamTrace('anthropic_messages') try { while (true) { const { done, value } = await readWithIdleTimeout(reader, streamIdleTimeoutMs, { signal, cancelReader: readerCanceller.cancel, - onTimeout: () => { - const elapsed = Math.round((Date.now() - lastDataTime) / 1000) + onTimeout: error => { + const elapsed = Math.round((performance.now() - lastDataTime) / 1000) logForDebugging( `Anthropic-compatible SSE stream idle for ${elapsed}s (limit: ${streamIdleTimeoutMs / 1000}s). Connection likely dropped.`, { level: 'error' }, ) + streamTrace.recordIdleTimeout(error) }, }) if (done) { streamComplete = true + streamTrace.recordEof() buffer += decoder.decode() } else { - if (value) lastDataTime = Date.now() + if (value) lastDataTime = performance.now() + streamTrace.recordRaw(value) throwIfStreamAborted(signal) buffer += decoder.decode(value, { stream: true }) } @@ -170,10 +255,14 @@ export async function* anthropicSsePassthrough( throwIfStreamAborted(signal) const lines = chunk.split(/\r\n|\n|\r/).map(line => line.trim()).filter(Boolean) const dataLines = lines.filter(line => line.startsWith('data:')) - if (dataLines.length === 0) continue + if (dataLines.length === 0) { + streamTrace.recordControl() + continue + } const rawData = dataLines.map(line => line.slice(5).replace(/^ /, '')).join('\n') if (rawData === '[DONE]') { - streamComplete = true + streamTrace.recordControl() + protocolComplete = true readerCanceller.cancel() return } @@ -182,17 +271,41 @@ export async function* anthropicSsePassthrough( parsed = JSON.parse(rawData) as T } catch { // Ignore malformed frames and continue parsing later frames. + streamTrace.recordIgnored() continue } if (parsed && typeof parsed === 'object' && 'type' in parsed) { + streamTrace.recordParsed() + if ((parsed as { type?: unknown }).type === 'message_stop') { + protocolComplete = true + } throwIfStreamAborted(signal) yield parsed as Awaited + } else { + streamTrace.recordIgnored() } } if (done) break } + } catch (error) { + readerFailed = true + throw error } finally { - if (!streamComplete || signal?.aborted) readerCanceller.cancel(createStreamAbortError()) + streamTrace.recordClosed( + signal?.aborted + ? 'root_aborted' + : readerFailed + ? 'failed' + : protocolComplete + ? 'complete' + : streamComplete + ? 'eof_without_terminal' + : 'incomplete', + signal, + ) + if (!streamComplete || signal?.aborted) { + readerCanceller.cancel(createStreamAbortError()) + } readerCanceller.cleanup() reader.releaseLock() } diff --git a/src/services/api/openaiShim/streamConversion.test.ts b/src/services/api/openaiShim/streamConversion.test.ts index 8281e6da6c..d341e060a5 100644 --- a/src/services/api/openaiShim/streamConversion.test.ts +++ b/src/services/api/openaiShim/streamConversion.test.ts @@ -9,6 +9,7 @@ import { geminiSseToAnthropic, type GeminiStreamDependencies, } from './geminiStreamConversion.js' +import { createProviderStreamTrace } from './streamControl.js' function makeSseResponse(frames: unknown[]): Response { const encoder = new TextEncoder() @@ -77,6 +78,7 @@ async function readWithSignal( } const commonControlDependencies: GeminiStreamDependencies = { + createProviderStreamTrace, createReaderCanceller, createStreamAbortError: () => new DOMException('Aborted', 'AbortError'), getStreamIdleTimeoutMs: () => 1_000, diff --git a/src/services/api/openaiShim/streamConversion.ts b/src/services/api/openaiShim/streamConversion.ts index 9a4be7626b..a709ca5304 100644 --- a/src/services/api/openaiShim/streamConversion.ts +++ b/src/services/api/openaiShim/streamConversion.ts @@ -16,6 +16,7 @@ import { getStreamStats, processStreamChunk, } from '../../../utils/streamingOptimizer.js' +import type { createProviderStreamTrace } from './streamControl.js' type ParsedRawToolCall = { id: string; name: string; argumentsJson: string } type ParsedTextToolCall = { id: string; name: string; arguments: unknown } @@ -65,6 +66,7 @@ export type StreamConversionDependencies = { reader: ReadableStreamDefaultReader, signal?: AbortSignal, ): ReaderCanceller + createProviderStreamTrace: typeof createProviderStreamTrace createStreamAbortError(): DOMException findXmlToolCallOpener(text: string, allowHy3: boolean): number geminiThoughtSignatureFromExtraContent(extraContent: unknown): string | undefined @@ -85,7 +87,7 @@ export type StreamConversionDependencies = { options?: { signal?: AbortSignal cancelReader?: (error?: unknown) => void - onTimeout?: () => void + onTimeout?: (error: unknown) => void }, ): Promise repairPossiblyTruncatedObjectJson(raw: string): string | null @@ -116,6 +118,7 @@ export async function* openaiStreamToAnthropic( convertNonStreamingResponseToAnthropicMessage, couldBeRawToolCallsRequestedPrefix, createReaderCanceller, + createProviderStreamTrace, createStreamAbortError, findXmlToolCallOpener, geminiThoughtSignatureFromExtraContent, @@ -154,6 +157,8 @@ export async function* openaiStreamToAnthropic( let lastStopReason: 'tool_use' | 'max_tokens' | 'end_turn' | null = null let hasEmittedFinalUsage = false let protocolComplete = false + let providerTerminalObserved = false + let readerFailed = false let hasProcessedFinishReason = false // Accumulated text for Ollama text-based tool call fallback parsing (#1053) let accumulatedText = '' @@ -299,8 +304,9 @@ export async function* openaiStreamToAnthropic( const decoder = new TextDecoder() let buffer = '' const streamIdleTimeoutMs = getStreamIdleTimeoutMs() - let lastDataTime = Date.now() + let lastDataTime = performance.now() let streamComplete = false + const streamTrace = createProviderStreamTrace('openai_chat_completions') const closeActiveContentBlock = async function* () { if (!hasEmittedContentStart) return @@ -463,19 +469,22 @@ export async function* openaiStreamToAnthropic( const { done, value } = await readWithIdleTimeout(reader, streamIdleTimeoutMs, { signal, cancelReader: readerCanceller.cancel, - onTimeout: () => { - const elapsed = Math.round((Date.now() - lastDataTime) / 1000) + onTimeout: error => { + const elapsed = Math.round((performance.now() - lastDataTime) / 1000) logForDebugging( `OpenAI-compatible SSE stream idle for ${elapsed}s (limit: ${streamIdleTimeoutMs / 1000}s). Connection likely dropped.`, { level: 'error' }, ) + streamTrace.recordIdleTimeout(error) }, }) if (done) { streamComplete = true + streamTrace.recordEof() break } - if (value) lastDataTime = Date.now() + if (value) lastDataTime = performance.now() + streamTrace.recordRaw(value) throwIfStreamAborted(signal) buffer += decoder.decode(value, { stream: true }) @@ -487,17 +496,32 @@ export async function* openaiStreamToAnthropic( const trimmed = line.trim() if (!trimmed) continue if (trimmed === 'data: [DONE]') { + streamTrace.recordControl() // `[DONE]` is protocol completion; do not wait for a transport EOF. // Leave streamComplete false so finally cancels an unread response body. protocolComplete = true break } - if (!trimmed.startsWith('data: ')) continue + if (!trimmed.startsWith('data: ')) { + if (trimmed.startsWith(':') || trimmed.startsWith('event:')) { + streamTrace.recordControl() + } else { + streamTrace.recordIgnored() + } + continue + } let chunk: OpenAIStreamChunk try { - chunk = JSON.parse(trimmed.slice(6)) + const parsed: unknown = JSON.parse(trimmed.slice(6)) + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + streamTrace.recordIgnored() + } else { + streamTrace.recordParsed() + } + chunk = parsed as OpenAIStreamChunk } catch { + streamTrace.recordIgnored() continue } @@ -752,6 +776,7 @@ export async function* openaiStreamToAnthropic( // Finish — guard ensures we only process finish_reason once even if // multiple chunks arrive with finish_reason set (some providers do this) if (choice.finish_reason && !hasProcessedFinishReason) { + providerTerminalObserved = true hasProcessedFinishReason = true // Close any open thinking block that wasn't closed by content transition @@ -1083,7 +1108,22 @@ export async function* openaiStreamToAnthropic( if (protocolComplete) break } yield* finalizeIncompleteStream() + } catch (error) { + readerFailed = true + throw error } finally { + streamTrace.recordClosed( + signal?.aborted + ? 'root_aborted' + : readerFailed + ? 'failed' + : protocolComplete || providerTerminalObserved + ? 'complete' + : streamComplete + ? 'eof_without_terminal' + : 'incomplete', + signal, + ) if (!streamComplete || signal?.aborted) { readerCanceller.cancel(createStreamAbortError()) } diff --git a/src/services/api/openaiShim/transport.ts b/src/services/api/openaiShim/transport.ts index 44ab700255..8bd4ab39aa 100644 --- a/src/services/api/openaiShim/transport.ts +++ b/src/services/api/openaiShim/transport.ts @@ -9,6 +9,16 @@ import { redactSecretSubstringsForDisplay, } from '../../../utils/providerSecrets.js' import { redactUrlForDisplay } from '../../../utils/redaction.js' +import { + getInterruptionSignalAbortEventId, + getInterruptionSignalId, + registerInterruptionController, + registerInterruptionSignal, + requestAbort, + traceCombinedAbortSignal, + traceCombinedSignal, + traceInterruptionEvent, +} from '../../../utils/interruptionTrace.js' import { buildOpenAICompatibilityErrorMessage, classifyOpenAINetworkFailure, @@ -82,28 +92,69 @@ function combineRequestSignals( } if (typeof AbortSignal.any === 'function') { + const signal = AbortSignal.any([callerSignal, deadlineSignal]) + traceCombinedAbortSignal(signal, [callerSignal, deadlineSignal], { + subsystem: 'openai_shim_transport', + controllerRole: 'request-combined', + }) return { - signal: AbortSignal.any([callerSignal, deadlineSignal]), + signal, cleanupAfterHeaders: () => {}, cleanup: () => {}, } } const combined = new AbortController() + const callerId = registerInterruptionSignal(callerSignal, { + subsystem: 'openai_shim_transport', + controllerRole: 'request-caller', + }) + const deadlineId = registerInterruptionSignal(deadlineSignal, { + subsystem: 'openai_shim_transport', + controllerRole: 'headers-deadline', + }) + const parentControllerIds = [callerId, deadlineId].filter( + (id): id is string => id !== undefined, + ) + traceCombinedSignal(combined, [callerSignal, deadlineSignal], { + subsystem: 'openai_shim_transport', + controllerRole: 'request-combined', + }) const abortFromCaller = () => { deadlineSignal.removeEventListener('abort', abortFromDeadline) - combined.abort(callerSignal.reason) + requestAbort(combined, callerSignal.reason, { + source: 'request_caller', + subsystem: 'openai_shim_transport', + controllerRole: 'request-combined', + parentControllerIds, + winningParentControllerId: getInterruptionSignalId(callerSignal), + causalEventId: getInterruptionSignalAbortEventId(callerSignal), + }) } const abortFromDeadline = () => { callerSignal.removeEventListener('abort', abortFromCaller) - combined.abort(deadlineSignal.reason) + requestAbort(combined, deadlineSignal.reason, { + source: 'headers_deadline', + subsystem: 'openai_shim_transport', + controllerRole: 'request-combined', + parentControllerIds, + winningParentControllerId: getInterruptionSignalId(deadlineSignal), + causalEventId: getInterruptionSignalAbortEventId(deadlineSignal), + }) } const cleanupAfterHeaders = () => { deadlineSignal.removeEventListener('abort', abortFromDeadline) } + let cleanedUp = false const cleanup = () => { + if (cleanedUp) return + cleanedUp = true callerSignal.removeEventListener('abort', abortFromCaller) cleanupAfterHeaders() + traceInterruptionEvent('combined_signal.cleanup', { + subsystem: 'openai_shim_transport', + controllerRole: 'request-combined', + }) } callerSignal.addEventListener('abort', abortFromCaller, { once: true }) @@ -187,6 +238,10 @@ export async function fetchWithHeadersDeadline( const redactedUrl = redactUrlForDiagnostics(url) const fetchWithAttemptDeadline: ProxyRetryFetcher = async (input, attemptInit) => { const deadlineController = new AbortController() + registerInterruptionController(deadlineController, { + subsystem: 'openai_shim_transport', + controllerRole: 'headers-deadline', + }) const timeoutReason = new ResponseHeadersTimeoutError( options.timeoutMs, redactedUrl, @@ -198,7 +253,12 @@ export async function fetchWithHeadersDeadline( cleanupAfterBody, } = combineRequestSignals(options.callerSignal, deadlineController.signal) const timer = setTimeout( - () => deadlineController.abort(timeoutReason), + () => + requestAbort(deadlineController, timeoutReason, { + source: 'headers_deadline_timer', + subsystem: 'openai_shim_transport', + controllerRole: 'headers-deadline', + }), options.timeoutMs, ) timer.unref?.() diff --git a/src/services/compact/compact.test.ts b/src/services/compact/compact.test.ts index 8e1d31891f..5b391c8210 100644 --- a/src/services/compact/compact.test.ts +++ b/src/services/compact/compact.test.ts @@ -7,6 +7,7 @@ import { expect, mock, test, + vi, } from 'bun:test' import { randomUUID } from 'crypto' import { unlink } from 'fs/promises' @@ -19,6 +20,11 @@ import { } from '../../test/sharedMutationLock.js' import type { Message } from '../../types/message.js' import * as realConfig from '../../utils/config.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, +} from '../../utils/interruptionTrace.js' // Several earlier test files in the smoke suite call // mock.module('../../utils/model/providers.js', ...) to stub getAPIProvider. @@ -815,6 +821,51 @@ afterAll(async () => { }) describe('compactConversation provider gate', () => { + test('attributes cache-sharing timeout aborts to the compact fork', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + vi.useFakeTimers() + try { + const runForkedAgent = mock(() => new Promise(() => {})) + const { compactConversation } = await importCompact({ runForkedAgent }) + const messages = [userMessage('Hello'), assistantMessage('Hi there!')] + const compactPromise = compactConversation( + messages, + toolUseContext(), + cacheSafeParams(messages), + false, + ) + + for (let attempt = 0; attempt < 10 && runForkedAgent.mock.calls.length === 0; attempt++) { + await Promise.resolve() + } + expect(runForkedAgent).toHaveBeenCalledTimes(1) + + vi.advanceTimersByTime(120_000) + await compactPromise + + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'abort.requested', + ), + ).toMatchObject({ + source: 'compact_timeout', + subsystem: 'compact', + controllerRole: 'compact-fork', + }) + } finally { + vi.useRealTimers() + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + } + }) + test('skips forked-agent cache-sharing for non-Anthropic providers', async () => { // Simulate a non-Anthropic provider (e.g. OpenAI) via env vars. // The real isAnthropicProvider() reads from process.env and returns false. diff --git a/src/services/compact/compact.ts b/src/services/compact/compact.ts index 0940ab410b..5357222aff 100644 --- a/src/services/compact/compact.ts +++ b/src/services/compact/compact.ts @@ -61,6 +61,7 @@ import { } from '../../utils/hooks.js' import { logError } from '../../utils/log.js' import { MEMORY_TYPE_VALUES } from '../../utils/memory/types.js' +import { requestAbort } from '../../utils/interruptionTrace.js' import { createCompactBoundaryMessage, createUserMessage, @@ -139,6 +140,14 @@ export const POST_COMPACT_SKILLS_TOKEN_BUDGET = 25_000 const MAX_COMPACT_STREAMING_RETRIES = 2 const COMPACT_TIMEOUT_MS = 120_000 +function requestCompactTimeoutAbort(controller: AbortController): void { + requestAbort(controller, undefined, { + source: 'compact_timeout', + subsystem: 'compact', + controllerRole: 'compact-fork', + }) +} + /** * Strip image blocks from user messages before sending for compaction. * Images are not needed for generating a conversation summary and can @@ -1268,7 +1277,10 @@ async function streamCompactSummary({ // Use a child AbortController that properly propagates parent aborts // (user ESC) and cleans up listeners automatically via createChildAbortController. const forkAbortController = context.abortController - ? createChildAbortController(context.abortController) + ? createChildAbortController(context.abortController, undefined, { + subsystem: 'compact', + controllerRole: 'compact-fork', + }) : new AbortController() let timeoutId: ReturnType | undefined @@ -1298,7 +1310,7 @@ async function streamCompactSummary({ }), new Promise((_, reject) => { timeoutId = setTimeout(() => { - forkAbortController.abort() + requestCompactTimeoutAbort(forkAbortController) reject(new Error('Compaction timed out')) }, COMPACT_TIMEOUT_MS) }), diff --git a/src/services/goal/controller.test.ts b/src/services/goal/controller.test.ts index ff217e5a6f..6eaf3cd5f5 100644 --- a/src/services/goal/controller.test.ts +++ b/src/services/goal/controller.test.ts @@ -8,6 +8,17 @@ import { } from './controller.js' import type { GoalState } from './types.js' import type { AssistantMessage } from '../../types/message.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, + registerInterruptionController, + requestAbort, +} from '../../utils/interruptionTrace.js' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../../test/sharedMutationLock.js' function assistant(uuid: string, text: string) { // Minimal fixture — cast rather than fabricate the full envelope. @@ -57,6 +68,66 @@ async function drain( } describe('goal continuation controller', () => { + test.serial('traces goal evaluation start and completion', async () => { + await acquireSharedMutationLock('goal/controller trace') + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + try { + const { context, abortController } = makeContext() + registerInterruptionController(abortController, { + controllerRole: 'query-root', + }) + await drain( + evaluateGoalAfterTurn({ + messagesForQuery: [], + assistantMessages: [assistant('assistant-trace', 'Done.')], + toolUseContext: context, + querySource: 'repl_main_thread', + deps: { + evaluateGoal: async () => { + requestAbort(abortController, 'user-cancel', { + source: 'cancel_keybinding', + controllerRole: 'query-root', + }) + return { + complete: false, + confidence: 0, + decision: 'error', + reason: 'cancelled', + nextInstruction: null, + } + }, + saveGoalState: async () => {}, + }, + }), + ) + + const trace = __getInterruptionTraceSnapshotForTests() + expect(trace.map(entry => entry.event)).toEqual( + expect.arrayContaining([ + 'goal.evaluation_started', + 'goal.evaluation_completed', + ]), + ) + const rootAbort = trace.find(entry => entry.event === 'abort.requested') + const completed = trace.find( + entry => entry.event === 'goal.evaluation_completed', + ) + expect(rootAbort).toBeDefined() + expect(completed).toBeDefined() + expect(typeof rootAbort!.eventId).toBe('string') + expect(typeof completed!.causalEventId).toBe('string') + expect(completed!.causalEventId).toBe(rootAbort!.eventId) + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + releaseSharedMutationLock() + } + }) + test('evaluator complete => no blocking error, goal achieved', async () => { const { context, getState } = makeContext() const deps: GoalEvaluationDeps = { diff --git a/src/services/goal/controller.ts b/src/services/goal/controller.ts index 42c4076676..7c6c9bdb39 100644 --- a/src/services/goal/controller.ts +++ b/src/services/goal/controller.ts @@ -5,6 +5,10 @@ import type { ToolUseContext } from '../../Tool.js' import type { Message } from '../../types/message.js' import { logForDebugging } from '../../utils/debug.js' import { logForDiagnosticsNoPII } from '../../utils/diagLogs.js' +import { + getInterruptionSignalAbortEventId, + traceInterruptionEvent, +} from '../../utils/interruptionTrace.js' import { createSystemMessage, createUserMessage } from '../../utils/messages.js' import { evaluateGoal as evaluateGoalDefault } from './evaluator.js' import { buildGoalContinuationInstruction } from './instructions.js' @@ -173,6 +177,12 @@ export async function* evaluateGoalAfterTurn({ return [] } + traceInterruptionEvent('goal.evaluation_started', { + subsystem: 'goal', + phase: 'evaluator', + querySource, + attemptId: goal.id, + }) const decision = await evaluateGoal({ goal, messages: getRecentGoalEvaluationMessages( @@ -183,6 +193,17 @@ export async function* evaluateGoalAfterTurn({ isNonInteractiveSession: toolUseContext.options.isNonInteractiveSession ?? false, }) + traceInterruptionEvent('goal.evaluation_completed', { + subsystem: 'goal', + phase: 'evaluator', + querySource, + attemptId: goal.id, + outcome: decision.decision, + reason: toolUseContext.abortController.signal.reason, + causalEventId: getInterruptionSignalAbortEventId( + toolUseContext.abortController.signal, + ), + }) if (toolUseContext.abortController.signal.aborted) return [] diff --git a/src/services/goal/evaluator.test.ts b/src/services/goal/evaluator.test.ts index 9a522ab9f7..c0c98c05be 100644 --- a/src/services/goal/evaluator.test.ts +++ b/src/services/goal/evaluator.test.ts @@ -6,6 +6,17 @@ import { evaluateGoal, type GoalModelCaller, } from './evaluator.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, + registerInterruptionController, + requestAbort, +} from '../../utils/interruptionTrace.js' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../../test/sharedMutationLock.js' function user(uuid: string, content: string) { return { @@ -27,6 +38,52 @@ function assistant(uuid: string, content: string) { } describe('goal evaluator', () => { + test.serial('traces provider failures without serializing the error message', async () => { + await acquireSharedMutationLock('goal/evaluator trace') + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + try { + const controller = new AbortController() + registerInterruptionController(controller, { + controllerRole: 'query-root', + }) + await evaluateGoal({ + goal: createGoalState('finish implementation'), + messages: [], + signal: controller.signal, + isNonInteractiveSession: false, + modelCaller: async () => { + requestAbort(controller, 'user-cancel', { + source: 'cancel_keybinding', + controllerRole: 'query-root', + }) + throw new Error('private provider detail') + }, + }) + + const trace = __getInterruptionTraceSnapshotForTests() + const serialized = JSON.stringify(trace) + expect(serialized).toContain('goal.evaluation_failed') + expect(serialized).not.toContain('private provider detail') + const rootAbort = trace.find(entry => entry.event === 'abort.requested') + const failed = trace.find( + entry => entry.event === 'goal.evaluation_failed', + ) + expect(rootAbort).toBeDefined() + expect(failed).toBeDefined() + expect(typeof rootAbort!.eventId).toBe('string') + expect(typeof failed!.causalEventId).toBe('string') + expect(failed!.causalEventId).toBe(rootAbort!.eventId) + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + releaseSharedMutationLock() + } + }) + test('valid complete JSON', async () => { const caller: GoalModelCaller = async () => JSON.stringify({ diff --git a/src/services/goal/evaluator.ts b/src/services/goal/evaluator.ts index a168858e62..d8f30ca053 100644 --- a/src/services/goal/evaluator.ts +++ b/src/services/goal/evaluator.ts @@ -1,5 +1,9 @@ import type { SystemPrompt } from '../../utils/systemPromptType.js' import { asSystemPrompt } from '../../utils/systemPromptType.js' +import { + getInterruptionSignalAbortEventId, + traceInterruptionEvent, +} from '../../utils/interruptionTrace.js' import { queryHaiku } from '../api/claude.js' import type { GoalEvaluatorDecision, GoalState } from './types.js' @@ -267,7 +271,14 @@ export async function evaluateGoal({ 'Goal evaluator returned malformed JSON; pausing automatic goal continuation.', nextInstruction: null, } - } catch { + } catch (error) { + traceInterruptionEvent('goal.evaluation_failed', { + subsystem: 'goal', + phase: 'evaluator', + error, + reason: signal.reason, + causalEventId: getInterruptionSignalAbortEventId(signal), + }) return { complete: false, confidence: 0, diff --git a/src/services/tools/StreamingToolExecutor.ts b/src/services/tools/StreamingToolExecutor.ts index 05fe4de52d..087dcd01ff 100644 --- a/src/services/tools/StreamingToolExecutor.ts +++ b/src/services/tools/StreamingToolExecutor.ts @@ -13,6 +13,10 @@ import { getMissingToolResultAbortMessage, shouldCreateUserInterruptionMessage, } from '../../utils/abortReasons.js' +import { + getInterruptionSignalAbortEventId, + requestAbort, +} from '../../utils/interruptionTrace.js' import { runToolUse } from './toolExecution.js' type MessageUpdate = { @@ -66,6 +70,11 @@ export class StreamingToolExecutor { this.toolUseContext = toolUseContext this.siblingAbortController = createChildAbortController( toolUseContext.abortController, + undefined, + { + subsystem: 'streaming_tool_executor', + controllerRole: 'sibling-tools', + }, ) } @@ -77,7 +86,11 @@ export class StreamingToolExecutor { discard(): void { if (this.discarded) return this.discarded = true - this.siblingAbortController.abort('streaming_fallback') + requestAbort(this.siblingAbortController, 'streaming_fallback', { + source: 'streaming_fallback', + subsystem: 'streaming_tool_executor', + controllerRole: 'sibling-tools', + }) const activeLifecycleToolUseIds = new Set( this.toolUseContext.queryLifecycle ?.snapshot() @@ -355,6 +368,11 @@ export class StreamingToolExecutor { // sends REJECT_MESSAGE to the model instead of aborting (#21056 regression). const toolAbortController = createChildAbortController( this.siblingAbortController, + undefined, + { + subsystem: 'streaming_tool_executor', + controllerRole: 'tool', + }, ) toolAbortController.signal.addEventListener( 'abort', @@ -364,8 +382,17 @@ export class StreamingToolExecutor { !this.toolUseContext.abortController.signal.aborted && !this.discarded ) { - this.toolUseContext.abortController.abort( + requestAbort( + this.toolUseContext.abortController, toolAbortController.signal.reason, + { + source: 'tool_abort_propagation', + causalEventId: getInterruptionSignalAbortEventId( + toolAbortController.signal, + ), + subsystem: 'streaming_tool_executor', + controllerRole: 'query-root', + }, ) } }, @@ -414,7 +441,11 @@ export class StreamingToolExecutor { if (tool.block.name === BASH_TOOL_NAME) { this.hasErrored = true this.erroredToolDescription = this.getToolDescription(tool) - this.siblingAbortController.abort('sibling_error') + requestAbort(this.siblingAbortController, 'sibling_error', { + source: 'sibling_error', + subsystem: 'streaming_tool_executor', + controllerRole: 'sibling-tools', + }) } } diff --git a/src/state/teammateViewHelpers.interruptionTrace.test.ts b/src/state/teammateViewHelpers.interruptionTrace.test.ts new file mode 100644 index 0000000000..928635ac4c --- /dev/null +++ b/src/state/teammateViewHelpers.interruptionTrace.test.ts @@ -0,0 +1,82 @@ +import { afterEach, beforeEach, expect, test } from 'bun:test' +import type { AppState } from './AppState.js' +import { getDefaultAppState } from './AppStateStore.js' +import { stopOrDismissAgent } from './teammateViewHelpers.js' +import type { LocalAgentTaskState } from '../tasks/LocalAgentTask/LocalAgentTask.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, +} from '../utils/interruptionTrace.js' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../test/sharedMutationLock.js' + +const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + +beforeEach(async () => { + await acquireSharedMutationLock( + 'state/teammateViewHelpers.interruptionTrace.test.ts', + ) +}) + +afterEach(() => { + try { + __resetInterruptionTraceForTests() + if (originalTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + } finally { + releaseSharedMutationLock() + } +}) + +test('records the panel stop source and causal input before aborting an agent', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const abortController = new AbortController() + const taskId = 'agent-task-1' + const task = { + id: taskId, + type: 'local_agent', + status: 'running', + description: 'test agent', + startTime: Date.now(), + outputFile: '/tmp/test-agent-output', + outputOffset: 0, + notified: false, + agentId: 'agent-1', + prompt: 'test', + agentType: 'general-purpose', + abortController, + retrieved: false, + lastReportedToolCount: 0, + lastReportedTokenCount: 0, + isBackgrounded: true, + pendingMessages: [], + retain: false, + diskLoaded: false, + } satisfies LocalAgentTaskState + let state: AppState = { + ...getDefaultAppState(), + tasks: { [taskId]: task }, + } + + stopOrDismissAgent(taskId, updater => { + state = updater(state) + }) + + const entries = __getInterruptionTraceSnapshotForTests() + const input = entries.find(entry => entry.event === 'input.agent_panel_stop') + const requested = entries.find(entry => entry.event === 'abort.requested') + expect(abortController.signal.aborted).toBe(true) + expect(input).toBeDefined() + expect(requested).toMatchObject({ + source: 'agent_panel_stop', + subsystem: 'local_agent_task', + controllerRole: 'background-agent', + subagentId: taskId, + causalEventId: input?.eventId, + }) +}) diff --git a/src/state/teammateViewHelpers.ts b/src/state/teammateViewHelpers.ts index 2c32f7b1f8..6cd4ee97a4 100644 --- a/src/state/teammateViewHelpers.ts +++ b/src/state/teammateViewHelpers.ts @@ -1,6 +1,10 @@ import { logEvent } from '../services/analytics/index.js' import { isTerminalTaskStatus } from '../Task.js' import type { LocalAgentTaskState } from '../tasks/LocalAgentTask/LocalAgentTask.js' +import { + requestAbort, + traceInterruptionEvent, +} from '../utils/interruptionTrace.js' // Inlined from framework.ts — importing creates a cycle through // BackgroundTasksDialog. Keep in sync with PANEL_GRACE_MS there. @@ -121,7 +125,20 @@ export function stopOrDismissAgent( const task = prev.tasks[taskId] if (!isLocalAgent(task)) return prev if (task.status === 'running') { - task.abortController?.abort() + const causalEventId = traceInterruptionEvent('input.agent_panel_stop', { + source: 'agent_panel_stop', + subsystem: 'local_agent_task', + subagentId: taskId, + }) + if (task.abortController) { + requestAbort(task.abortController, undefined, { + source: 'agent_panel_stop', + subsystem: 'local_agent_task', + controllerRole: 'background-agent', + subagentId: taskId, + causalEventId, + }) + } return prev } if (task.evictAfter === 0) return prev diff --git a/src/tasks/LocalAgentTask/LocalAgentTask.tsx b/src/tasks/LocalAgentTask/LocalAgentTask.tsx index 87847be38e..747d18ea32 100644 --- a/src/tasks/LocalAgentTask/LocalAgentTask.tsx +++ b/src/tasks/LocalAgentTask/LocalAgentTask.tsx @@ -16,6 +16,7 @@ import { registerCleanup } from '../../utils/cleanupRegistry.js'; import { getToolSearchOrReadInfo } from '../../utils/collapseReadSearch.js'; import { enqueuePendingNotification } from '../../utils/messageQueueManager.js'; import { getAgentTranscriptPath } from '../../utils/sessionStorage.js'; +import { requestAbort } from '../../utils/interruptionTrace.js'; import { evictTaskOutput, getTaskOutputPath, initTaskOutputAsSymlink } from '../../utils/task/diskOutput.js'; import { PANEL_GRACE_MS, registerTask, updateTaskState } from '../../utils/task/framework.js'; import { emitTaskProgress } from '../../utils/task/sdkProgress.js'; @@ -271,21 +272,34 @@ export const LocalAgentTask: Task = { name: 'LocalAgentTask', type: 'local_agent', async kill(taskId, setAppState) { - killAsyncAgent(taskId, setAppState); + killAsyncAgent(taskId, setAppState, { source: 'task_stop' }); } }; /** * Kill an agent task. No-op if already killed/completed. */ -export function killAsyncAgent(taskId: string, setAppState: SetAppState): void { +export type AgentKillTrace = { + source: string; + causalEventId?: string; +}; + +export function killAsyncAgent(taskId: string, setAppState: SetAppState, trace: AgentKillTrace = { source: 'agent_cleanup' }): void { let killed = false; updateTaskState(taskId, setAppState, task => { if (task.status !== 'running') { return task; } killed = true; - task.abortController?.abort(); + if (task.abortController && !task.abortController.signal.aborted) { + requestAbort(task.abortController, undefined, { + source: trace.source, + subsystem: 'local_agent_task', + controllerRole: 'background-agent', + subagentId: taskId, + causalEventId: trace.causalEventId + }); + } task.unregisterCleanup?.(); return { ...task, @@ -306,10 +320,10 @@ export function killAsyncAgent(taskId: string, setAppState: SetAppState): void { * Kill all running agent tasks. * Used by ESC cancellation in coordinator mode to stop all subagents. */ -export function killAllRunningAgentTasks(tasks: Record, setAppState: SetAppState): void { +export function killAllRunningAgentTasks(tasks: Record, setAppState: SetAppState, trace: AgentKillTrace): void { for (const [taskId, task] of Object.entries(tasks)) { if (task.type === 'local_agent' && task.status === 'running') { - killAsyncAgent(taskId, setAppState); + killAsyncAgent(taskId, setAppState, trace); } } } @@ -483,7 +497,13 @@ export function registerAsyncAgent({ void initTaskOutputAsSymlink(agentId, getAgentTranscriptPath(asAgentId(agentId))); // Create abort controller - if parent provided, create child that auto-aborts with parent - const abortController = parentAbortController ? createChildAbortController(parentAbortController) : createAbortController(); + const abortController = parentAbortController + ? createChildAbortController(parentAbortController, undefined, { + subsystem: 'local_agent_task', + controllerRole: 'background-agent', + subagentId: agentId, + }) + : createAbortController(); const taskState: LocalAgentTaskState = { ...createTaskStateBase(agentId, 'local_agent', description, toolUseId), type: 'local_agent', diff --git a/src/test/fixtures/gracefulShutdownTrace.fixture.ts b/src/test/fixtures/gracefulShutdownTrace.fixture.ts new file mode 100644 index 0000000000..b559dec502 --- /dev/null +++ b/src/test/fixtures/gracefulShutdownTrace.fixture.ts @@ -0,0 +1,61 @@ +import { getFsImplementation, setFsImplementation } from '../../utils/fsOperations.js' +import { gracefulShutdown, resetShutdownState } from '../../utils/gracefulShutdown.js' +import { + __resetInterruptionTraceForTests, + requestAbort, + traceInterruptionEvent, +} from '../../utils/interruptionTrace.js' + +process.env.NODE_ENV = 'test' +process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' +process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/virtual/trace.jsonl' + +const originalFs = getFsImplementation() +const originalExit = process.exit +let writeSettled = false +let exitObservedSettledWrite = false +let exitCalled = false + +setFsImplementation({ + ...originalFs, + mkdirSync: () => {}, + appendRegularFile: async () => { + if (process.env.TRACE_SHUTDOWN_BLOCK_WRITE === '1') { + await new Promise(() => {}) + } + await Bun.sleep(50) + writeSettled = true + }, +}) +process.exit = ((_code?: number) => { + exitCalled = true + exitObservedSettledWrite = writeSettled + throw new Error('mocked process exit') +}) as typeof process.exit + +try { + const controller = new AbortController() + if (process.env.TRACE_SHUTDOWN_PENDING_ONLY === '1') { + controller.abort('interrupt') + traceInterruptionEvent('shutdown.pending') + } else { + requestAbort(controller, 'interrupt', { + source: 'print_sigint', + controllerRole: 'query-root', + }) + } + await gracefulShutdown(0).catch(() => {}) + console.log( + `TRACE_SHUTDOWN_RESULT ${JSON.stringify({ + aborted: controller.signal.aborted, + exitCalled, + writeSettled, + exitObservedSettledWrite, + })}`, + ) +} finally { + process.exit = originalExit + setFsImplementation(originalFs) + resetShutdownState() + __resetInterruptionTraceForTests() +} diff --git a/src/tools/SendMessageTool/SendMessageTool.ts b/src/tools/SendMessageTool/SendMessageTool.ts index a0c4b9d014..a7e963511e 100644 --- a/src/tools/SendMessageTool/SendMessageTool.ts +++ b/src/tools/SendMessageTool/SendMessageTool.ts @@ -40,6 +40,7 @@ import { } from '../../utils/teammateMailbox.js' import { resumeAgentBackground } from '../AgentTool/resumeAgent.js' import { SEND_MESSAGE_TOOL_NAME } from './constants.js' +import { abortApprovedInProcessTeammate } from './shutdownInterruptionTrace.js' import { DESCRIPTION, getPrompt } from './prompt.js' import { renderToolResultMessage, renderToolUseMessage } from './UI.js' @@ -354,7 +355,9 @@ async function handleShutdownApproval( const appState = context.getAppState() const task = findTeammateTaskByAgentId(agentId, appState.tasks) if (task?.abortController) { - task.abortController.abort() + abortApprovedInProcessTeammate(task.abortController, { + agentId, + }) logForDebugging( `[SendMessageTool] Aborted controller for in-process teammate ${agentName}`, ) @@ -372,7 +375,9 @@ async function handleShutdownApproval( logForDebugging( `[SendMessageTool] Fallback: Found in-process task for ${agentName} via AppState, aborting`, ) - task.abortController.abort() + abortApprovedInProcessTeammate(task.abortController, { + agentId, + }) return { data: { diff --git a/src/tools/SendMessageTool/shutdownInterruptionTrace.test.ts b/src/tools/SendMessageTool/shutdownInterruptionTrace.test.ts new file mode 100644 index 0000000000..b84a0340ff --- /dev/null +++ b/src/tools/SendMessageTool/shutdownInterruptionTrace.test.ts @@ -0,0 +1,66 @@ +import { afterEach, beforeEach, expect, test } from 'bun:test' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../../test/sharedMutationLock.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, + registerInterruptionController, +} from '../../utils/interruptionTrace.js' +import { abortApprovedInProcessTeammate } from './shutdownInterruptionTrace.js' + +const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + +beforeEach(async () => { + await acquireSharedMutationLock('shutdownInterruptionTrace.test.ts') +}) + +afterEach(async () => { + try { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } finally { + releaseSharedMutationLock() + } +}) + +test('shutdown approval records its input before requesting the teammate abort', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const controller = new AbortController() + registerInterruptionController(controller, { + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: 'agent-1', + }) + + abortApprovedInProcessTeammate(controller, { + agentId: 'agent-1', + }) + + expect(controller.signal.aborted).toBe(true) + expect(controller.signal.reason).toBeInstanceOf(DOMException) + expect((controller.signal.reason as DOMException).name).toBe('AbortError') + const trace = __getInterruptionTraceSnapshotForTests() + const approved = trace.find( + entry => entry.event === 'teammate.shutdown_approved', + ) + const requested = trace.find(entry => entry.event === 'abort.requested') + expect(approved).toMatchObject({ + source: 'shutdown_approved', + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: 'agent-1', + }) + expect(requested).toMatchObject({ + source: 'shutdown_approved', + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: 'agent-1', + causalEventId: approved?.eventId, + }) + expect(trace.indexOf(approved!)).toBeLessThan(trace.indexOf(requested!)) +}) diff --git a/src/tools/SendMessageTool/shutdownInterruptionTrace.ts b/src/tools/SendMessageTool/shutdownInterruptionTrace.ts new file mode 100644 index 0000000000..15fbd4922c --- /dev/null +++ b/src/tools/SendMessageTool/shutdownInterruptionTrace.ts @@ -0,0 +1,23 @@ +import { + requestAbort, + traceInterruptionEvent, +} from '../../utils/interruptionTrace.js' + +export function abortApprovedInProcessTeammate( + controller: AbortController, + identity: { agentId: string }, +): void { + const causalEventId = traceInterruptionEvent('teammate.shutdown_approved', { + source: 'shutdown_approved', + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: identity.agentId, + }) + requestAbort(controller, undefined, { + source: 'shutdown_approved', + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: identity.agentId, + causalEventId, + }) +} diff --git a/src/utils/QueryGuard.ts b/src/utils/QueryGuard.ts index fc292d750b..710614cf45 100644 --- a/src/utils/QueryGuard.ts +++ b/src/utils/QueryGuard.ts @@ -29,6 +29,10 @@ * ) */ import { createSignal } from './signal.js' +import { + flushInterruptionTrace, + traceInterruptionEvent, +} from './interruptionTrace.js' import type { QueryActiveOperationSnapshot, QueryGuardMetadata, @@ -127,6 +131,8 @@ export class QueryGuard { private _suspendedAt = 0 private _totalSuspendedMs = 0 private _leaseCounter = 0 + private _traceActivityCount = 0 + private _lastTraceActivityAt = 0 private _activeLeases = new Map() private _context: QueryLifecycleContext | null = null private _lastContext: QueryLifecycleContext | null = null @@ -190,8 +196,17 @@ export class QueryGuard { this._totalSuspendedMs = 0 this._lastContext = null this._queryStartedAt = performance.now() + this._traceActivityCount = 0 + this._lastTraceActivityAt = 0 this._lastActivityAt = this._queryStartedAt this._context = this._createContext(metadata) + traceInterruptionEvent('query_guard.started', { + subsystem: 'query_guard', + phase: 'running', + queryId: this._context.queryId, + queryGeneration: this._generation, + querySource: this._context.querySource, + }) this._getActiveOperations = metadata?.getActiveOperations ?? null this._startTimeout() this._notify() @@ -221,7 +236,16 @@ export class QueryGuard { this._suspendCount = 0 this._suspendedAt = 0 this._totalSuspendedMs = 0 + const context = this._context this._completeContext(terminalReason, abortReason) + traceInterruptionEvent('query_guard.ended', { + subsystem: 'query_guard', + phase: terminalReason, + queryId: context?.queryId, + queryGeneration: generation, + querySource: context?.querySource, + reason: abortReason, + }) this._status = 'idle' this._getActiveOperations = null this._notify() @@ -244,7 +268,16 @@ export class QueryGuard { this._suspendCount = 0 this._suspendedAt = 0 this._totalSuspendedMs = 0 + const context = this._context this._completeContext(terminalReason, abortReason) + traceInterruptionEvent('query_guard.force_ended', { + subsystem: 'query_guard', + phase: terminalReason, + queryId: context?.queryId, + queryGeneration: this._generation, + querySource: context?.querySource, + reason: abortReason, + }) this._status = 'idle' this._getActiveOperations = null ++this._generation @@ -260,7 +293,22 @@ export class QueryGuard { void reason if (this._status !== 'running') return if (generation !== undefined && generation !== this._generation) return - this._lastActivityAt = performance.now() + const now = performance.now() + this._lastActivityAt = now + this._traceActivityCount++ + if ( + this._traceActivityCount === 1 || + now - this._lastTraceActivityAt >= 5_000 + ) { + this._lastTraceActivityAt = now + traceInterruptionEvent('query_guard.activity', { + subsystem: 'query_guard', + queryId: this._context?.queryId, + queryGeneration: this._generation, + querySource: this._context?.querySource, + yieldedEventCount: this._traceActivityCount, + }) + } this._scheduleTimeout() } @@ -314,6 +362,13 @@ export class QueryGuard { deadlineAt: leaseDeadlineAt, description: input.description, }) + traceInterruptionEvent('query_guard.lease_acquired', { + subsystem: 'query_guard', + queryId: this._context?.queryId, + queryGeneration: this._generation, + source: input.owner, + leaseCount: this._activeLeases.size, + }) this._lastActivityAt = now this._scheduleTimeout() @@ -331,6 +386,13 @@ export class QueryGuard { const lease = this._activeLeases.get(leaseId) if (!lease || lease.generation !== generation) return this._activeLeases.delete(leaseId) + traceInterruptionEvent('query_guard.lease_released', { + subsystem: 'query_guard', + queryId: this._context?.queryId, + queryGeneration: this._generation, + source: lease.owner, + leaseCount: this._activeLeases.size, + }) this._scheduleTimeout() } @@ -352,6 +414,12 @@ export class QueryGuard { this._suspendedAt = performance.now() } this._suspendCount++ + traceInterruptionEvent('query_guard.suspended', { + subsystem: 'query_guard', + queryId: this._context?.queryId, + queryGeneration: this._generation, + suspendCount: this._suspendCount, + }) this._scheduleTimeout() let resumed = false @@ -362,6 +430,12 @@ export class QueryGuard { if (generation !== this._generation) return if (this._suspendCount === 0) return this._suspendCount-- + traceInterruptionEvent('query_guard.resumed', { + subsystem: 'query_guard', + queryId: this._context?.queryId, + queryGeneration: this._generation, + suspendCount: this._suspendCount, + }) if (this._suspendCount === 0) { this._resumeAfterInteraction() } @@ -534,6 +608,21 @@ export class QueryGuard { context, activeOperations: this._snapshotActiveOperations(), } + const causalEventId = traceInterruptionEvent('query_guard.fired', { + subsystem: 'query_guard', + phase: terminalReason, + queryId: context.queryId, + queryGeneration: context.queryGeneration, + querySource: context.querySource, + trigger: reason, + elapsedQueryMs: timeout.elapsedMs, + sinceLastActivityMs: now - this._lastActivityAt, + activeApiCallCount: timeout.activeOperations.apiCalls.length, + activeToolUseCount: timeout.activeOperations.toolUses.length, + leaseCount: this._activeLeases.size, + suspendCount: this._suspendCount, + }) + if (causalEventId) timeout.causalEventId = causalEventId console.error( `[QueryGuard] Query ${reason} timeout - force-ending to prevent infinite spinner`, @@ -544,6 +633,7 @@ export class QueryGuard { console.error('[QueryGuard] Timeout handler failed', error) } finally { this.forceEnd(terminalReason, reason) + flushInterruptionTrace('query_guard_timeout') } } diff --git a/src/utils/abortController.ts b/src/utils/abortController.ts index f1401a156f..8048ee34b4 100644 --- a/src/utils/abortController.ts +++ b/src/utils/abortController.ts @@ -1,4 +1,11 @@ import { setMaxListeners } from 'events' +import { + getInterruptionSignalAbortEventId, + getInterruptionSignalId, + registerInterruptionController, + requestAbort, + type InterruptionTraceFields, +} from './interruptionTrace.js' /** * Default max listeners for standard operations @@ -32,7 +39,19 @@ function propagateAbort( weakChild: WeakRef, ): void { const parent = this.deref() - weakChild.deref()?.abort(parent?.signal.reason) + const child = weakChild.deref() + if (!child) return + requestAbort(child, parent?.signal.reason, { + source: 'parent_signal', + subsystem: 'abort_controller', + controllerRole: 'child', + ...(parent && { + causalEventId: getInterruptionSignalAbortEventId(parent.signal), + }), + ...(parent && { + parentControllerIds: [getInterruptionSignalId(parent.signal) ?? 'unregistered-parent'], + }), + }) } /** @@ -63,17 +82,40 @@ function removeAbortHandler( * * @param parent - The parent AbortController * @param maxListeners - Maximum number of listeners (default: 50) + * @param traceFields - Owning lifecycle identity for interruption diagnostics * @returns Child AbortController */ export function createChildAbortController( parent: AbortController, maxListeners?: number, + traceFields: InterruptionTraceFields = {}, ): AbortController { const child = createAbortController(maxListeners) + const parentId = registerInterruptionController(parent, { + subsystem: 'abort_controller', + controllerRole: 'parent', + }, { provisionalRole: true }) + const hasOwningRole = traceFields.controllerRole !== undefined + registerInterruptionController( + child, + { + subsystem: 'abort_controller', + controllerRole: 'child', + ...traceFields, + ...(parentId && { parentControllerIds: [parentId] }), + }, + { provisionalRole: !hasOwningRole }, + ) // Fast path: parent already aborted, no listener setup needed if (parent.signal.aborted) { - child.abort(parent.signal.reason) + requestAbort(child, parent.signal.reason, { + source: 'already_aborted_parent', + subsystem: 'abort_controller', + controllerRole: 'child', + causalEventId: getInterruptionSignalAbortEventId(parent.signal), + ...(parentId && { parentControllerIds: [parentId] }), + }) return child } diff --git a/src/utils/attachments.ts b/src/utils/attachments.ts index 6f5c1fcefe..3cd6cd451f 100644 --- a/src/utils/attachments.ts +++ b/src/utils/attachments.ts @@ -2699,7 +2699,14 @@ export function startRelevantMemoryPrefetch( // Chained to the turn-level abort so user Escape cancels the sideQuery // immediately, not just on [Symbol.dispose] when queryLoop exits. - const controller = createChildAbortController(toolUseContext.abortController) + const controller = createChildAbortController( + toolUseContext.abortController, + undefined, + { + subsystem: 'memory_attachments', + controllerRole: 'memory-prefetch', + }, + ) const firedAt = Date.now() const promise = getRelevantMemoryAttachments( input, diff --git a/src/utils/combinedAbortSignal.ts b/src/utils/combinedAbortSignal.ts index 65db1bf868..5ff558ebe0 100644 --- a/src/utils/combinedAbortSignal.ts +++ b/src/utils/combinedAbortSignal.ts @@ -1,4 +1,12 @@ import { createAbortController } from './abortController.js' +import { + getInterruptionSignalAbortEventId, + getInterruptionSignalId, + registerInterruptionSignal, + requestAbort, + traceCombinedSignal, + traceInterruptionEvent, +} from './interruptionTrace.js' /** * Creates a combined AbortSignal that aborts when the input signal aborts, @@ -14,17 +22,47 @@ import { createAbortController } from './abortController.js' */ export function createCombinedAbortSignal( signal: AbortSignal | undefined, - opts?: { signalB?: AbortSignal; timeoutMs?: number }, + opts?: { + signalB?: AbortSignal + timeoutMs?: number + trace?: { subsystem: string; controllerRole?: string } + }, ): { signal: AbortSignal; cleanup: () => void } { - const { signalB, timeoutMs } = opts ?? {} + const { signalB, timeoutMs, trace } = opts ?? {} const combined = createAbortController() + const traceFields = { + subsystem: trace?.subsystem ?? 'combined_abort_signal', + controllerRole: trace?.controllerRole ?? 'combined', + } + const parentIds = [signal, signalB] + .filter((parent): parent is AbortSignal => parent !== undefined) + .map(parent => + registerInterruptionSignal(parent, { + ...traceFields, + controllerRole: 'combined-parent', + }), + ) + .filter((id): id is string => id !== undefined) + traceCombinedSignal(combined, [signal, signalB], traceFields) if (signal?.aborted) { - combined.abort(signal.reason) + requestAbort(combined, signal.reason, { + ...traceFields, + source: 'signal_a_already_aborted', + parentControllerIds: parentIds, + winningParentControllerId: getInterruptionSignalId(signal), + causalEventId: getInterruptionSignalAbortEventId(signal), + }) return { signal: combined.signal, cleanup: () => {} } } if (signalB?.aborted) { - combined.abort(signalB.reason) + requestAbort(combined, signalB.reason, { + ...traceFields, + source: 'signal_b_already_aborted', + parentControllerIds: parentIds, + winningParentControllerId: getInterruptionSignalId(signalB), + causalEventId: getInterruptionSignalAbortEventId(signalB), + }) return { signal: combined.signal, cleanup: () => {} } } @@ -39,15 +77,35 @@ export function createCombinedAbortSignal( } signal?.removeEventListener('abort', abortFromSignal) signalB?.removeEventListener('abort', abortFromSignalB) + traceInterruptionEvent('combined_signal.cleanup', traceFields) } - const abortCombined = (reason?: unknown) => { + const abortCombined = ( + reason: unknown, + source: string, + winningParent?: AbortSignal, + ) => { cleanup() - combined.abort(reason) + requestAbort(combined, reason, { + ...traceFields, + source, + parentControllerIds: parentIds, + winningParentControllerId: winningParent + ? getInterruptionSignalId(winningParent) + : undefined, + causalEventId: winningParent + ? getInterruptionSignalAbortEventId(winningParent) + : undefined, + }) } - const abortFromSignal = () => abortCombined(signal?.reason) - const abortFromSignalB = () => abortCombined(signalB?.reason) + const abortFromSignal = () => + abortCombined(signal?.reason, 'signal_a', signal) + const abortFromSignalB = () => + abortCombined(signalB?.reason, 'signal_b', signalB) const abortFromTimeout = () => - abortCombined(new DOMException('The operation timed out.', 'TimeoutError')) + abortCombined( + new DOMException('The operation timed out.', 'TimeoutError'), + 'timeout', + ) if (timeoutMs !== undefined) { timer = setTimeout(abortFromTimeout, timeoutMs) diff --git a/src/utils/computerUse/wrapper.tsx b/src/utils/computerUse/wrapper.tsx index 58bd9fae7c..2777f7f2d7 100644 --- a/src/utils/computerUse/wrapper.tsx +++ b/src/utils/computerUse/wrapper.tsx @@ -22,6 +22,7 @@ import { getSessionId } from '../../bootstrap/state.js'; import { ComputerUseApproval } from '../../components/permissions/ComputerUseApproval/ComputerUseApproval.js'; import type { Tool, ToolUseContext } from '../../Tool.js'; import { logForDebugging } from '../debug.js'; +import { requestAbort } from '../interruptionTrace.js'; import { checkComputerUseLock, tryAcquireComputerUseLock } from './computerUseLock.js'; import { registerEscHotkey } from './escHotkey.js'; import { getChicagoCoordinateMode } from './gates.js'; @@ -219,7 +220,11 @@ export function buildSessionContext(): ComputerUseSessionContext { // holds a pump retain until unregisterEscHotkey() in cleanup.ts. const escRegistered = registerEscHotkey(() => { logForDebugging('[cu-esc] user escape, aborting turn'); - tuc().abortController.abort(); + requestAbort(tuc().abortController, undefined, { + source: 'computer_use_escape', + subsystem: 'computer_use', + controllerRole: 'tool', + }); }); tuc().sendOSNotification?.({ message: escRegistered ? 'Claude is using your computer · press Esc to stop' : 'Claude is using your computer · press Ctrl+C to stop', diff --git a/src/utils/diagLogs.ts b/src/utils/diagLogs.ts index a2a3d382ae..8b0051ab74 100644 --- a/src/utils/diagLogs.ts +++ b/src/utils/diagLogs.ts @@ -1,4 +1,5 @@ import { dirname } from 'path' +import { getErrnoCode } from './errors.js' import { getFsImplementation } from './fsOperations.js' import { jsonStringify } from './slowOperations.js' @@ -11,6 +12,40 @@ type DiagnosticLogEntry = { data: Record } +export type DiagnosticAppendResult = + | 'committed' + | 'unsupported' + | 'retryable_failure' + | 'uncertain_failure' + +/** + * Append already-sanitized diagnostic records to an explicit file. + * + * This is the shared filesystem boundary for opt-in diagnostic streams. Callers + * must rebuild records from an allowlist before using it; arbitrary runtime + * objects, messages, prompts, and file paths are not safe inputs. + */ +export async function appendDiagnosticsNoPII( + logFile: string, + entries: readonly Record[], +): Promise { + if (entries.length === 0) return 'committed' + // Node exposes the descriptor-relative namespace required by the secure + // append implementation through /proc/self/fd on Linux only. + if (process.platform !== 'linux') return 'unsupported' + + try { + const fs = getFsImplementation() + const lines = entries.map(entry => jsonStringify(entry)).join('\n') + '\n' + await fs.appendRegularFile(logFile, lines, { mode: 0o600 }) + return 'committed' + } catch (error) { + return getErrnoCode(error) === 'ERR_DIAGNOSTIC_APPEND_UNCERTAIN' + ? 'uncertain_failure' + : 'retryable_failure' + } +} + /** * Logs diagnostic information to a logfile. This information is sent * via the environment manager to session-ingress to monitor issues from @@ -46,12 +81,12 @@ export function logForDiagnosticsNoPII( try { fs.appendFileSync(logFile, line) } catch { - // If append fails, try creating the directory first + // Preserve the legacy diagnostic logger's append-through-symlink contract. try { fs.mkdirSync(dirname(logFile)) fs.appendFileSync(logFile, line) } catch { - // Silently fail if logging is not possible + // Silently fail if logging is not possible. } } } diff --git a/src/utils/forkedAgent.ts b/src/utils/forkedAgent.ts index 6db5903e43..c62c06fb4d 100644 --- a/src/utils/forkedAgent.ts +++ b/src/utils/forkedAgent.ts @@ -358,7 +358,11 @@ export function createSubagentContext( overrides?.abortController ?? (overrides?.shareAbortController ? parentContext.abortController - : createChildAbortController(parentContext.abortController)) + : createChildAbortController(parentContext.abortController, undefined, { + subsystem: 'forked_agent', + controllerRole: 'subagent-context', + subagentId: overrides?.agentId, + })) // Determine getAppState - wrap to set shouldAvoidPermissionPrompts unless sharing abortController // (if sharing abortController, it's an interactive agent that CAN show UI) diff --git a/src/utils/fsOperations.ts b/src/utils/fsOperations.ts index 635a9c0e65..673815e637 100644 --- a/src/utils/fsOperations.ts +++ b/src/utils/fsOperations.ts @@ -97,6 +97,18 @@ export type FsOperations = { } /** Appends string to file */ appendFileSync(path: string, data: string, options?: { mode?: number }): void + /** + * Opens a regular file without following path symlinks and appends data. + * + * Linux only and requires an absolute path; throws on unsupported platforms + * or relative paths. A partial write that cannot be rolled back rejects with + * `ERR_DIAGNOSTIC_APPEND_UNCERTAIN`. + */ + appendRegularFile( + path: string, + data: string, + options?: { mode?: number }, + ): Promise /** Copies file from source to destination */ copyFileSync(src: string, dest: string): void /** Deletes file */ @@ -546,6 +558,103 @@ export const NodeFsOperations: FsOperations = { fs.appendFileSync(path, data) }, + async appendRegularFile(path, data, options) { + if (process.platform !== 'linux') { + throw new Error('Secure diagnostic file output is available on Linux only') + } + if (!nodePath.isAbsolute(path)) { + throw new Error('Secure diagnostic file output requires an absolute path') + } + + const resolvedPath = nodePath.resolve(path) + const components = nodePath + .relative(nodePath.parse(resolvedPath).root, nodePath.dirname(resolvedPath)) + .split(nodePath.sep) + .filter(Boolean) + const directoryFlags = fs.constants.O_RDONLY | + fs.constants.O_DIRECTORY | + fs.constants.O_NOFOLLOW + const descriptorDirectory = '/proc/self/fd' + let directoryHandle = await open('/', directoryFlags) + let committed = false + let operationError: unknown + + try { + for (const component of components) { + const descriptorPath = `${descriptorDirectory}/${directoryHandle.fd}/${component}` + let nextDirectoryHandle + try { + nextDirectoryHandle = await open(descriptorPath, directoryFlags) + } catch (error) { + if (getErrnoCode(error) !== 'ENOENT') throw error + await mkdirPromise(descriptorPath, { mode: 0o700 }) + nextDirectoryHandle = await open(descriptorPath, directoryFlags) + } + try { + await directoryHandle.close() + } catch (error) { + await nextDirectoryHandle.close().catch(() => {}) + throw error + } + directoryHandle = nextDirectoryHandle + } + + const descriptorPath = `${descriptorDirectory}/${directoryHandle.fd}/${nodePath.basename(resolvedPath)}` + const flags = fs.constants.O_APPEND | + fs.constants.O_CREAT | + fs.constants.O_NONBLOCK | + fs.constants.O_WRONLY | + fs.constants.O_NOFOLLOW + const handle = await open(descriptorPath, flags, options?.mode ?? 0o600) + let fileOperationError: unknown + try { + const stats = await handle.stat() + if (!stats.isFile()) { + throw new Error('Diagnostics target is not a regular file') + } + if (options?.mode !== undefined) await handle.chmod(options.mode) + try { + await handle.writeFile(data, { encoding: 'utf8' }) + committed = true + } catch (error) { + try { + await handle.truncate(stats.size) + } catch (rollbackError) { + throw Object.assign( + new Error('Diagnostic append may have partially committed', { + cause: rollbackError, + }), + { code: 'ERR_DIAGNOSTIC_APPEND_UNCERTAIN' }, + ) + } + throw error + } + } catch (error) { + fileOperationError = error + } + try { + await handle.close() + } catch (error) { + // A close failure after writeFile completed cannot safely be interpreted + // as a failed append; replaying would duplicate the committed batch. + if (!committed && fileOperationError === undefined) { + fileOperationError = error + } + } + if (fileOperationError !== undefined) throw fileOperationError + } catch (error) { + operationError = error + } + try { + await directoryHandle.close() + } catch (error) { + // Preserve an earlier operation error so a cleanup failure cannot hide an + // uncertain commit and cause the caller to replay the same batch. + if (!committed && operationError === undefined) operationError = error + } + if (operationError !== undefined) throw operationError + }, + copyFileSync(src, dest) { using _ = slowLogging`fs.copyFileSync(${src} → ${dest})` fs.copyFileSync(src, dest) diff --git a/src/utils/gracefulShutdown.interruptionTrace.test.ts b/src/utils/gracefulShutdown.interruptionTrace.test.ts new file mode 100644 index 0000000000..689d726af6 --- /dev/null +++ b/src/utils/gracefulShutdown.interruptionTrace.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from 'bun:test' +import { spawnSync } from 'node:child_process' +import { resolve } from 'node:path' + +const testLinuxTraceFile = process.platform === 'linux' ? test : test.skip + +function runFixture(extraEnv: NodeJS.ProcessEnv = {}): { + aborted: boolean + exitCalled: boolean + writeSettled: boolean + exitObservedSettledWrite: boolean +} { + const fixture = resolve( + import.meta.dirname, + '../test/fixtures/gracefulShutdownTrace.fixture.ts', + ) + const result = spawnSync(process.execPath, [fixture], { + encoding: 'utf8', + timeout: 15_000, + env: { ...process.env, FORCE_COLOR: '0', ...extraEnv }, + }) + if (result.error) throw result.error + if (result.status !== 0) { + throw new Error( + `fixture exited with ${result.status}\nstderr:\n${result.stderr}`, + ) + } + const resultLine = result.stdout + .split('\n') + .find(line => line.startsWith('TRACE_SHUTDOWN_RESULT ')) + expect(resultLine).toBeDefined() + return JSON.parse(resultLine!.slice('TRACE_SHUTDOWN_RESULT '.length)) +} + +testLinuxTraceFile('graceful shutdown drains the interruption trace before process exit', () => { + expect(runFixture()).toEqual({ + aborted: true, + exitCalled: true, + writeSettled: true, + exitObservedSettledWrite: true, + }) +}, 20_000) + +testLinuxTraceFile('graceful shutdown queues otherwise-pending trace records', () => { + expect(runFixture({ TRACE_SHUTDOWN_PENDING_ONLY: '1' })).toEqual({ + aborted: true, + exitCalled: true, + writeSettled: true, + exitObservedSettledWrite: true, + }) +}, 20_000) + +testLinuxTraceFile('graceful shutdown bounds a blocked interruption trace drain', () => { + expect(runFixture({ TRACE_SHUTDOWN_BLOCK_WRITE: '1' })).toEqual({ + aborted: true, + exitCalled: true, + writeSettled: false, + exitObservedSettledWrite: false, + }) +}, 20_000) diff --git a/src/utils/gracefulShutdown.ts b/src/utils/gracefulShutdown.ts index 9a5e397a64..8f1241d151 100644 --- a/src/utils/gracefulShutdown.ts +++ b/src/utils/gracefulShutdown.ts @@ -39,6 +39,10 @@ import { runCleanupFunctions } from './cleanupRegistry.js' import { createCombinedAbortSignal } from './combinedAbortSignal.js' import { logForDebugging } from './debug.js' import { logForDiagnosticsNoPII } from './diagLogs.js' +import { + flushInterruptionTrace, + waitForInterruptionTraceFlush, +} from './interruptionTrace.js' import { isEnvTruthy } from './envUtils.js' import { getCurrentSessionTitle, sessionIdExists } from './sessionStorage.js' import { sleep } from './sleep.js' @@ -534,6 +538,12 @@ export async function gracefulShutdown( } } + // Root interruption traces are queued off the cancellation path so they + // cannot delay abort. Drain that queue before the final process exit; the + // drain has its own budget so a blocked target cannot stall normal exit. + flushInterruptionTrace('graceful_shutdown') + await Promise.race([waitForInterruptionTraceFlush(), sleep(500)]) + forceExit(exitCode) } diff --git a/src/utils/handlePromptSubmit.test.ts b/src/utils/handlePromptSubmit.test.ts index fa3d8dd3f5..e2f33e5132 100644 --- a/src/utils/handlePromptSubmit.test.ts +++ b/src/utils/handlePromptSubmit.test.ts @@ -7,6 +7,12 @@ import * as realAnalyticsModule from 'src/services/analytics/index.js' import { getCommandQueue, resetCommandQueue } from './messageQueueManager.js' import { createUserMessage } from './messages.js' import * as realProcessUserInputModule from './processUserInput/processUserInput.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, +} from './interruptionTrace.js' +import type { HandlePromptSubmitParams } from './handlePromptSubmit.js' const realAnalytics = { ...realAnalyticsModule } const realProcessUserInput = { ...realProcessUserInputModule } @@ -666,4 +672,61 @@ describe('handlePromptSubmit', () => { }, ]) }) + + it('traces the explicit submit-interrupt path at runtime', async () => { + const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const controller = new AbortController() + const { handlePromptSubmit } = await import('./handlePromptSubmit.js') + try { + const params: HandlePromptSubmitParams = { + input: 'echo ready', + mode: 'bash', + pastedContents: {}, + helpers: { + setCursorOffset: () => {}, + clearBuffer: () => {}, + resetHistory: () => {}, + }, + onInputChange: () => {}, + setPastedContents: () => {}, + abortController: controller, + hasInterruptibleToolInProgress: true, + streamMode: 'requesting', + queryGuard: { isActive: true } as never, + isExternalLoading: false, + commands: [], + messages: [], + mainLoopModel: 'sonnet', + ideSelection: undefined, + querySource: 'repl' as never, + setToolJSX: () => {}, + getToolUseContext: () => ({}) as never, + setUserInputOnProcessing: () => {}, + setAbortController: () => {}, + onQuery: async () => {}, + setAppState: () => ({}) as never, + } + await handlePromptSubmit(params) + + expect(controller.signal.aborted).toBe(true) + const trace = __getInterruptionTraceSnapshotForTests() + const inputEvent = trace.find( + entry => entry.event === 'input.submit_interrupt', + ) + const abortEvent = trace.find(entry => entry.event === 'abort.requested') + expect(abortEvent?.source).toBe('interrupt_on_submit') + expect(inputEvent).toBeDefined() + expect(abortEvent).toBeDefined() + expect(typeof inputEvent!.eventId).toBe('string') + expect(typeof abortEvent!.causalEventId).toBe('string') + expect(abortEvent!.causalEventId).toBe(inputEvent!.eventId) + } finally { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + }) }) diff --git a/src/utils/handlePromptSubmit.ts b/src/utils/handlePromptSubmit.ts index d226e34da1..85ab84567a 100644 --- a/src/utils/handlePromptSubmit.ts +++ b/src/utils/handlePromptSubmit.ts @@ -24,6 +24,7 @@ import type { EffortValue } from './effort.js' import type { FileHistoryState } from './fileHistory.js' import { fileHistoryEnabled, fileHistoryMakeSnapshot } from './fileHistory.js' import { gracefulShutdownSync } from './gracefulShutdown.js' +import { requestAbort, traceInterruptionEvent } from './interruptionTrace.js' import { enqueue } from './messageQueueManager.js' import { resolveSkillModelOverride } from './model/model.js' import type { ProcessUserInputContext } from './processUserInput/processUserInput.js' @@ -376,7 +377,18 @@ export async function handlePromptSubmit( streamMode: params.streamMode as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, }) - params.abortController?.abort('interrupt') + if (params.abortController) { + const causalEventId = traceInterruptionEvent('input.submit_interrupt', { + source: 'interrupt_on_submit', + subsystem: 'prompt_submit', + }) + requestAbort(params.abortController, 'interrupt', { + source: 'interrupt_on_submit', + subsystem: 'prompt_submit', + controllerRole: 'query-root', + causalEventId, + }) + } } // Enqueue with string value + raw pastedContents. Images will be resized diff --git a/src/utils/interruptionTrace.test.ts b/src/utils/interruptionTrace.test.ts new file mode 100644 index 0000000000..3ad44a7480 --- /dev/null +++ b/src/utils/interruptionTrace.test.ts @@ -0,0 +1,923 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test' +import { mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import * as interruptionTraceModule from './interruptionTrace.js' +import { + __getInterruptionTraceSnapshotForTests, + __INTERRUPTION_TRACE_CAPACITY_FOR_TESTS, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, + flushInterruptionTrace, + getInterruptionSignalAbortEventId, + registerInterruptionController, + requestAbort, + traceInterruptionEvent, +} from './interruptionTrace.js' +import { createCombinedAbortSignal } from './combinedAbortSignal.js' +import { createChildAbortController } from './abortController.js' +import { logForDiagnosticsNoPII } from './diagLogs.js' +import { + getFsImplementation, + setFsImplementation, + type FsOperations, +} from './fsOperations.js' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../test/sharedMutationLock.js' + +const originalEnabled = process.env.OPENCLAUDE_INTERRUPT_TRACE +const originalFile = process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE +const originalDiagnosticsFile = process.env.CLAUDE_CODE_DIAGNOSTICS_FILE +let tempDirectory: string | undefined +let originalFs: FsOperations +const testLinuxTraceFile = process.platform === 'linux' ? test : test.skip +const testPosixSymlink = process.platform === 'win32' ? test.skip : test + +beforeEach(async () => { + await acquireSharedMutationLock('utils/interruptionTrace.test.ts') + originalFs = getFsImplementation() + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + delete process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE + delete process.env.CLAUDE_CODE_DIAGNOSTICS_FILE +}) + +afterEach(async () => { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + setFsImplementation(originalFs) + if (originalEnabled === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalEnabled + if (originalFile === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE + else process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = originalFile + if (originalDiagnosticsFile === undefined) { + delete process.env.CLAUDE_CODE_DIAGNOSTICS_FILE + } else { + process.env.CLAUDE_CODE_DIAGNOSTICS_FILE = originalDiagnosticsFile + } + if (tempDirectory) { + await rm(tempDirectory, { recursive: true, force: true }) + tempDirectory = undefined + } + releaseSharedMutationLock() +}) + +describe('interruptionTrace', () => { + test('is a true no-op while disabled and preserves native abort behavior', () => { + const controller = new AbortController() + registerInterruptionController(controller, { controllerRole: 'root' }) + traceInterruptionEvent('query.started', { queryId: 'query-1' }) + requestAbort(controller, 'query-timeout', { + source: 'query_guard', + controllerRole: 'root', + }) + + expect(controller.signal.reason).toBe('query-timeout') + expect(__getInterruptionTraceSnapshotForTests()).toEqual([]) + }) + + test('correlates controllers and records first-wins plus repeated requests', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const controller = new AbortController() + const controllerId = registerInterruptionController(controller, { + controllerRole: 'query-root', + queryId: 'query-1', + }) + + requestAbort(controller, 'query-timeout', { + source: 'query_guard', + queryId: 'query-1', + }) + requestAbort(controller, 'user-cancel', { + source: 'cancel_keybinding', + queryId: 'query-1', + }) + + const entries = __getInterruptionTraceSnapshotForTests() + const requested = entries.find(entry => entry.event === 'abort.requested') + const observed = entries.find(entry => entry.event === 'signal.observed') + const repeated = entries.find(entry => entry.event === 'abort.repeated') + expect(controller.signal.reason).toBe('query-timeout') + expect(requested?.controllerId).toBe(controllerId) + expect(requested?.normalizedReason).toBe('query-timeout') + expect(requested?.abortStackFingerprint).toMatch(/^[a-f0-9]{16}$/) + expect(requested).not.toHaveProperty('abortCallSites') + expect(typeof requested?.eventId).toBe('string') + expect(typeof observed?.firstAbortEventId).toBe('string') + expect(typeof repeated?.firstAbortEventId).toBe('string') + expect(observed!.firstAbortEventId).toBe(requested!.eventId) + expect(repeated!.firstAbortEventId).toBe(requested!.eventId) + expect(repeated?.existingNormalizedReason).toBe('query-timeout') + expect(repeated?.attemptedNormalizedReason).toBe('user-abort') + expect(repeated?.outcome).toBe('ignored_first_abort_wins') + expect(repeated?.repeatedCount).toBe(1) + }) + + test('links a combined signal abort to the winning parent request', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const parent = new AbortController() + registerInterruptionController(parent, { + controllerRole: 'query-root', + }) + const combined = createCombinedAbortSignal(parent.signal, { + trace: { + subsystem: 'trace-test', + controllerRole: 'combined', + }, + }) + + requestAbort(parent, 'query-timeout', { + source: 'query_guard', + subsystem: 'trace-test', + controllerRole: 'query-root', + }) + + const requested = __getInterruptionTraceSnapshotForTests().filter( + entry => entry.event === 'abort.requested', + ) + expect(combined.signal.reason).toBe('query-timeout') + expect(requested).toHaveLength(2) + expect(requested[1]?.causalEventId).toBe(requested[0]?.eventId) + expect(requested[1]?.winningParentControllerId).toBe( + requested[0]?.controllerId, + ) + combined.cleanup() + }) + + test('assigns a causal event id when a registered signal aborts natively', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const controller = new AbortController() + registerInterruptionController(controller, { controllerRole: 'external' }) + + controller.abort('external-abort') + + const observed = __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'signal.observed', + ) + expect(observed).toBeDefined() + expect(typeof observed?.eventId).toBe('string') + expect(getInterruptionSignalAbortEventId(controller.signal)).toBe( + observed!.eventId, + ) + + requestAbort(controller, 'second-abort', { + source: 'native-abort-test', + controllerRole: 'external', + }) + const repeated = __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'abort.repeated', + ) + expect(repeated).toBeDefined() + expect(typeof repeated?.firstAbortEventId).toBe('string') + expect(repeated!.firstAbortEventId).toBe(observed!.eventId) + }) + + test('links a native AbortSignal.any result to its winning parent', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const caller = new AbortController() + const deadline = new AbortController() + const combined = AbortSignal.any([caller.signal, deadline.signal]) + interruptionTraceModule.traceCombinedAbortSignal( + combined, + [caller.signal, deadline.signal], { + subsystem: 'native-any-test', + controllerRole: 'request-combined', + }, + ) + + caller.abort('user-cancel') + + const entries = __getInterruptionTraceSnapshotForTests() + const parentObserved = entries.find( + entry => + entry.event === 'signal.observed' && + entry.controllerRole === 'combined-parent', + ) + const combinedObserved = entries.find( + entry => + entry.event === 'signal.observed' && + entry.controllerRole === 'request-combined', + ) + expect(parentObserved).toBeDefined() + expect(combinedObserved).toMatchObject({ + subsystem: 'native-any-test', + causalEventId: parentObserved!.eventId, + winningParentControllerId: parentObserved!.controllerId, + }) + }) + + test('records permission abort resolution with the input causal edge', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const inputEventId = traceInterruptionEvent('input.ctrl_c') + + interruptionTraceModule.tracePermissionAbortResolution( + 'ctrl_c', + inputEventId, + 'remote-permission', + ) + + expect(__getInterruptionTraceSnapshotForTests().at(-1)).toMatchObject({ + event: 'permission.abort_resolved', + source: 'ctrl_c', + subsystem: 'remote-permission', + causalEventId: inputEventId, + outcome: 'denied', + }) + }) + + testLinuxTraceFile('observes and flushes a query-root controller already aborted when registered', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl' + let writes = 0 + setFsImplementation({ + ...originalFs, + appendRegularFile: async () => { + writes++ + }, + }) + const controller = new AbortController() + controller.abort('external-abort') + + registerInterruptionController(controller, { controllerRole: 'query-root' }) + await __waitForInterruptionTraceFlushForTests() + + const observed = __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'signal.observed', + ) + expect(observed?.controllerRole).toBe('query-root') + expect(getInterruptionSignalAbortEventId(controller.signal)).toBe( + observed?.eventId, + ) + expect(writes).toBe(1) + }) + + testLinuxTraceFile('preserves an established query-root role when creating a child', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl' + let writes = 0 + setFsImplementation({ + ...originalFs, + appendRegularFile: async () => { + writes++ + }, + }) + const parent = new AbortController() + registerInterruptionController(parent, { + controllerRole: 'query-root', + queryId: 'query-root-1', + queryGeneration: 3, + }) + createChildAbortController(parent) + + requestAbort(parent, 'user-cancel', { + source: 'cancel_keybinding', + controllerRole: 'tool', + }) + await __waitForInterruptionTraceFlushForTests() + + const observed = __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'signal.observed' && + entry.normalizedReason === 'user-abort', + ) + const requested = __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'abort.requested', + ) + expect(requested).toMatchObject({ + source: 'cancel_keybinding', + controllerRole: 'query-root', + queryId: 'query-root-1', + queryGeneration: 3, + }) + expect(observed?.controllerRole).toBe('query-root') + expect(writes).toBe(1) + }) + + test('replaces provisional parent and child roles with concrete lifecycle roles', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const parent = new AbortController() + const child = createChildAbortController(parent) + + requestAbort(child, 'sibling_error', { + source: 'sibling_error', + subsystem: 'streaming_tool_executor', + controllerRole: 'sibling-tools', + }) + requestAbort(parent, 'task_stop', { + source: 'task_stop', + subsystem: 'local_agent_task', + controllerRole: 'background-agent', + subagentId: 'agent-1', + }) + + const requested = __getInterruptionTraceSnapshotForTests().filter( + entry => entry.event === 'abort.requested', + ) + expect(requested).toHaveLength(2) + expect(requested[0]).toMatchObject({ + source: 'sibling_error', + controllerRole: 'sibling-tools', + }) + expect(requested[1]).toMatchObject({ + source: 'task_stop', + controllerRole: 'background-agent', + subagentId: 'agent-1', + }) + const observed = __getInterruptionTraceSnapshotForTests().filter( + entry => entry.event === 'signal.observed', + ) + expect(observed).toHaveLength(2) + expect(observed[0]?.controllerRole).toBe('sibling-tools') + expect(observed[1]).toMatchObject({ + controllerRole: 'background-agent', + subagentId: 'agent-1', + }) + }) + + test('retains a child owner when the parent aborts before a later request', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const parent = new AbortController() + const child = createChildAbortController(parent, undefined, { + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: 'agent-1', + }) + + requestAbort(parent, undefined, { + source: 'cancel_keybinding', + subsystem: 'query_engine', + controllerRole: 'query-root', + }) + + const childObserved = __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'signal.observed' && + entry.controllerRole === 'subagent-lifecycle', + ) + expect(child.signal.aborted).toBe(true) + expect(childObserved).toMatchObject({ + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: 'agent-1', + }) + }) + + test('throwing abort-reason accessors cannot block native or combined aborts', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const reason = new Proxy( + {}, + { + get() { + throw new Error('reason getter must stay isolated') + }, + }, + ) + const direct = new AbortController() + expect(() => + requestAbort(direct, reason, { + source: 'throwing-reason-test', + controllerRole: 'query-root', + }), + ).not.toThrow() + expect(direct.signal.aborted).toBe(true) + expect(direct.signal.reason).toBe(reason) + + const parent = new AbortController() + const combined = createCombinedAbortSignal(parent.signal) + expect(() => parent.abort(reason)).not.toThrow() + expect(combined.signal.aborted).toBe(true) + expect(combined.signal.reason).toBe(reason) + combined.cleanup() + }) + + test('does not persist arbitrary custom error names', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const reason = new Error('private reason message') + reason.name = 'private customer prompt' + + traceInterruptionEvent('custom.error', { reason, error: reason }) + + const serialized = JSON.stringify(__getInterruptionTraceSnapshotForTests()) + expect(serialized).not.toContain('private customer prompt') + expect(serialized).not.toContain('private reason message') + const entry = __getInterruptionTraceSnapshotForTests().at(-1) + expect(entry?.rawReasonType).toBe('Error:Error') + expect(entry?.safeErrorIdentity).toBe('Error') + }) + + test('preserves only standardized DOMException names', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const standard = new DOMException('private message', 'AbortError') + const custom = new DOMException('private message', 'private customer prompt') + + traceInterruptionEvent('standard.dom.error', { + reason: standard, + error: standard, + }) + traceInterruptionEvent('custom.dom.error', { + reason: custom, + error: custom, + }) + + const entries = __getInterruptionTraceSnapshotForTests() + expect(entries.at(-2)?.rawReasonType).toBe('DOMException:AbortError') + expect(entries.at(-2)?.safeErrorIdentity).toBe('DOMException:AbortError') + expect(entries.at(-1)?.rawReasonType).toBe('DOMException') + expect(entries.at(-1)?.safeErrorIdentity).toBe('DOMException') + expect(JSON.stringify(entries)).not.toContain('private customer prompt') + expect(JSON.stringify(entries)).not.toContain('private message') + }) + + test('keeps only the newest allowlisted records up to capacity', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = 'true' + const emitted = __INTERRUPTION_TRACE_CAPACITY_FOR_TESTS + 88 + for (let index = 0; index < emitted; index++) { + traceInterruptionEvent('stream.progress', { + rawByteCount: index, + // Runtime extras are deliberately ignored by the allowlist boundary. + ...({ prompt: 'must-not-appear' } as Record), + }) + } + + const entries = __getInterruptionTraceSnapshotForTests() + expect(entries).toHaveLength(__INTERRUPTION_TRACE_CAPACITY_FOR_TESTS) + expect(entries[0]?.sequence).toBe(emitted - __INTERRUPTION_TRACE_CAPACITY_FOR_TESTS + 1) + expect(entries.at(-1)?.sequence).toBe(emitted) + expect(JSON.stringify(entries)).not.toContain('must-not-appear') + expect(JSON.stringify(entries)).not.toContain('prompt') + }) + + testLinuxTraceFile('flushes valid JSONL once to an explicit absolute path', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-interrupt-trace-')) + const traceFile = join(tempDirectory, 'trace.jsonl') + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = traceFile + + traceInterruptionEvent('query.started', { + queryId: 'query-1', + model: 'gpt-test', + }) + flushInterruptionTrace('test') + flushInterruptionTrace('test-repeat') + await __waitForInterruptionTraceFlushForTests() + + const lines = (await readFile(traceFile, 'utf8')).trim().split('\n') + const entries = lines.map(line => JSON.parse(line) as { event: string }) + expect(entries.map(entry => entry.event)).toEqual([ + 'query.started', + 'trace.flush', + ]) + }) + + testLinuxTraceFile('flushes every pending record when the ring is at capacity', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-interrupt-trace-')) + const traceFile = join(tempDirectory, 'trace.jsonl') + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = traceFile + for (let index = 0; index < __INTERRUPTION_TRACE_CAPACITY_FOR_TESTS; index++) { + traceInterruptionEvent('stream.progress', { rawByteCount: index }) + } + + flushInterruptionTrace('capacity') + await __waitForInterruptionTraceFlushForTests() + + const entries = (await readFile(traceFile, 'utf8')) + .trim() + .split('\n') + .map(line => JSON.parse(line) as { sequence: number; event: string }) + expect(entries).toHaveLength(__INTERRUPTION_TRACE_CAPACITY_FOR_TESTS + 1) + expect(entries[0]?.sequence).toBe(1) + expect(entries.at(-1)?.event).toBe('trace.flush') + }) + + testLinuxTraceFile('retains pending records after a failed write and retries them', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl' + let failWrites = true + let writeAttempts = 0 + const successfulWrites: string[] = [] + setFsImplementation({ + ...originalFs, + mkdirSync: () => {}, + appendRegularFile: async (_path, data) => { + writeAttempts++ + if (failWrites) throw new Error('synthetic write failure') + successfulWrites.push(data) + }, + }) + + traceInterruptionEvent('first') + flushInterruptionTrace('failed') + await __waitForInterruptionTraceFlushForTests() + expect(writeAttempts).toBe(1) + failWrites = false + traceInterruptionEvent('second') + flushInterruptionTrace('retry') + await __waitForInterruptionTraceFlushForTests() + + expect(writeAttempts).toBe(3) + expect(successfulWrites).toHaveLength(2) + const events = successfulWrites.flatMap(write => + write + .trim() + .split('\n') + .map(line => (JSON.parse(line) as { event: string }).event), + ) + expect(events).toEqual(['first', 'trace.flush', 'second', 'trace.flush']) + }) + + testLinuxTraceFile('captures the enabled output target before a detached flush starts', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/captured/trace.jsonl' + const writes: Array<{ path: string; data: string }> = [] + setFsImplementation({ + ...originalFs, + appendRegularFile: async (path, data) => { + writes.push({ path, data }) + }, + }) + + traceInterruptionEvent('before_restore') + flushInterruptionTrace('captured-target') + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + delete process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE + await __waitForInterruptionTraceFlushForTests() + + expect(writes).toHaveLength(1) + expect(writes[0]?.path).toBe('/captured/trace.jsonl') + expect(writes[0]?.data).toContain('"event":"before_restore"') + expect(writes[0]?.data).toContain('"event":"trace.flush"') + }) + + testLinuxTraceFile('keeps a retry batch bound to its original output target', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/first/trace.jsonl' + const writes: Array<{ path: string; data: string }> = [] + setFsImplementation({ + ...originalFs, + appendRegularFile: async (path, data) => { + writes.push({ path, data }) + if (writes.length === 1) throw new Error('synthetic retryable failure') + }, + }) + + traceInterruptionEvent('first_target') + flushInterruptionTrace('first') + await __waitForInterruptionTraceFlushForTests() + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/second/trace.jsonl' + traceInterruptionEvent('second_target') + flushInterruptionTrace('second') + await __waitForInterruptionTraceFlushForTests() + + expect(writes.map(write => write.path)).toEqual([ + '/first/trace.jsonl', + '/first/trace.jsonl', + '/second/trace.jsonl', + ]) + expect(writes[1]?.data).toContain('"event":"first_target"') + expect(writes[2]?.data).toContain('"event":"second_target"') + expect(writes[2]?.data).not.toContain('"event":"first_target"') + }) + + testLinuxTraceFile('retains an in-flight failed batch outside the bounded ring', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl' + let rejectFirstWrite!: (error: Error) => void + let markFirstWriteStarted!: () => void + const firstWriteStarted = new Promise(resolve => { + markFirstWriteStarted = resolve + }) + const firstWriteBlocked = new Promise((_resolve, reject) => { + rejectFirstWrite = reject + }) + const writes: string[] = [] + setFsImplementation({ + ...originalFs, + appendRegularFile: async (_path, data) => { + if (writes.length === 0) { + writes.push('failed') + markFirstWriteStarted() + await firstWriteBlocked + return + } + writes.push(data) + }, + }) + + traceInterruptionEvent('must_survive') + flushInterruptionTrace('blocked') + await firstWriteStarted + for (let index = 0; index < __INTERRUPTION_TRACE_CAPACITY_FOR_TESTS + 32; index++) { + traceInterruptionEvent('newer', { rawByteCount: index }) + } + rejectFirstWrite(new Error('synthetic write failure')) + await __waitForInterruptionTraceFlushForTests() + flushInterruptionTrace('retry') + await __waitForInterruptionTraceFlushForTests() + + expect(writes).toHaveLength(3) + expect(writes[1]).toContain('"event":"must_survive"') + }) + + testLinuxTraceFile('does not replay a batch after an uncertain append failure', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl' + const writes: string[] = [] + setFsImplementation({ + ...originalFs, + appendRegularFile: async (_path, data) => { + writes.push(data) + if (writes.length === 1) { + throw Object.assign(new Error('partial append'), { + code: 'ERR_DIAGNOSTIC_APPEND_UNCERTAIN', + }) + } + }, + }) + + traceInterruptionEvent('first') + flushInterruptionTrace('uncertain') + await __waitForInterruptionTraceFlushForTests() + traceInterruptionEvent('second') + flushInterruptionTrace('later') + await __waitForInterruptionTraceFlushForTests() + + expect(writes).toHaveLength(2) + expect(writes[0]).toContain('"event":"first"') + expect(writes[1]).not.toContain('"event":"first"') + expect(writes[1]).toContain('"event":"second"') + }) + + testLinuxTraceFile('rejects an existing non-regular trace target', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-interrupt-trace-')) + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = tempDirectory + traceInterruptionEvent('pending') + + flushInterruptionTrace('non-regular') + await __waitForInterruptionTraceFlushForTests() + + expect( + __getInterruptionTraceSnapshotForTests().some( + entry => entry.event === 'trace.flush', + ), + ).toBe(false) + }) + + testLinuxTraceFile('rejects symlink targets and creates private files and directories', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-interrupt-trace-')) + const target = join(tempDirectory, 'target.jsonl') + const link = join(tempDirectory, 'trace-link.jsonl') + await writeFile(target, '') + await symlink(target, link) + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = link + traceInterruptionEvent('symlink-pending') + flushInterruptionTrace('symlink') + await __waitForInterruptionTraceFlushForTests() + expect(await readFile(target, 'utf8')).toBe('') + + // The rejected batch remains bound to its original target for retry. + // Reset before exercising independent creation and permission behavior. + __resetInterruptionTraceForTests() + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + + const privateDirectory = join(tempDirectory, 'private') + const privateTrace = join(privateDirectory, 'trace.jsonl') + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = privateTrace + traceInterruptionEvent('private-target-pending') + flushInterruptionTrace('private-target') + await __waitForInterruptionTraceFlushForTests() + expect((await stat(privateDirectory)).mode & 0o777).toBe(0o700) + expect((await stat(privateTrace)).mode & 0o777).toBe(0o600) + + const existingTrace = join(tempDirectory, 'existing.jsonl') + await writeFile(existingTrace, '', { mode: 0o644 }) + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = existingTrace + traceInterruptionEvent('existing-private-target') + flushInterruptionTrace('existing-private-target') + await __waitForInterruptionTraceFlushForTests() + expect((await stat(existingTrace)).mode & 0o777).toBe(0o600) + }) + + testLinuxTraceFile('rejects a trace target beneath a symlinked parent directory', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-interrupt-trace-')) + const realDirectory = join(tempDirectory, 'real-parent') + const linkedDirectory = join(tempDirectory, 'linked-parent') + const realTrace = join(realDirectory, 'trace.jsonl') + await mkdir(realDirectory) + await symlink(realDirectory, linkedDirectory, 'dir') + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = join( + linkedDirectory, + 'trace.jsonl', + ) + + traceInterruptionEvent('symlinked-parent-pending') + flushInterruptionTrace('symlinked-parent') + await __waitForInterruptionTraceFlushForTests() + + const result = await readFile(realTrace, 'utf8').catch( + error => (error as NodeJS.ErrnoException).code, + ) + expect(result).toBe('ENOENT') + }) + + testPosixSymlink('preserves legacy diagnostics append-through-symlink behavior', async () => { + tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-diagnostics-')) + const target = join(tempDirectory, 'target.jsonl') + const link = join(tempDirectory, 'diagnostics-link.jsonl') + await writeFile(target, '') + await symlink(target, link) + process.env.CLAUDE_CODE_DIAGNOSTICS_FILE = link + + logForDiagnosticsNoPII('info', 'symlink-test') + + expect(await readFile(target, 'utf8')).toContain('symlink-test') + delete process.env.CLAUDE_CODE_DIAGNOSTICS_FILE + }) + + test('does not block native abort while a trace append is pending', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl' + let releaseWrite!: () => void + const writeBlocked = new Promise(resolve => { + releaseWrite = resolve + }) + setFsImplementation({ + ...originalFs, + appendRegularFile: async () => writeBlocked, + }) + const controller = new AbortController() + + try { + requestAbort(controller, 'user-cancel', { + source: 'cancel_keybinding', + controllerRole: 'query-root', + }) + + expect(controller.signal.aborted).toBe(true) + } finally { + releaseWrite() + await __waitForInterruptionTraceFlushForTests() + } + }) + + testLinuxTraceFile('keeps sequence IDs unique and drains later events during an async flush', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl' + let releaseFirstWrite!: () => void + let markFirstWriteStarted!: () => void + const firstWriteStarted = new Promise(resolve => { + markFirstWriteStarted = resolve + }) + const firstWriteBlocked = new Promise(resolve => { + releaseFirstWrite = resolve + }) + const writes: string[] = [] + setFsImplementation({ + ...originalFs, + appendRegularFile: async (_path, data) => { + writes.push(data) + if (writes.length === 1) { + markFirstWriteStarted() + await firstWriteBlocked + } + }, + }) + + traceInterruptionEvent('before') + flushInterruptionTrace('first') + try { + await firstWriteStarted + traceInterruptionEvent('during_one') + traceInterruptionEvent('during_two') + } finally { + releaseFirstWrite() + await __waitForInterruptionTraceFlushForTests() + } + const persistedEvents = writes.flatMap(write => + write + .trim() + .split('\n') + .map(line => (JSON.parse(line) as { event: string }).event), + ) + expect(persistedEvents.filter(event => event === 'before')).toHaveLength(1) + expect(persistedEvents.filter(event => event === 'during_one')).toHaveLength(1) + expect(persistedEvents.filter(event => event === 'during_two')).toHaveLength(1) + const snapshot = __getInterruptionTraceSnapshotForTests() + expect(new Set(snapshot.map(entry => entry.eventId)).size).toBe(snapshot.length) + expect(snapshot.map(entry => entry.sequence)).toEqual( + [...snapshot.map(entry => entry.sequence)].sort((left, right) => left - right), + ) + }) + + test('never persists dynamic function names from abort stacks', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const controller = new AbortController() + const customerSpecificHandlerName = () => { + requestAbort(controller, 'user-cancel', { source: 'test' }) + } + + customerSpecificHandlerName() + + const serialized = JSON.stringify(__getInterruptionTraceSnapshotForTests()) + expect(serialized).not.toContain('customerSpecificHandlerName') + expect(serialized).not.toContain('abortCallSites') + }) + + test('does not write for relative paths and isolates write failures', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + traceInterruptionEvent('query.started') + + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = 'relative-trace.jsonl' + expect(() => flushInterruptionTrace('relative')).not.toThrow() + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/proc/openclaude/trace.jsonl' + expect(() => flushInterruptionTrace('unwritable')).not.toThrow() + await __waitForInterruptionTraceFlushForTests() + }) + + test('discards trace-file batches without retrying on unsupported platforms', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl' + const writtenData: string[] = [] + setFsImplementation({ + ...originalFs, + appendRegularFile: async (_path, data) => { + writtenData.push(data) + }, + }) + const platformDescriptor = Object.getOwnPropertyDescriptor(process, 'platform') + Object.defineProperty(process, 'platform', { + value: 'darwin', + configurable: true, + }) + try { + traceInterruptionEvent('pending') + flushInterruptionTrace('posix-platform') + await __waitForInterruptionTraceFlushForTests() + traceInterruptionEvent('later') + flushInterruptionTrace('posix-platform-later') + await __waitForInterruptionTraceFlushForTests() + expect(writtenData).toEqual([]) + } finally { + if (platformDescriptor) { + Object.defineProperty(process, 'platform', platformDescriptor) + } + } + if (process.platform === 'linux') { + traceInterruptionEvent('after_restore') + flushInterruptionTrace('restored-platform') + await __waitForInterruptionTraceFlushForTests() + expect(writtenData).toHaveLength(1) + const restoredWrite = writtenData[0] ?? '' + expect(restoredWrite).toContain('"event":"after_restore"') + expect(restoredWrite).not.toContain('"event":"pending"') + expect(restoredWrite).not.toContain('"event":"later"') + } + }) + + test('redacts secret-shaped values and absolute local paths', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + traceInterruptionEvent('provider.failed', { + model: 'AKIA1234567890ABCDEF', + providerRoute: 'github_pat_1234567890abcdef', + attemptId: '\\\\server\\share\\private', + queryId: 'prefix(/srv/private/project)', + error: new Error('message content is never serialized'), + }) + + const serialized = JSON.stringify(__getInterruptionTraceSnapshotForTests()) + expect(serialized).toContain('[redacted]') + expect(serialized).toContain('Error') + expect(serialized).not.toContain('AKIA1234567890ABCDEF') + expect(serialized).not.toContain('github_pat_1234567890abcdef') + expect(serialized).not.toContain('server') + expect(serialized).not.toContain('/srv/private/project') + expect(serialized).not.toContain('message content') + }) + + test('never serializes secret-shaped or path-shaped abort reasons', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const secretReason = 'github_pat_1234567890abcdef' + const pathReason = '/srv/private/project/abort-reason' + + traceInterruptionEvent('abort.requested', { reason: secretReason }) + traceInterruptionEvent('abort.repeated', { + existingReason: pathReason, + attemptedReason: secretReason, + }) + + const snapshot = __getInterruptionTraceSnapshotForTests() + const serialized = JSON.stringify(snapshot) + expect(snapshot).toHaveLength(2) + expect(snapshot[0]?.normalizedReason).toBe('unknown-abort') + expect(snapshot[1]?.existingNormalizedReason).toBe('unknown-abort') + expect(snapshot[1]?.attemptedNormalizedReason).toBe('unknown-abort') + expect(serialized).not.toContain(secretReason) + expect(serialized).not.toContain(pathReason) + }) +}) diff --git a/src/utils/interruptionTrace.ts b/src/utils/interruptionTrace.ts new file mode 100644 index 0000000000..3999505d98 --- /dev/null +++ b/src/utils/interruptionTrace.ts @@ -0,0 +1,790 @@ +import { createHash, randomUUID } from 'node:crypto' +import { isAbsolute, posix, win32 } from 'node:path' +import { + monitorEventLoopDelay, + type IntervalHistogram, +} from 'node:perf_hooks' +import { normalizeAbortReason } from './abortReasons.js' +import { appendDiagnosticsNoPII } from './diagLogs.js' +import { redactHomePath, redactLikelySecrets } from './redaction.js' + +const TRACE_CAPACITY = 512 +export const __INTERRUPTION_TRACE_CAPACITY_FOR_TESTS = TRACE_CAPACITY +const TRACE_SCHEMA_VERSION = 1 +const TRACE_ENABLED_ENV = 'OPENCLAUDE_INTERRUPT_TRACE' +const TRACE_FILE_ENV = 'OPENCLAUDE_INTERRUPT_TRACE_FILE' + +export type InterruptionTraceFields = { + source?: string + subsystem?: string + phase?: string + queryId?: string + queryGeneration?: number + querySource?: string + parentQueryId?: string + subagentId?: string + providerRoute?: string + transport?: string + model?: string + attemptId?: string + controllerRole?: string + parentControllerIds?: readonly string[] + winningParentControllerId?: string + causalEventId?: string + trigger?: string + outcome?: string + reason?: unknown + existingReason?: unknown + attemptedReason?: unknown + error?: unknown + elapsedQueryMs?: number + sinceLastActivityMs?: number + sinceLastRawByteMs?: number + sinceLastParsedFrameMs?: number + sinceLastYieldMs?: number + rawByteCount?: number + parsedFrameCount?: number + controlFrameCount?: number + ignoredFrameCount?: number + ignoredParsedFrameCount?: number + yieldedEventCount?: number + activeApiCallCount?: number + activeToolUseCount?: number + leaseCount?: number + suspendCount?: number + repeatedCount?: number + eventLoopDelayMaxMs?: number + eventLoopDelayMeanMs?: number +} + +type SafeTraceFields = Omit< + InterruptionTraceFields, + | 'reason' + | 'existingReason' + | 'attemptedReason' + | 'error' + | 'parentControllerIds' +> & { + normalizedReason?: string + rawReasonType?: string + existingNormalizedReason?: string + existingRawReasonType?: string + attemptedNormalizedReason?: string + attemptedRawReasonType?: string + safeErrorIdentity?: string + parentControllerIds?: string[] +} + +export type InterruptionTraceEntry = SafeTraceFields & { + schemaVersion: number + sequence: number + eventId: string + traceSessionId: string + timestamp: string + monotonicMs: number + clockDeltaMs: number + event: string + controllerId?: string + firstAbortEventId?: string + abortStackFingerprint?: string +} + +type ControllerTraceState = { + id: string + firstAbortEventId?: string + repeatedCount: number + fields: InterruptionTraceFields + provisionalRole: boolean +} + +type SignalTraceState = { + id: string + fields: InterruptionTraceFields + getAbortFields?: () => InterruptionTraceFields +} + +let traceSessionId = '' +let sequence = 0 +let startedWallMs = Date.now() +let startedMonotonicMs = performance.now() +let ring: InterruptionTraceEntry[] = [] +let flushedThroughSequence = 0 +let controllerCounter = 0 +let signalCounter = 0 +let controllerStates = new WeakMap() +let signalIds = new WeakMap() +let signalAbortEventIds = new WeakMap() +let signalAbortSources = new WeakMap() +let signalStates = new WeakMap() +let errorCausalEventIds = new WeakMap() +let eventLoopDelay: IntervalHistogram | undefined +let flushQueue: Promise = Promise.resolve() +let retryBatch: + | { + entries: readonly InterruptionTraceEntry[] + marker: InterruptionTraceEntry + logFile: string + } + | undefined + +function isEnabled(): boolean { + const value = process.env[TRACE_ENABLED_ENV]?.toLowerCase() + return value === '1' || value === 'true' +} + +export function isInterruptionTraceEnabled(): boolean { + return isEnabled() +} + +function safeString(value: unknown): string | undefined { + if (typeof value !== 'string' || value.length === 0) return undefined + const clipped = value.slice(0, 160) + const redacted = redactLikelySecrets(redactHomePath(clipped)) + if ( + redacted !== clipped || + posix.isAbsolute(clipped) || + win32.isAbsolute(clipped) || + /(?:^|[^a-z0-9_])(?:[a-z]:[\\/]|[/\\]{1,2})/i.test(clipped) + ) { + return '[redacted]' + } + return clipped +} + +function safeEventName(value: string): string { + return /^[a-z0-9][a-z0-9._-]{0,79}$/i.test(value) ? value : 'unknown' +} + +function safeFiniteNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined +} + +const BUILTIN_ERROR_NAMES = new Set([ + 'AggregateError', + 'Error', + 'EvalError', + 'RangeError', + 'ReferenceError', + 'SyntaxError', + 'TypeError', + 'URIError', +]) + +const STANDARD_DOM_EXCEPTION_NAMES = new Set([ + 'AbortError', + 'DataCloneError', + 'DataError', + 'EncodingError', + 'HierarchyRequestError', + 'InUseAttributeError', + 'IndexSizeError', + 'InvalidAccessError', + 'InvalidCharacterError', + 'InvalidModificationError', + 'InvalidNodeTypeError', + 'InvalidStateError', + 'NamespaceError', + 'NetworkError', + 'NoDataAllowedError', + 'NoModificationAllowedError', + 'NotAllowedError', + 'NotFoundError', + 'NotReadableError', + 'NotSupportedError', + 'OperationError', + 'QuotaExceededError', + 'ReadOnlyError', + 'SecurityError', + 'TimeoutError', + 'TransactionInactiveError', + 'UnknownError', + 'URLMismatchError', + 'VersionError', + 'WrongDocumentError', +]) + +function getSafeErrorName(error: Error): string { + if (error instanceof DOMException) { + return STANDARD_DOM_EXCEPTION_NAMES.has(error.name) + ? `DOMException:${error.name}` + : 'DOMException' + } + return BUILTIN_ERROR_NAMES.has(error.name) ? error.name : 'Error' +} + +function getRawReasonType(reason: unknown): string { + if (reason === null) return 'null' + if (reason instanceof DOMException) return getSafeErrorName(reason) + if (reason instanceof Error) return `Error:${getSafeErrorName(reason)}` + if (Array.isArray(reason)) return 'array' + return typeof reason +} + +function getSafeErrorIdentity(error: unknown): string | undefined { + if (error instanceof Error) return getSafeErrorName(error) + return error === undefined ? undefined : typeof error +} + +function toSafeFields(fields: InterruptionTraceFields): SafeTraceFields { + const reason = fields.reason + const safe: SafeTraceFields = {} + const stringFields = [ + 'source', + 'subsystem', + 'phase', + 'queryId', + 'querySource', + 'parentQueryId', + 'subagentId', + 'providerRoute', + 'transport', + 'model', + 'attemptId', + 'controllerRole', + 'winningParentControllerId', + 'causalEventId', + 'trigger', + 'outcome', + ] as const + for (const key of stringFields) { + const value = safeString(fields[key]) + if (value !== undefined) safe[key] = value + } + const numberFields = [ + 'queryGeneration', + 'elapsedQueryMs', + 'sinceLastActivityMs', + 'sinceLastRawByteMs', + 'sinceLastParsedFrameMs', + 'sinceLastYieldMs', + 'rawByteCount', + 'parsedFrameCount', + 'controlFrameCount', + 'ignoredFrameCount', + 'ignoredParsedFrameCount', + 'yieldedEventCount', + 'activeApiCallCount', + 'activeToolUseCount', + 'leaseCount', + 'suspendCount', + 'repeatedCount', + 'eventLoopDelayMaxMs', + 'eventLoopDelayMeanMs', + ] as const + for (const key of numberFields) { + const value = safeFiniteNumber(fields[key]) + if (value !== undefined) safe[key] = value + } + if (fields.parentControllerIds) { + safe.parentControllerIds = fields.parentControllerIds + .map(safeString) + .filter((value): value is string => value !== undefined) + .slice(0, 4) + } + if (reason !== undefined) { + safe.normalizedReason = safeString(normalizeAbortReason(reason)) + safe.rawReasonType = getRawReasonType(reason) + } + if (fields.existingReason !== undefined) { + safe.existingNormalizedReason = safeString( + normalizeAbortReason(fields.existingReason), + ) + safe.existingRawReasonType = getRawReasonType(fields.existingReason) + } + if (fields.attemptedReason !== undefined) { + safe.attemptedNormalizedReason = safeString( + normalizeAbortReason(fields.attemptedReason), + ) + safe.attemptedRawReasonType = getRawReasonType(fields.attemptedReason) + } + const safeErrorIdentity = getSafeErrorIdentity(fields.error) + if (safeErrorIdentity !== undefined) safe.safeErrorIdentity = safeErrorIdentity + return safe +} + +function getAbortStackEvidence(): { abortStackFingerprint?: string } { + const rawStack = new Error().stack + if (!rawStack) return {} + const sites = rawStack + .split('\n') + .slice(2, 7) + .map(line => { + const functionMatch = line.match(/^\s*at\s+([^\s(]+)/) + const candidate = safeString(functionMatch?.[1]) + return candidate && !/[\\/:]/.test(candidate) + ? candidate + : '' + }) + const fingerprint = createHash('sha256').update(sites.join('>')).digest('hex').slice(0, 16) + return { abortStackFingerprint: fingerprint } +} + +function buildEntry( + event: string, + fields: InterruptionTraceFields, + nextSequence: number, + extra: Partial = {}, + allowDisabled = false, +): InterruptionTraceEntry | undefined { + if (!allowDisabled && !isEnabled()) return undefined + if (!traceSessionId) traceSessionId = randomUUID() + if (!eventLoopDelay) { + eventLoopDelay = monitorEventLoopDelay({ resolution: 20 }) + eventLoopDelay.enable() + } + const monotonicMs = performance.now() - startedMonotonicMs + const wallElapsedMs = Date.now() - startedWallMs + const entry: InterruptionTraceEntry = { + schemaVersion: TRACE_SCHEMA_VERSION, + sequence: nextSequence, + eventId: `${traceSessionId}:${nextSequence}`, + traceSessionId, + timestamp: new Date().toISOString(), + monotonicMs, + clockDeltaMs: wallElapsedMs - monotonicMs, + event: safeEventName(event), + ...toSafeFields({ + ...fields, + eventLoopDelayMaxMs: Number.isFinite(eventLoopDelay.max) + ? eventLoopDelay.max / 1_000_000 + : 0, + eventLoopDelayMeanMs: Number.isFinite(eventLoopDelay.mean) + ? eventLoopDelay.mean / 1_000_000 + : 0, + }), + ...extra, + } + return entry +} + +function addEntry( + event: string, + fields: InterruptionTraceFields, + extra: Partial = {}, +): InterruptionTraceEntry | undefined { + if (!isEnabled()) return undefined + try { + const entry = buildEntry(event, fields, sequence + 1, extra) + if (!entry) return undefined + sequence = entry.sequence + ring.push(entry) + if (ring.length > TRACE_CAPACITY) ring = ring.slice(-TRACE_CAPACITY) + return entry + } catch { + // Diagnostics must be total for arbitrary abort reasons and metadata. + return undefined + } +} + +export function traceInterruptionEvent( + event: string, + fields: InterruptionTraceFields = {}, +): string | undefined { + return addEntry(event, fields)?.eventId +} + +export function setInterruptionErrorCausalEventId( + error: unknown, + eventId: string | undefined, +): void { + if ( + eventId && + ((typeof error === 'object' && error !== null) || + typeof error === 'function') + ) { + errorCausalEventIds.set(error, eventId) + } +} + +export function getInterruptionErrorCausalEventId( + error: unknown, +): string | undefined { + return (typeof error === 'object' && error !== null) || + typeof error === 'function' + ? errorCausalEventIds.get(error) + : undefined +} + +export function tracePermissionAbortResolution( + source: string | undefined, + causalEventId: string | undefined, + subsystem: string, +): string | undefined { + return traceInterruptionEvent('permission.abort_resolved', { + source: source ?? 'unknown', + subsystem, + causalEventId, + outcome: 'denied', + }) +} + +export function registerInterruptionController( + controller: AbortController, + fields: InterruptionTraceFields = {}, + options: { + provisionalRole?: boolean + refreshQueryContext?: boolean + } = {}, +): string | undefined { + if (!isEnabled()) return undefined + const existing = controllerStates.get(controller) + if (existing) { + // Registration metadata describes controller identity. Later callers may + // add missing context, but must not relabel an established query root as a + // parent/tool/controller role merely because they are observing it there. + existing.fields = { ...fields, ...existing.fields } + if ( + existing.provisionalRole && + fields.controllerRole !== undefined && + !options.provisionalRole + ) { + existing.fields.controllerRole = fields.controllerRole + existing.provisionalRole = false + } + if (options.refreshQueryContext) { + existing.fields = { + ...existing.fields, + ...(fields.queryId !== undefined && { queryId: fields.queryId }), + ...(fields.queryGeneration !== undefined && { + queryGeneration: fields.queryGeneration, + }), + ...(fields.querySource !== undefined && { + querySource: fields.querySource, + }), + ...(fields.parentQueryId !== undefined && { + parentQueryId: fields.parentQueryId, + }), + } + } + return existing.id + } + + const id = `controller-${++controllerCounter}` + controllerStates.set(controller, { + id, + repeatedCount: 0, + fields, + provisionalRole: options.provisionalRole === true, + }) + signalIds.set(controller.signal, id) + addEntry('controller.registered', fields, { controllerId: id }) + const observeAbort = () => { + const state = controllerStates.get(controller) + const observed = addEntry( + 'signal.observed', + { ...state?.fields, reason: controller.signal.reason }, + { + controllerId: id, + ...(state?.firstAbortEventId && { + firstAbortEventId: state.firstAbortEventId, + }), + }, + ) + if (observed && !signalAbortEventIds.has(controller.signal)) { + signalAbortEventIds.set(controller.signal, observed.eventId) + } + if ( + observed && + state?.fields.source && + !signalAbortSources.has(controller.signal) + ) { + signalAbortSources.set(controller.signal, state.fields.source) + } + if (observed && state && !state.firstAbortEventId) { + state.firstAbortEventId = observed.eventId + } + if (state?.fields.controllerRole === 'query-root') { + flushInterruptionTrace('root_abort_observed') + } + } + if (controller.signal.aborted) observeAbort() + else controller.signal.addEventListener('abort', observeAbort, { once: true }) + return id +} + +function preserveControllerIdentity( + fields: InterruptionTraceFields, + state: ControllerTraceState | undefined, +): InterruptionTraceFields { + const identity = state?.fields + if (!identity) return fields + return { + ...fields, + ...(identity.queryId !== undefined && { queryId: identity.queryId }), + ...(identity.queryGeneration !== undefined && { + queryGeneration: identity.queryGeneration, + }), + ...(identity.querySource !== undefined && { + querySource: identity.querySource, + }), + ...(identity.parentQueryId !== undefined && { + parentQueryId: identity.parentQueryId, + }), + ...(identity.subagentId !== undefined && { + subagentId: identity.subagentId, + }), + ...(identity.controllerRole !== undefined && { + controllerRole: identity.controllerRole, + }), + ...(identity.parentControllerIds !== undefined && { + parentControllerIds: identity.parentControllerIds, + }), + } +} + +export function getInterruptionSignalId(signal: AbortSignal): string | undefined { + return isEnabled() ? signalIds.get(signal) : undefined +} + +export function getInterruptionSignalAbortEventId( + signal: AbortSignal, +): string | undefined { + return isEnabled() ? signalAbortEventIds.get(signal) : undefined +} + +export function getInterruptionSignalAbortTrace( + signal: AbortSignal, +): { source?: string; causalEventId?: string } { + if (!isEnabled()) return {} + return { + source: signalAbortSources.get(signal), + causalEventId: signalAbortEventIds.get(signal), + } +} + +export function registerInterruptionSignal( + signal: AbortSignal, + fields: InterruptionTraceFields = {}, + getAbortFields?: () => InterruptionTraceFields, +): string | undefined { + if (!isEnabled()) return undefined + const existing = signalIds.get(signal) + if (existing) return existing + const id = `signal-${++signalCounter}` + signalIds.set(signal, id) + signalStates.set(signal, { id, fields, getAbortFields }) + addEntry('signal.registered', fields, { controllerId: id }) + const observeAbort = () => observeRegisteredSignal(signal) + if (signal.aborted) observeAbort() + else signal.addEventListener('abort', observeAbort, { once: true }) + return id +} + +function observeRegisteredSignal(signal: AbortSignal): string | undefined { + const existingEventId = signalAbortEventIds.get(signal) + if (existingEventId) return existingEventId + const state = signalStates.get(signal) + if (!state) return undefined + const observed = addEntry( + 'signal.observed', + { ...state.fields, ...state.getAbortFields?.(), reason: signal.reason }, + { controllerId: state.id }, + ) + if (observed) signalAbortEventIds.set(signal, observed.eventId) + return observed?.eventId +} + +export function traceCombinedAbortSignal( + combinedSignal: AbortSignal, + parents: readonly AbortSignal[], + fields: InterruptionTraceFields = {}, +): string | undefined { + if (!isEnabled()) return undefined + const parentControllerIds = parents + .map(parent => + registerInterruptionSignal(parent, { + subsystem: fields.subsystem, + controllerRole: 'combined-parent', + }), + ) + .filter((value): value is string => value !== undefined) + return registerInterruptionSignal( + combinedSignal, + { + ...fields, + parentControllerIds, + }, + () => { + const winner = parents.find(parent => parent.aborted) + if (!winner) return {} + // AbortSignal.any can abort the combined signal before ordinary listeners + // on the winning parent run. Record the parent synchronously here so the + // combined observation carries the causal edge on the native path. + observeRegisteredSignal(winner) + return { + winningParentControllerId: getInterruptionSignalId(winner), + causalEventId: getInterruptionSignalAbortEventId(winner), + } + }, + ) +} + +export function requestAbort( + controller: AbortController, + reason: unknown, + fields: InterruptionTraceFields, +): void { + if (!isEnabled()) { + controller.abort(reason) + return + } + + let shouldFlushRoot = fields.controllerRole === 'query-root' + try { + const controllerId = + registerInterruptionController(controller, fields) ?? 'controller-unknown' + const state = controllerStates.get(controller) + const requestFields = preserveControllerIdentity(fields, state) + shouldFlushRoot ||= state?.fields.controllerRole === 'query-root' + if (controller.signal.aborted) { + if (state) state.repeatedCount++ + addEntry( + 'abort.repeated', + { + ...requestFields, + existingReason: controller.signal.reason, + attemptedReason: reason, + outcome: 'ignored_first_abort_wins', + repeatedCount: state?.repeatedCount ?? 1, + }, + { + controllerId, + ...(state?.firstAbortEventId && { + firstAbortEventId: state.firstAbortEventId, + }), + }, + ) + } else { + const entry = addEntry( + 'abort.requested', + { ...requestFields, reason }, + { controllerId, ...getAbortStackEvidence() }, + ) + if (entry && state) { + state.firstAbortEventId = entry.eventId + signalAbortEventIds.set(controller.signal, entry.eventId) + if (fields.source) { + signalAbortSources.set(controller.signal, fields.source) + } + } + } + } catch { + // Best-effort diagnostics must never interfere with the native abort. + } finally { + controller.abort(reason) + } + if (shouldFlushRoot) flushInterruptionTrace('root_abort_observed') +} + +export function traceCombinedSignal( + combinedController: AbortController, + parents: readonly (AbortSignal | undefined)[], + fields: InterruptionTraceFields = {}, +): string | undefined { + if (!isEnabled()) return undefined + const parentControllerIds = parents + .map(parent => + parent + ? registerInterruptionSignal(parent, { + subsystem: fields.subsystem, + controllerRole: 'combined-parent', + }) + : undefined, + ) + .filter((value): value is string => value !== undefined) + return registerInterruptionController(combinedController, { + ...fields, + parentControllerIds, + }) +} + +async function performInterruptionTraceFlush( + trigger: string, + logFile: string, +): Promise { + try { + while (true) { + let batch = retryBatch + if (!batch) { + const pending = ring.filter( + entry => entry.sequence > flushedThroughSequence, + ) + if (pending.length === 0) return + const marker = buildEntry( + 'trace.flush', + { trigger, repeatedCount: pending.length }, + sequence + 1, + {}, + true, + ) + if (!marker) return + // Reserve the marker before awaiting. Later events get larger IDs, and + // this immutable batch remains reachable even if the ring wraps. + sequence = marker.sequence + batch = { entries: [...pending, marker], marker, logFile } + } + + const result = await appendDiagnosticsNoPII(batch.logFile, batch.entries) + if (result === 'retryable_failure') { + retryBatch = batch + return + } + + retryBatch = undefined + flushedThroughSequence = batch.marker.sequence + if (result === 'committed') { + ring = [...ring, batch.marker] + .sort((left, right) => left.sequence - right.sequence) + .slice(-TRACE_CAPACITY) + } + // Keep draining: events can be recorded while an append is in flight, + // including during graceful shutdown after the caller requested a flush. + } + } catch { + // A trace flush is never allowed to affect request cleanup. + } +} + +export function flushInterruptionTrace(trigger: string): void { + if (!isEnabled()) return + const logFile = process.env[TRACE_FILE_ENV] + if (!logFile || !isAbsolute(logFile)) return + flushQueue = flushQueue + .then(() => performInterruptionTraceFlush(trigger, logFile)) + .catch(() => { + // Diagnostics are deliberately detached from request cancellation. + }) +} + +export async function waitForInterruptionTraceFlush(): Promise { + await flushQueue +} + +export async function __waitForInterruptionTraceFlushForTests(): Promise { + await waitForInterruptionTraceFlush() +} + +export function __getInterruptionTraceSnapshotForTests(): readonly InterruptionTraceEntry[] { + return [...ring] +} + +export function __resetInterruptionTraceForTests(): void { + traceSessionId = '' + sequence = 0 + startedWallMs = Date.now() + startedMonotonicMs = performance.now() + ring = [] + flushedThroughSequence = 0 + controllerCounter = 0 + signalCounter = 0 + controllerStates = new WeakMap() + signalIds = new WeakMap() + signalAbortEventIds = new WeakMap() + signalAbortSources = new WeakMap() + signalStates = new WeakMap() + errorCausalEventIds = new WeakMap() + retryBatch = undefined + flushQueue = Promise.resolve() + eventLoopDelay?.disable() + eventLoopDelay = undefined +} diff --git a/src/utils/permissions/PermissionPromptToolResultSchema.ts b/src/utils/permissions/PermissionPromptToolResultSchema.ts index acd0b2e71f..8989976b45 100644 --- a/src/utils/permissions/PermissionPromptToolResultSchema.ts +++ b/src/utils/permissions/PermissionPromptToolResultSchema.ts @@ -2,6 +2,7 @@ import type { Tool, ToolUseContext } from 'src/Tool.js' import z from 'zod/v4' import { logForDebugging } from '../debug.js' import { lazySchema } from '../lazySchema.js' +import { requestAbort } from '../interruptionTrace.js' import type { PermissionDecision, PermissionDecisionReason, @@ -159,7 +160,11 @@ export async function permissionPromptToolResultToPermissionDecision( logForDebugging( `SDK permission prompt deny+interrupt: tool=${tool.name} message=${result.message}`, ) - toolUseContext.abortController.abort('interrupt') + requestAbort(toolUseContext.abortController, 'interrupt', { + source: 'sdk_permission_interrupt', + subsystem: 'tool_permission', + controllerRole: 'query-root', + }) } return { ...result, diff --git a/src/utils/permissions/permissions.headlessPlanHooks.test.ts b/src/utils/permissions/permissions.headlessPlanHooks.test.ts index dbbe9bf47d..d30fc84247 100644 --- a/src/utils/permissions/permissions.headlessPlanHooks.test.ts +++ b/src/utils/permissions/permissions.headlessPlanHooks.test.ts @@ -1,4 +1,12 @@ -import { afterAll, beforeAll, describe, expect, mock, test } from 'bun:test' +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + mock, + test, +} from 'bun:test' import { z } from 'zod/v4' import type { ToolPermissionContext, ToolUseContext } from '../../Tool.js' import { createToolFixture } from '../../test/toolFixtures.js' @@ -6,13 +14,15 @@ import { acquireSharedMutationLock, releaseSharedMutationLock, } from '../../test/sharedMutationLock.js' -import type { PermissionDecision } from './PermissionResult.js' +import type { PermissionRequestResult } from '../../types/hooks.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, +} from '../interruptionTrace.js' import { permissionPromptToolResultToPermissionDecision } from './PermissionPromptToolResultSchema.js' -import type { PermissionUpdate } from './PermissionUpdateSchema.js' -type HookDecision = PermissionDecision & { - updatedPermissions?: PermissionUpdate[] -} +type HookDecision = PermissionRequestResult let hookDecision: HookDecision let hasPermissionsToUseTool: typeof import('./permissions.js').hasPermissionsToUseTool @@ -20,6 +30,7 @@ let createPermissionContext: typeof import('../../hooks/toolPermission/Permissio let StructuredIO: typeof import('../../cli/structuredIO.js').StructuredIO let actualHooks: typeof import('../hooks.js') let beforeHookDecision: (() => void) | undefined +const originalInterruptionTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE beforeAll(async () => { await acquireSharedMutationLock( @@ -55,6 +66,16 @@ afterAll(() => { } }) +afterEach(async () => { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalInterruptionTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalInterruptionTrace + } +}) + function planContext( overrides: Partial = {}, ): { @@ -367,6 +388,82 @@ describe('headless plan-mode PermissionRequest hooks', () => { expect(state.getPermissionContext().mode).toBe('plan') }) + test('labels SDK permission prompt interrupts as query-root aborts', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const tool = createToolFixture(z.object({}), { + name: 'SDKPromptInterruptTool', + isReadOnly: () => true, + }) + const state = planContext({ mode: 'default' }) + + const result = await permissionPromptToolResultToPermissionDecision( + { + behavior: 'deny', + message: 'Stop the query', + interrupt: true, + }, + tool, + {}, + state.context, + ) + + expect(result.behavior).toBe('deny') + expect(state.context.abortController.signal.aborted).toBe(true) + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'abort.requested' && + entry.source === 'sdk_permission_interrupt', + ), + ).toMatchObject({ + subsystem: 'tool_permission', + controllerRole: 'query-root', + }) + }) + + test('labels headless permission hook interrupts as query-root aborts', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const tool = createToolFixture(z.object({}), { + name: 'HeadlessHookInterruptTool', + isReadOnly: () => true, + async checkPermissions() { + return { behavior: 'ask' as const, message: 'Review read' } + }, + }) + const state = planContext({ + mode: 'default', + shouldAvoidPermissionPrompts: true, + }) + hookDecision = { + behavior: 'deny', + message: 'Stop the query', + interrupt: true, + } + + const result = await hasPermissionsToUseTool( + tool, + {}, + state.context, + assistantMessage, + 'headless-hook-interrupt', + ) + + expect(result.behavior).toBe('deny') + expect(state.context.abortController.signal.aborted).toBe(true) + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'abort.requested' && + entry.source === 'permission_hook', + ), + ).toMatchObject({ + subsystem: 'tool_permission', + controllerRole: 'query-root', + }) + }) + test('SDK permission prompt rechecks plan mode after approval normalization', async () => { const conditionalTool = createToolFixture( z.object({ operation: z.enum(['read', 'write']) }), @@ -597,6 +694,47 @@ describe('headless plan-mode PermissionRequest hooks', () => { ]) }) + test('labels interactive permission hook interrupts as query-root aborts', async () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() + const tool = createToolFixture(z.object({}), { + name: 'InteractiveHookInterruptTool', + isReadOnly: () => true, + }) + const state = planContext({ mode: 'default' }) + const permissionContext = createPermissionContext( + tool, + {}, + state.context, + { message: { id: 'assistant-message' } } as never, + 'interactive-hook-interrupt', + state.setPermissionContext, + ) + hookDecision = { + behavior: 'deny', + message: 'Stop the query', + interrupt: true, + } + + const result = await permissionContext.runHooks(undefined, undefined) + + expect(result).toMatchObject({ + behavior: 'deny', + message: 'Stop the query', + }) + expect(state.context.abortController.signal.aborted).toBe(true) + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => + entry.event === 'abort.requested' && + entry.source === 'permission_hook', + ), + ).toMatchObject({ + subsystem: 'tool_permission', + controllerRole: 'query-root', + }) + }) + test('interactive PermissionRequest hooks cannot rewrite a read into a mutation', async () => { const conditionalTool = createToolFixture( z.object({ operation: z.enum(['read', 'write']) }), diff --git a/src/utils/permissions/permissions.ts b/src/utils/permissions/permissions.ts index 32b557ea73..7eae7e3b0a 100644 --- a/src/utils/permissions/permissions.ts +++ b/src/utils/permissions/permissions.ts @@ -25,6 +25,7 @@ import { extractOutputRedirections } from '../bash/commands.js' import { logForDebugging } from '../debug.js' import { AbortError, toError } from '../errors.js' import { logError } from '../log.js' +import { requestAbort } from '../interruptionTrace.js' import { SandboxManager } from '../sandbox/sandbox-adapter.js' import { getSettingSourceDisplayNameLowercase, @@ -524,7 +525,11 @@ async function runPermissionRequestHooksForHeadlessAgent( logForDebugging( `Hook interrupt: tool=${tool.name} hookMessage=${decision.message}`, ) - context.abortController.abort('interrupt') + requestAbort(context.abortController, 'interrupt', { + source: 'permission_hook', + subsystem: 'tool_permission', + controllerRole: 'query-root', + }) } return { behavior: 'deny', diff --git a/src/utils/queryEventDriver.test.ts b/src/utils/queryEventDriver.test.ts new file mode 100644 index 0000000000..1176469d31 --- /dev/null +++ b/src/utils/queryEventDriver.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, test } from 'bun:test' +import { driveQueryEvents } from './queryEventDriver.js' + +describe('driveQueryEvents', () => { + test('registers activity only for yielded events and returns terminal state', async () => { + async function* query(): AsyncGenerator< + { type: string }, + { reason: string } + > { + yield { type: 'stream_request_start' } + yield { type: 'stream_event' } + return { reason: 'completed' } + } + const activity: string[] = [] + const events: string[] = [] + + const terminal = await driveQueryEvents( + query(), + reason => activity.push(reason), + event => events.push(event.type), + ) + + expect(activity).toEqual([ + 'query_event:stream_request_start', + 'query_event:stream_event', + ]) + expect(events).toEqual(['stream_request_start', 'stream_event']) + expect(terminal).toEqual({ reason: 'completed' }) + }) + + test('closes the generator if event handling throws', async () => { + let finalized = false + async function* query(): AsyncGenerator<{ type: string }, void> { + try { + yield { type: 'stream_event' } + yield { type: 'must_not_be_seen' } + } finally { + finalized = true + } + } + + await expect( + driveQueryEvents( + query(), + () => {}, + () => { + throw new Error('consumer failed') + }, + ), + ).rejects.toThrow('consumer failed') + expect(finalized).toBe(true) + }) + + test('does not let generator cleanup mask an event-handler failure', async () => { + const generator: AsyncGenerator<{ type: string }, void> = { + next: async () => ({ done: false, value: { type: 'stream_event' } }), + return: async () => { + throw new Error('generator cleanup failed') + }, + throw: async error => { + throw error + }, + [Symbol.asyncIterator]() { + return this + }, + [Symbol.asyncDispose]: async () => {}, + } + + await expect( + driveQueryEvents( + generator, + () => {}, + () => { + throw new Error('consumer failed') + }, + ), + ).rejects.toThrow('consumer failed') + }) +}) diff --git a/src/utils/queryEventDriver.ts b/src/utils/queryEventDriver.ts new file mode 100644 index 0000000000..58d1385b28 --- /dev/null +++ b/src/utils/queryEventDriver.ts @@ -0,0 +1,30 @@ +/** + * Consume a query event generator while preserving the REPL's activity policy: + * only events yielded to the consumer count as QueryGuard progress. + */ +export async function driveQueryEvents( + queryGenerator: AsyncGenerator, + registerActivity: (reason: string) => void, + onEvent: (event: TEvent) => void, +): Promise { + let generatorDone = false + try { + while (true) { + const next = await queryGenerator.next() + if (next.done) { + generatorDone = true + return next.value + } + registerActivity(`query_event:${next.value.type}`) + onEvent(next.value) + } + } finally { + if (!generatorDone) { + try { + await queryGenerator.return(undefined as never) + } catch { + // Preserve the generator or event-handler failure that triggered cleanup. + } + } + } +} diff --git a/src/utils/queryLifecycle.test.ts b/src/utils/queryLifecycle.test.ts index 15fb41414d..6cb41e4174 100644 --- a/src/utils/queryLifecycle.test.ts +++ b/src/utils/queryLifecycle.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'vitest' import { formatQueryLifecycleAbortSignalReason, formatQueryLifecycleLogMessage, + getQueryTerminalOutcome, getQueryTerminalReason, type QueryLifecycleContext, } from './queryLifecycle.js' @@ -47,6 +48,13 @@ describe('query lifecycle log formatting', () => { }) describe('query terminal reason classification', () => { + test('classifies coarse terminal outcomes with abort precedence', () => { + expect(getQueryTerminalOutcome({ aborted: false }, false)).toBe('completed') + expect(getQueryTerminalOutcome({ aborted: false }, true)).toBe('failed') + expect(getQueryTerminalOutcome({ aborted: true }, false)).toBe('aborted') + expect(getQueryTerminalOutcome({ aborted: true }, true)).toBe('aborted') + }) + test('classifies non-aborted completion from throw state', () => { expect( getQueryTerminalReason({ aborted: false, reason: undefined }, false), diff --git a/src/utils/queryLifecycle.ts b/src/utils/queryLifecycle.ts index 01f00358ac..be53dc450a 100644 --- a/src/utils/queryLifecycle.ts +++ b/src/utils/queryLifecycle.ts @@ -13,6 +13,8 @@ export type QueryTerminalReason = export type QueryGuardTimeoutReason = 'idle' | 'hard_max' | 'lease_expired' +export type QueryTerminalOutcome = 'aborted' | 'failed' | 'completed' + export type QueryActiveApiCall = { clientRequestId?: string requestId?: string | null @@ -67,6 +69,7 @@ export type QueryGuardTimeoutInfo = { elapsedMs: number context: QueryLifecycleContext activeOperations: QueryActiveOperationSnapshot + causalEventId?: string } export function getQueryTerminalReason( @@ -93,6 +96,14 @@ export function getQueryTerminalReason( } } +export function getQueryTerminalOutcome( + signal: Pick, + didThrow: boolean, +): QueryTerminalOutcome { + if (signal.aborted) return 'aborted' + return didThrow ? 'failed' : 'completed' +} + export function formatQueryLifecycleAbortSignalReason(reason: string): string { return `abortSignalReason=${reason}` } diff --git a/src/utils/replInterruption.test.ts b/src/utils/replInterruption.test.ts new file mode 100644 index 0000000000..0a0df354d5 --- /dev/null +++ b/src/utils/replInterruption.test.ts @@ -0,0 +1,79 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, +} from './interruptionTrace.js' +import { + requestBackgroundHandoffAbort, + requestBridgeInterrupt, + requestPriorityNowAbort, +} from './replInterruption.js' + +const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + +beforeEach(async () => { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' +}) + +afterEach(async () => { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE + else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace +}) + +describe('REPL interruption source wiring', () => { + test('bridge interruption reads the current controller and links its input event', () => { + const stale = new AbortController() + const current = new AbortController() + const ref = { current: stale } + ref.current = current + + requestBridgeInterrupt(ref) + + expect(stale.signal.aborted).toBe(false) + expect(current.signal.aborted).toBe(true) + const trace = __getInterruptionTraceSnapshotForTests() + const input = trace.find(entry => entry.event === 'input.bridge_interrupt') + const abort = trace.find(entry => entry.event === 'abort.requested') + expect(abort?.source).toBe('bridge_interrupt') + expect(input).toBeDefined() + expect(abort).toBeDefined() + expect(typeof input!.eventId).toBe('string') + expect(typeof abort!.causalEventId).toBe('string') + expect(abort!.causalEventId).toBe(input!.eventId) + }) + + test('background handoff aborts the foreground query with its source', () => { + const controller = new AbortController() + + requestBackgroundHandoffAbort(controller) + + expect(controller.signal.reason).toBe('background') + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'abort.requested', + )?.source, + ).toBe('background_handoff') + }) + + test('priority-now reads and aborts the current query controller', () => { + const stale = new AbortController() + const current = new AbortController() + const ref = { current: stale } + ref.current = current + + requestPriorityNowAbort(ref) + + expect(stale.signal.aborted).toBe(false) + expect(current.signal.reason).toBe('interrupt') + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'abort.requested', + )?.source, + ).toBe('priority_now') + }) +}) diff --git a/src/utils/replInterruption.ts b/src/utils/replInterruption.ts new file mode 100644 index 0000000000..e167637bb8 --- /dev/null +++ b/src/utils/replInterruption.ts @@ -0,0 +1,39 @@ +import { requestAbort, traceInterruptionEvent } from './interruptionTrace.js' + +type AbortControllerRef = { readonly current: AbortController | null } + +export function requestBridgeInterrupt(ref: AbortControllerRef): void { + const controller = ref.current + const causalEventId = traceInterruptionEvent('input.bridge_interrupt', { + source: 'bridge_interrupt', + subsystem: 'repl_bridge', + }) + if (!controller) return + requestAbort(controller, 'interrupt', { + source: 'bridge_interrupt', + subsystem: 'repl_bridge', + controllerRole: 'query-root', + causalEventId, + }) +} + +export function requestBackgroundHandoffAbort( + controller: AbortController | null, +): void { + if (!controller) return + requestAbort(controller, 'background', { + source: 'background_handoff', + subsystem: 'repl', + controllerRole: 'query-root', + }) +} + +export function requestPriorityNowAbort(ref: AbortControllerRef): void { + const controller = ref.current + if (!controller) return + requestAbort(controller, 'interrupt', { + source: 'priority_now', + subsystem: 'repl', + controllerRole: 'query-root', + }) +} diff --git a/src/utils/swarm/inProcessPermissionAbort.test.ts b/src/utils/swarm/inProcessPermissionAbort.test.ts new file mode 100644 index 0000000000..4b95a7f386 --- /dev/null +++ b/src/utils/swarm/inProcessPermissionAbort.test.ts @@ -0,0 +1,138 @@ +import { afterEach, beforeEach, expect, test } from 'bun:test' +import type { ToolUseConfirm } from '../../components/permissions/PermissionRequest.js' +import { getDefaultAppState } from '../../state/AppStateStore.js' +import type { Tool } from '../../Tool.js' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../../test/sharedMutationLock.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, + __waitForInterruptionTraceFlushForTests, + requestAbort, +} from '../interruptionTrace.js' +import { createInProcessPermissionAbortCompleter } from './inProcessPermissionAbort.js' +import { createInProcessCanUseTool } from './inProcessRunner.js' +import { + registerLeaderToolUseConfirmQueue, + unregisterLeaderToolUseConfirmQueue, +} from './leaderPermissionBridge.js' + +const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + +beforeEach(async () => { + await acquireSharedMutationLock('inProcessPermissionAbort.test.ts') + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + __resetInterruptionTraceForTests() +}) + +afterEach(async () => { + try { + await __waitForInterruptionTraceFlushForTests() + __resetInterruptionTraceForTests() + unregisterLeaderToolUseConfirmQueue() + if (originalTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + } finally { + releaseSharedMutationLock() + } +}) + +test('links signal-driven permission denial to the winning abort request once', () => { + const controller = new AbortController() + let deniedCount = 0 + const completion = createInProcessPermissionAbortCompleter( + controller.signal, + () => { + deniedCount++ + }, + ) + + requestAbort(controller, undefined, { + source: 'query_guard', + subsystem: 'query_engine', + controllerRole: 'query-root', + }) + + const trace = __getInterruptionTraceSnapshotForTests() + const abortRequest = trace.find(entry => entry.event === 'abort.requested') + const permissionResolution = trace.find( + entry => entry.event === 'permission.abort_resolved', + ) + expect(deniedCount).toBe(1) + expect(abortRequest?.eventId).toBeString() + expect(permissionResolution).toMatchObject({ + source: 'query_guard', + subsystem: 'in_process_permission_bridge', + outcome: 'denied', + causalEventId: abortRequest?.eventId, + }) + expect(completion.completeAbort('ui_cancel', 'later-event')).toBe(false) + expect(deniedCount).toBe(1) +}) + +test('aborts a queued in-process permission through the production signal path', async () => { + const controller = new AbortController() + let queue: ToolUseConfirm[] = [] + registerLeaderToolUseConfirmQueue(updater => { + queue = updater(queue) + }) + const canUseTool = createInProcessCanUseTool( + { + agentId: 'worker-1', + agentName: 'worker', + teamName: 'test-team', + planModeRequired: false, + parentSessionId: 'parent-session', + }, + controller, + ) + const appState = getDefaultAppState() + const decisionPromise = canUseTool( + { + name: 'TraceTestTool', + description: async () => 'test permission', + } as unknown as Tool, + {}, + { + getAppState: () => appState, + options: { + isNonInteractiveSession: false, + tools: [], + }, + } as never, + {} as never, + 'tool-use-1', + { behavior: 'ask', message: 'approval required' }, + ) + + for (let attempt = 0; attempt < 10 && queue.length === 0; attempt++) { + await Promise.resolve() + } + expect(queue).toHaveLength(1) + + requestAbort(controller, undefined, { + source: 'query_guard', + subsystem: 'query_engine', + controllerRole: 'query-root', + }) + + await expect(decisionPromise).resolves.toMatchObject({ + behavior: 'ask', + }) + expect(queue).toHaveLength(0) + const trace = __getInterruptionTraceSnapshotForTests() + const abortRequest = trace.find(entry => entry.event === 'abort.requested') + expect( + trace.find(entry => entry.event === 'permission.abort_resolved'), + ).toMatchObject({ + source: 'query_guard', + subsystem: 'in_process_permission_bridge', + outcome: 'denied', + causalEventId: abortRequest?.eventId, + }) +}) diff --git a/src/utils/swarm/inProcessPermissionAbort.ts b/src/utils/swarm/inProcessPermissionAbort.ts new file mode 100644 index 0000000000..8e9c85b0cf --- /dev/null +++ b/src/utils/swarm/inProcessPermissionAbort.ts @@ -0,0 +1,61 @@ +import { + getInterruptionSignalAbortTrace, + tracePermissionAbortResolution, +} from '../interruptionTrace.js' + +export type InProcessPermissionAbortCompleter = { + claim(): boolean + completeAbort( + source: string | undefined, + causalEventId: string | undefined, + ): boolean + completeSignalAbort(): boolean + isSettled(): boolean +} + +export function createInProcessPermissionAbortCompleter( + signal: AbortSignal, + onAbortSettled: () => void, +): InProcessPermissionAbortCompleter { + let settled = false + + const claim = (): boolean => { + if (settled) return false + settled = true + signal.removeEventListener('abort', onSignalAbort) + return true + } + + const completeAbort = ( + source: string | undefined, + causalEventId: string | undefined, + ): boolean => { + if (!claim()) return false + tracePermissionAbortResolution( + source, + causalEventId, + 'in_process_permission_bridge', + ) + onAbortSettled() + return true + } + + const completeSignalAbort = (): boolean => { + const trace = getInterruptionSignalAbortTrace(signal) + return completeAbort(trace.source, trace.causalEventId) + } + + const onSignalAbort = () => { + completeSignalAbort() + } + + signal.addEventListener('abort', onSignalAbort, { once: true }) + if (signal.aborted) onSignalAbort() + + return { + claim, + completeAbort, + completeSignalAbort, + isSettled: () => settled, + } +} diff --git a/src/utils/swarm/inProcessRunner.ts b/src/utils/swarm/inProcessRunner.ts index cf6ae8c1b0..85bbda2418 100644 --- a/src/utils/swarm/inProcessRunner.ts +++ b/src/utils/swarm/inProcessRunner.ts @@ -49,6 +49,9 @@ import { } from '../../tasks/LocalAgentTask/LocalAgentTask.js' import type { CustomAgentDefinition } from '../../tools/AgentTool/loadAgentsDir.js' import { runAgent } from '../../tools/AgentTool/runAgent.js' +import { + registerInterruptionController, +} from '../interruptionTrace.js' import { awaitClassifierAutoApproval } from '../../tools/BashTool/bashPermissions.js' import { BASH_TOOL_NAME } from '../../tools/BashTool/toolName.js' import { SEND_MESSAGE_TOOL_NAME } from '../../tools/SendMessageTool/constants.js' @@ -126,6 +129,7 @@ import { sendPermissionRequestViaMailbox, } from './permissionSync.js' import { TEAMMATE_SYSTEM_PROMPT_ADDENDUM } from './teammatePromptAddendum.js' +import { createInProcessPermissionAbortCompleter } from './inProcessPermissionAbort.js' type SetAppStateFn = (updater: (prev: AppState) => AppState) => void @@ -143,7 +147,7 @@ const PERMISSION_POLL_INTERVAL_MS = 500 * sends a permission request to the leader's inbox, waits for the response * in the teammate's own mailbox. */ -function createInProcessCanUseTool( +export function createInProcessCanUseTool( identity: TeammateIdentity, abortController: AbortController, onPermissionWaitMs?: (waitMs: number) => void, @@ -259,7 +263,6 @@ function createInProcessCanUseTool( // Standard path: use ToolUseConfirm dialog with worker badge if (setToolUseConfirmQueue) { return new Promise(resolve => { - let decisionMade = false const permissionStartMs = Date.now() // Report permission wait time to the caller so it can be @@ -268,19 +271,20 @@ function createInProcessCanUseTool( onPermissionWaitMs?.(Date.now() - permissionStartMs) } - const onAbortListener = () => { - if (decisionMade) return - decisionMade = true - reportPermissionWait() - resolve({ behavior: 'ask', message: SUBAGENT_REJECT_MESSAGE }) - setToolUseConfirmQueue(queue => - queue.filter(item => item.toolUseID !== toolUseID), - ) - } + const completion = createInProcessPermissionAbortCompleter( + abortController.signal, + () => { + reportPermissionWait() + resolve({ behavior: 'ask', message: SUBAGENT_REJECT_MESSAGE }) + setToolUseConfirmQueue(queue => + queue.filter(item => item.toolUseID !== toolUseID), + ) + }, + ) - abortController.signal.addEventListener('abort', onAbortListener, { - once: true, - }) + if (completion.isSettled()) { + return + } setToolUseConfirmQueue(queue => [ ...queue, @@ -299,15 +303,8 @@ function createInProcessCanUseTool( onUserInteraction() { // No-op for teammates (no classifier auto-approval) }, - onAbort() { - if (decisionMade) return - decisionMade = true - abortController.signal.removeEventListener( - 'abort', - onAbortListener, - ) - reportPermissionWait() - resolve({ behavior: 'ask', message: SUBAGENT_REJECT_MESSAGE }) + onAbort(source, causalEventId) { + completion.completeAbort(source, causalEventId) }, async onAllow( updatedInput: Record, @@ -315,12 +312,7 @@ function createInProcessCanUseTool( feedback?: string, contentBlocks?: ContentBlockParam[], ) { - if (decisionMade) return - decisionMade = true - abortController.signal.removeEventListener( - 'abort', - onAbortListener, - ) + if (!completion.claim()) return reportPermissionWait() const approval = await guardExternalApproval( updatedInput, @@ -365,12 +357,7 @@ function createInProcessCanUseTool( }) }, onReject(feedback?: string, contentBlocks?: ContentBlockParam[]) { - if (decisionMade) return - decisionMade = true - abortController.signal.removeEventListener( - 'abort', - onAbortListener, - ) + if (!completion.claim()) return reportPermissionWait() const message = feedback ? `${SUBAGENT_REJECT_MESSAGE_WITH_REASON_PREFIX}${feedback}` @@ -378,7 +365,7 @@ function createInProcessCanUseTool( resolve({ behavior: 'ask', message, contentBlocks }) }, async recheckPermission() { - if (decisionMade) return + if (completion.isSettled()) return const freshResult = await hasPermissionsToUseTool( tool, input, @@ -386,12 +373,10 @@ function createInProcessCanUseTool( assistantMessage, toolUseID, ) - if (freshResult.behavior === 'allow') { - decisionMade = true - abortController.signal.removeEventListener( - 'abort', - onAbortListener, - ) + if ( + freshResult.behavior === 'allow' && + completion.claim() + ) { reportPermissionWait() setToolUseConfirmQueue(queue => queue.filter(item => item.toolUseID !== toolUseID), @@ -421,6 +406,26 @@ function createInProcessCanUseTool( workerColor: identity.color, teamName: identity.teamName, }) + let pollInterval: ReturnType | undefined + + function cleanup() { + if (pollInterval !== undefined) { + clearInterval(pollInterval) + } + unregisterPermissionCallback(request.id) + } + + const completion = createInProcessPermissionAbortCompleter( + abortController.signal, + () => { + cleanup() + resolve({ behavior: 'ask', message: SUBAGENT_REJECT_MESSAGE }) + }, + ) + + if (completion.isSettled()) { + return + } // Register callback to be invoked when the leader responds registerPermissionCallback({ @@ -432,6 +437,7 @@ function createInProcessCanUseTool( _feedback?: string, contentBlocks?: ContentBlockParam[], ) { + if (!completion.claim()) return cleanup() const finalInput = updatedInput && Object.keys(updatedInput).length > 0 @@ -460,6 +466,7 @@ function createInProcessCanUseTool( }) }, onReject(feedback?: string, contentBlocks?: ContentBlockParam[]) { + if (!completion.claim()) return cleanup() const message = feedback ? `${SUBAGENT_REJECT_MESSAGE_WITH_REASON_PREFIX}${feedback}` @@ -472,11 +479,10 @@ function createInProcessCanUseTool( void sendPermissionRequestViaMailbox(request) // Poll teammate's mailbox for the response - const pollInterval = setInterval( - async (abortController, cleanup, resolve, identity, request) => { + pollInterval = setInterval( + async (completion, identity, request) => { if (abortController.signal.aborted) { - cleanup() - resolve({ behavior: 'ask', message: SUBAGENT_REJECT_MESSAGE }) + completion.completeSignalAbort() return } @@ -514,27 +520,10 @@ function createInProcessCanUseTool( } }, PERMISSION_POLL_INTERVAL_MS, - abortController, - cleanup, - resolve, + completion, identity, request, ) - - const onAbortListener = () => { - cleanup() - resolve({ behavior: 'ask', message: SUBAGENT_REJECT_MESSAGE }) - } - - abortController.signal.addEventListener('abort', onAbortListener, { - once: true, - }) - - function cleanup() { - clearInterval(pollInterval) - unregisterPermissionCallback(request.id) - abortController.signal.removeEventListener('abort', onAbortListener) - } }) } } @@ -1168,6 +1157,12 @@ export async function runInProcessTeammate( // This allows Escape to stop current work without killing the whole teammate. // The lifecycle abortController still kills the whole teammate if needed. const currentWorkAbortController = createAbortController() + registerInterruptionController(currentWorkAbortController, { + subsystem: 'in_process_teammate', + controllerRole: 'subagent-turn', + subagentId: identity.agentId, + querySource: 'agent:custom', + }) // Store the work controller in task state so UI can abort it updateTaskState( diff --git a/src/utils/swarm/spawnInProcess.interruptionTrace.test.ts b/src/utils/swarm/spawnInProcess.interruptionTrace.test.ts new file mode 100644 index 0000000000..0b4a72a68d --- /dev/null +++ b/src/utils/swarm/spawnInProcess.interruptionTrace.test.ts @@ -0,0 +1,88 @@ +import { afterEach, beforeEach, expect, test } from 'bun:test' +import type { AppState } from '../../state/AppState.js' +import { getDefaultAppState } from '../../state/AppStateStore.js' +import type { InProcessTeammateTaskState } from '../../tasks/InProcessTeammateTask/types.js' +import { + __getInterruptionTraceSnapshotForTests, + __resetInterruptionTraceForTests, +} from '../interruptionTrace.js' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../../test/sharedMutationLock.js' +import { killInProcessTeammate } from './spawnInProcess.js' + +const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE + +beforeEach(async () => { + await acquireSharedMutationLock( + 'utils/swarm/spawnInProcess.interruptionTrace.test.ts', + ) +}) + +afterEach(() => { + try { + __resetInterruptionTraceForTests() + if (originalTrace === undefined) { + delete process.env.OPENCLAUDE_INTERRUPT_TRACE + } else { + process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace + } + } finally { + releaseSharedMutationLock() + } +}) + +test('records the teammate lifecycle source before a kill abort', () => { + process.env.OPENCLAUDE_INTERRUPT_TRACE = '1' + const abortController = new AbortController() + const taskId = 'teammate-task-1' + const task = { + id: taskId, + type: 'in_process_teammate', + status: 'running', + description: 'test teammate', + startTime: Date.now(), + outputFile: '/tmp/test-teammate-output', + outputOffset: 0, + notified: false, + identity: { + agentId: 'researcher', + agentName: 'researcher', + teamName: '', + planModeRequired: false, + parentSessionId: 'parent-session', + }, + prompt: 'test', + abortController, + awaitingPlanApproval: false, + permissionMode: 'default', + isIdle: false, + shutdownRequested: false, + pendingUserMessages: [], + lastReportedToolCount: 0, + lastReportedTokenCount: 0, + } satisfies InProcessTeammateTaskState + let state: AppState = { + ...getDefaultAppState(), + tasks: { [taskId]: task }, + } + + expect( + killInProcessTeammate(taskId, updater => { + state = updater(state) + }), + ).toBe(true) + + expect(abortController.signal.aborted).toBe(true) + expect( + __getInterruptionTraceSnapshotForTests().find( + entry => entry.event === 'abort.requested', + ), + ).toMatchObject({ + source: 'task_stop', + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: 'researcher', + }) +}) diff --git a/src/utils/swarm/spawnInProcess.ts b/src/utils/swarm/spawnInProcess.ts index 7f548f28aa..b1c3a8e28c 100644 --- a/src/utils/swarm/spawnInProcess.ts +++ b/src/utils/swarm/spawnInProcess.ts @@ -27,6 +27,10 @@ import { createAbortController } from '../abortController.js' import { formatAgentId } from '../agentId.js' import { registerCleanup } from '../cleanupRegistry.js' import { logForDebugging } from '../debug.js' +import { + registerInterruptionController, + requestAbort, +} from '../interruptionTrace.js' import { emitTaskTerminatedSdk } from '../sdkEventQueue.js' import { evictTaskOutput } from '../task/diskOutput.js' import { @@ -44,6 +48,11 @@ import { removeMemberByAgentId } from './teamHelpers.js' type SetAppStateFn = (updater: (prev: AppState) => AppState) => void +export type InProcessTeammateKillTrace = { + source: string + causalEventId?: string +} + /** * Minimal context required for spawning an in-process teammate. * This is a subset of ToolUseContext - only what spawnInProcessTeammate actually uses. @@ -120,6 +129,11 @@ export async function spawnInProcessTeammate( // Create independent AbortController for this teammate // Teammates should not be aborted when the leader's query is interrupted const abortController = createAbortController() + registerInterruptionController(abortController, { + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: agentId, + }) // Get parent session ID for transcript correlation const parentSessionId = getSessionId() @@ -182,7 +196,12 @@ export async function spawnInProcessTeammate( // Register cleanup handler for graceful shutdown const unregisterCleanup = registerCleanup(async () => { logForDebugging(`[spawnInProcessTeammate] Cleanup called for ${agentId}`) - abortController.abort() + requestAbort(abortController, undefined, { + source: 'graceful_shutdown', + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: agentId, + }) // Task state will be updated by the execution loop when it detects abort }) taskState.unregisterCleanup = unregisterCleanup @@ -227,6 +246,7 @@ export async function spawnInProcessTeammate( export function killInProcessTeammate( taskId: string, setAppState: SetAppStateFn, + trace: InProcessTeammateKillTrace = { source: 'task_stop' }, ): boolean { let killed = false let teamName: string | null = null @@ -253,7 +273,15 @@ export function killInProcessTeammate( description = teammateTask.description // Abort the controller to stop execution - teammateTask.abortController?.abort() + if (teammateTask.abortController) { + requestAbort(teammateTask.abortController, undefined, { + source: trace.source, + subsystem: 'in_process_teammate', + controllerRole: 'subagent-lifecycle', + subagentId: teammateTask.identity.agentId, + causalEventId: trace.causalEventId, + }) + } // Call cleanup handler teammateTask.unregisterCleanup?.() diff --git a/tests/sdk/query-lifecycle.test.ts b/tests/sdk/query-lifecycle.test.ts index 1292ca0a31..ed8f802814 100644 --- a/tests/sdk/query-lifecycle.test.ts +++ b/tests/sdk/query-lifecycle.test.ts @@ -128,6 +128,21 @@ describe('Engine lazy-init guard (COR-1)', () => { expect(() => q.close()).not.toThrow() }) + test('QueryImpl close() aborts its wrapper controller after an engine override', () => { + const abortController = new AbortController() + const q = query({ + prompt: 'test', + options: { cwd: process.cwd(), abortController }, + }) + ;(q as unknown as { + setEngine(engine: { interrupt(): void }): void + }).setEngine({ interrupt() {} }) + + q.close() + + expect(abortController.signal.aborted).toBe(true) + }) + test('SDKSession getMessages() works after construction', async () => { const { unstable_v2_createSession } = await import('../../src/entrypoints/sdk/index.js') const session = unstable_v2_createSession({ diff --git a/tests/sdk/sdk-v2-lifecycle.test.ts b/tests/sdk/sdk-v2-lifecycle.test.ts index 662f973106..200993016a 100644 --- a/tests/sdk/sdk-v2-lifecycle.test.ts +++ b/tests/sdk/sdk-v2-lifecycle.test.ts @@ -151,6 +151,19 @@ describe('V2: session interrupt', () => { expect(() => session.interrupt()).not.toThrow() }) + test('session.close() aborts its wrapper controller after an engine override', () => { + const abortController = new AbortController() + const session = unstable_v2_createSession({ + cwd: process.cwd(), + abortController, + }) + attachMockEngine(session, new MockQueryEngine()) + + session.close() + + expect(abortController.signal.aborted).toBe(true) + }) + test('session with external abortController — abort signal propagates', async () => { const ac = new AbortController() const session = unstable_v2_createSession({