From f164cacfd43fd549c6dfee4a36a8d3fb0e4c2dba Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 10:59:46 -0700 Subject: [PATCH 01/13] fix(fallback): converge main-chat and workflow model fallback (#2170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main chat and workflow stages resolved fallback candidates through the same helper but disagreed on when a chain advances, how long a candidate is retried first, and how long a switch lasts. Converge them in three directions. Share the failure classifier. The single implementation now lives in packages/coding-agent/src/core/model-fallback-failures.ts; the workflows and subagents modules re-export it. Main chat splits the one predicate in two: _isRetryableError answers "request this model again", _isFallbackableError answers "spend the next candidate". Auth, model-unavailable, and request-incompatible failures now advance the chain, and openai-codex token invalidation is terminal for the current model while remaining fallbackable. Scope a main-chat switch to the failing turn. A fallback records the user-selected origin model and reasoning level, and restores them at the next turn boundary — before the next idle prompt, before queued follow-ups, and after a compaction continuation settles. Restores are recorded in model history and in the model_fallback_* lifecycle. An explicit /model, model cycle, or thinking-level choice cancels the pending restore so the user's selection wins. Give the workflow candidate walk retry-then-advance timing. Thrown auth, model-resolution, and transport failures now get a bounded same-model retry with exponential backoff from settings.retry before handleCandidateFailure advances, and retry.enabled: false keeps immediate advancement. The retry honours pause/resume and abort, and keeps the admitted stage prompt when it resumes the existing turn with _runAgentContinue(), which the agent rejects on a transcript that does not end in the prompt being resumed. Context overflow keeps compaction first. Only once compaction is disabled, fails, or reports the overflow unresolved does the chain advance, so a compactable first overflow still costs no candidate but an unrecoverable one can reach a larger-context model. The retry decision and its backoff curve now live once, in packages/coding-agent/src/core/retry-policy.ts, used by both the main-chat retry loop and the workflow prompt and session-creation retries. Refs: #2170 Assistant-model: Claude Opus 5 --- packages/coding-agent/CHANGELOG.md | 6 +- packages/coding-agent/docs/settings.md | 4 +- packages/coding-agent/docs/subagents.md | 2 +- packages/coding-agent/docs/workflows.md | 2 +- .../src/core/agent-session-auto-compaction.ts | 72 +- .../src/core/agent-session-events.ts | 66 +- .../src/core/agent-session-methods.ts | 10 + .../src/core/agent-session-models.ts | 7 +- .../src/core/agent-session-prompt.ts | 3 + .../src/core/agent-session-retry.ts | 374 ++++++---- .../src/core/agent-session-tool-hooks.ts | 14 + .../coding-agent/src/core/agent-session.ts | 7 + .../src/core/model-fallback-failures.ts | 655 ++++++++++++++++++ .../coding-agent/src/core/retry-policy.ts | 42 ++ packages/coding-agent/src/index.ts | 12 + packages/subagents/CHANGELOG.md | 4 + .../src/runs/shared/model-fallback.ts | 499 +------------ packages/workflows/CHANGELOG.md | 5 + .../workflows/src/durable/tool-primitive.ts | 32 +- .../src/extension/atomic-stage-session.ts | 3 + packages/workflows/src/extension/wiring.ts | 64 +- .../runs/foreground/stage-runner-context.ts | 2 +- .../foreground/stage-runner-controller.ts | 488 ++++++++++++- .../src/runs/foreground/stage-runner-types.ts | 4 +- .../runs/shared/model-fallback-failures.ts | 532 +------------- packages/workflows/src/runs/shared/retry.ts | 44 ++ test/unit/main-chat-model-fallback.test.ts | 315 ++++++++- ...el-fallback-classifier-conformance.test.ts | 42 +- test/unit/stage-runner-thrown-retry.test.ts | 589 ++++++++++++++++ 29 files changed, 2636 insertions(+), 1263 deletions(-) create mode 100644 packages/coding-agent/src/core/model-fallback-failures.ts create mode 100644 packages/coding-agent/src/core/retry-policy.ts create mode 100644 packages/workflows/src/runs/shared/retry.ts create mode 100644 test/unit/stage-runner-thrown-retry.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 527a56637..0486a714a 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,7 +4,12 @@ ### Fixed +<<<<<<< HEAD - Workflow runs awaiting human input now use the blue `？` indicator in the BACKGROUND panel, `/workflow connect` picker, and `/workflow status` listing, including prompts raised by hidden nested workflow children; the indicator returns to the run's current state when the prompt resolves. +======= +- Fixed main-chat model fallback to classify provider failures consistently with workflows, advance rejected credentials and incompatible or unavailable models to the next candidate, and restore the user-selected model at the next turn without overriding an explicit `/model` choice ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed a context overflow that compaction cannot resolve to advance the configured `fallbackModels` chain instead of ending the turn, so a larger-context candidate can answer. Compaction still runs first, and a compactable overflow spends no fallback candidate ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +>>>>>>> d51dd122a (fix(fallback): converge main-chat and workflow model fallback (#2170)) ## [0.9.12] - 2026-08-04 @@ -30,7 +35,6 @@ Cumulative release of the `0.9.12-alpha.1` prerelease. The summary below covers - The compaction relevance query is no longer truncated to 1,000 characters. Truncation made prompt section order the retention policy: for a long structured prompt only the leading section reached the planner, so a constraint stated later could not influence what was kept, while the objective it qualified survived and was acted on. Long queries are safe — an oversized planner request surfaces as an explicit provider-overflow failure rather than silent truncation ([#2172](https://github.com/bastani-inc/atomic/issues/2172)). - Cleared three advisories in the shipped dependency tree. `undici` moves 8.5.0 → 8.9.0, which covers five advisories against 8.0.0–8.8.0 — response desynchronization via the retry interceptor, two cross-user cache-directive disclosures, CRLF injection through a blob body `type`, and cookie-attribute injection — and it is the dispatcher behind `fetch_url` and every agent HTTP request. Transitively under `@modelcontextprotocol/sdk`, `ip-address` is pinned to 10.3.1 for three SSRF and trust-boundary bypasses (octal-decoded leading-zero octets, CIDR suffixes suppressing special-use classification, and misclassified IPv4-mapped/NAT64 addresses), and `hono` to 4.12.34 for a CORS-middleware ReDoS. `npm audit` is clean. - - Fixed bundled builtin extensions failing to load on Windows when shared modules were evaluated twice, so bundled workflows and tools initialize correctly. ## [0.9.11] - 2026-08-03 diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index b0fe65046..2edee9f84 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -56,7 +56,9 @@ Settings and trust JSON files may start with a UTF-8 BOM, as commonly written by #### fallbackModels -`fallbackModels` gives ordinary main-chat turns an ordered model fallback chain. Atomic starts with the selected/default model. If that model exhausts the normal same-model auto-retry loop for a retryable provider/model failure — including rate limits and quota/usage-limit exhaustion such as a provider reporting `The usage limit has been reached` — Atomic switches to the next configured fallback model and continues the same turn. If `retry.enabled` is `false`, Atomic skips same-model retries and moves directly to the next fallback for retryable failures. Non-retryable task failures and cancellations do not trigger model fallback. +`fallbackModels` gives ordinary main-chat turns an ordered model fallback chain. Atomic starts with the selected/default model. If that model exhausts the normal same-model auto-retry loop for a retryable provider/model failure — including rate limits and quota/usage-limit exhaustion such as a provider reporting `The usage limit has been reached` — Atomic switches to the next configured fallback model and continues the same turn. If `retry.enabled` is `false`, Atomic skips same-model retries and moves directly to the next fallback for retryable failures. Non-retryable task failures and cancellations do not trigger model fallback. After a successful or exhausted fallback turn, Atomic restores the user-selected primary before the next user turn; an explicit `/model` choice during fallback cancels that restore. + +Context overflow keeps its normal recovery order: compaction runs first, and a compactable overflow costs no fallback candidate. Only once compaction is disabled, fails, or reports the overflow unresolved does Atomic advance to the next configured candidate, which is how a larger-context model gets a chance at the turn. The same list is also **borrowed by compaction**. When the compaction range planner cannot produce a usable plan on the current model — a rate limit, quota exhaustion, provider error, context overflow, or an empty plan — Atomic runs one planner request against the next configured candidate, using that candidate's own credentials. **A configured fallback model may therefore receive the compaction transcript.** Borrowing is planner-only: it never changes the session model, thinking level, or model history, it appends no model-change entry, and it emits no fallback status. See [Compaction](/compaction#planning-rungs-and-failure-behavior). diff --git a/packages/coding-agent/docs/subagents.md b/packages/coding-agent/docs/subagents.md index 2c3e64ff3..bcd0a2389 100644 --- a/packages/coding-agent/docs/subagents.md +++ b/packages/coding-agent/docs/subagents.md @@ -234,7 +234,7 @@ Dynamic fanout `collect.outputSchema` validates the collected result array after ## Fallback models -Agents can define ordered `fallbackModels` for retryable provider or model failures such as rate limits, quota/usage-limit exhaustion (for example a provider reporting `The usage limit has been reached`, or `usage_limit_reached`/`insufficient_quota` codes), auth problems, unavailable models, network timeouts, or 5xx errors. Atomic tries the requested primary model first, then configured fallbacks, and finally appends the current user-selected model as the last fallback candidate when available. Cancellations, safety refusals, and task/tool failures are never retried on another model. +Agents can define ordered `fallbackModels` for retryable provider or model failures such as rate limits, quota/usage-limit exhaustion (for example a provider reporting `The usage limit has been reached`, or `usage_limit_reached`/`insufficient_quota` codes), auth problems, unavailable models, network timeouts, or 5xx errors. Atomic tries the requested primary model first, then configured fallbacks, and finally appends the current user-selected model as the last fallback candidate when available. The main chat and workflow stages share one failure classifier, so auth, model-availability, request-incompatibility, and transport signals are handled consistently. Cancellations, safety refusals, and task/tool failures are never retried on another model. A candidate that cannot serve the current request — for example an HTTP 400/413/422 bad/unprocessable/payload-too-large request, an unsupported tool or parameter, a context-length/context-window overflow, or a `too large` / `invalid_request` error — is treated as request/context incompatible and the chain advances to the next candidate rather than stopping. This means that if none of the configured candidates are applicable to the request, Atomic falls back to the currently selected user model instead of failing outright. diff --git a/packages/coding-agent/docs/workflows.md b/packages/coding-agent/docs/workflows.md index 6f56d5141..d0bc2c7cd 100644 --- a/packages/coding-agent/docs/workflows.md +++ b/packages/coding-agent/docs/workflows.md @@ -2295,7 +2295,7 @@ readonly fallbackModels?: readonly string[]; readonly fallbackThinkingLevels?: readonly string[]; ``` -`fallbackModels` tries the primary first, each fallback in order, and then the current Atomic-selected model when available. It advances for rate limits and quota or usage-limit exhaustion, including messages such as `The usage limit has been reached` and codes such as `usage_limit_reached` or `insufficient_quota`. Auth/provider outages, unavailable models, network timeouts, generic transport errors such as `Connection error.` or `fetch failed`, and 5xx responses also advance the chain. +`fallbackModels` tries the primary first, each fallback in order, and then the current Atomic-selected model when available. It advances for rate limits and quota or usage-limit exhaustion, including messages such as `The usage limit has been reached` and codes such as `usage_limit_reached` or `insufficient_quota`. Auth/provider outages, unavailable models, network timeouts, generic transport errors such as `Connection error.` or `fetch failed`, and 5xx responses also advance the chain. A thrown retryable failure is retried on the same candidate with exponential backoff from `settings.retry` before the chain advances; `retry.enabled: false` keeps immediate advancement. Request/context incompatibility also advances it, including HTTP 400/413/422 bad, unprocessable, or payload-too-large requests; unsupported tools or parameters; context-length or context-window overflow; and `too large`, `invalid_request`, or `bad_request` errors. This lets the chain reach the current selected user model when no configured candidate can serve the request. diff --git a/packages/coding-agent/src/core/agent-session-auto-compaction.ts b/packages/coding-agent/src/core/agent-session-auto-compaction.ts index 09683d73e..878c7811f 100644 --- a/packages/coding-agent/src/core/agent-session-auto-compaction.ts +++ b/packages/coding-agent/src/core/agent-session-auto-compaction.ts @@ -57,19 +57,22 @@ export async function _checkCompaction( // response — before a fresh user prompt we must not resume the old turn. const isLiveTurnCompletion = skipAbortedCheck; const settings = this.settingsManager.getCompactionSettings(); - if (!settings.enabled) return; - - // Skip if message was aborted (user cancelled) - unless skipAbortedCheck is false - if (skipAbortedCheck && assistantMessage.stopReason === "aborted") return; - const contextWindow = this.model?.contextWindow ?? 0; - - // Skip overflow check if the message came from a different model. + // Skip overflow handling if the message came from a different model. // This handles the case where user switched from a smaller-context model (e.g. opus) // to a larger-context model (e.g. codex) - the overflow error from the old model // shouldn't trigger compaction for the new model. const sameModel = this.model && assistantMessage.provider === this.model.provider && assistantMessage.model === this.model.id; + if (!settings.enabled) { + // Compaction cannot recover this turn, so a configured fallback chain may + // advance to a larger-context candidate instead of dead-ending. + if (sameModel && isContextOverflow(assistantMessage, contextWindow)) this._contextOverflowUnresolved = true; + return; + } + + // Skip if message was aborted (user cancelled) - unless skipAbortedCheck is false + if (skipAbortedCheck && assistantMessage.stopReason === "aborted") return; // Skip compaction checks if this assistant message is older than the latest // compaction boundary. This prevents a stale pre-compaction usage/error @@ -91,6 +94,9 @@ export async function _checkCompaction( } if (this._overflowRecoveryAttempted) { + // One compact-and-retry has already been spent on this turn; a configured + // fallback chain may now advance to a larger-context candidate. + this._contextOverflowUnresolved = true; this._emit({ type: "compaction_end", reason: "overflow", @@ -266,10 +272,25 @@ export function _schedulePostAutoCompactionContinuationProbe( ): void { const token = this._postCompactionContinuationToken + 1; this._postCompactionContinuationToken = token; + const fallbackScopeGeneration = this._fallbackOriginGeneration; let pending: Promise; pending = new Promise((resolve) => { setTimeout(() => { void (async () => { + const restoreIfOwned = async (): Promise => { + if ( + fallbackScopeGeneration === undefined || + this._fallbackOriginGeneration !== fallbackScopeGeneration || + typeof this._restoreFallbackModel !== "function" + ) + return; + try { + await this._restoreFallbackModel(); + } catch { + // A listener must not strand the continuation waiter. The model + // state was already restored before lifecycle notifications ran. + } + }; try { if (willRetry) { if (this._postCompactionContinuationToken !== token) return; @@ -278,9 +299,24 @@ export function _schedulePostAutoCompactionContinuationProbe( await this.agent.waitForIdle(); if (this._postCompactionContinuationToken !== token) return; if (this.isCompacting || this.isStreaming) return; - if (!this.agent.hasQueuedMessages()) return; + if (!this.agent.hasQueuedMessages()) { + await restoreIfOwned(); + return; + } + // A queued message starts the next user turn. Restore before + // Agent snapshots the next request's model. + await restoreIfOwned(); } + + if (this._pendingPostCompactionContinuation !== pending) return; + // Clear this probe before entering the next run. Its promise is + // still awaited by _awaitPendingPostCompactionContinuation, but + // the nested agent_end must be free to schedule a new probe. + this._pendingPostCompactionContinuation = undefined; await this._resumeAfterAutoCompaction(); + if (willRetry && this._pendingPostCompactionContinuation === undefined) { + await restoreIfOwned(); + } } finally { if (this._pendingPostCompactionContinuation === pending) { this._pendingPostCompactionContinuation = undefined; @@ -334,8 +370,18 @@ export function _resumeAfterLengthTruncation(this: AgentSession): void { this._schedulePostAutoCompactionContinuationProbe("threshold", true); } -function overflowUnresolved(reason: "overflow" | "threshold", aborted = false): boolean | undefined { - return reason === "overflow" && !aborted ? true : undefined; +/** + * Whether an overflow turn is now unrecoverable by compaction, recording it on + * the session so a configured fallback chain may advance to another candidate. + */ +function overflowUnresolved( + this: AgentSession, + reason: "overflow" | "threshold", + aborted = false, +): boolean | undefined { + if (reason !== "overflow" || aborted) return undefined; + this._contextOverflowUnresolved = true; + return true; } export async function _runAutoCompaction( @@ -367,7 +413,7 @@ export async function _runAutoCompaction( result: undefined, aborted: false, willRetry: false, - unresolvedOverflow: overflowUnresolved(reason), + unresolvedOverflow: overflowUnresolved.call(this, reason), }); return; } @@ -398,7 +444,7 @@ export async function _runAutoCompaction( result: undefined, aborted: false, willRetry: false, - unresolvedOverflow: overflowUnresolved(reason), + unresolvedOverflow: overflowUnresolved.call(this, reason), }); return; } @@ -419,7 +465,7 @@ export async function _runAutoCompaction( result: undefined, aborted, willRetry: false, - unresolvedOverflow: overflowUnresolved(reason, aborted), + unresolvedOverflow: overflowUnresolved.call(this, reason, aborted), errorMessage: aborted ? undefined : reason === "overflow" diff --git a/packages/coding-agent/src/core/agent-session-events.ts b/packages/coding-agent/src/core/agent-session-events.ts index 4fcece5d0..ab5648aa5 100644 --- a/packages/coding-agent/src/core/agent-session-events.ts +++ b/packages/coding-agent/src/core/agent-session-events.ts @@ -67,11 +67,14 @@ export function _handleAgentEvent(this: AgentSession, event: AgentEvent): Promis this._agentEventQueue = processing; // Keep queue alive if an event handler fails. Agent-core must additionally - // await the hidden reconciliation's persistence boundary before its provider - // request, but an extension-listener failure retains the legacy nonblocking - // event behavior. + // await protected persistence and fallback reconciliation before the next + // provider request; other listener work stays nonblocking. processing.catch(() => {}); - if (awaitProtectedPersistence) return processing.catch(() => {}); + if ( + awaitProtectedPersistence || + (event.type === "agent_end" && (this._fallbackModels.length > 0 || this._fallbackOriginModel !== undefined)) + ) + return processing.catch(() => {}); } export function _createRetryPromiseForAgentEnd(this: AgentSession, event: AgentEvent): void { @@ -89,10 +92,13 @@ export function _createRetryPromiseForAgentEnd(this: AgentSession, event: AgentE return; } + const fallbackable = + typeof this._isFallbackableError === "function" + ? this._isFallbackableError(lastAssistant) + : this._isRetryableError(lastAssistant); + const retryable = this._isRetryableError(lastAssistant); const shouldRetry = - this._isRetryableError(lastAssistant) || - this._isEmptyCompletion(lastAssistant) || - this._isSafetyRefusal(lastAssistant); + retryable || fallbackable || this._isEmptyCompletion(lastAssistant) || this._isSafetyRefusal(lastAssistant); if (!shouldRetry) { return; } @@ -206,10 +212,13 @@ export async function _processAgentEvent(this: AgentSession, event: AgentEvent): this._isEmptyCompletion(assistantMsg) || this._isSafetyRefusal(assistantMsg); if (!assistantFailed) { - this._fallbackAttemptedKeys.clear(); this._overflowRecoveryAttempted = false; + this._contextOverflowUnresolved = false; this._outputBudgetErrorContinuationAttempts = 0; } + if (!assistantFailed && assistantMsg.stopReason === "stop") { + this._fallbackAttemptedKeys.clear(); + } // A non-truncated assistant response means the length-continuation loop // made progress (or the turn completed cleanly), so reset the bounded @@ -243,14 +252,26 @@ export async function _processAgentEvent(this: AgentSession, event: AgentEvent): this._postToolCompactionPreflightError !== undefined && msg.errorMessage === this._postToolCompactionPreflightError; - // Check for retryable errors first (overloaded, rate limit, server errors, - // transient provider finish_reason errors, degenerate empty completions, - // or intercepted canned safety refusals) + // Check provider/model failures before compaction. Fallback eligibility is + // broader than same-model retry eligibility: auth and request-incompatible + // failures advance without re-requesting the failed model. + const fallbackableError = + !postToolPreflightFailed && + (typeof this._isFallbackableError === "function" + ? this._isFallbackableError(msg) + : this._isRetryableError(msg)); const retryableError = !postToolPreflightFailed && this._isRetryableError(msg); - const emptyCompletion = !postToolPreflightFailed && !retryableError && this._isEmptyCompletion(msg); + const emptyCompletion = + !postToolPreflightFailed && !retryableError && !fallbackableError && this._isEmptyCompletion(msg); const safetyRefusal = - !postToolPreflightFailed && !retryableError && !emptyCompletion && this._isSafetyRefusal(msg); - if (retryableError || emptyCompletion || safetyRefusal) { + !postToolPreflightFailed && + !retryableError && + !fallbackableError && + !emptyCompletion && + this._isSafetyRefusal(msg); + const modelFailure = retryableError || fallbackableError || emptyCompletion || safetyRefusal; + let restoreAfterTurn = !modelFailure; + if (modelFailure) { if (emptyCompletion && !msg.errorMessage) { // Surface a clear reason in the retry banner; empty completions carry no // provider error message of their own. @@ -260,10 +281,27 @@ export async function _processAgentEvent(this: AgentSession, event: AgentEvent): } const didRetry = await this._handleRetryableError(msg); if (didRetry) return; // Retry was initiated, don't proceed to compaction + restoreAfterTurn = true; } this._resolveRetry(); + this._contextOverflowUnresolved = false; await this._checkCompaction(msg); + // Compaction owns context overflow first. Only once it is disabled, fails, + // or reports the overflow unresolved may the chain spend a candidate on a + // larger-context model, so a compactable first overflow costs nothing. + if (this._contextOverflowUnresolved) { + this._contextOverflowUnresolved = false; + if (typeof this._trySwitchToFallbackModel === "function" && (await this._trySwitchToFallbackModel(msg))) + return; + restoreAfterTurn = true; + } + // Keep a fallback active across the compact-and-continue probe that belongs + // to this same user turn. The probe restores it before a queued next turn, + // or after a continuation settles. + if (restoreAfterTurn && this._pendingPostCompactionContinuation === undefined) { + if (typeof this._restoreFallbackModel === "function") await this._restoreFallbackModel(); + } } } diff --git a/packages/coding-agent/src/core/agent-session-methods.ts b/packages/coding-agent/src/core/agent-session-methods.ts index 2f0dbdc49..42d4b4910 100644 --- a/packages/coding-agent/src/core/agent-session-methods.ts +++ b/packages/coding-agent/src/core/agent-session-methods.ts @@ -253,10 +253,14 @@ export interface AgentSessionMethodSurface extends AgentSessionQueuePauseControl reload(options?: AgentSessionReloadOptions): Promise; _isRetryableError(message: AssistantMessage): boolean; + _isFallbackableError(message: AssistantMessage): boolean; _isEmptyCompletion(message: AssistantMessage): boolean; _isSafetyRefusal(message: AssistantMessage): boolean; _handleRetryableError(message: AssistantMessage): Promise; _trySwitchToFallbackModel(message: AssistantMessage): Promise; + _beginFallbackModelScope(): void; + _clearFallbackModelScope(): void; + _restoreFallbackModel(): Promise; abortRetry(): void; waitForRetry(): Promise; setAutoRetryEnabled(enabled: boolean): void; @@ -389,6 +393,11 @@ export interface AgentSessionInternalSurface extends AgentSessionMethodSurface, _scopedModels: Array<{ model: Model; thinkingLevel?: ThinkingLevel }>; _fallbackModels: string[]; _fallbackAttemptedKeys: Set; + _fallbackOriginModel: Model | undefined; + _fallbackOriginThinkingLevel: ThinkingLevel | undefined; + _fallbackScopeGeneration: number; + _fallbackOriginGeneration: number | undefined; + _fallbackRestoreError: string | undefined; _unsubscribeAgent?: () => void; _eventListeners: AgentSessionEventListener[]; @@ -420,6 +429,7 @@ export interface AgentSessionInternalSurface extends AgentSessionMethodSurface, _autoCompactionAbortController: AbortController | undefined; _compactionReason: import("./agent-session-types.ts").CompactionReason | undefined; _overflowRecoveryAttempted: boolean; + _contextOverflowUnresolved: boolean; _branchSummaryAbortController: AbortController | undefined; _retryAbortController: AbortController | undefined; _retryAttempt: number; diff --git a/packages/coding-agent/src/core/agent-session-models.ts b/packages/coding-agent/src/core/agent-session-models.ts index e7c46b6ba..22a876378 100644 --- a/packages/coding-agent/src/core/agent-session-models.ts +++ b/packages/coding-agent/src/core/agent-session-models.ts @@ -90,6 +90,7 @@ export async function setModel(this: AgentSession, model: Model): Promise | undefined): string { return model ? `${model.provider}/${model.id}` : "unknown model"; } -function resolveFallbackModel( - this: AgentSession, - value: string, -): { model: Model; thinkingLevel?: ThinkingLevel } | undefined { - return resolveConfiguredFallbackModel( - value, - this._modelRuntime, - this.model?.provider ?? this.settingsManager.getDefaultProvider(), - ); -} - -function hasProviderTransportDiagnostic( - value: unknown, - seen = new Set(), - includeMessageFields = false, -): boolean { +function containsCodexTokenInvalidation(provider: string, value: unknown, seen = new Set()): boolean { if (value === null || value === undefined || seen.has(value)) return false; - if (typeof value !== "object") { - return /provider_transport_failure|websocket.*error|sse.*404/i.test(String(value)); - } + if (typeof value === "string") return isCodexTokenInvalidationError(provider, value); + if (typeof value !== "object") return false; seen.add(value); const record = value as Record; - const fields = includeMessageFields - ? [record.type, record.code, record.name, record.message, record.errorMessage, record.status, record.statusCode] - : [record.type, record.code, record.name, record.status, record.statusCode]; - for (const field of fields) { - if (typeof field === "string" || typeof field === "number") { - if (/provider_transport_failure|websocket.*error|sse.*404|\b404\b/i.test(String(field))) return true; - } + for (const field of [record.errorMessage, record.message, record.statusText]) { + if (typeof field === "string" && isCodexTokenInvalidationError(provider, field)) return true; } for (const nested of [ record.error, record.cause, ...(Array.isArray(record.diagnostics) ? record.diagnostics : []), ]) { - if (hasProviderTransportDiagnostic(nested, seen, true)) return true; + if (containsCodexTokenInvalidation(provider, nested, seen)) return true; } return false; } -function hasProviderModelUnavailableDiagnostic( - value: unknown, - seen = new Set(), - includeMessageFields = false, -): boolean { - if (value === null || value === undefined || seen.has(value)) return false; - if (typeof value !== "object") return false; - seen.add(value); - const record = value as Record; - const fields = includeMessageFields - ? [record.type, record.code, record.name, record.message, record.errorMessage] - : [record.type, record.code, record.name, record.errorMessage]; - for (const field of fields) { - if ( - typeof field === "string" && - /model(?:[_\s-].*)?(?:not[_\s-]?found|unavailable|unknown|disabled)|model[_-]?not[_-]?found/i.test(field) - ) - return true; - } - for (const nested of [ - record.error, - record.cause, - ...(Array.isArray(record.diagnostics) ? record.diagnostics : []), - ]) { - if (hasProviderModelUnavailableDiagnostic(nested, seen, true)) return true; - } - return false; +function resolveFallbackModel( + this: AgentSession, + value: string, +): { model: Model; thinkingLevel?: ThinkingLevel } | undefined { + return resolveConfiguredFallbackModel( + value, + this._modelRuntime, + this.model?.provider ?? this.settingsManager.getDefaultProvider(), + ); } + +/** Whether an assistant error may advance the configured fallback chain. */ +export function _isFallbackableError(this: AgentSession, message: AssistantMessage): boolean { + if (message.stopReason !== "error") return false; + const contextWindow = this.model?.contextWindow ?? 0; + if (isContextOverflow(message, contextWindow)) return false; + const signal = normalizeModelFailureSignal(message); + if (signal.kind === "cancelled" || signal.kind === "task_failure") return false; + const provider = message.provider || this.model?.provider; + if (provider && containsCodexTokenInvalidation(provider, message)) return true; + return isRetryableModelFailure(message); +} + +/** + * Whether the current model should be requested again. + * + * Fallback eligibility is broader than same-model retry eligibility. Auth and + * request-incompatible failures should spend a fallback candidate, but cannot + * be repaired by sending the same request again. Codex token invalidation is + * also terminal for the current model while remaining fallbackable. + */ export function _isRetryableError(this: AgentSession, message: AssistantMessage): boolean { if (message.stopReason !== "error") return false; - // Context overflow is handled by compaction, not retry + // Context overflow is handled by compaction, not retry. const contextWindow = this.model?.contextWindow ?? 0; if (isContextOverflow(message, contextWindow)) return false; - // A definitive Codex authentication rejection is terminal. It takes - // precedence over transport diagnostics recorded during WebSocket-to-SSE - // fallback so those earlier diagnostics cannot trigger a pointless retry. const provider = message.provider || this.model?.provider; - if (provider && isCodexTokenInvalidationError(provider, message.errorMessage)) return false; - - if (hasProviderTransportDiagnostic(message) || hasProviderModelUnavailableDiagnostic(message)) return true; - // Keep Atomic-specific terminal exclusions above, then inherit pi-ai's - // transport classifier (including getaddrinfo, ENOTFOUND, and EAI_AGAIN). - if (isRetryableAssistantError(message)) return true; - if (!message.errorMessage) return false; - - const err = message.errorMessage; - - // Safety triggers surface through structured API signals that pi-ai maps to - // stopReason "error": - // - Anthropic `refusal` stops become pi-ai's canned "The model refused to - // complete the request" error message; - // - OpenAI-style APIs (and github-copilot CAPI, which also maps spurious - // Gemini RECITATION/safety blocks this way) surface - // `finish_reason: content_filter`. - // Spurious safety triggers are common in agentic settings, so these are - // re-requested like transient failures, bounded by maxRetries (issue #1608). - if (/refused to complete the request|finish.?reason:?\s*content.?filter/i.test(err)) { - return true; - } + if (provider && containsCodexTokenInvalidation(provider, message)) return false; - // Match: overloaded_error, provider returned error, rate limit, quota/usage-limit exhaustion (e.g. "Codex error: The usage limit has been reached" — retryable so configured fallbackModels can advance to a provider/model with remaining headroom), 429, 500, 502, 503, 504, service unavailable, network/connection errors (including connection lost), WebSocket transport closes/errors, fetch failed, premature stream endings, HTTP/2 closed before response, terminated, retry delay exceeded, and a bare/transient provider finish_reason "error" (e.g. github-copilot Gemini's CAPI mapping of MALFORMED_FUNCTION_CALL/OTHER/UNEXPECTED_TOOL_CALL). These are provider-agnostic transient failures. - return /overloaded|provider.?returned.?error|rate.?limit|usage.?limit|quota|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|network.?error|connection.?error|connection.?refused|connection.?lost|websocket.?closed|websocket.?error|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|ended without|stream ended before message_stop|http2 request did not get a response|timed? out|timeout|terminated|retry delay|finish.?reason:?\s*error/i.test( - err, - ); + // Safety/refusal errors remain bounded same-model retries, but do not spend + // another provider candidate. The refusal grammar lives in the shared + // classifier so this decision cannot drift from workflow fallback handling. + if (isSafetyRefusalFailure(message)) return true; + return isRetryableSameModelFailure(message); } /** @@ -216,30 +181,153 @@ export function _isSafetyRefusal(this: AgentSession, message: AssistantMessage): return (message.usage?.output ?? 0) === 0; } -/** - * Handle retryable errors with exponential backoff. - * @returns true if retry was initiated, false if max retries exceeded or disabled - */ +function applyFallbackThinkingLevel(this: AgentSession, level: ThinkingLevel): boolean { + const previousLevel = this.agent.state.thinkingLevel ?? this.thinkingLevel; + this.agent.state.thinkingLevel = level; + if (previousLevel === level) return false; + this.sessionManager.appendThinkingLevelChange(level); + this._emit({ type: "thinking_level_changed", level }); + void this._extensionRunner?.emit({ + type: "thinking_level_select", + level, + previousLevel, + }); + return true; +} +/** Capture the user-selected model before the first fallback in a turn. */ +export function _beginFallbackModelScope(this: AgentSession): void { + const currentModel = this.agent.state.model ?? this.model; + if (this._fallbackOriginModel !== undefined || currentModel === undefined) return; + this._fallbackScopeGeneration = (this._fallbackScopeGeneration ?? 0) + 1; + this._fallbackOriginGeneration = this._fallbackScopeGeneration; + this._fallbackOriginModel = currentModel; + this._fallbackOriginThinkingLevel = this.agent.state.thinkingLevel ?? this.thinkingLevel; + this._fallbackRestoreError = undefined; +} + +function clearFallbackModelScopeState(this: AgentSession): void { + this._fallbackOriginGeneration = undefined; + this._fallbackOriginModel = undefined; + this._fallbackOriginThinkingLevel = undefined; + this._fallbackRestoreError = undefined; + this._fallbackAttemptedKeys.clear(); +} + +function finishFallbackModelScope( + this: AgentSession, + options: { + readonly success: boolean; + readonly from?: string; + readonly to?: string; + readonly finalError?: string; + }, +): boolean { + if (this._fallbackOriginModel === undefined || this._fallbackOriginGeneration === undefined) return false; + this._fallbackScopeGeneration = (this._fallbackScopeGeneration ?? 0) + 1; + clearFallbackModelScopeState.call(this); + this._emit({ + type: "model_fallback_end", + success: options.success, + ...(options.from === undefined ? {} : { from: options.from }), + ...(options.to === undefined ? {} : { to: options.to }), + ...(options.finalError === undefined ? {} : { finalError: options.finalError }), + }); + return true; +} + +/** Cancel a pending fallback restore after an explicit model/effort choice. */ +export function _clearFallbackModelScope(this: AgentSession): void { + const currentModel = this.agent.state.model ?? this.model; + const finalError = this._fallbackRestoreError; + if (this._fallbackOriginModel !== undefined && this._fallbackOriginGeneration !== undefined) { + finishFallbackModelScope.call(this, { + success: finalError === undefined, + from: modelLabel(currentModel), + ...(finalError === undefined ? {} : { finalError }), + }); + return; + } + this._fallbackScopeGeneration = (this._fallbackScopeGeneration ?? 0) + 1; + clearFallbackModelScopeState.call(this); +} + +/** Restore the model that was active before this turn spent a fallback. */ +export async function _restoreFallbackModel(this: AgentSession): Promise { + const originModel = this._fallbackOriginModel; + const originGeneration = this._fallbackOriginGeneration; + const finalError = this._fallbackRestoreError; + if (originModel === undefined || originGeneration === undefined) { + clearFallbackModelScopeState.call(this); + return false; + } + + const previousModel = this.agent.state.model ?? this.model; + const previousThinkingLevel = this.agent.state.thinkingLevel ?? this.thinkingLevel; + const originThinkingLevel = clampThinkingLevel( + originModel, + this._fallbackOriginThinkingLevel ?? DEFAULT_THINKING_LEVEL, + ) as ThinkingLevel; + const modelChanged = !modelsAreEqual(previousModel, originModel); + const thinkingChanged = previousThinkingLevel !== originThinkingLevel; + if (!modelChanged && !thinkingChanged) { + return finishFallbackModelScope.call(this, { + success: finalError === undefined, + from: modelLabel(previousModel), + to: modelLabel(originModel), + ...(finalError === undefined ? {} : { finalError }), + }); + } + + const stillOwnsRestore = (): boolean => + this._fallbackOriginGeneration === originGeneration && + this._fallbackOriginModel === originModel && + modelsAreEqual(this.agent.state.model ?? this.model, originModel) && + (this.agent.state.thinkingLevel ?? this.thinkingLevel) === originThinkingLevel; + + this.agent.state.model = originModel; + if (modelChanged) this.sessionManager.appendModelChange(originModel.provider, originModel.id); + applyFallbackThinkingLevel.call(this, originThinkingLevel); + this._refreshBaseSystemPromptFromActiveTools(); + this._emitModelChanged(originModel, previousModel, "restore"); + if (!stillOwnsRestore()) return false; + await this._emitModelSelect(originModel, previousModel, "restore"); + if (!stillOwnsRestore()) return false; + + return finishFallbackModelScope.call(this, { + success: finalError === undefined, + from: modelLabel(previousModel), + to: modelLabel(originModel), + ...(finalError === undefined ? {} : { finalError }), + }); +} + +/** Handle retryable errors with exponential backoff. */ export async function _trySwitchToFallbackModel(this: AgentSession, message: AssistantMessage): Promise { - if (this._fallbackModels.length === 0 || !this.model) return false; + const currentModel = this.agent.state.model ?? this.model; + if (this._fallbackModels.length === 0 || currentModel === undefined) return false; - this._fallbackAttemptedKeys.add(fallbackKey(this.model, this.thinkingLevel)); - const fromModel = this.model; + const currentThinkingLevel = this.agent.state.thinkingLevel ?? this.thinkingLevel; + const fromModel = currentModel; for (const rawCandidate of this._fallbackModels) { const candidate = resolveFallbackModel.call(this, rawCandidate); if (!candidate) continue; - const key = fallbackKey(candidate.model, candidate.thinkingLevel); - if (this._fallbackAttemptedKeys.has(key)) continue; const nextModel = candidate.model; const nextLevel = clampThinkingLevel( nextModel, candidate.thinkingLevel ?? this.settingsManager.getDefaultThinkingLevel() ?? - this.thinkingLevel ?? + currentThinkingLevel ?? DEFAULT_THINKING_LEVEL, ) as ThinkingLevel; - if (modelsAreEqual(candidate.model, fromModel) && nextLevel === this.thinkingLevel) continue; + const key = fallbackKey(nextModel, nextLevel); + if (this._fallbackAttemptedKeys.has(key)) continue; + if (modelsAreEqual(nextModel, fromModel) && nextLevel === currentThinkingLevel) continue; + + // Do not create a fallback lifecycle until a candidate can actually be + // selected. An exhausted or unresolvable chain has no start to close. + _beginFallbackModelScope.call(this); + this._fallbackAttemptedKeys.add(fallbackKey(currentModel, currentThinkingLevel)); if (this._retryAttempt > 0) { this._emit({ type: "auto_retry_end", @@ -262,51 +350,65 @@ export async function _trySwitchToFallbackModel(this: AgentSession, message: Ass } this.agent.state.model = nextModel; this.sessionManager.appendModelChange(nextModel.provider, nextModel.id); - this.agent.state.thinkingLevel = nextLevel; - this.sessionManager.appendThinkingLevelChange(nextLevel); + applyFallbackThinkingLevel.call(this, nextLevel); this._refreshBaseSystemPromptFromActiveTools(); this._emitModelChanged(nextModel, fromModel, "fallback"); await this._emitModelSelect(nextModel, fromModel, "fallback"); this._retryAttempt = 0; + const fallbackGeneration = this._fallbackOriginGeneration; setTimeout(() => { - this.agent.continue().then( - () => { - // A resolved continuation may still have produced an assistant - // error that will be classified by agent_end and may advance to - // the next fallback. Do not emit a successful fallback end here; - // agent_end/turn_end clear UI state for successful turns, while - // fallback exhaustion emits the failure end event. - }, - (error: unknown) => { - const finalError = error instanceof Error ? error.message : String(error); - this._emit({ - type: "model_fallback_end", - success: false, - from: modelLabel(fromModel), - to: modelLabel(nextModel), - finalError, - }); - this._retryAttempt = 0; - this._resolveRetry(); - }, - ); + void this.agent.continue().catch(async (error: unknown) => { + const finalError = error instanceof Error ? error.message : String(error); + if (this._fallbackOriginGeneration === fallbackGeneration) { + this._fallbackRestoreError = finalError; + try { + await this._restoreFallbackModel(); + } catch { + if (this._fallbackOriginGeneration === fallbackGeneration) { + finishFallbackModelScope.call(this, { + success: false, + from: modelLabel(this.agent.state.model ?? this.model), + finalError, + }); + } + } + } + this._retryAttempt = 0; + this._resolveRetry(); + }); }, 0); return true; } - this._emit({ - type: "model_fallback_end", - success: false, - from: modelLabel(fromModel), - finalError: message.errorMessage, - }); + if (this._fallbackOriginModel !== undefined) { + this._fallbackRestoreError = message.errorMessage || "Model fallback exhausted"; + } return false; } export async function _handleRetryableError(this: AgentSession, message: AssistantMessage): Promise { const settings = this.settingsManager.getRetrySettings(); - if (!settings.enabled) { - return this._trySwitchToFallbackModel(message); + const retryableError = this._isRetryableError?.(message) ?? true; + const fallbackableError = this._isFallbackableError?.(message) ?? retryableError; + const emptyCompletion = this._isEmptyCompletion?.(message) ?? false; + const safetyRefusal = this._isSafetyRefusal?.(message) ?? false; + const canRetrySameModel = retryableError || emptyCompletion || safetyRefusal; + // Empty completions have historically been allowed to spend a fallback after + // same-model retries. Provider safety refusals stay on the same model. + const canAdvanceToFallback = fallbackableError || emptyCompletion; + if (!settings.enabled || !canRetrySameModel) { + if (canAdvanceToFallback && (await this._trySwitchToFallbackModel(message))) return true; + if (this._retryAttempt > 0) { + this._emit({ + type: "auto_retry_end", + success: false, + attempt: this._retryAttempt, + finalError: message.errorMessage, + }); + } + this._retryAttempt = 0; + this._resolveRetry(); + return false; } // Retry promise is created synchronously in _handleAgentEvent for agent_end. @@ -317,10 +419,14 @@ export async function _handleRetryableError(this: AgentSession, message: Assista }); } - this._retryAttempt++; - - if (this._retryAttempt > settings.maxRetries) { - if (await this._trySwitchToFallbackModel(message)) { + // One shared policy decides the bounded same-model budget and its backoff, so + // main chat and workflow stages cannot drift apart on the same settings. The + // counter still advances on the exhausting attempt, so `_trySwitchToFallbackModel` + // closes the retry lifecycle before the fallback lifecycle opens. + const decision = nextRetryDecision(settings, this._retryAttempt, canRetrySameModel); + this._retryAttempt += 1; + if (decision === undefined) { + if (canAdvanceToFallback && (await this._trySwitchToFallbackModel(message))) { return true; } // Max retries exceeded, emit final failure and reset @@ -335,7 +441,7 @@ export async function _handleRetryableError(this: AgentSession, message: Assista return false; } - const delayMs = settings.baseDelayMs * 2 ** (this._retryAttempt - 1); + const delayMs = decision.delayMs; this._emit({ type: "auto_retry_start", @@ -426,10 +532,14 @@ export function setAutoRetryEnabled(this: AgentSession, enabled: boolean): void export const agentSessionRetryMethods = { _isRetryableError, + _isFallbackableError, _isEmptyCompletion, _isSafetyRefusal, _handleRetryableError, _trySwitchToFallbackModel, + _beginFallbackModelScope, + _clearFallbackModelScope, + _restoreFallbackModel, abortRetry, waitForRetry, setAutoRetryEnabled, diff --git a/packages/coding-agent/src/core/agent-session-tool-hooks.ts b/packages/coding-agent/src/core/agent-session-tool-hooks.ts index 791bab6a4..beae45a7c 100644 --- a/packages/coding-agent/src/core/agent-session-tool-hooks.ts +++ b/packages/coding-agent/src/core/agent-session-tool-hooks.ts @@ -105,6 +105,20 @@ export function _installAgentNextTurnRefresh(this: AgentSession): void { ? await this._preflightPostToolContext(previousContext.messages, signal) : previousContext.messages; + // Restore before queued follow-up messages are polled, but keep the + // fallback for deceptive completions that event processing must retry on + // the same model (safety refusal, empty completion, or length truncation). + const preserveFallbackForFailure = + turn.message.role === "assistant" && + !this.agent.hasQueuedMessages() && + (turn.message.stopReason === "length" || + this._isEmptyCompletion?.(turn.message) === true || + this._isSafetyRefusal?.(turn.message) === true); + if (!preserveFallbackForFailure && (turn.toolResults.length === 0 || terminatingBatch)) { + await this._agentEventQueue; + await this._restoreFallbackModel(); + } + return { ...previousSnapshot, context: { diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index fa2bd3f53..23df42f87 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -75,6 +75,11 @@ class AgentSessionBase { protected _scopedModels: Array<{ model: Model; thinkingLevel?: ThinkingLevel }>; protected _fallbackModels: string[]; protected _fallbackAttemptedKeys: Set = new Set(); + protected _fallbackOriginModel: Model | undefined; + protected _fallbackOriginThinkingLevel: ThinkingLevel | undefined; + protected _fallbackScopeGeneration = 0; + protected _fallbackOriginGeneration: number | undefined; + protected _fallbackRestoreError: string | undefined; protected _unsubscribeAgent?: () => void; protected _eventListeners: AgentSessionEventListener[] = []; protected _agentEventQueue: Promise = Promise.resolve(); @@ -98,6 +103,8 @@ class AgentSessionBase { protected _manualCompactionPromise: Promise | undefined = undefined; protected _autoCompactionAbortController: AbortController | undefined = undefined; protected _overflowRecoveryAttempted = false; + /** Set when compaction cannot recover a context overflow on the current model. */ + protected _contextOverflowUnresolved = false; protected _pendingPostCompactionContinuation: Promise | undefined = undefined; protected _postCompactionContinuationToken = 0; protected _lengthContinuationAttempts = 0; diff --git a/packages/coding-agent/src/core/model-fallback-failures.ts b/packages/coding-agent/src/core/model-fallback-failures.ts new file mode 100644 index 000000000..672434f4d --- /dev/null +++ b/packages/coding-agent/src/core/model-fallback-failures.ts @@ -0,0 +1,655 @@ +/** + * Shared model/provider failure classification. + * + * Workflows, subagents, and the main chat all need the same decision about + * whether a provider failure may spend another model candidate. Keep this + * module free of AgentSession and workflow imports so the published SDK can + * expose it to the raw-TypeScript companion packages without a cycle. + */ + +const RETRYABLE_MODEL_FAILURE_PATTERNS: readonly RegExp[] = [ + /rate\s*limit/i, + /too\s*many\s*requests/i, + /\b429\b/, + /quota/i, + /usage[\s_-]*limit/i, + /billing/i, + /credit/i, + /auth(?:entication|orization)?/i, + /unauthori[sz]ed/i, + /\b40[13]\b/, + /api[_\s-]*key/i, + /token[_\s-]*expired/i, + /token[_\s-]*(?:revoked|invalidated)/i, + /invalidated[_\s-]+(?:oauth|auth)[_\s-]+token/i, + /forbidden/i, + /invalid\s*key/i, + /model.*(?:unavailable|disabled|not\s*found|unknown)/i, + /(?:unavailable|disabled|not\s*found|unknown).*model/i, + /overloaded/i, + /temporarily\s*unavailable/i, + /service\s*unavailable/i, + /network/i, + /fetch/i, + /getaddrinfo/i, + /ENOTFOUND/i, + /EAI_AGAIN/i, + /socket/i, + /connection\s*refused/i, + /upstream/i, + /timeout/i, + /timed\s*out/i, + /\b50[0-4]\b/, + /\b524\b/, + /provider.?returned.?error/i, + /server.?error/i, + /internal.?error/i, + /connection.?error/i, + /connection.?lost/i, + /other side closed/i, + /websocket.?closed/i, + /websocket.?error/i, + /reset before headers/i, + /socket hang up/i, + /ended without/i, + /stream ended before message_stop/i, + /stream ended before a terminal response event/i, + /http2 request did not get a response/i, + /retry delay/i, + /finish.?reason:?\s*error/i, + /terminated/i, + /you can retry your request/i, + /try your request again/i, + /please retry your request/i, +]; + +const NON_RETRYABLE_FAILURE_PATTERNS: readonly RegExp[] = [ + /command failed/i, + /tests? failed/i, + /shell/i, + /missing file/i, + /no such file/i, + /cancel/i, + /abort/i, + /interrupted/i, +]; + +const CANCELLED_FAILURE_PATTERNS: readonly RegExp[] = [/cancel/i, /abort/i, /interrupted/i]; + +const TRANSPORT_CANCELLATION_FAILURE_PATTERNS: readonly RegExp[] = [/getaddrinfo|ENOTFOUND|EAI_AGAIN/i]; + +export type ModelFallbackFailureKind = + | "auth_on_candidate_provider" + | "rate_limit" + | "provider_unavailable" + | "network_timeout" + | "transport_error" + | "model_unavailable" + | "request_incompatible" + | "cancelled" + | "task_failure" + | "unknown"; + +export type ModelFallbackFailureSource = + | "assistant_message" + | "diagnostic" + | "throw" + | "structured" + | "string_fallback"; + +export interface ModelFallbackFailureSignal { + readonly kind: ModelFallbackFailureKind; + readonly message: string; + readonly source: ModelFallbackFailureSource; + readonly stopReason?: string; + readonly status?: number; + readonly code?: string | number; + readonly name?: string; +} + +const FALLBACKABLE_FAILURE_KINDS: ReadonlySet = new Set([ + "auth_on_candidate_provider", + "rate_limit", + "provider_unavailable", + "network_timeout", + "transport_error", + "model_unavailable", + "request_incompatible", +]); + +function asRecord(value: unknown): Record | undefined { + return value !== null && typeof value === "object" ? (value as Record) : undefined; +} + +function field(value: unknown, key: string): unknown { + return asRecord(value)?.[key]; +} + +function stringField(value: unknown, key: string): string | undefined { + const raw = field(value, key); + return typeof raw === "string" && raw.trim().length > 0 ? raw : undefined; +} + +function errorName(value: unknown): string | undefined { + return value instanceof Error ? value.name : stringField(value, "name"); +} + +function directMessageFrom(value: unknown): string | undefined { + return stringField(value, "errorMessage") ?? stringField(value, "message") ?? stringField(value, "statusText"); +} + +function integerFrom(value: unknown): number | undefined { + if (typeof value === "number" && Number.isInteger(value)) return value; + if (typeof value !== "string" || value.trim().length === 0) return undefined; + const parsed = Number(value.trim()); + return Number.isInteger(parsed) ? parsed : undefined; +} + +function statusFrom(value: unknown): number | undefined { + return ( + integerFrom(field(value, "status")) ?? + integerFrom(field(value, "statusCode")) ?? + integerFrom(field(value, "httpStatus")) + ); +} + +function codeFrom(value: unknown): string | number | undefined { + const rawCode = field(value, "code"); + return typeof rawCode === "string" || typeof rawCode === "number" ? rawCode : undefined; +} + +function stopReasonFrom(value: unknown): string | undefined { + return stringField(value, "stopReason"); +} + +function finishReasonFrom(value: unknown): string | undefined { + return stringField(value, "finish_reason") ?? stringField(value, "finishReason"); +} + +function causeOf(value: unknown): unknown { + return value instanceof Error ? value.cause : field(value, "cause"); +} + +function diagnosticErrors(value: unknown): readonly unknown[] { + const diagnostics = field(value, "diagnostics"); + if (!Array.isArray(diagnostics)) return []; + const errors: unknown[] = []; + for (const diagnostic of diagnostics) { + const diagnosticType = stringField(diagnostic, "type"); + const diagnosticError = field(diagnostic, "error"); + errors.push( + diagnosticType !== undefined && /provider[_\s-]?transport[_\s-]?failure/i.test(diagnosticType) + ? diagnostic + : (diagnosticError ?? diagnostic), + ); + } + return errors; +} + +function normalizeCode(value: string | number | undefined): string | undefined { + if (value === undefined) return undefined; + const normalized = String(value) + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, ""); + return normalized.length > 0 ? normalized : undefined; +} + +function kindFromStatus(status: number | undefined): ModelFallbackFailureKind | undefined { + switch (status) { + case 400: + case 413: + case 422: + return "request_incompatible"; + case 401: + case 403: + return "auth_on_candidate_provider"; + case 408: + return "network_timeout"; + case 404: + return "model_unavailable"; + case 429: + return "rate_limit"; + default: + if (status !== undefined && status >= 500 && status <= 599) return "provider_unavailable"; + return undefined; + } +} + +function refusalKindFromCode(code: string | number | undefined): ModelFallbackFailureKind | undefined { + const normalizedCode = normalizeCode(code); + if (normalizedCode === undefined) return undefined; + if (normalizedCode.includes("content_filter") || normalizedCode.includes("contentfilter")) return "task_failure"; + if (normalizedCode.includes("safety") || normalizedCode.includes("policy")) return "task_failure"; + switch (normalizedCode) { + case "blocked": + case "blocked_by_provider": + case "blocked_by_safety": + case "blocked_by_policy": + case "provider_refusal": + case "refusal": + case "tool_refusal": + case "tool_call_refusal": + case "tool_use_refusal": + return "task_failure"; + default: + return undefined; + } +} + +const REQUEST_INCOMPATIBLE_CODES: ReadonlySet = new Set([ + "invalid_request", + "invalid_request_error", + "bad_request", + "context_length_exceeded", + "request_too_large", + "too_large", + "request_entity_too_large", + "max_tokens", + "max_context_length", + "context_window_exceeded", +]); + +function requestIncompatibleKindFromCode(code: string | number | undefined): ModelFallbackFailureKind | undefined { + const normalized = normalizeCode(code); + return normalized !== undefined && REQUEST_INCOMPATIBLE_CODES.has(normalized) ? "request_incompatible" : undefined; +} + +function kindFromCode(code: string | number | undefined): ModelFallbackFailureKind | undefined { + const normalizedCode = normalizeCode(code); + if (normalizedCode === undefined) return undefined; + const refusalKind = refusalKindFromCode(code); + if (refusalKind !== undefined) return refusalKind; + const httpStatusKind = kindFromStatus(integerFrom(code)); + if (httpStatusKind !== undefined) return httpStatusKind; + const requestIncompatibleKind = requestIncompatibleKindFromCode(code); + if (requestIncompatibleKind !== undefined) return requestIncompatibleKind; + + switch (normalizedCode) { + case "auth": + case "auth_required": + case "authentication_required": + case "unauthorized": + case "forbidden": + case "invalid_api_key": + case "missing_api_key": + case "invalid_key": + case "token_expired": + case "token_revoked": + case "token_invalidated": + case "oauth_token_invalidated": + return "auth_on_candidate_provider"; + case "etimedout": + case "econnreset": + case "econnrefused": + case "enotfound": + case "eai_again": + case "fetch_failed": + case "network_error": + case "timeout": + case "timeout_error": + case "und_err_connect_timeout": + return "network_timeout"; + case "rate_limit": + case "rate_limit_exceeded": + case "too_many_requests": + case "quota_exceeded": + case "insufficient_quota": + case "usage_limit": + case "usage_limit_reached": + case "usage_limit_exceeded": + return "rate_limit"; + case "aborterror": + case "aborted": + case "cancelled": + case "canceled": + return "cancelled"; + case "model_not_found": + case "model_unavailable": + case "model_disabled": + case "unknown_model": + return "model_unavailable"; + case "provider_error": + case "api_error": + case "service_unavailable": + case "temporarily_unavailable": + case "overloaded": + return "provider_unavailable"; + default: + return undefined; + } +} + +const REQUEST_INCOMPATIBLE_FAILURE_PATTERNS: readonly RegExp[] = [ + /\bcontext[_\s-]?length(?:[_\s-]?exceeded)?\b/i, + /\bcontext[_\s-]?window(?:[_\s-]?exceeded)?\b/i, + /\bmax[_\s-]?context\b/i, + /\bmax[_\s-]?tokens?\b/i, + /\brequest(?:[_\s-]?entity)?[_\s-]?too[_\s-]?large\b/i, + /\btoo[_\s-]?large\b/i, + /\b(?:unsupported|unknown|invalid)\s+(?:tool|parameter|function)\b/i, + /\b(?:tool|parameter|function)\s+(?:not\s+(?:supported|found|allowed)|unknown|invalid)\b/i, + /\binvalid[_\s-]?request(?:[_\s-]?error)?\b/i, + /\bbad[_\s-]?request\b/i, +]; + +const PROVIDER_REFUSAL_FAILURE_PATTERNS: readonly RegExp[] = [ + /\bthe\s+model\s+refused\s+to\s+complete\s+the\s+request\b/i, + /\bfinish[_\s-]?reason\b[^\n]*\bcontent[_\s-]?filter\b/i, + /\bcontent[_\s-]?filter(?:ed|ing)?\b/i, + /\b(?:safety|policy)\b[^\n]*\b(?:refus(?:e|al|ed|es|ing)?|block(?:ed|ing)?|filter(?:ed|ing)?|violat(?:e|ion|ed|ing)?|disallow(?:ed|ing)?|reject(?:ed|ion|ing)?)\b/i, + /\b(?:refus(?:e|al|ed|es|ing)?|block(?:ed|ing)?|filter(?:ed|ing)?|violat(?:e|ion|ed|ing)?|disallow(?:ed|ing)?|reject(?:ed|ion|ing)?)\b[^\n]*\b(?:safety|policy)\b/i, + /\btool[_\s-]?(?:call|use)?[_\s-]?refus(?:e|al|ed|es|ing)?\b/i, + /\btool(?:\s+call|\s+use)?\b[^\n]*\brefus(?:e|al|ed|es|ing)?\b/i, + /\brefus(?:e|al|ed|es|ing)?\b[^\n]*\btool(?:\s+call|\s+use)?\b/i, + /\bprovider[_\s-]?refus(?:e|al|ed|es|ing)?\b/i, + /\bprovider\b[^\n]*\brefus(?:e|al|ed|es|ing)?\b[^\n]*\b(?:prompt|request|content|policy|safety)\b/i, +]; + +const TRANSPORT_OUTAGE_FAILURE_PATTERNS: readonly RegExp[] = [/^connection\s+error\.?$/i, /^fetch\s+failed\.?$/i]; + +function transportOutageKindFromMessage(message: string): ModelFallbackFailureKind | undefined { + return TRANSPORT_OUTAGE_FAILURE_PATTERNS.some((pattern) => pattern.test(message.trim())) + ? "transport_error" + : undefined; +} + +function refusalKindFromMessage(message: string): ModelFallbackFailureKind | undefined { + if (TRANSPORT_CANCELLATION_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "network_timeout"; + if (CANCELLED_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "cancelled"; + if (NON_RETRYABLE_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "task_failure"; + if (PROVIDER_REFUSAL_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "task_failure"; + return undefined; +} + +function fallbackKindFromMessage(message: string, name: string | undefined): ModelFallbackFailureKind | undefined { + const refusalKind = refusalKindFromMessage(message); + if (refusalKind !== undefined) return refusalKind; + const transportOutageKind = transportOutageKindFromMessage(message); + if (transportOutageKind !== undefined) return transportOutageKind; + if (REQUEST_INCOMPATIBLE_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "request_incompatible"; + const nameKind = kindFromCode(name); + if (nameKind !== undefined) return nameKind; + if (!RETRYABLE_MODEL_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return undefined; + if (/rate\s*limit|too\s*many\s*requests|\b429\b|quota|usage[\s_-]*limit|billing|credit/i.test(message)) + return "rate_limit"; + if ( + /auth|unauthori[sz]ed|\b40[13]\b|api[_\s-]*key|token[_\s-]*(?:expired|revoked|invalidated|rejected)|forbidden|invalid[_\s-]*key/i.test( + message, + ) + ) + return "auth_on_candidate_provider"; + if ( + /model.*(?:unavailable|disabled|not\s*found|unknown)|(?:unavailable|disabled|not\s*found|unknown).*model/i.test( + message, + ) + ) + return "model_unavailable"; + if (/network|fetch|socket|connection\s*refused|getaddrinfo|ENOTFOUND|EAI_AGAIN|timeout|timed\s*out/i.test(message)) + return "network_timeout"; + return "provider_unavailable"; +} + +function signalSource(value: unknown, fallback: ModelFallbackFailureSource | undefined): ModelFallbackFailureSource { + if (fallback !== undefined) return fallback; + if (stopReasonFrom(value) !== undefined || diagnosticErrors(value).length > 0) return "assistant_message"; + if (value instanceof Error) return "throw"; + return "structured"; +} + +function makeSignal( + kind: ModelFallbackFailureKind, + value: unknown, + source: ModelFallbackFailureSource | undefined, +): ModelFallbackFailureSignal { + const status = statusFrom(value); + const code = codeFrom(value); + const name = errorName(value); + const stopReason = stopReasonFrom(value); + return { + kind, + message: errorMessage(value), + source: signalSource(value, source), + ...(stopReason !== undefined ? { stopReason } : {}), + ...(status !== undefined ? { status } : {}), + ...(code !== undefined ? { code } : {}), + ...(name !== undefined ? { name } : {}), + }; +} + +function fallbackSignalFromDirectMessage( + value: unknown, + source: ModelFallbackFailureSource | undefined, +): ModelFallbackFailureSignal | undefined { + const message = directMessageFrom(value); + if (message === undefined) return undefined; + const kind = fallbackKindFromMessage(message, errorName(value)); + return kind === undefined ? undefined : makeSignal(kind, value, source); +} + +function fallbackSignalFromMessage( + value: unknown, + source: ModelFallbackFailureSource | undefined, +): ModelFallbackFailureSignal | undefined { + const message = errorMessage(value); + if (!message.trim()) return undefined; + const kind = fallbackKindFromMessage(message, errorName(value)); + return kind === undefined ? undefined : makeSignal(kind, value, source); +} + +function classifyAssistantRefusalSignal( + value: unknown, + source: ModelFallbackFailureSource | undefined, +): ModelFallbackFailureSignal | undefined { + const codeRefusalKind = + refusalKindFromCode(codeFrom(value)) ?? + refusalKindFromCode(errorName(value)) ?? + refusalKindFromCode(finishReasonFrom(value)); + if (codeRefusalKind !== undefined) return makeSignal(codeRefusalKind, value, source); + + const messageRefusalKind = refusalKindFromMessage(directMessageFrom(value) ?? ""); + return messageRefusalKind === undefined ? undefined : makeSignal(messageRefusalKind, value, source); +} + +function isRefusalSignal(signal: ModelFallbackFailureSignal): boolean { + return signal.kind === "cancelled" || signal.kind === "task_failure"; +} + +function nestedSignalPriority(signal: ModelFallbackFailureSignal): number { + switch (signal.kind) { + case "cancelled": + case "task_failure": + return 100; + case "auth_on_candidate_provider": + return 90; + case "request_incompatible": + return 85; + case "model_unavailable": + return 80; + case "rate_limit": + return 70; + case "provider_unavailable": + return 60; + case "network_timeout": + return 50; + case "transport_error": + return 82; + case "unknown": + return 0; + } +} +function structuredSignal( + value: unknown, + seen: Set, + source?: ModelFallbackFailureSource, +): ModelFallbackFailureSignal | undefined { + if (value === undefined || value === null || seen.has(value)) return undefined; + if (typeof value === "object") seen.add(value); + + const stopReason = stopReasonFrom(value)?.toLowerCase(); + const diagnosticType = stringField(value, "type"); + const providerTransportSignal = + diagnosticType !== undefined && /provider[_\s-]?transport[_\s-]?failure/i.test(diagnosticType) + ? makeSignal("transport_error", value, source) + : undefined; + if (stopReason === "aborted") return makeSignal("cancelled", value, source); + + // Defer direct refusal/cancellation signals until nested diagnostics and + // causes have been inspected. A transport wrapper can carry a cancellation + // or provider refusal that must win over the wrapper's transport label. + const directRefusalSignal = classifyAssistantRefusalSignal(value, source); + const codeKind = kindFromCode(codeFrom(value)); + const nameKind = kindFromCode(errorName(value)); + const directCancellationSignal = + codeKind === "cancelled" || nameKind === "cancelled" ? makeSignal("cancelled", value, source) : undefined; + + let bestNestedFallbackSignal: ModelFallbackFailureSignal | undefined; + const nestedSeen = new Set(seen); + const nestedValues: readonly { value: unknown; source?: ModelFallbackFailureSource }[] = [ + ...diagnosticErrors(value).map((diagnosticError) => ({ value: diagnosticError, source: "diagnostic" as const })), + ...(field(value, "error") === undefined ? [] : [{ value: field(value, "error"), source }]), + ]; + for (const nested of nestedValues) { + const nestedSignal = + structuredSignal(nested.value, nestedSeen, nested.source) ?? + fallbackSignalFromMessage(nested.value, nested.source); + if (nestedSignal === undefined) continue; + if (isRefusalSignal(nestedSignal)) return nestedSignal; + if ( + bestNestedFallbackSignal === undefined || + nestedSignalPriority(nestedSignal) > nestedSignalPriority(bestNestedFallbackSignal) + ) { + bestNestedFallbackSignal = nestedSignal; + } + } + const cause = causeOf(value); + const causeSignal = structuredSignal(cause, nestedSeen, source) ?? fallbackSignalFromMessage(cause, source); + if (causeSignal !== undefined) { + if (isRefusalSignal(causeSignal)) return causeSignal; + if ( + bestNestedFallbackSignal === undefined || + nestedSignalPriority(causeSignal) > nestedSignalPriority(bestNestedFallbackSignal) + ) { + bestNestedFallbackSignal = causeSignal; + } + } + + const directSignals = [ + directRefusalSignal, + directCancellationSignal, + providerTransportSignal, + fallbackSignalFromDirectMessage(value, source), + kindFromStatus(statusFrom(value)) === undefined + ? undefined + : makeSignal(kindFromStatus(statusFrom(value))!, value, source), + kindFromCode(codeFrom(value)) === undefined + ? undefined + : makeSignal(kindFromCode(codeFrom(value))!, value, source), + kindFromCode(errorName(value)) === undefined + ? undefined + : makeSignal(kindFromCode(errorName(value))!, value, source), + ].filter((signal): signal is ModelFallbackFailureSignal => signal !== undefined); + for (const directSignal of directSignals) { + if (isRefusalSignal(directSignal)) return directSignal; + if ( + bestNestedFallbackSignal === undefined || + nestedSignalPriority(directSignal) > nestedSignalPriority(bestNestedFallbackSignal) + ) { + bestNestedFallbackSignal = directSignal; + } + } + + if (bestNestedFallbackSignal !== undefined) return bestNestedFallbackSignal; + if (stopReason === "error") return makeSignal("provider_unavailable", value, source); + return undefined; +} + +function messageFromUnknown(value: unknown, seen: Set): string | undefined { + if (value === undefined || value === null || seen.has(value)) return undefined; + if (typeof value === "string") return value.trim().length > 0 ? value : undefined; + if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return String(value); + if (typeof value === "symbol" || typeof value === "function") return undefined; + seen.add(value); + + if (value instanceof Error && value.message.trim().length > 0) return value.message; + const directMessage = directMessageFrom(value); + if (directMessage !== undefined) return directMessage; + for (const diagnosticError of diagnosticErrors(value)) { + const diagnosticMessage = messageFromUnknown(diagnosticError, seen); + if (diagnosticMessage !== undefined) return diagnosticMessage; + } + const causeMessage = messageFromUnknown(causeOf(value), seen); + if (causeMessage !== undefined) return causeMessage; + const finishReason = finishReasonFrom(value); + if (finishReason !== undefined) return `Model request finished with finish_reason:${finishReason}`; + const stopReason = stopReasonFrom(value); + if (stopReason !== undefined) return `Assistant message ended with stopReason:${stopReason}`; + const status = statusFrom(value); + if (status !== undefined) return `Model request failed with status ${status}`; + const code = codeFrom(value); + if (code !== undefined) return `Model request failed with code ${String(code)}`; + return undefined; +} + +export function errorMessage(error: unknown): string { + const structuredMessage = messageFromUnknown(error, new Set()); + if (structuredMessage !== undefined) return structuredMessage; + const rendered = String(error); + return rendered === "[object Object]" ? "Model request failed" : rendered; +} + +/** Alias used by the subagent package's older public helper name. */ +export const modelFailureMessage = errorMessage; + +export function normalizeModelFailureSignal(error: unknown): ModelFallbackFailureSignal { + const structured = structuredSignal(error, new Set()); + if (structured !== undefined) return structured; + const message = errorMessage(error); + const name = errorName(error); + const fallbackKind = message.trim().length > 0 ? fallbackKindFromMessage(message, name) : undefined; + return { + kind: fallbackKind ?? "unknown", + message, + source: "string_fallback", + ...(name !== undefined ? { name } : {}), + }; +} + +/** Whether a task-failure signal is a provider safety/refusal response. */ +export function isSafetyRefusalFailure(error: unknown): boolean { + const signal = normalizeModelFailureSignal(error); + if (signal.kind !== "task_failure") return false; + const text = `${signal.message}\n${signal.code ?? ""}\n${signal.name ?? ""}`; + return PROVIDER_REFUSAL_FAILURE_PATTERNS.some((pattern) => pattern.test(text)); +} + +/** True when this failure may spend a configured model fallback candidate. */ +export function isRetryableModelFailure(error: unknown): boolean { + if (error === undefined) return false; + return FALLBACKABLE_FAILURE_KINDS.has(normalizeModelFailureSignal(error).kind); +} + +const SAME_MODEL_RETRYABLE_FAILURE_KINDS: ReadonlySet = new Set([ + "rate_limit", + "provider_unavailable", + "network_timeout", + "transport_error", +]); + +/** True when the same model may be requested again before fallback advances. */ +export function isRetryableSameModelFailure(error: unknown): boolean { + if (!isRetryableModelFailure(error)) return false; + const signal = normalizeModelFailureSignal(error); + if (!SAME_MODEL_RETRYABLE_FAILURE_KINDS.has(signal.kind)) return false; + if ( + signal.kind === "provider_unavailable" && + signal.source === "assistant_message" && + signal.status === undefined && + signal.code === undefined && + signal.name === undefined && + !RETRYABLE_MODEL_FAILURE_PATTERNS.some((pattern) => pattern.test(signal.message)) + ) + return false; + return true; +} diff --git a/packages/coding-agent/src/core/retry-policy.ts b/packages/coding-agent/src/core/retry-policy.ts new file mode 100644 index 000000000..8fde8dbc8 --- /dev/null +++ b/packages/coding-agent/src/core/retry-policy.ts @@ -0,0 +1,42 @@ +/** + * The one bounded same-target retry policy. + * + * Main chat and workflow stages both retry a failing model before spending a + * fallback candidate. Keeping the gate and the backoff curve here means the two + * cannot drift into different budgets for the same `settings.retry` values. + * + * Callers keep their own concerns: attempt state, lifecycle events, abort + * handling, sleeping, and whether the chain may advance afterwards. + */ + +export interface RetryPolicySettings { + readonly enabled: boolean; + readonly maxRetries: number; + readonly baseDelayMs: number; +} + +export interface RetryDecision { + /** 1-based number of the retry this decision authorizes. */ + readonly attempt: number; + /** Exponential backoff to wait before that retry. */ + readonly delayMs: number; +} + +/** + * Decide whether one more attempt against the same target is allowed. + * + * @param settings retry settings, or `undefined` when the caller has none + * @param retriesSpent retries already made for this failure + * @param eligible whether this failure can be repaired by requesting again + */ +export function nextRetryDecision( + settings: RetryPolicySettings | undefined, + retriesSpent: number, + eligible: boolean, +): RetryDecision | undefined { + if (settings === undefined || !settings.enabled || !eligible || retriesSpent >= settings.maxRetries) { + return undefined; + } + const attempt = retriesSpent + 1; + return { attempt, delayMs: settings.baseDelayMs * 2 ** (attempt - 1) }; +} diff --git a/packages/coding-agent/src/index.ts b/packages/coding-agent/src/index.ts index 97d27878e..a2ce93fd5 100644 --- a/packages/coding-agent/src/index.ts +++ b/packages/coding-agent/src/index.ts @@ -125,6 +125,17 @@ export { // Footer data provider (git branch + extension statuses - data not otherwise available to extensions) export type { ReadonlyFooterDataProvider } from "./core/footer-data-provider.ts"; export { convertToLlm } from "./core/messages.ts"; +export type { + ModelFallbackFailureKind, + ModelFallbackFailureSignal, + ModelFallbackFailureSource, +} from "./core/model-fallback-failures.ts"; +export { + errorMessage, + isRetryableModelFailure, + modelFailureMessage, + normalizeModelFailureSignal, +} from "./core/model-fallback-failures.ts"; export { ModelRegistry } from "./core/model-registry.ts"; export { type CreateModelRuntimeOptions, ModelRuntime, type ModelRuntimeAuthOverrides } from "./core/model-runtime.ts"; export type { @@ -143,6 +154,7 @@ export type { ResourceLoader, } from "./core/resource-loader.ts"; export { DefaultResourceLoader, loadProjectContextFiles } from "./core/resource-loader.ts"; +export { nextRetryDecision, type RetryDecision, type RetryPolicySettings } from "./core/retry-policy.ts"; // SDK for programmatic usage export { AgentSessionRuntime, diff --git a/packages/subagents/CHANGELOG.md b/packages/subagents/CHANGELOG.md index 3e3fc4697..ac101c1bf 100644 --- a/packages/subagents/CHANGELOG.md +++ b/packages/subagents/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed model-failure classification sharing so subagent fallback decisions stay aligned with main chat and workflow candidates for auth, unavailable-model, request-incompatible, quota, and transport failures ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). + ## [0.9.12] - 2026-08-04 ### Fixed diff --git a/packages/subagents/src/runs/shared/model-fallback.ts b/packages/subagents/src/runs/shared/model-fallback.ts index 301eca527..2a244427f 100644 --- a/packages/subagents/src/runs/shared/model-fallback.ts +++ b/packages/subagents/src/runs/shared/model-fallback.ts @@ -1,3 +1,10 @@ +import type { ModelFallbackFailureKind, ModelFallbackFailureSignal, ModelFallbackFailureSource } from "@bastani/atomic"; +import { + errorMessage, + isRetryableModelFailure, + modelFailureMessage, + normalizeModelFailureSignal, +} from "@bastani/atomic"; import { type ModelInfo as AvailableModelInfo, splitKnownThinkingSuffix, @@ -5,7 +12,8 @@ import { } from "../../shared/model-info.ts"; import type { Usage } from "../../shared/types.ts"; -export type { AvailableModelInfo }; +export type { AvailableModelInfo, ModelFallbackFailureKind, ModelFallbackFailureSignal, ModelFallbackFailureSource }; +export { errorMessage, isRetryableModelFailure, modelFailureMessage, normalizeModelFailureSignal }; interface ModelAttemptSummary { model: string; @@ -68,495 +76,6 @@ export function currentModelFullId(model: { provider: string; id: string } | und return `${String(model.provider)}/${model.id}`; } -const RETRYABLE_MODEL_FAILURE_PATTERNS: readonly RegExp[] = [ - /rate\s*limit/i, - /too many requests/i, - /\b429\b/, - /quota/i, - /usage[\s_-]*limit/i, // usage-limit exhaustion is a quota condition (accept space/underscore/hyphen/joined forms); next candidate may have headroom - /billing/i, - /credit/i, - /auth(?:entication)?/i, - /unauthori[sz]ed/i, - /\b40[13]\b/, - /forbidden/i, - /api key/i, - /token expired/i, - /invalid key/i, - /provider.*unavailable/i, - /model.*unavailable/i, - /model.*disabled/i, - /model.*not found/i, - /unknown model/i, - /overloaded/i, - /service unavailable/i, - /temporar(?:ily)? unavailable/i, - /connection refused/i, - /fetch failed/i, - /network error/i, - /socket hang up/i, - /upstream/i, - /timed? out/i, - /timeout/i, - /\b50[0-4]\b/, -]; - -const NON_RETRYABLE_FAILURE_PATTERNS: readonly RegExp[] = [ - /command failed/i, - /tests? failed/i, - /shell/i, - /missing file/i, - /no such file/i, - /cancel/i, - /abort/i, - /interrupted/i, -]; - -const CANCELLED_FAILURE_PATTERNS: readonly RegExp[] = [/cancel/i, /abort/i, /interrupted/i]; - -export type ModelFallbackFailureKind = - | "auth_on_candidate_provider" - | "rate_limit" - | "provider_unavailable" - | "network_timeout" - | "model_unavailable" - | "request_incompatible" - | "cancelled" - | "task_failure" - | "unknown"; - -export type ModelFallbackFailureSource = - | "assistant_message" - | "diagnostic" - | "throw" - | "structured" - | "string_fallback"; - -export interface ModelFallbackFailureSignal { - readonly kind: ModelFallbackFailureKind; - readonly message: string; - readonly source: ModelFallbackFailureSource; - readonly stopReason?: string; - readonly status?: number; - readonly code?: string | number; - readonly name?: string; -} - -const FALLBACKABLE_FAILURE_KINDS: ReadonlySet = new Set([ - "auth_on_candidate_provider", - "rate_limit", - "provider_unavailable", - "network_timeout", - "model_unavailable", - "request_incompatible", -]); - -function asRecord(value: unknown): Record | undefined { - return value !== null && typeof value === "object" ? (value as Record) : undefined; -} - -function field(value: unknown, key: string): unknown { - return asRecord(value)?.[key]; -} - -function stringField(value: unknown, key: string): string | undefined { - const raw = field(value, key); - return typeof raw === "string" && raw.trim().length > 0 ? raw : undefined; -} - -function errorName(value: unknown): string | undefined { - return value instanceof Error ? value.name : stringField(value, "name"); -} - -function directMessageFrom(value: unknown): string | undefined { - return stringField(value, "errorMessage") ?? stringField(value, "message") ?? stringField(value, "statusText"); -} - -function integerFrom(value: unknown): number | undefined { - if (typeof value === "number" && Number.isInteger(value)) return value; - if (typeof value !== "string" || value.trim().length === 0) return undefined; - const parsed = Number(value.trim()); - return Number.isInteger(parsed) ? parsed : undefined; -} - -function statusFrom(value: unknown): number | undefined { - return ( - integerFrom(field(value, "status")) ?? - integerFrom(field(value, "statusCode")) ?? - integerFrom(field(value, "httpStatus")) - ); -} - -function codeFrom(value: unknown): string | number | undefined { - const rawCode = field(value, "code"); - return typeof rawCode === "string" || typeof rawCode === "number" ? rawCode : undefined; -} - -function stopReasonFrom(value: unknown): string | undefined { - return stringField(value, "stopReason"); -} - -function finishReasonFrom(value: unknown): string | undefined { - return stringField(value, "finish_reason") ?? stringField(value, "finishReason"); -} - -function causeOf(value: unknown): unknown { - return value instanceof Error ? value.cause : field(value, "cause"); -} - -function diagnosticErrors(value: unknown): readonly unknown[] { - const diagnostics = field(value, "diagnostics"); - if (!Array.isArray(diagnostics)) return []; - const errors: unknown[] = []; - for (const diagnostic of diagnostics) { - const diagnosticError = field(diagnostic, "error"); - errors.push(diagnosticError ?? diagnostic); - } - return errors; -} - -function normalizeCode(value: string | number | undefined): string | undefined { - if (value === undefined) return undefined; - const normalized = String(value) - .trim() - .toLowerCase() - .replace(/[^a-z0-9]+/g, "_") - .replace(/^_+|_+$/g, ""); - return normalized.length > 0 ? normalized : undefined; -} - -function kindFromStatus(status: number | undefined): ModelFallbackFailureKind | undefined { - switch (status) { - case 400: - case 413: - case 422: - return "request_incompatible"; - case 401: - case 403: - return "auth_on_candidate_provider"; - case 408: - return "network_timeout"; - case 404: - return "model_unavailable"; - case 429: - return "rate_limit"; - default: - if (status !== undefined && status >= 500 && status <= 599) return "provider_unavailable"; - return undefined; - } -} - -const REQUEST_INCOMPATIBLE_CODES: ReadonlySet = new Set([ - "invalid_request", - "invalid_request_error", - "bad_request", - "context_length_exceeded", - "request_too_large", - "too_large", - "request_entity_too_large", - "max_tokens", - "max_context_length", - "context_window_exceeded", -]); -function requestIncompatibleKindFromCode(code: string | number | undefined): ModelFallbackFailureKind | undefined { - const normalizedCode = normalizeCode(code); - return normalizedCode !== undefined && REQUEST_INCOMPATIBLE_CODES.has(normalizedCode) - ? "request_incompatible" - : undefined; -} -function refusalKindFromCode(code: string | number | undefined): ModelFallbackFailureKind | undefined { - const normalizedCode = normalizeCode(code); - if (normalizedCode === undefined) return undefined; - if (normalizedCode.includes("content_filter") || normalizedCode.includes("contentfilter")) return "task_failure"; - if (normalizedCode.includes("safety") || normalizedCode.includes("policy")) return "task_failure"; - switch (normalizedCode) { - case "blocked": - case "blocked_by_provider": - case "blocked_by_safety": - case "blocked_by_policy": - case "provider_refusal": - case "refusal": - case "tool_refusal": - case "tool_call_refusal": - case "tool_use_refusal": - return "task_failure"; - default: - return undefined; - } -} - -const CODE_KINDS_BY_KIND: ReadonlyArray]> = [ - [ - "auth_on_candidate_provider", - new Set([ - "auth", - "auth_required", - "authentication_required", - "unauthorized", - "forbidden", - "invalid_api_key", - "missing_api_key", - "invalid_key", - ]), - ], - [ - "network_timeout", - new Set([ - "etimedout", - "econnreset", - "econnrefused", - "enotfound", - "eai_again", - "fetch_failed", - "network_error", - "timeout", - "timeout_error", - "und_err_connect_timeout", - ]), - ], - [ - "rate_limit", - new Set([ - "rate_limit", - "rate_limit_exceeded", - "too_many_requests", - "quota_exceeded", - "insufficient_quota", - "usage_limit", - "usage_limit_reached", - "usage_limit_exceeded", - ]), - ], - ["cancelled", new Set(["aborterror", "aborted", "cancelled", "canceled"])], - ["model_unavailable", new Set(["model_not_found", "model_unavailable", "model_disabled", "unknown_model"])], - [ - "provider_unavailable", - new Set(["provider_error", "api_error", "service_unavailable", "temporarily_unavailable", "overloaded"]), - ], -]; -function kindFromCode(code: string | number | undefined): ModelFallbackFailureKind | undefined { - const normalizedCode = normalizeCode(code); - if (normalizedCode === undefined) return undefined; - const refusalKind = refusalKindFromCode(code); - if (refusalKind !== undefined) return refusalKind; - const httpStatusKind = kindFromStatus(integerFrom(code)); - if (httpStatusKind !== undefined) return httpStatusKind; - const requestIncompatibleKind = requestIncompatibleKindFromCode(code); - if (requestIncompatibleKind !== undefined) return requestIncompatibleKind; - return CODE_KINDS_BY_KIND.find(([_, codes]) => codes.has(normalizedCode))?.[0]; -} - -const REQUEST_INCOMPATIBLE_FAILURE_PATTERNS: readonly RegExp[] = [ - /\bcontext[_\s-]?(?:length|window)(?:[_\s-]?exceeded)?\b/i, - /\bmax[_\s-]?(?:context|tokens?)\b/i, - /\b(?:request(?:[_\s-]?entity)?[_\s-]?too|too)[_\s-]?large\b/i, - /\b(?:unsupported|unknown|invalid)\s+(?:tool|parameter|function)\b/i, - /\b(?:tool|parameter|function)\s+(?:not\s+(?:supported|found|allowed)|unknown|invalid)\b/i, - /\b(?:invalid[_\s-]?request(?:[_\s-]?error)?|bad[_\s-]?request)\b/i, -]; -const PROVIDER_REFUSAL_FAILURE_PATTERNS: readonly RegExp[] = [ - /\bfinish[_\s-]?reason\b[^\n]*\bcontent[_\s-]?filter\b/i, - /\bcontent[_\s-]?filter(?:ed|ing)?\b/i, - /\b(?:safety|policy)\b[^\n]*\b(?:refus(?:e|al|ed|es|ing)?|block(?:ed|ing)?|filter(?:ed|ing)?|violat(?:e|ion|ed|ing)?|disallow(?:ed|ing)?|reject(?:ed|ion|ing)?)\b/i, - /\b(?:refus(?:e|al|ed|es|ing)?|block(?:ed|ing)?|filter(?:ed|ing)?|violat(?:e|ion|ed|ing)?|disallow(?:ed|ing)?|reject(?:ed|ion|ing)?)\b[^\n]*\b(?:safety|policy)\b/i, - /\btool[_\s-]?(?:call|use)?[_\s-]?refus(?:e|al|ed|es|ing)?\b/i, - /\btool(?:\s+call|\s+use)?\b[^\n]*\brefus(?:e|al|ed|es|ing)?\b/i, - /\brefus(?:e|al|ed|es|ing)?\b[^\n]*\btool(?:\s+call|\s+use)?\b/i, - /\bprovider[_\s-]?refus(?:e|al|ed|es|ing)?\b/i, - /\bprovider\b[^\n]*\brefus(?:e|al|ed|es|ing)?\b[^\n]*\b(?:prompt|request|content|policy|safety)\b/i, -]; - -function refusalKindFromMessage(message: string): ModelFallbackFailureKind | undefined { - if (CANCELLED_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "cancelled"; - if (NON_RETRYABLE_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "task_failure"; - if (PROVIDER_REFUSAL_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "task_failure"; - return undefined; -} - -function fallbackKindFromMessage(message: string, name: string | undefined): ModelFallbackFailureKind | undefined { - const refusalKind = refusalKindFromMessage(message); - if (refusalKind !== undefined) return refusalKind; - if (REQUEST_INCOMPATIBLE_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "request_incompatible"; - const nameKind = kindFromCode(name); - if (nameKind !== undefined) return nameKind; - if (!RETRYABLE_MODEL_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return undefined; - if (/rate\s*limit|too many requests|\b429\b|quota|usage[\s_-]*limit|billing|credit/i.test(message)) - return "rate_limit"; - if (/auth|unauthori[sz]ed|\b40[13]\b|api key|token expired|forbidden|invalid key/i.test(message)) - return "auth_on_candidate_provider"; - if ( - /model.*(?:unavailable|disabled|not found|unknown)|(?:unavailable|disabled|not found|unknown).*model/i.test( - message, - ) - ) - return "model_unavailable"; - if (/network|fetch failed|socket|connection refused|timeout|timed? out/i.test(message)) return "network_timeout"; - return "provider_unavailable"; -} - -function signalSource(value: unknown, fallback: ModelFallbackFailureSource | undefined): ModelFallbackFailureSource { - if (fallback !== undefined) return fallback; - if (stopReasonFrom(value) !== undefined || diagnosticErrors(value).length > 0) return "assistant_message"; - if (value instanceof Error) return "throw"; - return "structured"; -} - -function makeSignal( - kind: ModelFallbackFailureKind, - value: unknown, - source: ModelFallbackFailureSource | undefined, -): ModelFallbackFailureSignal { - const status = statusFrom(value); - const code = codeFrom(value); - const name = errorName(value); - const stopReason = stopReasonFrom(value); - return { - kind, - message: modelFailureMessage(value), - source: signalSource(value, source), - ...(stopReason !== undefined ? { stopReason } : {}), - ...(status !== undefined ? { status } : {}), - ...(code !== undefined ? { code } : {}), - ...(name !== undefined ? { name } : {}), - }; -} - -function fallbackSignalForMessage( - message: string | undefined, - value: unknown, - source: ModelFallbackFailureSource | undefined, -): ModelFallbackFailureSignal | undefined { - if (message === undefined || message.trim().length === 0) return undefined; - const kind = fallbackKindFromMessage(message, errorName(value)); - return kind === undefined ? undefined : makeSignal(kind, value, source); -} -function fallbackSignalFromMessage( - value: unknown, - source: ModelFallbackFailureSource | undefined, -): ModelFallbackFailureSignal | undefined { - return fallbackSignalForMessage(modelFailureMessage(value), value, source); -} -function fallbackSignalFromDirectMessage( - value: unknown, - source: ModelFallbackFailureSource | undefined, -): ModelFallbackFailureSignal | undefined { - return fallbackSignalForMessage(directMessageFrom(value), value, source); -} - -function classifyAssistantRefusalSignal( - value: unknown, - source: ModelFallbackFailureSource | undefined, -): ModelFallbackFailureSignal | undefined { - const codeRefusalKind = - refusalKindFromCode(codeFrom(value)) ?? - refusalKindFromCode(errorName(value)) ?? - refusalKindFromCode(finishReasonFrom(value)); - if (codeRefusalKind !== undefined) return makeSignal(codeRefusalKind, value, source); - - const messageRefusalKind = refusalKindFromMessage(directMessageFrom(value) ?? ""); - return messageRefusalKind === undefined ? undefined : makeSignal(messageRefusalKind, value, source); -} - -function isRefusalSignal(signal: ModelFallbackFailureSignal): boolean { - return signal.kind === "cancelled" || signal.kind === "task_failure"; -} - -function structuredSignal( - value: unknown, - seen: Set, - source?: ModelFallbackFailureSource, -): ModelFallbackFailureSignal | undefined { - if (value === undefined || value === null || seen.has(value)) return undefined; - if (typeof value === "object") seen.add(value); - - const stopReason = stopReasonFrom(value)?.toLowerCase(); - if (stopReason === "aborted") return makeSignal("cancelled", value, source); - - const directRefusalSignal = classifyAssistantRefusalSignal(value, source); - if (directRefusalSignal !== undefined) return directRefusalSignal; - - const codeKind = kindFromCode(codeFrom(value)); - const nameKind = kindFromCode(errorName(value)); - if (codeKind === "cancelled" || nameKind === "cancelled") return makeSignal("cancelled", value, source); - - let firstNestedFallbackSignal: ModelFallbackFailureSignal | undefined; - const nestedSeen = new Set(seen); - for (const diagnosticError of diagnosticErrors(value)) { - const diagnosticSignal = - structuredSignal(diagnosticError, nestedSeen, "diagnostic") ?? - fallbackSignalFromMessage(diagnosticError, "diagnostic"); - if (diagnosticSignal === undefined) continue; - if (isRefusalSignal(diagnosticSignal)) return diagnosticSignal; - firstNestedFallbackSignal ??= diagnosticSignal; - } - const cause = causeOf(value); - const causeSignal = structuredSignal(cause, nestedSeen, source) ?? fallbackSignalFromMessage(cause, source); - if (causeSignal !== undefined) { - if (isRefusalSignal(causeSignal)) return causeSignal; - firstNestedFallbackSignal ??= causeSignal; - } - // Direct-message classification runs after nested traversal so a generic wrapper - // ("invalid request"/"400 bad request") cannot mask a non-retryable nested signal. - // Kept in the same position as the workflows classifier so the two copies stay - // behaviorally parallel (see test/unit/model-fallback-classifier-conformance.test.ts). - const directMessageSignal = fallbackSignalFromDirectMessage(value, source); - if (directMessageSignal !== undefined) return directMessageSignal; - const statusKind = kindFromStatus(statusFrom(value)); - if (statusKind !== undefined) return makeSignal(statusKind, value, source); - if (codeKind !== undefined) return makeSignal(codeKind, value, source); - if (nameKind !== undefined) return makeSignal(nameKind, value, source); - if (firstNestedFallbackSignal !== undefined) return firstNestedFallbackSignal; - if (stopReason === "error") return makeSignal("provider_unavailable", value, source); - return undefined; -} -function messageFromUnknown(value: unknown, seen: Set): string | undefined { - if (value === undefined || value === null || seen.has(value)) return undefined; - if (typeof value === "string") return value.trim().length > 0 ? value : undefined; - if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return String(value); - if (typeof value === "symbol" || typeof value === "function") return undefined; - seen.add(value); - if (value instanceof Error && value.message.trim().length > 0) return value.message; - const directMessage = directMessageFrom(value); - if (directMessage !== undefined) return directMessage; - for (const diagnosticError of diagnosticErrors(value)) { - const diagnosticMessage = messageFromUnknown(diagnosticError, seen); - if (diagnosticMessage !== undefined) return diagnosticMessage; - } - const causeMessage = messageFromUnknown(causeOf(value), seen); - if (causeMessage !== undefined) return causeMessage; - const stopReason = stopReasonFrom(value); - if (stopReason !== undefined) return `Assistant message ended with stopReason:${stopReason}`; - const finishReason = finishReasonFrom(value); - if (finishReason !== undefined) return `Model request finished with finish_reason:${finishReason}`; - const status = statusFrom(value); - if (status !== undefined) return `Model request failed with status ${status}`; - const code = codeFrom(value); - return code !== undefined ? `Model request failed with code ${String(code)}` : undefined; -} -export function modelFailureMessage(error: unknown): string { - const structuredMessage = messageFromUnknown(error, new Set()); - if (structuredMessage !== undefined) return structuredMessage; - const rendered = String(error); - return rendered === "[object Object]" ? "Model request failed" : rendered; -} -export function normalizeModelFailureSignal(error: unknown): ModelFallbackFailureSignal { - const structured = structuredSignal(error, new Set()); - if (structured !== undefined) return structured; - const message = modelFailureMessage(error); - const name = errorName(error); - const fallbackKind = message.trim().length > 0 ? fallbackKindFromMessage(message, name) : undefined; - return { - kind: fallbackKind ?? "unknown", - message, - source: "string_fallback", - ...(name !== undefined ? { name } : {}), - }; -} -export function isRetryableModelFailure(error: unknown): boolean { - if (error === undefined) return false; - const signal = normalizeModelFailureSignal(error); - return FALLBACKABLE_FAILURE_KINDS.has(signal.kind); -} export function formatModelAttemptNote(attempt: ModelAttemptSummary, nextModel?: string): string { const failure = attempt.error?.trim() || `exit ${attempt.exitCode ?? 1}`; return nextModel diff --git a/packages/workflows/CHANGELOG.md b/packages/workflows/CHANGELOG.md index 320deb7e4..216c411f2 100644 --- a/packages/workflows/CHANGELOG.md +++ b/packages/workflows/CHANGELOG.md @@ -13,6 +13,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Lifecycle notices now carry two attributions. *Origin* — who launched the run — renders on every kind as "which you started" or "which the user started", is recorded once at dispatch, survives session restore and durable resume, and is inherited by a continuation from the run it continues rather than recomputed. *Actor* — who performed this one event — renders as "The user paused". A run with no recorded origin, including a legacy or restored snapshot, omits the clause entirely instead of guessing ([#2177](https://github.com/bastani-inc/atomic/issues/2177)). - One request now produces exactly one notice. A whole-run pause or resume reports at run scope; a stage-scoped pause or resume that leaves siblings paused reports at stage scope, and one that stops or restarts the whole run reports the run instead, never both. A quit reports the quit alone rather than the pause it publishes on the way. Because control actions are reversible, they are deduplicated by run id together with the occurrence timestamp, so pause → resume → pause → resume notifies four times while repeated snapshot invalidations at one unchanged state notify once. Resuming reports a resume and never a start, whoever requested it: resuming a failed or blocked run launches a continuation under a fresh id whose notice names both run ids, while resuming a quit run reuses the original workflow id so durable checkpoints replay and its notice names the one id. A run already started, paused, or quit when notifications install (restore, replay, `/reload`, session-preserving reinstall) stays silent, and nested child runs never notify at top level. The default `workflowNotifications.notifyOn` becomes `["started", "completed", "failed", "blocked", "awaiting_input", "paused", "quit", "resumed"]`; a config that pins `notifyOn` explicitly keeps exactly the kinds it lists ([#2177](https://github.com/bastani-inc/atomic/issues/2177)). +### Fixed + +- Fixed thrown retryable stage failures to retry the current model with bounded exponential backoff from `settings.retry` before advancing through `fallbackModels`; disabling retry keeps immediate candidate advancement ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed a same-candidate stage retry losing the stage prompt. The retry resumed the existing session turn with a transcript that no longer ended in the prompt it was resuming, which the agent rejects; the prompt is now retained for that path and still re-sent on the re-prompt path ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). + ## [0.9.12] - 2026-08-04 Cumulative release of the `0.9.12-alpha.1` prerelease. The summary below covers the user-visible outcome of that work; the per-change detail remains in the prerelease section below. diff --git a/packages/workflows/src/durable/tool-primitive.ts b/packages/workflows/src/durable/tool-primitive.ts index 5b1822cf3..dafa99e70 100644 --- a/packages/workflows/src/durable/tool-primitive.ts +++ b/packages/workflows/src/durable/tool-primitive.ts @@ -16,6 +16,7 @@ */ import { runCallback } from "@bastani/atomic"; +import { sleepOrAbort } from "../runs/shared/retry.js"; import { flattenTruncatedString } from "../shared/flat-string.js"; import type { ToolNodeSnapshot } from "../shared/store-types.js"; import type { @@ -26,6 +27,9 @@ import type { WorkflowToolPrimitive, } from "../shared/types.js"; import { field, hasProcessFailureEvidence, normalizeCode } from "../shared/workflow-failures-signals.js"; + +export { sleepOrAbort } from "../runs/shared/retry.js"; + import type { DurableWorkflowBackend } from "./backend.js"; import { durableHash } from "./backend.js"; import { recordThrowingToolFailure } from "./tool-failure-checkpoint.js"; @@ -660,34 +664,6 @@ async function executeWithRetries( throw lastError ?? new Error("ctx.tool: retries exhausted"); } -export function sleepOrAbort(ms: number, signal?: AbortSignal): Promise { - if (signal?.aborted) - return Promise.reject( - signal.reason instanceof Error ? signal.reason : new Error("atomic-workflows: workflow cancelled"), - ); - return new Promise((resolve, reject) => { - let settled = false; - const cleanup = (): void => signal?.removeEventListener("abort", onAbort); - const finish = (): void => { - if (settled) return; - settled = true; - cleanup(); - resolve(); - }; - const fail = (err: Error): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - cleanup(); - reject(err); - }; - const timer = setTimeout(finish, ms); - const onAbort = (): void => - fail(signal?.reason instanceof Error ? signal.reason : new Error("atomic-workflows: workflow cancelled")); - signal?.addEventListener("abort", onAbort, { once: true }); - }); -} - /** * Create a monotonic checkpoint id generator for a workflow. */ diff --git a/packages/workflows/src/extension/atomic-stage-session.ts b/packages/workflows/src/extension/atomic-stage-session.ts index 7571b66de..33c514789 100644 --- a/packages/workflows/src/extension/atomic-stage-session.ts +++ b/packages/workflows/src/extension/atomic-stage-session.ts @@ -8,6 +8,7 @@ import type { StageSessionRuntime } from "../runs/foreground/stage-runner.js"; export interface PiSdkSettingsManager { getCodexFastModeSettings(): { readonly chat: boolean; readonly workflow: boolean }; + getRetrySettings?(): { readonly enabled: boolean; readonly maxRetries: number; readonly baseDelayMs: number }; } export interface PiSdkResourceLoader { @@ -45,6 +46,7 @@ export type AtomicCreateAgentSessionOptions = Omit< export interface PrepareAtomicStageSessionOptions { resourceLoaderInheritanceSnapshot?: DefaultResourceLoaderInheritanceSnapshot; + onSettingsManager?: (settingsManager: PiSdkSettingsManager) => void; } function resolveSessionCwd(options: AtomicCreateAgentSessionOptions | undefined): string { @@ -87,6 +89,7 @@ export async function prepareAtomicStageSessionOptions( ? undefined : { projectTrusted: inheritanceSnapshot.projectTrusted }, ); + prepareOptions.onSettingsManager?.(settingsManager); const inheritedBuiltinPackagePaths = inheritanceSnapshot?.builtinPackagePaths; const builtinPackagePaths = inheritedBuiltinPackagePaths === undefined diff --git a/packages/workflows/src/extension/wiring.ts b/packages/workflows/src/extension/wiring.ts index ff4d3fff8..1d6a512ed 100644 --- a/packages/workflows/src/extension/wiring.ts +++ b/packages/workflows/src/extension/wiring.ts @@ -134,22 +134,64 @@ function isTestContext(): boolean { return process.env.NODE_TEST_CONTEXT !== undefined || process.env.NODE_ENV === "test"; } +type StageSettingsManager = ReturnType; + +function attachSettingsManager(error: unknown, settingsManager: StageSettingsManager): unknown { + if (error !== null && (typeof error === "object" || typeof error === "function")) { + try { + if (Object.isExtensible(error)) { + Object.defineProperty(error, "settingsManager", { + configurable: true, + enumerable: false, + value: settingsManager, + writable: false, + }); + return error; + } + } catch { + // Frozen or proxy errors cannot carry the private retry hint. Wrap them + // without replacing the original failure as the cause. + } + } + const wrapped = new Error(error instanceof Error ? error.message : String(error), { cause: error }); + Object.defineProperty(wrapped, "settingsManager", { + configurable: true, + enumerable: false, + value: settingsManager, + writable: false, + }); + return wrapped; +} + async function createPiSdkAgentSession( options?: CreateAgentSessionOptions, prepareOptions?: PrepareAtomicStageSessionOptions, ): Promise { const sdk = (await import("@bastani/atomic")) as PiCodingAgentSdk; - const sessionOptions = await prepareAtomicStageSessionOptions(options, sdk, prepareOptions); - const result = await sdk.createAgentSession(sessionOptions); - // `CreateAgentSessionResult` is `{ session, extensionsResult, modelFallbackMessage? }`; - // workflow stages only consume `.session` (structurally an `AgentSession`, - // which is a superset of our `StageSessionRuntime` projection). - const resultSettingsManager = result.session.settingsManager; - const settingsManager = sessionOptions?.settingsManager ?? resultSettingsManager; - return { - session: result.session, - ...(settingsManager?.getCodexFastModeSettings !== undefined ? { settingsManager } : {}), - }; + let settingsManager: ReturnType | undefined; + try { + const sessionOptions = await prepareAtomicStageSessionOptions(options, sdk, { + ...prepareOptions, + onSettingsManager: (manager) => { + settingsManager = manager; + prepareOptions?.onSettingsManager?.(manager); + }, + }); + settingsManager = sessionOptions?.settingsManager ?? settingsManager; + const result = await sdk.createAgentSession(sessionOptions); + // `CreateAgentSessionResult` is `{ session, extensionsResult, modelFallbackMessage? }`; + // workflow stages only consume `.session` (structurally an `AgentSession`, + // which is a superset of our `StageSessionRuntime` projection). + const resultSettingsManager = result.session.settingsManager; + settingsManager = sessionOptions?.settingsManager ?? resultSettingsManager ?? settingsManager; + return { + session: result.session, + ...(settingsManager?.getCodexFastModeSettings !== undefined ? { settingsManager } : {}), + }; + } catch (error) { + if (settingsManager !== undefined) throw attachSettingsManager(error, settingsManager); + throw error; + } } async function createTestAgentSession(_options?: CreateAgentSessionOptions): Promise { diff --git a/packages/workflows/src/runs/foreground/stage-runner-context.ts b/packages/workflows/src/runs/foreground/stage-runner-context.ts index 7766ca0e9..95e9f8e84 100644 --- a/packages/workflows/src/runs/foreground/stage-runner-context.ts +++ b/packages/workflows/src/runs/foreground/stage-runner-context.ts @@ -233,7 +233,7 @@ export function createStageContext(opts: StageRunnerOpts): InternalStageContext }, async abort() { - await controller.currentSession?.abort(); + await controller.abort(); }, async __dispose() { diff --git a/packages/workflows/src/runs/foreground/stage-runner-controller.ts b/packages/workflows/src/runs/foreground/stage-runner-controller.ts index bc5f3887c..a2f869d5c 100644 --- a/packages/workflows/src/runs/foreground/stage-runner-controller.ts +++ b/packages/workflows/src/runs/foreground/stage-runner-controller.ts @@ -21,6 +21,7 @@ import { type WorkflowResolvedModelCandidate, workflowModelId, } from "../shared/model-fallback.js"; +import { nextRetryDecision, sleepOrAbort } from "../shared/retry.js"; import { StageDeliveryActivity, type StageDeliveryActivityListener } from "./stage-delivery-activity.js"; import { stageSessionQueueUpdateEvent } from "./stage-queued-user-messages.js"; import { candidateLabel, effectiveCandidateReasoning, modelAttemptReasoning } from "./stage-runner-candidate.js"; @@ -52,6 +53,7 @@ import type { StageSessionRuntime, StageUserMessagePreparation, WorkflowFastModeSettingsManager, + WorkflowRetrySettings, } from "./stage-runner-types.js"; import { nextResumedContextOverflowFallbackIndex, @@ -60,9 +62,73 @@ import { unresolvedContextOverflowMessage, } from "./stage-runner-unresolved-overflow.js"; +type RetryPauseResume = Promise<{ readonly message?: string }>; + +interface ThrownErrorRetryState { + readonly controller: AbortController; + pauseResume?: RetryPauseResume; +} + +interface SessionCreationPauseResult { + readonly kind: "paused"; + readonly resumeMessage?: string; +} + +function isSessionCreationPauseResult( + value: StageSessionRuntime | SessionCreationPauseResult, +): value is SessionCreationPauseResult { + return "kind" in value && value.kind === "paused"; +} + +class StageSessionCreationCancelled extends Error { + constructor() { + super("atomic-workflows: stage session creation was cancelled while paused"); + this.name = "StageSessionCreationCancelled"; + } +} + +function stageUserMessageText(message: StageSessionRuntime["messages"][number]): string | undefined { + if (message.role !== "user") return undefined; + if (typeof message.content === "string") return message.content; + return message.content + .filter((part) => part.type === "text") + .map((part) => part.text) + .join(""); +} +function retrySettingsManagerFromError(error: unknown): WorkflowFastModeSettingsManager | undefined { + if (error === null || typeof error !== "object") return undefined; + const manager = (error as { readonly settingsManager?: unknown }).settingsManager; + if (manager === null || typeof manager !== "object") return undefined; + const candidate = manager as Partial; + return typeof candidate.getCodexFastModeSettings === "function" + ? (candidate as WorkflowFastModeSettingsManager) + : undefined; +} + +type RetryableAgentSession = AgentSession & { + _runAgentContinue(): Promise; +}; + +function retryableAgentSession(activeSession: StageSessionRuntime): RetryableAgentSession | undefined { + const session = asAgentSession(activeSession); + if (session === undefined) return undefined; + const candidate = session as AgentSession & { readonly _runAgentContinue?: unknown }; + return typeof candidate._runAgentContinue === "function" ? (session as RetryableAgentSession) : undefined; +} + +class ThrownErrorRetryPaused extends Error { + constructor(readonly resume: RetryPauseResume) { + super("atomic-workflows: thrown-error retry paused"); + this.name = "ThrownErrorRetryPaused"; + } +} + export class StageSessionController { private session: StageSessionRuntime | undefined; private activeCreation: Promise | undefined; + private abortGeneration = 0; + private abortReason: Error | DOMException | string | undefined; + private abortReasonGeneration = 0; private sessionPromise: Promise | undefined; private reattachSessionFile: string | undefined; private lastPromptStartIndex: number | undefined; @@ -88,7 +154,9 @@ export class StageSessionController { private readonly pendingFallbackWarnings: string[] = []; private readonly modelCatalog: WorkflowModelCatalogPort | undefined; private sessionSettingsManager: WorkflowFastModeSettingsManager | undefined; + private readonly thrownErrorRetryStates = new Set(); private readonly replacement = new StageSessionReplacement(); + private pendingCreationResumeMessage: string | undefined; private readonly messageAdmission = new StageMessageAdmission(); private readonly deliveryActivity = new StageDeliveryActivity(); @@ -186,11 +254,28 @@ export class StageSessionController { } preparation?.beforePreparation?.(); const sessionFile = preparation?.sessionFile; - const deliver = async (activity?: StageDeliveryActivity) => - sendStageUserMessage( + const deliver = async (activity?: StageDeliveryActivity) => { + const activeSession = sessionFile === undefined ? await this.ensureSession("prompt") - : await this.ensureSessionFromFile(sessionFile, "prompt"), + : await this.ensureSessionFromFile(sessionFile, "prompt"); + const pausedDelivery = this.pauseControl.deferRunnerOwnedDelivery(() => + sendStageUserMessage( + activeSession, + content, + options, + beforeDelivery, + release, + this.messageAdmission, + activity, + ), + ); + if (pausedDelivery !== undefined) { + release(); + return pausedDelivery; + } + return sendStageUserMessage( + activeSession, content, options, beforeDelivery, @@ -198,6 +283,7 @@ export class StageSessionController { this.messageAdmission, activity, ); + }; if (this.session === undefined || sessionFile !== undefined) return this.deliveryActivity.runWithLease(() => deliver()); return deliver(this.deliveryActivity); @@ -226,29 +312,56 @@ export class StageSessionController { consumer: AgentSessionConsumer = "prompt", ): Promise { if (!this.hasExplicitModelFallbackConfig) { - await this.promptWithPauseResume(await this.ensureSession(consumer), text, sdkOptions); + try { + const activeSession = await this.ensureSession(consumer); + const resumedText = this.pendingCreationResumeMessage; + this.pendingCreationResumeMessage = undefined; + await this.promptWithThrownErrorRetry(activeSession, resumedText ?? text, sdkOptions); + } catch (error) { + if (error instanceof StageSessionCreationCancelled) return; + throw error; + } return; } const candidates = await this.modelCandidates(); if (candidates.length === 0) { - await this.promptWithPauseResume(await this.ensureSession(consumer), text, sdkOptions); + try { + const activeSession = await this.ensureSession(consumer); + const resumedText = this.pendingCreationResumeMessage; + this.pendingCreationResumeMessage = undefined; + await this.promptWithThrownErrorRetry(activeSession, resumedText ?? text, sdkOptions); + } catch (error) { + if (error instanceof StageSessionCreationCancelled) return; + throw error; + } return; } if (await this.tryResumeCurrentSession(text, sdkOptions, candidates)) return; let index = this.activeCandidateIndex ?? 0; + let promptText = text; while (index < candidates.length) { const candidate = candidates[index]!; - const activeSession = - this.session && this.activeCandidateIndex === index - ? this.session - : await this.createSession(candidate, consumer); - this.activeCandidateIndex = index; - this.selectedModel = candidate.id; - this.notifyModelFallbackMetaChange(); try { - const { terminalScanStartIndex } = await this.promptWithPauseResume(activeSession, text, sdkOptions); + const created = + this.session && this.activeCandidateIndex === index + ? this.session + : await this.createSessionWithThrownErrorRetry(candidate, consumer); + if (isSessionCreationPauseResult(created)) { + if (created.resumeMessage === undefined) return; + promptText = created.resumeMessage; + continue; + } + const activeSession = created; + this.activeCandidateIndex = index; + this.selectedModel = candidate.id; + this.notifyModelFallbackMetaChange(); + const { terminalScanStartIndex } = await this.promptWithThrownErrorRetry( + activeSession, + promptText, + sdkOptions, + ); const terminalFailure = latestTerminalAssistantFailureSince(activeSession.messages, terminalScanStartIndex); if (terminalFailure !== undefined) { if (this.capturedStructuredOutputForAttempt()) { @@ -283,6 +396,9 @@ export class StageSessionController { async disposeAll(): Promise { this.disposed = true; + const reason = new Error(`atomic-workflows: stage "${this.opts.stageName}" session has been disposed`); + this.markAbort(reason); + this.pauseControl.reject(reason); for (const unsubscribe of this.listenerUnsubscribes.values()) unsubscribe(); this.listenerUnsubscribes.clear(); this.pendingListeners.clear(); @@ -294,8 +410,18 @@ export class StageSessionController { await this.replacement.dispose(); await disposeStageSession(this.session); } + + async abort(): Promise { + const reason = new DOMException("stage aborted", "AbortError"); + this.markAbort(reason); + this.pauseControl.reject(reason); + await this.session?.abort(); + } requestPause(): Promise { - return this.pauseControl.requestPause(); + const pause = this.pauseControl.requestPause(); + const resume = this.pauseControl.currentResume(); + if (resume !== undefined) this.pauseThrownErrorRetries(resume); + return pause; } resume( message?: string, @@ -326,13 +452,206 @@ export class StageSessionController { return new DOMException("workflow killed", "AbortError"); }; const onAbort = (): void => { + const reason = abortReason(); + this.markAbort(reason); void this.session?.abort().catch(() => {}); - this.pauseControl.reject(abortReason()); + this.pauseControl.reject(reason); }; if (signal.aborted) onAbort(); else signal.addEventListener("abort", onAbort, { once: true }); } + private markAbort(reason: Error | DOMException | string): void { + this.abortGeneration += 1; + this.abortReason = reason; + this.abortReasonGeneration = this.abortGeneration; + this.abortThrownErrorRetries(reason); + } + + private pauseThrownErrorRetries(resume: RetryPauseResume): void { + for (const state of this.thrownErrorRetryStates) { + if (state.pauseResume !== undefined) continue; + state.pauseResume = resume; + state.controller.abort(new ThrownErrorRetryPaused(resume)); + } + } + + private abortThrownErrorRetries(reason?: Error | DOMException | string): void { + for (const state of this.thrownErrorRetryStates) state.controller.abort(reason); + this.thrownErrorRetryStates.clear(); + } + + private retrySettings(): WorkflowRetrySettings | undefined { + const managers = [ + this.sessionSettingsManager, + this.session?.settingsManager, + this.effectiveStageOptions?.settingsManager, + ]; + for (const manager of managers) { + if (manager === undefined || typeof manager.getRetrySettings !== "function") continue; + return manager.getRetrySettings(); + } + return undefined; + } + /** + * Drop this attempt's failed input from live state before a same-candidate + * retry, keeping unrelated concurrent messages. The durable transcript keeps + * everything; this mirrors main-chat retry, which also only edits live state. + * + * `keepPrompt` retains the admitted stage prompt because the continuation + * path resumes the existing turn with `_runAgentContinue()`, and pi-agent-core + * rejects a transcript that does not end in a user or tool-result message. + * Returns the retained prompt so a later re-`prompt()` can drop it. + */ + private restoreSessionMessages( + session: StageSessionRuntime, + snapshot: StageSessionRuntime["messages"], + promptText: string, + keepPrompt: boolean, + ): StageSessionRuntime["messages"][number] | undefined { + const snapshotMessages = new Set(snapshot); + const admitted = session.messages.filter((message) => !snapshotMessages.has(message)); + const failedAssistantIndex = admitted.findLastIndex( + (message) => message.role === "assistant" && message.stopReason === "error", + ); + const promptUser = admitted + .slice(0, failedAssistantIndex < 0 ? admitted.length : failedAssistantIndex) + .findLast((message) => message.role === "user" && stageUserMessageText(message) === promptText); + const retainedMessages = admitted.filter((message) => { + if (message === promptUser) return keepPrompt; + if (message.role === "assistant") return message.stopReason !== "error"; + return ["user", "toolResult", "custom", "bashExecution", "branchSummary"].includes(message.role); + }); + session.messages.splice(0, session.messages.length, ...snapshot, ...retainedMessages); + return keepPrompt ? promptUser : undefined; + } + + /** Remove a prompt retained for a continuation that will not happen. */ + private dropRetainedPrompt( + session: StageSessionRuntime, + retained: StageSessionRuntime["messages"][number] | undefined, + ): void { + if (retained === undefined) return; + const index = session.messages.indexOf(retained); + if (index >= 0) session.messages.splice(index, 1); + } + private async sleepForThrownErrorRetry(delayMs: number, state: ThrownErrorRetryState): Promise { + this.thrownErrorRetryStates.add(state); + const currentResume = this.pauseControl.currentResume(); + if (currentResume !== undefined) { + state.pauseResume = currentResume; + state.controller.abort(new ThrownErrorRetryPaused(currentResume)); + } + try { + await sleepOrAbort(delayMs, state.controller.signal); + } finally { + this.thrownErrorRetryStates.delete(state); + } + } + + private async promptWithThrownErrorRetry( + activeSession: StageSessionRuntime, + text: string, + sdkOptions: PromptOptions | undefined, + ): Promise<{ readonly terminalScanStartIndex: number }> { + let retryAttempt = 0; + let nextText = text; + let retryAdmittedPrompt = false; + let retainedPrompt: StageSessionRuntime["messages"][number] | undefined; + let terminalScanStartIndex: number | undefined; + while (true) { + const messagesBeforeAttempt = [...activeSession.messages]; + try { + if (retryAdmittedPrompt) { + const continuationSession = retryableAgentSession(activeSession); + if (continuationSession !== undefined) { + await continuationSession._runAgentContinue(); + return { + terminalScanStartIndex: + terminalScanStartIndex ?? this.lastPromptStartIndex ?? messagesBeforeAttempt.length, + }; + } + // No continuation is possible after all, so the retained prompt + // must not survive into the re-prompt below. + this.dropRetainedPrompt(activeSession, retainedPrompt); + retainedPrompt = undefined; + retryAdmittedPrompt = false; + } + const result = await this.promptWithPauseResume(activeSession, nextText, sdkOptions); + return { + terminalScanStartIndex: terminalScanStartIndex ?? result.terminalScanStartIndex, + }; + } catch (error) { + const errorSettingsManager = retrySettingsManagerFromError(error); + if (errorSettingsManager !== undefined) this.sessionSettingsManager = errorSettingsManager; + const decision = nextRetryDecision(this.retrySettings(), retryAttempt, isRetryableModelFailure(error)); + const continuationSession = retryableAgentSession(activeSession); + const admittedMessages = activeSession.messages.length > messagesBeforeAttempt.length; + const retryableFailure = isRetryableModelFailure(error); + const willRetry = + decision !== undefined && + !this.disposed && + this.opts.signal?.aborted !== true && + !this.capturedStructuredOutputForAttempt(); + // The continuation path resumes the same turn, so it needs the + // admitted prompt to stay; the re-prompt path re-sends it. + const willContinue = continuationSession !== undefined && admittedMessages; + if (retryableFailure && willRetry) { + retainedPrompt = + this.restoreSessionMessages(activeSession, messagesBeforeAttempt, nextText, willContinue) ?? + retainedPrompt; + } + if (!willRetry) throw error; + terminalScanStartIndex ??= this.lastPromptStartIndex ?? messagesBeforeAttempt.length; + retryAttempt = decision.attempt; + const state: ThrownErrorRetryState = { controller: new AbortController() }; + let pauseResume: RetryPauseResume | undefined; + try { + await this.sleepForThrownErrorRetry(decision.delayMs, state); + } catch (sleepError) { + if (sleepError instanceof ThrownErrorRetryPaused) pauseResume = sleepError.resume; + else { + if (this.opts.signal?.aborted) throw this.workflowAbortReason(); + if (this.disposed) + throw new Error(`atomic-workflows: stage "${this.opts.stageName}" session has been disposed`); + throw sleepError; + } + } + if (pauseResume !== undefined) { + const resumed = await pauseResume; + retryAttempt = 0; + retryAdmittedPrompt = false; + terminalScanStartIndex = undefined; + // A resumed prompt re-sends its text, so the retained input would + // otherwise be duplicated. + this.dropRetainedPrompt(activeSession, retainedPrompt); + retainedPrompt = undefined; + if (resumed.message === undefined) { + return { terminalScanStartIndex: activeSession.messages.length }; + } + nextText = resumed.message; + continue; + } + if (this.disposed) + throw new Error(`atomic-workflows: stage "${this.opts.stageName}" session has been disposed`); + if (this.opts.signal?.aborted) throw this.workflowAbortReason(); + retryAdmittedPrompt = willContinue; + } + } + } + + private workflowAbortReason(): Error | DOMException | string { + const reason = this.opts.signal?.reason; + if (reason instanceof Error || reason instanceof DOMException || typeof reason === "string") return reason; + return new DOMException("workflow killed", "AbortError"); + } + + private staleCreationReason(startGeneration: number): Error | DOMException | string { + if (this.opts.signal?.aborted) return this.workflowAbortReason(); + if (this.abortReasonGeneration > startGeneration && this.abortReason !== undefined) return this.abortReason; + return new DOMException("stage aborted", "AbortError"); + } + private modelCandidates(): Promise { if (!this.candidatesPromise) { this.candidatesPromise = buildModelCandidatesFromCatalog({ @@ -346,10 +665,18 @@ export class StageSessionController { } private async createInitialSession(consumer: AgentSessionConsumer): Promise { - if (!this.hasExplicitModelFallbackConfig) return this.createSession(undefined, consumer); + if (!this.hasExplicitModelFallbackConfig) { + return this.createSession(undefined, consumer).catch((error) => + this.createInitialSessionWithRetry(undefined, consumer, { error }), + ); + } const candidates = await this.modelCandidates(); const first = candidates[0]; - if (first === undefined) return this.createSession(undefined, consumer); + if (first === undefined) { + return this.createSession(undefined, consumer).catch((error) => + this.createInitialSessionWithRetry(undefined, consumer, { error }), + ); + } if (this.reattachSessionFile !== undefined) { const resumed = await this.createSession(undefined, consumer, { restoreSavedModel: true }); const restoredId = workflowModelId(resumed.model); @@ -361,7 +688,81 @@ export class StageSessionController { } this.activeCandidateIndex = 0; this.selectedModel = first.id; - return this.createSession(first, consumer); + return this.createSession(first, consumer).catch((error) => + this.createInitialSessionWithRetry(first, consumer, { error }), + ); + } + + private async createInitialSessionWithRetry( + candidate: WorkflowResolvedModelCandidate | undefined, + consumer: AgentSessionConsumer, + initialFailure?: { readonly error: unknown }, + ): Promise { + let pendingFailure = initialFailure; + while (true) { + const created = await this.createSessionWithThrownErrorRetry(candidate, consumer, pendingFailure); + pendingFailure = undefined; + if (!isSessionCreationPauseResult(created)) return created; + if (created.resumeMessage === undefined) { + // A pause without a replacement objective cancels this pending + // creation. Let the next prompt start a fresh creation attempt. + this.pendingCreationResumeMessage = undefined; + this.sessionPromise = undefined; + throw new StageSessionCreationCancelled(); + } + this.pendingCreationResumeMessage = created.resumeMessage; + } + } + + private async createSessionWithThrownErrorRetry( + candidate: WorkflowResolvedModelCandidate | undefined, + consumer: AgentSessionConsumer, + initialFailure?: { readonly error: unknown }, + ): Promise { + let retryAttempt = 0; + let pendingFailure = initialFailure; + while (true) { + try { + if (pendingFailure !== undefined) { + const failure = pendingFailure; + pendingFailure = undefined; + throw failure.error; + } + return await this.createSession(candidate, consumer); + } catch (error) { + const errorSettingsManager = retrySettingsManagerFromError(error); + if (errorSettingsManager !== undefined) this.sessionSettingsManager = errorSettingsManager; + const decision = nextRetryDecision(this.retrySettings(), retryAttempt, isRetryableModelFailure(error)); + if ( + decision === undefined || + this.disposed || + this.opts.signal?.aborted === true || + this.capturedStructuredOutputForAttempt() + ) { + throw error; + } + retryAttempt = decision.attempt; + const state: ThrownErrorRetryState = { controller: new AbortController() }; + try { + await this.sleepForThrownErrorRetry(decision.delayMs, state); + } catch (sleepError) { + if (sleepError instanceof ThrownErrorRetryPaused) { + const resumed = await sleepError.resume; + return { + kind: "paused", + ...(resumed.message === undefined ? {} : { resumeMessage: resumed.message }), + }; + } + if (this.opts.signal?.aborted) throw this.workflowAbortReason(); + if (this.disposed) + throw new Error(`atomic-workflows: stage "${this.opts.stageName}" session has been disposed`); + throw sleepError; + } + if (this.disposed) + throw new Error(`atomic-workflows: stage "${this.opts.stageName}" session has been disposed`); + if (this.opts.signal?.aborted) throw this.workflowAbortReason(); + } + } } private createSession( @@ -384,6 +785,7 @@ export class StageSessionController { consumer: AgentSessionConsumer, resumeOptions?: { restoreSavedModel?: boolean }, ): Promise { + const startGeneration = this.abortGeneration; this.applyCandidateThinking(candidate); const stageOptions = buildStageSessionOptions({ effectiveStageOptions: this.effectiveStageOptions, @@ -392,22 +794,37 @@ export class StageSessionController { reattachSessionFile: this.reattachSessionFile, sharedModelRuntime: this.sharedModelRuntime, }); - const created = this.opts.adapters.agentSession - ? await this.opts.adapters.agentSession.create( - stripWorkflowOnlyOptions( - stageOptions, - this.opts.defaultSessionDir, - this.meta, - ) as StageSessionCreateOptions, - { - ...this.meta, - stageOptions, - ...(this.sharedOrchestrationContext !== undefined - ? { orchestrationContext: this.sharedOrchestrationContext } - : {}), - }, - ) - : missingAdapter(consumer); + let created: StageSessionRuntime | StageSessionCreateResult; + try { + created = this.opts.adapters.agentSession + ? await this.opts.adapters.agentSession.create( + stripWorkflowOnlyOptions( + stageOptions, + this.opts.defaultSessionDir, + this.meta, + ) as StageSessionCreateOptions, + { + ...this.meta, + stageOptions, + ...(this.sharedOrchestrationContext !== undefined + ? { orchestrationContext: this.sharedOrchestrationContext } + : {}), + }, + ) + : missingAdapter(consumer); + } catch (error) { + if (this.disposed || this.opts.signal?.aborted === true || this.abortGeneration !== startGeneration) + throw this.disposed + ? new Error(`atomic-workflows: stage "${this.opts.stageName}" session has been disposed`) + : this.staleCreationReason(startGeneration); + throw error; + } + if (this.disposed || this.opts.signal?.aborted === true || this.abortGeneration !== startGeneration) { + await disposeStageSession(normalizeSessionCreateResult(created).session).catch(() => {}); + if (this.disposed) + throw new Error(`atomic-workflows: stage "${this.opts.stageName}" session has been disposed`); + throw this.staleCreationReason(startGeneration); + } return attachCreatedStageSession(created, this.disposed, this.opts.stageName, (result) => this.attachSession(result), ); @@ -453,6 +870,7 @@ export class StageSessionController { } private async disposeCurrentSession(): Promise { + this.abortThrownErrorRetries(new Error(`atomic-workflows: stage "${this.opts.stageName}" session was replaced`)); const current = this.session; this.messageAdmission.reset(); this.replacement.retire(current); @@ -526,7 +944,7 @@ export class StageSessionController { const resumedLabel = this.selectedModel ?? workflowModelId(resumedSession.model) ?? candidates[0]!.id; this.notifyModelFallbackMetaChange(); try { - const { terminalScanStartIndex } = await this.promptWithPauseResume(resumedSession, text, sdkOptions); + const { terminalScanStartIndex } = await this.promptWithThrownErrorRetry(resumedSession, text, sdkOptions); const terminalFailure = latestTerminalAssistantFailureSince(resumedSession.messages, terminalScanStartIndex); if (terminalFailure === undefined || this.capturedStructuredOutputForAttempt()) { this.modelAttempts.push({ model: resumedLabel, success: true }); diff --git a/packages/workflows/src/runs/foreground/stage-runner-types.ts b/packages/workflows/src/runs/foreground/stage-runner-types.ts index 9bd0da55e..de868303d 100644 --- a/packages/workflows/src/runs/foreground/stage-runner-types.ts +++ b/packages/workflows/src/runs/foreground/stage-runner-types.ts @@ -1,4 +1,4 @@ -import type { AgentSession, CreateAgentSessionOptions, PromptOptions } from "@bastani/atomic"; +import type { AgentSession, CreateAgentSessionOptions, PromptOptions, SettingsManager } from "@bastani/atomic"; import type { CompleteStageOpts, StageContext, @@ -18,9 +18,11 @@ export type WorkflowFastModeSettings = { readonly chat: boolean; readonly workflow: boolean; }; +export type WorkflowRetrySettings = ReturnType; export type WorkflowFastModeSettingsManager = { getCodexFastModeSettings(): WorkflowFastModeSettings; + getRetrySettings?(): WorkflowRetrySettings; }; export type StageUserMessageDeliveryAction = "prompt" | "steer" | "followUp" | "handled"; diff --git a/packages/workflows/src/runs/shared/model-fallback-failures.ts b/packages/workflows/src/runs/shared/model-fallback-failures.ts index b9828875a..dd2ae1994 100644 --- a/packages/workflows/src/runs/shared/model-fallback-failures.ts +++ b/packages/workflows/src/runs/shared/model-fallback-failures.ts @@ -1,514 +1,18 @@ -const RETRYABLE_MODEL_FAILURE_PATTERNS: readonly RegExp[] = [ - /rate\s*limit/i, - /too\s*many\s*requests/i, - /\b429\b/, - /quota/i, - /usage[\s_-]*limit/i, // usage-limit exhaustion is a quota condition (accept space/underscore/hyphen/joined forms); next candidate may have headroom - /billing/i, - /credit/i, - /auth(?:entication|orization)?/i, - /unauthori[sz]ed/i, - /\b40[13]\b/, - /api\s*key/i, - /token\s*expired/i, - /forbidden/i, - /invalid\s*key/i, - /model.*(?:unavailable|disabled|not\s*found|unknown)/i, - /(?:unavailable|disabled|not\s*found|unknown).*model/i, - /overloaded/i, - /temporarily\s*unavailable/i, - /service\s*unavailable/i, - /network/i, - /fetch/i, - /socket/i, - /connection\s*refused/i, - /upstream/i, - /timeout/i, - /timed\s*out/i, - /\b50[0-4]\b/, -]; - -const NON_RETRYABLE_FAILURE_PATTERNS: readonly RegExp[] = [ - /command failed/i, - /tests? failed/i, - /shell/i, - /missing file/i, - /no such file/i, - /cancel/i, - /abort/i, - /interrupted/i, -]; - -const CANCELLED_FAILURE_PATTERNS: readonly RegExp[] = [/cancel/i, /abort/i, /interrupted/i]; - -export type ModelFallbackFailureKind = - | "auth_on_candidate_provider" - | "rate_limit" - | "provider_unavailable" - | "network_timeout" - | "transport_error" - | "model_unavailable" - | "request_incompatible" - | "cancelled" - | "task_failure" - | "unknown"; - -export type ModelFallbackFailureSource = - | "assistant_message" - | "diagnostic" - | "throw" - | "structured" - | "string_fallback"; - -export interface ModelFallbackFailureSignal { - readonly kind: ModelFallbackFailureKind; - readonly message: string; - readonly source: ModelFallbackFailureSource; - readonly stopReason?: string; - readonly status?: number; - readonly code?: string | number; - readonly name?: string; -} - -const FALLBACKABLE_FAILURE_KINDS: ReadonlySet = new Set([ - "auth_on_candidate_provider", - "rate_limit", - "provider_unavailable", - "network_timeout", - "transport_error", - "model_unavailable", - "request_incompatible", -]); - -function asRecord(value: unknown): Record | undefined { - return value !== null && typeof value === "object" ? (value as Record) : undefined; -} - -function field(value: unknown, key: string): unknown { - return asRecord(value)?.[key]; -} - -function stringField(value: unknown, key: string): string | undefined { - const raw = field(value, key); - return typeof raw === "string" && raw.trim().length > 0 ? raw : undefined; -} - -function errorName(value: unknown): string | undefined { - return value instanceof Error ? value.name : stringField(value, "name"); -} - -function directMessageFrom(value: unknown): string | undefined { - return stringField(value, "errorMessage") ?? stringField(value, "message") ?? stringField(value, "statusText"); -} - -function integerFrom(value: unknown): number | undefined { - if (typeof value === "number" && Number.isInteger(value)) return value; - if (typeof value !== "string" || value.trim().length === 0) return undefined; - const parsed = Number(value.trim()); - return Number.isInteger(parsed) ? parsed : undefined; -} - -function statusFrom(value: unknown): number | undefined { - return ( - integerFrom(field(value, "status")) ?? - integerFrom(field(value, "statusCode")) ?? - integerFrom(field(value, "httpStatus")) - ); -} - -function codeFrom(value: unknown): string | number | undefined { - const rawCode = field(value, "code"); - return typeof rawCode === "string" || typeof rawCode === "number" ? rawCode : undefined; -} - -function stopReasonFrom(value: unknown): string | undefined { - return stringField(value, "stopReason"); -} - -function finishReasonFrom(value: unknown): string | undefined { - return stringField(value, "finish_reason") ?? stringField(value, "finishReason"); -} - -function causeOf(value: unknown): unknown { - return value instanceof Error ? value.cause : field(value, "cause"); -} - -function diagnosticErrors(value: unknown): readonly unknown[] { - const diagnostics = field(value, "diagnostics"); - if (!Array.isArray(diagnostics)) return []; - const errors: unknown[] = []; - for (const diagnostic of diagnostics) { - const diagnosticError = field(diagnostic, "error"); - errors.push(diagnosticError ?? diagnostic); - } - return errors; -} - -function normalizeCode(value: string | number | undefined): string | undefined { - if (value === undefined) return undefined; - const normalized = String(value) - .trim() - .toLowerCase() - .replace(/[^a-z0-9]+/g, "_") - .replace(/^_+|_+$/g, ""); - return normalized.length > 0 ? normalized : undefined; -} - -function kindFromStatus(status: number | undefined): ModelFallbackFailureKind | undefined { - switch (status) { - case 400: - case 413: - case 422: - return "request_incompatible"; - case 401: - case 403: - return "auth_on_candidate_provider"; - case 408: - return "network_timeout"; - case 404: - return "model_unavailable"; - case 429: - return "rate_limit"; - default: - if (status !== undefined && status >= 500 && status <= 599) return "provider_unavailable"; - return undefined; - } -} - -function refusalKindFromCode(code: string | number | undefined): ModelFallbackFailureKind | undefined { - const normalizedCode = normalizeCode(code); - if (normalizedCode === undefined) return undefined; - if (normalizedCode.includes("content_filter") || normalizedCode.includes("contentfilter")) return "task_failure"; - if (normalizedCode.includes("safety") || normalizedCode.includes("policy")) return "task_failure"; - switch (normalizedCode) { - case "blocked": - case "blocked_by_provider": - case "blocked_by_safety": - case "blocked_by_policy": - case "provider_refusal": - case "refusal": - case "tool_refusal": - case "tool_call_refusal": - case "tool_use_refusal": - return "task_failure"; - default: - return undefined; - } -} - -const REQUEST_INCOMPATIBLE_CODES: ReadonlySet = new Set([ - "invalid_request", - "invalid_request_error", - "bad_request", - "context_length_exceeded", - "request_too_large", - "too_large", - "request_entity_too_large", - "max_tokens", - "max_context_length", - "context_window_exceeded", -]); -function requestIncompatibleKindFromCode(code: string | number | undefined): ModelFallbackFailureKind | undefined { - const normalizedCode = normalizeCode(code); - return normalizedCode !== undefined && REQUEST_INCOMPATIBLE_CODES.has(normalizedCode) - ? "request_incompatible" - : undefined; -} -function kindFromCode(code: string | number | undefined): ModelFallbackFailureKind | undefined { - const normalizedCode = normalizeCode(code); - if (normalizedCode === undefined) return undefined; - const refusalKind = refusalKindFromCode(code); - if (refusalKind !== undefined) return refusalKind; - const httpStatusKind = kindFromStatus(integerFrom(code)); - if (httpStatusKind !== undefined) return httpStatusKind; - const requestIncompatibleKind = requestIncompatibleKindFromCode(code); - if (requestIncompatibleKind !== undefined) return requestIncompatibleKind; - - switch (normalizedCode) { - case "auth": - case "auth_required": - case "authentication_required": - case "unauthorized": - case "forbidden": - case "invalid_api_key": - case "missing_api_key": - case "invalid_key": - return "auth_on_candidate_provider"; - case "etimedout": - case "econnreset": - case "econnrefused": - case "enotfound": - case "eai_again": - case "fetch_failed": - case "network_error": - case "timeout": - case "timeout_error": - case "und_err_connect_timeout": - return "network_timeout"; - case "rate_limit": - case "rate_limit_exceeded": - case "too_many_requests": - case "quota_exceeded": - case "insufficient_quota": - case "usage_limit": - case "usage_limit_reached": - case "usage_limit_exceeded": - return "rate_limit"; - case "aborterror": - case "aborted": - case "cancelled": - case "canceled": - return "cancelled"; - case "model_not_found": - case "model_unavailable": - case "model_disabled": - case "unknown_model": - return "model_unavailable"; - case "provider_error": - case "api_error": - case "service_unavailable": - case "temporarily_unavailable": - case "overloaded": - return "provider_unavailable"; - default: - return undefined; - } -} - -const REQUEST_INCOMPATIBLE_FAILURE_PATTERNS: readonly RegExp[] = [ - /\bcontext[_\s-]?length(?:[_\s-]?exceeded)?\b/i, - /\bcontext[_\s-]?window(?:[_\s-]?exceeded)?\b/i, - /\bmax[_\s-]?context\b/i, - /\bmax[_\s-]?tokens?\b/i, - /\brequest(?:[_\s-]?entity)?[_\s-]?too[_\s-]?large\b/i, - /\btoo[_\s-]?large\b/i, - /\b(?:unsupported|unknown|invalid)\s+(?:tool|parameter|function)\b/i, - /\b(?:tool|parameter|function)\s+(?:not\s+(?:supported|found|allowed)|unknown|invalid)\b/i, - /\binvalid[_\s-]?request(?:[_\s-]?error)?\b/i, - /\bbad[_\s-]?request\b/i, -]; -const PROVIDER_REFUSAL_FAILURE_PATTERNS: readonly RegExp[] = [ - /\bfinish[_\s-]?reason\b[^\n]*\bcontent[_\s-]?filter\b/i, - /\bcontent[_\s-]?filter(?:ed|ing)?\b/i, - /\b(?:safety|policy)\b[^\n]*\b(?:refus(?:e|al|ed|es|ing)?|block(?:ed|ing)?|filter(?:ed|ing)?|violat(?:e|ion|ed|ing)?|disallow(?:ed|ing)?|reject(?:ed|ion|ing)?)\b/i, - /\b(?:refus(?:e|al|ed|es|ing)?|block(?:ed|ing)?|filter(?:ed|ing)?|violat(?:e|ion|ed|ing)?|disallow(?:ed|ing)?|reject(?:ed|ion|ing)?)\b[^\n]*\b(?:safety|policy)\b/i, - /\btool[_\s-]?(?:call|use)?[_\s-]?refus(?:e|al|ed|es|ing)?\b/i, - /\btool(?:\s+call|\s+use)?\b[^\n]*\brefus(?:e|al|ed|es|ing)?\b/i, - /\brefus(?:e|al|ed|es|ing)?\b[^\n]*\btool(?:\s+call|\s+use)?\b/i, - /\bprovider[_\s-]?refus(?:e|al|ed|es|ing)?\b/i, - /\bprovider\b[^\n]*\brefus(?:e|al|ed|es|ing)?\b[^\n]*\b(?:prompt|request|content|policy|safety)\b/i, -]; - -const TRANSPORT_OUTAGE_FAILURE_PATTERNS: readonly RegExp[] = [/^connection\s+error\.?$/i, /^fetch\s+failed\.?$/i]; - -function transportOutageKindFromMessage(message: string): ModelFallbackFailureKind | undefined { - return TRANSPORT_OUTAGE_FAILURE_PATTERNS.some((pattern) => pattern.test(message.trim())) - ? "transport_error" - : undefined; -} - -function refusalKindFromMessage(message: string): ModelFallbackFailureKind | undefined { - if (CANCELLED_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "cancelled"; - if (NON_RETRYABLE_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "task_failure"; - if (PROVIDER_REFUSAL_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "task_failure"; - return undefined; -} - -function fallbackKindFromMessage(message: string, name: string | undefined): ModelFallbackFailureKind | undefined { - const refusalKind = refusalKindFromMessage(message); - if (refusalKind !== undefined) return refusalKind; - const transportOutageKind = transportOutageKindFromMessage(message); - if (transportOutageKind !== undefined) return transportOutageKind; - if (REQUEST_INCOMPATIBLE_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "request_incompatible"; - const nameKind = kindFromCode(name); - if (nameKind !== undefined) return nameKind; - if (!RETRYABLE_MODEL_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return undefined; - if (/rate\s*limit|too\s*many\s*requests|\b429\b|quota|usage[\s_-]*limit|billing|credit/i.test(message)) - return "rate_limit"; - if (/auth|unauthori[sz]ed|\b40[13]\b|api\s*key|token\s*expired|forbidden|invalid\s*key/i.test(message)) - return "auth_on_candidate_provider"; - if ( - /model.*(?:unavailable|disabled|not\s*found|unknown)|(?:unavailable|disabled|not\s*found|unknown).*model/i.test( - message, - ) - ) - return "model_unavailable"; - if (/network|fetch|socket|connection\s*refused|timeout|timed\s*out/i.test(message)) return "network_timeout"; - return "provider_unavailable"; -} - -function signalSource(value: unknown, fallback: ModelFallbackFailureSource | undefined): ModelFallbackFailureSource { - if (fallback !== undefined) return fallback; - if (stopReasonFrom(value) !== undefined || diagnosticErrors(value).length > 0) return "assistant_message"; - if (value instanceof Error) return "throw"; - return "structured"; -} - -function makeSignal( - kind: ModelFallbackFailureKind, - value: unknown, - source: ModelFallbackFailureSource | undefined, -): ModelFallbackFailureSignal { - const status = statusFrom(value); - const code = codeFrom(value); - const name = errorName(value); - const stopReason = stopReasonFrom(value); - return { - kind, - message: errorMessage(value), - source: signalSource(value, source), - ...(stopReason !== undefined ? { stopReason } : {}), - ...(status !== undefined ? { status } : {}), - ...(code !== undefined ? { code } : {}), - ...(name !== undefined ? { name } : {}), - }; -} - -function fallbackSignalFromDirectMessage( - value: unknown, - source: ModelFallbackFailureSource | undefined, -): ModelFallbackFailureSignal | undefined { - const message = directMessageFrom(value); - if (message === undefined) return undefined; - const kind = fallbackKindFromMessage(message, errorName(value)); - return kind === undefined ? undefined : makeSignal(kind, value, source); -} - -function fallbackSignalFromMessage( - value: unknown, - source: ModelFallbackFailureSource | undefined, -): ModelFallbackFailureSignal | undefined { - const message = errorMessage(value); - if (!message.trim()) return undefined; - const kind = fallbackKindFromMessage(message, errorName(value)); - return kind === undefined ? undefined : makeSignal(kind, value, source); -} - -function classifyAssistantRefusalSignal( - value: unknown, - source: ModelFallbackFailureSource | undefined, -): ModelFallbackFailureSignal | undefined { - const codeRefusalKind = - refusalKindFromCode(codeFrom(value)) ?? - refusalKindFromCode(errorName(value)) ?? - refusalKindFromCode(finishReasonFrom(value)); - if (codeRefusalKind !== undefined) return makeSignal(codeRefusalKind, value, source); - - const messageRefusalKind = refusalKindFromMessage(directMessageFrom(value) ?? ""); - return messageRefusalKind === undefined ? undefined : makeSignal(messageRefusalKind, value, source); -} - -function isRefusalSignal(signal: ModelFallbackFailureSignal): boolean { - return signal.kind === "cancelled" || signal.kind === "task_failure"; -} - -function structuredSignal( - value: unknown, - seen: Set, - source?: ModelFallbackFailureSource, -): ModelFallbackFailureSignal | undefined { - if (value === undefined || value === null || seen.has(value)) return undefined; - if (typeof value === "object") seen.add(value); - - const stopReason = stopReasonFrom(value)?.toLowerCase(); - if (stopReason === "aborted") return makeSignal("cancelled", value, source); - - const directRefusalSignal = classifyAssistantRefusalSignal(value, source); - if (directRefusalSignal !== undefined) return directRefusalSignal; - - const codeKind = kindFromCode(codeFrom(value)); - const nameKind = kindFromCode(errorName(value)); - if (codeKind === "cancelled" || nameKind === "cancelled") return makeSignal("cancelled", value, source); - - let firstNestedFallbackSignal: ModelFallbackFailureSignal | undefined; - const nestedSeen = new Set(seen); - for (const diagnosticError of diagnosticErrors(value)) { - const diagnosticSignal = - structuredSignal(diagnosticError, nestedSeen, "diagnostic") ?? - fallbackSignalFromMessage(diagnosticError, "diagnostic"); - if (diagnosticSignal === undefined) continue; - if (isRefusalSignal(diagnosticSignal)) return diagnosticSignal; - firstNestedFallbackSignal ??= diagnosticSignal; - } - const cause = causeOf(value); - const causeSignal = structuredSignal(cause, nestedSeen, source) ?? fallbackSignalFromMessage(cause, source); - if (causeSignal !== undefined) { - if (isRefusalSignal(causeSignal)) return causeSignal; - firstNestedFallbackSignal ??= causeSignal; - } - - // Direct-message classification runs after nested traversal so a generic wrapper - // ("invalid request"/"400 bad request") cannot mask a non-retryable nested signal. - const directMessageSignal = fallbackSignalFromDirectMessage(value, source); - if (directMessageSignal !== undefined) return directMessageSignal; - - const statusKind = kindFromStatus(statusFrom(value)); - if (statusKind !== undefined) return makeSignal(statusKind, value, source); - if (codeKind !== undefined) return makeSignal(codeKind, value, source); - if (nameKind !== undefined) return makeSignal(nameKind, value, source); - - if (firstNestedFallbackSignal !== undefined) return firstNestedFallbackSignal; - - if (stopReason === "error") return makeSignal("provider_unavailable", value, source); - - return undefined; -} - -function messageFromUnknown(value: unknown, seen: Set): string | undefined { - if (value === undefined || value === null || seen.has(value)) return undefined; - if (typeof value === "string") return value.trim().length > 0 ? value : undefined; - if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return String(value); - if (typeof value === "symbol" || typeof value === "function") return undefined; - seen.add(value); - - if (value instanceof Error && value.message.trim().length > 0) return value.message; - const directMessage = directMessageFrom(value); - if (directMessage !== undefined) return directMessage; - - for (const diagnosticError of diagnosticErrors(value)) { - const diagnosticMessage = messageFromUnknown(diagnosticError, seen); - if (diagnosticMessage !== undefined) return diagnosticMessage; - } - - const causeMessage = messageFromUnknown(causeOf(value), seen); - if (causeMessage !== undefined) return causeMessage; - - const stopReason = stopReasonFrom(value); - if (stopReason !== undefined) return `Assistant message ended with stopReason:${stopReason}`; - const finishReason = finishReasonFrom(value); - if (finishReason !== undefined) return `Model request finished with finish_reason:${finishReason}`; - const status = statusFrom(value); - if (status !== undefined) return `Model request failed with status ${status}`; - const code = codeFrom(value); - if (code !== undefined) return `Model request failed with code ${String(code)}`; - - return undefined; -} - -export function errorMessage(error: unknown): string { - const structuredMessage = messageFromUnknown(error, new Set()); - if (structuredMessage !== undefined) return structuredMessage; - const rendered = String(error); - return rendered === "[object Object]" ? "Model request failed" : rendered; -} - -export function normalizeModelFailureSignal(error: unknown): ModelFallbackFailureSignal { - const structured = structuredSignal(error, new Set()); - if (structured !== undefined) return structured; - - const message = errorMessage(error); - const name = errorName(error); - const fallbackKind = message.trim().length > 0 ? fallbackKindFromMessage(message, name) : undefined; - return { - kind: fallbackKind ?? "unknown", - message, - source: "string_fallback", - ...(name !== undefined ? { name } : {}), - }; -} - -export function isRetryableModelFailure(error: unknown): boolean { - if (error === undefined) return false; - const signal = normalizeModelFailureSignal(error); - return FALLBACKABLE_FAILURE_KINDS.has(signal.kind); -} +/** + * Compatibility exports for the shared Atomic model-failure classifier. + * + * The implementation lives in @bastani/atomic so the main chat, workflows, + * and subagents cannot drift into separate fallback decisions. + */ + +export type { + ModelFallbackFailureKind, + ModelFallbackFailureSignal, + ModelFallbackFailureSource, +} from "@bastani/atomic"; +export { + errorMessage, + isRetryableModelFailure, + modelFailureMessage, + normalizeModelFailureSignal, +} from "@bastani/atomic"; diff --git a/packages/workflows/src/runs/shared/retry.ts b/packages/workflows/src/runs/shared/retry.ts new file mode 100644 index 000000000..fda524a7e --- /dev/null +++ b/packages/workflows/src/runs/shared/retry.ts @@ -0,0 +1,44 @@ +/** + * Compatibility exports for the shared Atomic retry policy, plus the workflow's + * abort-aware sleep. + * + * The decision/backoff policy is implemented in @bastani/atomic so main chat and + * workflow stages cannot drift into different budgets for the same + * `settings.retry` values. + */ + +export type { RetryDecision, RetryPolicySettings } from "@bastani/atomic"; +export { nextRetryDecision } from "@bastani/atomic"; + +type WorkflowAbortReason = Error | DOMException | string; + +function abortReason(signal: AbortSignal | undefined): WorkflowAbortReason { + const reason = signal?.reason; + if (reason instanceof Error || reason instanceof DOMException || typeof reason === "string") return reason; + return new Error("atomic-workflows: workflow cancelled"); +} + +/** Sleep for a bounded interval and reject promptly when the signal aborts. */ +export function sleepOrAbort(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.reject(abortReason(signal)); + return new Promise((resolve, reject) => { + let settled = false; + const cleanup = (): void => signal?.removeEventListener("abort", onAbort); + const finish = (): void => { + if (settled) return; + settled = true; + cleanup(); + resolve(); + }; + const fail = (error: WorkflowAbortReason): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + cleanup(); + reject(error); + }; + const timer = setTimeout(finish, Math.max(0, ms)); + const onAbort = (): void => fail(abortReason(signal)); + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} diff --git a/test/unit/main-chat-model-fallback.test.ts b/test/unit/main-chat-model-fallback.test.ts index 801e4982e..4aa1f7c51 100644 --- a/test/unit/main-chat-model-fallback.test.ts +++ b/test/unit/main-chat-model-fallback.test.ts @@ -2,10 +2,17 @@ import assert from "node:assert/strict"; import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; import type { Api, AssistantMessage, Model } from "@earendil-works/pi-ai/compat"; import { test } from "vitest"; -import { _createRetryPromiseForAgentEnd } from "../../packages/coding-agent/src/core/agent-session-events.js"; +import { _checkCompaction } from "../../packages/coding-agent/src/core/agent-session-auto-compaction.js"; import { + _createRetryPromiseForAgentEnd, + _processAgentEvent, +} from "../../packages/coding-agent/src/core/agent-session-events.js"; +import { + _clearFallbackModelScope, _handleRetryableError, + _isFallbackableError, _isRetryableError, + _restoreFallbackModel, _trySwitchToFallbackModel, } from "../../packages/coding-agent/src/core/agent-session-retry.js"; import type { CreateAgentSessionFromServicesOptions } from "../../packages/coding-agent/src/core/agent-session-services.js"; @@ -49,15 +56,43 @@ test("main-chat retry classifies structured provider transport diagnostics", () assert.equal(_isRetryableError.call(session as never, message), true); assert.equal(_isRetryableError.call(session as never, diagnosticOnlyMessage), true); assert.equal(_isRetryableError.call(session as never, retryableMessage({ errorMessage: "Tool not found" })), false); - assert.equal(_isRetryableError.call(session as never, retryableMessage({ errorMessage: "model not found" })), true); assert.equal( _isRetryableError.call( + session as never, + retryableMessage({ + errorMessage: "The model refused to complete the request", + diagnostics: [{ type: "provider_transport_failure", error: { name: "AbortError", message: "aborted" } }], + } as unknown as Partial), + ), + false, + ); + assert.equal(_isRetryableError.call(session as never, retryableMessage({ errorMessage: "model not found" })), false); + assert.equal( + _isFallbackableError.call(session as never, retryableMessage({ errorMessage: "model not found" })), + true, + ); + assert.equal( + _isRetryableError.call( + session as never, + retryableMessage({ errorMessage: undefined, code: "model_not_found" } as unknown as Partial), + ), + false, + ); + assert.equal( + _isFallbackableError.call( session as never, retryableMessage({ errorMessage: undefined, code: "model_not_found" } as unknown as Partial), ), true, ); }); +test("main chat treats rejected credentials as fallbackable but not same-model retryable", () => { + const session = { model: model("openai-codex", "gpt-5.5") }; + const message = retryableMessage({ errorMessage: "OAuth token invalidated" }); + + assert.equal(_isFallbackableError.call(session as never, message), true); + assert.equal(_isRetryableError.call(session as never, message), false); +}); test("main-chat fallback switches models after same-model retry exhaustion", async () => { const primary = model("openai-codex", "gpt-5.5"); @@ -127,6 +162,54 @@ test("main-chat fallback switches models after same-model retry exhaustion", asy assert.ok(events.some((event) => event.type === "session_model" && event.provider === "anthropic")); }); +test("main-chat fallback restores the primary model at the next turn boundary", async () => { + const primary = model("openai-codex", "gpt-5.5"); + const fallback = model("anthropic", "claude-opus-4-8"); + const events: Array<{ type: string; [key: string]: unknown }> = []; + const session = { + model: primary, + thinkingLevel: "high" as ThinkingLevel, + _fallbackModels: ["anthropic/claude-opus-4-8:high"], + _fallbackAttemptedKeys: new Set(), + _retryAttempt: 0, + settingsManager: { + getDefaultThinkingLevel: () => "high" as ThinkingLevel, + getDefaultProvider: () => "openai-codex", + }, + _modelRuntime: { + getAvailableSnapshot: () => [primary, fallback], + getModel: (provider: string, id: string) => + provider === fallback.provider && id === fallback.id ? fallback : undefined, + hasConfiguredAuth: () => true, + }, + agent: { + state: { model: primary, thinkingLevel: "high" as ThinkingLevel, messages: [retryableMessage()] }, + continue: async () => undefined, + }, + sessionManager: { + appendModelChange: (provider: string, id: string) => events.push({ type: "session_model", provider, id }), + appendThinkingLevelChange: (level: ThinkingLevel) => events.push({ type: "session_thinking", level }), + }, + _withContextWindowForModelSwitch: (candidate: Model) => candidate, + _refreshBaseSystemPromptFromActiveTools: () => undefined, + _emitModelChanged: (next: Model, previous: Model | undefined, source: string) => + events.push({ type: "model_changed", next: next.id, previous: previous?.id, source }), + _emitModelSelect: async () => undefined, + _emit: (event: { type: string; [key: string]: unknown }) => events.push(event), + }; + + assert.equal(await _trySwitchToFallbackModel.call(session as never, retryableMessage()), true); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(session.agent.state.model, fallback); + + assert.equal(await _restoreFallbackModel.call(session as never), true); + assert.equal(session.agent.state.model, primary); + assert.equal(session.agent.state.thinkingLevel, "high"); + assert.ok(events.some((event) => event.type === "model_changed" && event.source === "restore")); + assert.ok(events.some((event) => event.type === "model_fallback_end" && event.success === true)); + assert.equal(await _restoreFallbackModel.call(session as never), false); +}); + test("main-chat fallback can change reasoning on the same provider/model", async () => { const primary = model("openai", "gpt-5-mini"); const events: Array<{ type: string; [key: string]: unknown }> = []; @@ -362,3 +445,231 @@ test("main-chat fallback continuation resolution does not mark assistant errors false, ); }); + +test("a rejected codex credential advances to the next candidate without re-requesting the same model", async () => { + const primary = model("openai-codex", "gpt-5.5"); + const fallback = model("anthropic", "claude-opus-4-8"); + const invalidated = retryableMessage({ errorMessage: "OAuth token invalidated" }); + const events: Array<{ type: string; [key: string]: unknown }> = []; + let continued = 0; + const session = { + model: primary, + thinkingLevel: "high" as ThinkingLevel, + _fallbackModels: ["anthropic/claude-opus-4-8:high"], + _fallbackAttemptedKeys: new Set(), + _retryAttempt: 0, + settingsManager: { + // Same-model retry is enabled and generously budgeted; a dead credential + // must still skip it entirely. + getRetrySettings: () => ({ enabled: true, maxRetries: 3, baseDelayMs: 1000 }), + getDefaultThinkingLevel: () => "high" as ThinkingLevel, + getDefaultProvider: () => "openai-codex", + }, + _modelRuntime: { + getAvailableSnapshot: () => [primary, fallback], + getModel: (provider: string, id: string) => + provider === fallback.provider && id === fallback.id ? fallback : undefined, + hasConfiguredAuth: () => true, + }, + agent: { + state: { model: primary, thinkingLevel: "high" as ThinkingLevel, messages: [invalidated] }, + continue: async () => { + continued += 1; + }, + }, + sessionManager: { + appendModelChange: (provider: string, id: string) => events.push({ type: "session_model", provider, id }), + appendThinkingLevelChange: () => undefined, + }, + _withContextWindowForModelSwitch: (candidate: Model) => candidate, + _refreshBaseSystemPromptFromActiveTools: () => undefined, + _emitModelChanged: () => undefined, + _emitModelSelect: async () => undefined, + _emit: (event: { type: string; [key: string]: unknown }) => events.push(event), + _resolveRetry: () => undefined, + _isRetryableError, + _isFallbackableError, + _trySwitchToFallbackModel, + }; + + const handled = await _handleRetryableError.call(session as never, invalidated); + await new Promise((resolve) => setTimeout(resolve, 5)); + + assert.equal(handled, true); + assert.equal( + events.some((event) => event.type === "auto_retry_start"), + false, + "a revoked credential must not be re-requested on the same model", + ); + assert.ok(events.some((event) => event.type === "model_fallback_start" && event.to === "anthropic/claude-opus-4-8")); + assert.equal(session.agent.state.model, fallback); + assert.equal(continued, 1); +}); + +test("an explicit /model choice during a fallback is not overwritten by the restore", async () => { + const primary = model("openai-codex", "gpt-5.5"); + const fallback = model("anthropic", "claude-opus-4-8"); + const chosen = model("openai", "gpt-5-mini"); + const events: Array<{ type: string; [key: string]: unknown }> = []; + const session = { + model: primary, + thinkingLevel: "high" as ThinkingLevel, + _fallbackModels: ["anthropic/claude-opus-4-8:high"], + _fallbackAttemptedKeys: new Set(), + _retryAttempt: 0, + settingsManager: { + getDefaultThinkingLevel: () => "high" as ThinkingLevel, + getDefaultProvider: () => "openai-codex", + }, + _modelRuntime: { + getAvailableSnapshot: () => [primary, fallback, chosen], + getModel: (provider: string, id: string) => + provider === fallback.provider && id === fallback.id ? fallback : undefined, + hasConfiguredAuth: () => true, + }, + agent: { + state: { model: primary, thinkingLevel: "high" as ThinkingLevel, messages: [retryableMessage()] }, + continue: async () => undefined, + }, + sessionManager: { + appendModelChange: (provider: string, id: string) => events.push({ type: "session_model", provider, id }), + appendThinkingLevelChange: () => undefined, + }, + _withContextWindowForModelSwitch: (candidate: Model) => candidate, + _refreshBaseSystemPromptFromActiveTools: () => undefined, + _emitModelChanged: (next: Model, previous: Model | undefined, source: string) => + events.push({ type: "model_changed", next: next.id, previous: previous?.id, source }), + _emitModelSelect: async () => undefined, + _emit: (event: { type: string; [key: string]: unknown }) => events.push(event), + }; + + assert.equal(await _trySwitchToFallbackModel.call(session as never, retryableMessage()), true); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(session.agent.state.model, fallback); + + // setModel()/cycleModel() cancel the pending restore before applying the + // user's explicit choice. + _clearFallbackModelScope.call(session as never); + session.agent.state.model = chosen; + + assert.equal(await _restoreFallbackModel.call(session as never), false); + assert.equal(session.agent.state.model, chosen, "the explicit choice must survive the turn boundary"); + assert.equal( + events.some((event) => event.type === "model_changed" && event.source === "restore"), + false, + ); + assert.ok(events.some((event) => event.type === "model_fallback_end")); +}); + +function overflowMessage(): AssistantMessage { + return retryableMessage({ + errorMessage: "context_length_exceeded: prompt is too long for this model", + provider: "openai-codex", + model: "gpt-5.5", + timestamp: Date.now(), + } as unknown as Partial); +} + +/** Session double for the `agent_end` branch of `_processAgentEvent`. */ +function overflowTurnSession(checkCompaction: (session: Record) => void): { + session: Record; + switched: AssistantMessage[]; + restored: number; +} { + const switched: AssistantMessage[] = []; + const counters = { restored: 0 }; + const session: Record = { + model: model("openai-codex", "gpt-5.5"), + _lastAssistantMessage: overflowMessage(), + _protectedStreamingCustomMessages: [], + _postToolCompactionPreflightError: undefined, + _pendingPostCompactionContinuation: undefined, + _contextOverflowUnresolved: false, + _applyInterruptAbortMessage: () => undefined, + _applyProviderErrorGuidance: () => undefined, + _emitExtensionEvent: async () => undefined, + _emit: () => undefined, + _isFallbackableError, + _isRetryableError, + _isEmptyCompletion: () => false, + _isSafetyRefusal: () => false, + _handleRetryableError: async () => false, + _resolveRetry: () => undefined, + _checkCompaction: async () => checkCompaction(session), + _trySwitchToFallbackModel: async (message: AssistantMessage) => { + switched.push(message); + return true; + }, + _restoreFallbackModel: async () => { + counters.restored += 1; + return false; + }, + }; + return { + session, + switched, + get restored() { + return counters.restored; + }, + }; +} + +test("a compactable context overflow does not spend a fallback candidate", async () => { + // Compaction recovers the turn, so it must keep the current model. + const probe = overflowTurnSession(() => undefined); + + await _processAgentEvent.call(probe.session as never, { type: "agent_end" } as never); + + assert.deepEqual(probe.switched, []); + assert.equal(probe.session._contextOverflowUnresolved, false); +}); + +test("an unresolved context overflow advances to the next fallback candidate", async () => { + const probe = overflowTurnSession((session) => { + session._contextOverflowUnresolved = true; + }); + + await _processAgentEvent.call(probe.session as never, { type: "agent_end" } as never); + + assert.equal(probe.switched.length, 1, "an overflow compaction cannot fix must reach the fallback chain"); + assert.equal(probe.switched[0]?.errorMessage, overflowMessage().errorMessage); + assert.equal(probe.session._contextOverflowUnresolved, false); + assert.equal(probe.restored, 0, "a successful switch owns the turn and must not restore yet"); +}); + +test("compaction disabled marks a same-model context overflow unresolved", async () => { + const session = { + model: model("openai-codex", "gpt-5.5"), + _contextOverflowUnresolved: false, + _pendingPostToolCompactionGuard: undefined, + _postToolCompactionPreflightError: undefined, + settingsManager: { getCompactionSettings: () => ({ enabled: false }) }, + _emit: () => undefined, + }; + + await _checkCompaction.call(session as never, overflowMessage()); + + assert.equal(session._contextOverflowUnresolved, true); +}); + +test("compaction disabled leaves an ordinary provider error resolvable by same-model retry", async () => { + const session = { + model: model("openai-codex", "gpt-5.5"), + _contextOverflowUnresolved: false, + _pendingPostToolCompactionGuard: undefined, + _postToolCompactionPreflightError: undefined, + settingsManager: { getCompactionSettings: () => ({ enabled: false }) }, + _emit: () => undefined, + }; + + await _checkCompaction.call( + session as never, + retryableMessage({ + errorMessage: "rate limit", + provider: "openai-codex", + model: "gpt-5.5", + } as unknown as Partial), + ); + + assert.equal(session._contextOverflowUnresolved, false); +}); diff --git a/test/unit/model-fallback-classifier-conformance.test.ts b/test/unit/model-fallback-classifier-conformance.test.ts index 1903f5241..643d64a97 100644 --- a/test/unit/model-fallback-classifier-conformance.test.ts +++ b/test/unit/model-fallback-classifier-conformance.test.ts @@ -9,23 +9,9 @@ import { normalizeModelFailureSignal as workflowsNormalize, } from "../../packages/workflows/src/runs/shared/model-fallback.js"; -// The subagents and workflows model-failure classifiers are maintained as parallel -// copies (packages/subagents/src/runs/shared/model-fallback.ts and -// packages/workflows/src/runs/shared/model-fallback-failures.ts). This conformance -// suite runs a shared corpus of failure fixtures through both and asserts they -// agree, so a change to one copy that silently diverges the other fails here. -// -// Limitations: this enforces BEHAVIORAL parity over a finite corpus only — the two -// copies are structurally different (e.g. the subagents copy combines regex -// alternations that the workflows copy keeps separate), and a divergence on an -// input outside this corpus would pass CI silently. When adding or changing a -// classification rule, update BOTH copies and add a fixture here that exercises -// the new rule. (Extracting a single shared module is blocked by the mandated -// package split; the copies must not import each other.) -// -// Known intentional difference (not covered by the shared corpus): the workflows -// classifier has an extra `transport_error` kind for bare "connection error." / -// "fetch failed." wrapper messages that the subagents classifier does not model. +// Both companion packages re-export the classifier implemented by the published +// Atomic package. The identity assertion below protects the single-source +// contract; the corpus keeps the public wrappers covered as well. type Fixture = { label: string; failure: unknown; kind: string; retryable: boolean }; @@ -204,6 +190,24 @@ const CONFORMANCE_FIXTURES: readonly Fixture[] = [ kind: "task_failure", retryable: false, }, + { + label: "provider transport diagnostic wrapping cancellation", + failure: { + diagnostics: [ + { type: "provider_transport_failure", error: { name: "AbortError", message: "aborted by user" } }, + ], + }, + kind: "cancelled", + retryable: false, + }, + { + label: "provider transport diagnostic wrapping safety refusal", + failure: { + diagnostics: [{ type: "provider_transport_failure", error: { message: "blocked by safety policy" } }], + }, + kind: "task_failure", + retryable: false, + }, { label: "abort cause under invalid request", failure: abortWrappedError(), kind: "cancelled", retryable: false }, { label: "cancel cause under 400 wrapper", @@ -369,6 +373,10 @@ const CONFORMANCE_FIXTURES: readonly Fixture[] = [ ]; describe("model fallback classifier conformance (subagents vs workflows)", () => { + test("companion packages expose the one shared classifier implementation", () => { + assert.strictEqual(subagentsNormalize, workflowsNormalize); + assert.strictEqual(subagentsIsRetryable, workflowsIsRetryable); + }); test("both classifiers agree on the shared failure corpus", () => { for (const fixture of CONFORMANCE_FIXTURES) { const subagentsSignal = subagentsNormalize(fixture.failure); diff --git a/test/unit/stage-runner-thrown-retry.test.ts b/test/unit/stage-runner-thrown-retry.test.ts new file mode 100644 index 000000000..34fc69409 --- /dev/null +++ b/test/unit/stage-runner-thrown-retry.test.ts @@ -0,0 +1,589 @@ +import { describe, test } from "vitest"; +import { nextRetryDecision as codingAgentNextRetryDecision } from "../../packages/coding-agent/src/core/retry-policy.js"; +import type { + AgentSessionAdapter, + InternalStageContext, + StageSessionCreateOptions, + StageSessionRuntime, +} from "../../packages/workflows/src/runs/foreground/stage-runner.js"; +import type { WorkflowFastModeSettingsManager } from "../../packages/workflows/src/runs/foreground/stage-runner-types.js"; +import { nextRetryDecision as workflowsNextRetryDecision } from "../../packages/workflows/src/runs/shared/retry.js"; +import { + assert, + createStageContext, + flushMicrotasks, + makeMockSession, + makeOpts, + Type, +} from "./stage-runner-helpers.js"; + +const retrySettings = ( + overrides: Partial>> = {}, +) => ({ + enabled: true, + maxRetries: 2, + baseDelayMs: 0, + ...overrides, +}); + +function sessionWithSettings( + settings: ReturnType, + prompt: StageSessionRuntime["prompt"], + overrides: Partial = {}, +): { readonly session: StageSessionRuntime; readonly settingsManager: WorkflowFastModeSettingsManager } { + const { session } = makeMockSession({ prompt, ...overrides }); + return { + session, + settingsManager: { + getCodexFastModeSettings: () => ({ chat: false, workflow: false }), + getRetrySettings: () => settings, + }, + }; +} + +function modelFor(options: StageSessionCreateOptions): string { + return typeof options.model === "string" ? options.model : "object-model"; +} + +describe("createStageContext — thrown model failure retry", () => { + test("retries a transient thrown failure on the same session before succeeding", async () => { + let promptCalls = 0; + let created = 0; + const settings = retrySettings(); + const agentSession: AgentSessionAdapter = { + async create(options) { + created += 1; + const result = sessionWithSettings( + settings, + async () => { + promptCalls += 1; + if (promptCalls < 3) throw new Error("503 service unavailable"); + return "ok"; + }, + { getLastAssistantText: () => "ok" }, + ); + assert.equal(modelFor(options), "anthropic/primary"); + return result; + }, + }; + + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { model: "anthropic/primary" }, + }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("go"), "ok"); + assert.equal(promptCalls, 3); + assert.equal(created, 1); + assert.deepEqual( + ctx.__modelFallbackMeta().modelAttempts?.map(({ model, success }) => ({ model, success })), + [{ model: "anthropic/primary", success: true }], + ); + }); + + test("restores admitted messages before retrying a thrown provider failure", async () => { + let promptCalls = 0; + let activeSession: StageSessionRuntime | undefined; + const settings = retrySettings(); + const agentSession: AgentSessionAdapter = { + async create() { + const result = sessionWithSettings( + settings, + async () => { + promptCalls += 1; + activeSession?.messages.push({} as never); + if (promptCalls < 3) throw new Error("503 service unavailable"); + return "ok"; + }, + { getLastAssistantText: () => "ok" }, + ); + activeSession = result.session; + return result; + }, + }; + const ctx = createStageContext( + makeOpts({ adapters: { agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("go"), "ok"); + assert.equal(promptCalls, 3); + assert.equal(activeSession?.messages.length, 1); + }); + + test("keeps the admitted stage prompt when the retry continues the real session turn", async () => { + // pi-agent-core's Agent.continue() rejects an empty transcript and a + // transcript ending in an assistant message, so the continuation path must + // see the stage prompt it is resuming. + const messages: StageSessionRuntime["messages"] = []; + const continuedTranscripts: Array = []; + let promptCalls = 0; + const settings = retrySettings(); + const agentSession: AgentSessionAdapter = { + async create() { + return sessionWithSettings( + settings, + async (text) => { + promptCalls += 1; + messages.push({ role: "user", content: text, timestamp: Date.now() } as never); + messages.push({ + role: "assistant", + stopReason: "error", + errorMessage: "503 service unavailable", + content: [], + } as never); + throw new Error("503 service unavailable"); + }, + { + messages, + getLastAssistantText: () => "ok", + // Shape recognized by asAgentSession()/retryableAgentSession(). + state: { messages }, + sessionManager: {}, + modelRuntime: {}, + getContextUsage: () => ({}), + _runAgentContinue: async () => { + continuedTranscripts.push([...messages]); + messages.push({ + role: "assistant", + stopReason: "stop", + content: [{ type: "text", text: "ok" }], + } as never); + }, + } as unknown as Partial, + ); + }, + }; + const ctx = createStageContext( + makeOpts({ adapters: { agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("do it"), "ok"); + assert.equal(promptCalls, 1, "the continuation path must not re-prompt"); + assert.equal(continuedTranscripts.length, 1); + const observed = continuedTranscripts[0]!; + assert.ok(observed.length > 0, "continue() must not observe an empty transcript"); + const last = observed[observed.length - 1]!; + assert.equal(last.role, "user"); + assert.equal(last.content, "do it"); + assert.equal( + observed.some((message) => message.role === "assistant" && message.stopReason === "error"), + false, + "the failed assistant error must still be dropped from live state", + ); + }); + + test("preserves a concurrent user message admitted before the retry-owned prompt", async () => { + let promptCalls = 0; + let activeSession: StageSessionRuntime | undefined; + const settings = retrySettings(); + const agentSession: AgentSessionAdapter = { + async create() { + const result = sessionWithSettings( + settings, + async () => { + promptCalls += 1; + if (promptCalls === 1) { + activeSession?.messages.push({ + role: "user", + content: [{ type: "text", text: "concurrent" }], + timestamp: Date.now(), + } as never); + } + activeSession?.messages.push({ + role: "user", + content: [{ type: "text", text: "go" }], + timestamp: Date.now(), + } as never); + if (promptCalls < 3) throw new Error("503 service unavailable"); + return "ok"; + }, + { getLastAssistantText: () => "ok" }, + ); + activeSession = result.session; + return result; + }, + }; + const ctx = createStageContext( + makeOpts({ adapters: { agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("go"), "ok"); + assert.deepEqual( + activeSession?.messages.map((message) => { + if (message.role !== "user" || !Array.isArray(message.content)) return undefined; + const first = message.content[0]; + return first?.type === "text" ? first.text : undefined; + }), + ["concurrent", "go"], + ); + }); + + test("records one failed attempt after retry exhaustion, then advances to a fallback", async () => { + const promptCalls: string[] = []; + const created: string[] = []; + const disposed: string[] = []; + const settings = retrySettings(); + const agentSession: AgentSessionAdapter = { + async create(options) { + const model = modelFor(options); + created.push(model); + return sessionWithSettings( + settings, + async () => { + promptCalls.push(model); + if (model === "anthropic/primary") throw new Error("503 service unavailable"); + return "fallback answer"; + }, + { + dispose: () => { + disposed.push(model); + }, + getLastAssistantText: () => (model === "openai/fallback" ? "fallback answer" : undefined), + }, + ); + }, + }; + + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary", + fallbackModels: ["openai/fallback"], + }, + }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("go"), "fallback answer"); + assert.deepEqual(promptCalls, ["anthropic/primary", "anthropic/primary", "anthropic/primary", "openai/fallback"]); + assert.deepEqual(created, ["anthropic/primary", "openai/fallback"]); + assert.deepEqual(disposed, ["anthropic/primary"]); + assert.deepEqual( + ctx.__modelFallbackMeta().modelAttempts?.map(({ model, success }) => ({ model, success })), + [ + { model: "anthropic/primary", success: false }, + { model: "openai/fallback", success: true }, + ], + ); + }); + + test("retries session creation on the same candidate before advancing", async () => { + const created: string[] = []; + const settings = retrySettings(); + const settingsManager = { + getCodexFastModeSettings: () => ({ chat: false, workflow: false }), + getRetrySettings: () => settings, + }; + const agentSession: AgentSessionAdapter = { + async create(options) { + const model = modelFor(options); + created.push(model); + if (model === "anthropic/primary") throw new Error("503 service unavailable during create"); + return sessionWithSettings(settings, async () => "fallback answer", { + getLastAssistantText: () => "fallback answer", + }); + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary", + fallbackModels: ["openai/fallback"], + settingsManager: settingsManager as never, + }, + }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("go"), "fallback answer"); + assert.deepEqual(created, ["anthropic/primary", "anthropic/primary", "anthropic/primary", "openai/fallback"]); + assert.deepEqual( + ctx.__modelFallbackMeta().modelAttempts?.map(({ model, success, error }) => ({ model, success, error })), + [ + { model: "anthropic/primary", success: false, error: "503 service unavailable during create" }, + { model: "openai/fallback", success: true, error: undefined }, + ], + ); + }); + + test("disabled retry advances immediately without another prompt", async () => { + const calls: string[] = []; + const settings = retrySettings({ enabled: false }); + const agentSession: AgentSessionAdapter = { + async create(options) { + const model = modelFor(options); + return sessionWithSettings( + settings, + async () => { + calls.push(model); + if (model === "anthropic/primary") throw new Error("503 service unavailable"); + return "fallback answer"; + }, + { getLastAssistantText: () => "fallback answer" }, + ); + }, + }; + + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { model: "anthropic/primary", fallbackModels: ["openai/fallback"] }, + }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("go"), "fallback answer"); + assert.deepEqual(calls, ["anthropic/primary", "openai/fallback"]); + }); + + test("retries auth and request-incompatible thrown failures before advancing", async () => { + for (const failure of ["401 unauthorized", "400 bad request"]) { + const calls: string[] = []; + const settings = retrySettings(); + const agentSession: AgentSessionAdapter = { + async create(options) { + const model = modelFor(options); + return sessionWithSettings( + settings, + async () => { + calls.push(model); + if (model === "anthropic/primary") throw new Error(failure); + return "fallback answer"; + }, + { getLastAssistantText: () => "fallback answer" }, + ); + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { model: "anthropic/primary", fallbackModels: ["openai/fallback"] }, + }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("go"), "fallback answer"); + assert.deepEqual(calls, ["anthropic/primary", "anthropic/primary", "anthropic/primary", "openai/fallback"]); + } + }); + + test("does not retry a non-retryable thrown failure", async () => { + let calls = 0; + const settings = retrySettings(); + const agentSession: AgentSessionAdapter = { + async create() { + return sessionWithSettings(settings, async () => { + calls += 1; + throw new Error("command failed: bun test"); + }); + }, + }; + const ctx = createStageContext( + makeOpts({ adapters: { agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + + await assert.rejects(ctx.prompt("go"), /command failed/); + assert.equal(calls, 1); + }); + + test("structured output capture suppresses thrown retry", async () => { + let createOptions: StageSessionCreateOptions | undefined; + let promptCalls = 0; + const settings = retrySettings(); + const agentSession: AgentSessionAdapter = { + async create(options) { + createOptions = options; + return sessionWithSettings( + settings, + async () => { + promptCalls += 1; + const tool = createOptions?.customTools?.find((entry) => entry.name === "structured_output"); + assert.ok(tool); + await tool.execute("structured-call", { ok: true }, undefined, undefined, undefined as never); + throw new Error("503 service unavailable after structured output"); + }, + { getLastAssistantText: () => undefined }, + ); + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary", + schema: Type.Object({ ok: Type.Boolean() }, { additionalProperties: false }), + }, + }), + ) as InternalStageContext; + + assert.deepEqual(await ctx.prompt("go"), { ok: true }); + assert.equal(promptCalls, 1); + }); + + test("workflow abort during backoff prevents the next prompt and preserves its reason", async () => { + const signalController = new AbortController(); + const workflowError = new Error("workflow killed"); + let calls = 0; + const settings = retrySettings({ baseDelayMs: 1000 }); + const agentSession: AgentSessionAdapter = { + async create() { + return sessionWithSettings(settings, async () => { + calls += 1; + throw new Error("503 service unavailable"); + }); + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { model: "anthropic/primary" }, + signal: signalController.signal, + }), + ) as InternalStageContext; + const prompt = ctx.prompt("go"); + await flushMicrotasks(); + assert.equal(calls, 1); + signalController.abort(workflowError); + await assert.rejects(prompt, workflowError); + assert.equal(calls, 1); + }); + + test("ctx.abort cancels thrown-error backoff", async () => { + let calls = 0; + const settings = retrySettings({ baseDelayMs: 1000 }); + const agentSession: AgentSessionAdapter = { + async create() { + return sessionWithSettings(settings, async () => { + calls += 1; + throw new Error("503 service unavailable"); + }); + }, + }; + const ctx = createStageContext( + makeOpts({ adapters: { agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + const prompt = ctx.prompt("go"); + await flushMicrotasks(); + await ctx.abort(); + await assert.rejects(prompt, /stage aborted/); + assert.equal(calls, 1); + }); + + test("pause during backoff defers the retry until resume and uses resumed text", async () => { + const promptTexts: string[] = []; + let calls = 0; + const settings = retrySettings({ baseDelayMs: 1000 }); + const agentSession: AgentSessionAdapter = { + async create() { + return sessionWithSettings( + settings, + async (text) => { + calls += 1; + promptTexts.push(text); + if (calls === 1) throw new Error("503 service unavailable"); + return "resumed answer"; + }, + { getLastAssistantText: () => "resumed answer" }, + ); + }, + }; + const ctx = createStageContext( + makeOpts({ adapters: { agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + const prompt = ctx.prompt("first"); + await flushMicrotasks(); + await ctx.__requestPause(); + await new Promise((resolve) => setTimeout(resolve, 15)); + assert.equal(calls, 1); + await ctx.__resume("resumed"); + await flushMicrotasks(); + assert.equal(calls, 2); + assert.equal(await prompt, "resumed answer"); + assert.deepEqual(promptTexts, ["first", "resumed"]); + }); + + test("a paused session creation can be cancelled and retried by a later prompt", async () => { + let creates = 0; + const settings = retrySettings({ baseDelayMs: 1000 }); + const settingsManager = { + getCodexFastModeSettings: () => ({ chat: false, workflow: false }), + getRetrySettings: () => settings, + }; + const agentSession: AgentSessionAdapter = { + async create() { + creates += 1; + if (creates === 1) throw new Error("503 service unavailable during create"); + return sessionWithSettings(settings, async () => "fresh answer", { + getLastAssistantText: () => "fresh answer", + }); + }, + }; + const ctx = createStageContext( + makeOpts({ adapters: { agentSession }, stageOptions: { settingsManager: settingsManager as never } }), + ) as InternalStageContext; + const firstPrompt = ctx.prompt("first"); + await flushMicrotasks(); + await ctx.__requestPause(); + await ctx.__resume(); + assert.equal(await firstPrompt, ""); + assert.equal(await ctx.prompt("second"), "fresh answer"); + assert.equal(creates, 2); + }); + + test("a cancelled paused retry cannot leak its resume text into a later prompt", async () => { + const promptTexts: string[] = []; + let calls = 0; + const settings = retrySettings({ baseDelayMs: 1000 }); + const agentSession: AgentSessionAdapter = { + async create() { + return sessionWithSettings( + settings, + async (text) => { + calls += 1; + promptTexts.push(text); + if (calls < 3) throw new Error("503 service unavailable"); + return "new answer"; + }, + { getLastAssistantText: () => "new answer" }, + ); + }, + }; + const ctx = createStageContext( + makeOpts({ adapters: { agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + const firstPrompt = ctx.prompt("first"); + await flushMicrotasks(); + await ctx.__requestPause(); + await ctx.__resume("stale-resume"); + await flushMicrotasks(); + assert.equal(calls, 2); + await ctx.abort(); + await assert.rejects(firstPrompt, /stage aborted/); + + assert.equal(await ctx.prompt("new-objective"), "new answer"); + assert.deepEqual(promptTexts, ["first", "stale-resume", "new-objective"]); + }); +}); + +describe("shared retry policy", () => { + test("companion packages expose the one shared policy implementation", () => { + assert.strictEqual(workflowsNextRetryDecision, codingAgentNextRetryDecision); + }); + + test("honours enabled, maxRetries, and exponential baseDelayMs", () => { + const settings = { enabled: true, maxRetries: 3, baseDelayMs: 250 }; + assert.deepEqual(workflowsNextRetryDecision(settings, 0, true), { attempt: 1, delayMs: 250 }); + assert.deepEqual(workflowsNextRetryDecision(settings, 1, true), { attempt: 2, delayMs: 500 }); + assert.deepEqual(workflowsNextRetryDecision(settings, 2, true), { attempt: 3, delayMs: 1000 }); + assert.equal(workflowsNextRetryDecision(settings, 3, true), undefined); + }); + + test("disabled retry and missing settings advance immediately", () => { + assert.equal(workflowsNextRetryDecision({ enabled: false, maxRetries: 3, baseDelayMs: 250 }, 0, true), undefined); + assert.equal(workflowsNextRetryDecision(undefined, 0, true), undefined); + }); + + test("an ineligible failure is never retried on the same target", () => { + assert.equal(workflowsNextRetryDecision({ enabled: true, maxRetries: 3, baseDelayMs: 250 }, 0, false), undefined); + }); +}); From 7aeaa51a4aa920cc3d0b865e7098ff8baacdf409 Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 11:01:28 -0700 Subject: [PATCH 02/13] docs(research): record the #2170 model-fallback convergence research Captures the fallback seams, file references, contracts, and risks that guided the convergence work, alongside the state of the inherited implementation it was handed. Refs: #2170 Assistant-model: Claude Opus 5 --- ...6-08-04-2170-model-fallback-convergence.md | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 research/tickets/2026-08-04-2170-model-fallback-convergence.md diff --git a/research/tickets/2026-08-04-2170-model-fallback-convergence.md b/research/tickets/2026-08-04-2170-model-fallback-convergence.md new file mode 100644 index 000000000..3530752e4 --- /dev/null +++ b/research/tickets/2026-08-04-2170-model-fallback-convergence.md @@ -0,0 +1,190 @@ +--- +date: 2026-08-04 09:27:05 PDT +researcher: GPT-5.6 Luna +git_commit: 226eacfa71793cd561c9e86724f67ecbf00a2fdc +branch: atomic-issue-2170 +repository: atomic-issue-2170 +topic: "Issue #2170 model fallback convergence" +tags: [research, codebase, model-fallback, main-chat, workflows, retries] +status: complete +last_updated: 2026-08-04 +last_updated_by: GPT-5.6 Luna +breaking_changes_allowed: false +compatibility_context: "The repository publishes @bastani/atomic and raw-TypeScript companion packages. Preserve existing public APIs, session history behavior, workflow controls, and provider retry semantics unless the issue explicitly changes them." +--- + +# Research + +## Research Question + +Document the seams and current implementation for issue #2170: + +1. Share the model-failure classifier between main chat, workflows, and subagents. Main chat must classify auth-on-candidate-provider, `model_unavailable`, and `request_incompatible` as fallbackable without retrying the same model. Codex token invalidation must be non-retryable on the same model but fallbackable to the next candidate. +2. Scope a main-chat fallback switch to the failing turn. Restore the user-selected primary model for the next turn, record the departure and restore in model history/events, and do not overwrite an explicit `/model` choice made during fallback. +3. Add bounded same-candidate retry with `settings.retry` to workflow thrown failures before advancing the candidate walk. `retry.enabled: false` keeps immediate advancement. + +The hard constraints are shared classifier and retry policy locations, raw TypeScript in `packages/workflows`, npm-only dependency installation, Vitest tests using `node:assert/strict`, required checks, user-facing docs and changelogs, and no PR creation by this stage. + +## Compatibility Context + +`@bastani/atomic` is independently published and companion packages import its public types/helpers. The coding-agent session format is append-only and can be reopened by older and newer code. Workflow context and pause/abort controls have existing callers and test doubles. Therefore `breaking_changes_allowed: false`: downstream behavior and compatibility shims must be checked before accepting the inherited implementation. + +## Contract amendments received + +The supervisor's course correction is authoritative: + +> You are the RESEARCH stage of this workflow, not the implementer. You have already made extensive uncommitted edits in the worktree (~30 files across packages/coding-agent, packages/workflows, packages/subagents, docs, changelogs, tests). STOP making further code edits now. Do NOT revert or discard the existing changes — leave the working tree exactly as it is. + +> Finish your stage by producing the research artifact only, and make it serve two purposes: +> 1. The normal codebase research the orchestrator needs (fallback seams, file/line references, contracts, risks). +> 2. An explicit "PRE-EXISTING IMPLEMENTATION IN WORKTREE" section: list every modified file, what change it contains, which of the three issue directions (shared classifier / turn-scoped switch restore / retry-then-advance in the candidate walk) it belongs to, what is complete, what is incomplete or untested, and which checks (npm run check, npm run test:unit, npm run test --workspace=@bastani/atomic) have NOT yet been run or are failing. + +The downstream orchestrator must treat these uncommitted changes as inherited work to validate, complete, test, and commit — not as its own starting point to redo from scratch. + +No source or test files were changed after this amendment. The existing worktree changes are intentionally preserved for the downstream orchestrator. + +## Summary + +The inherited implementation places the classifier in `packages/coding-agent/src/core/model-fallback-failures.ts` and makes the workflows and subagents classifier modules compatibility re-exports. It exposes separate predicates for any fallback, same-model retry, and provider safety refusal. Main chat now has two decisions: `_isRetryableError` for another request to the current model and `_isFallbackableError` for moving to a configured candidate. Codex token invalidation is checked separately and takes the former path only as a terminal failure while remaining eligible for the latter. + +Main-chat fallback state now records an origin model, origin reasoning level, generation, and pending restore error. Candidate selection starts that scope; prompt/event/tool-hook/compaction boundaries can restore it; explicit model or thinking-level selection clears it. Restore writes model history and emits model-change/model-select and fallback-end lifecycle events. The event handler also treats fallbackable errors as model failures before compaction. + +Workflow stage prompting now uses a shared `thrownRetryDecision` and `sleepOrAbort` helper. Prompt and session-creation attempts retry thrown classifier-positive failures on the same candidate with exponential delay, honor `settings.retry`, react to pause/resume and abort, and advance only after the retry budget is exhausted. The implementation also adds rollback support for failed prompt messages and a private settings-manager handoff for session creation failures. + +The targeted implementation tests pass, and `npm run check` passed in the recorded run. The full root unit suite is not green: the review recorded 5,836 passed, 2 skipped, and 16 failures, with all 16 reproduced on a clean `HEAD` worktree and judged pre-existing. The full workspace test command was not successfully completed: an attempted invocation supplied a duplicate `--run` flag and Vitest stopped before running. Direct targeted Vitest runs in `packages/coding-agent` passed 27 tests. The downstream orchestrator must rerun the required commands from the inherited state and separate baseline failures from regressions. + +## Detailed Findings + +### Shared classifier seam + +- `packages/coding-agent/src/core/model-fallback-failures.ts:1-654` is the proposed single implementation. It normalizes status, code, name, stop reason, finish reason, nested diagnostics, causes, and messages into `ModelFallbackFailureSignal` values. +- The classifier recognizes `auth_on_candidate_provider`, `rate_limit`, `provider_unavailable`, `network_timeout`, `transport_error`, `model_unavailable`, `request_incompatible`, `cancelled`, `task_failure`, and `unknown` (`:81-118`). HTTP 400/413/422 are request-incompatible; 401/403 are auth; 404 is model unavailable; 429 is rate limit; 5xx is provider unavailable (`:199-217`). +- `isRetryableModelFailure` (`:627-630`) answers whether a failure may spend a fallback candidate. `isRetryableSameModelFailure` (`:633-653`) limits same-model retries to rate-limit/provider/network/transport kinds. `isSafetyRefusalFailure` (`:619-625`) identifies task-failure signals matching provider refusal patterns. +- `packages/coding-agent/src/index.ts:128-140` exports the classifier types and helpers from the published package. `packages/workflows/src/runs/shared/model-fallback-failures.ts:1-19` re-exports them for raw workflows. `packages/subagents/src/runs/shared/model-fallback.ts:1-16` re-exports them while retaining subagent model-candidate formatting helpers. +- `test/unit/model-fallback-classifier-conformance.test.ts:12-16, 24-373` checks a shared fixture corpus and identity of the two companion exports. Fixtures cover status/code/message classification, nested cancellation/refusal precedence, transport wrappers, and request incompatibility. +- Current review risk: the new shared implementation is more than a mechanical move from both copies. It adds nested signal priority, transport wrapper handling, extra token-invalidated codes, and changed message precedence. `isRetryableSameModelFailure` also contains a conservative provider-unavailable carve-out based on `signal.source`, status, code, name, and regex internals (`:645-653`). The downstream stage should decide whether these behavior changes are part of #2170 and add focused tests if they are retained. + +### Main-chat retry and fallback seam + +- `packages/coding-agent/src/core/agent-session-retry.ts:50-85` splits fallbackability from same-model retryability. `_isFallbackableError` rejects non-error, context-overflow, cancellation, and task-failure signals, then accepts Codex invalidation or the shared fallback classifier. `_isRetryableError` rejects context overflow and Codex invalidation, allows shared safety refusals, and then uses the same-model predicate. +- The Codex scan at `:20-37` walks error message, cause, and diagnostics and calls `isCodexTokenInvalidationError`. This makes invalidation terminal for the current model while allowing `_trySwitchToFallbackModel` to select the next model. +- `_handleRetryableError` at `:388-479` uses separate `canRetrySameModel` and `canAdvanceToFallback` values. Auth, model-unavailable, request-incompatible, and Codex invalidation skip same-model retry and can advance. Rate/network/provider failures use the existing bounded exponential retry before fallback. Empty completions retain their existing special behavior. +- `_beginFallbackModelScope`, `_clearFallbackModelScope`, and `_restoreFallbackModel` at `:197-302` own the per-turn lifetime. `_trySwitchToFallbackModel` at `:304-386` records the origin, candidate departure, history entries, thinking-level changes, model events, and asynchronous continuation. A restore emits a model change/select and `model_fallback_end`, then clears attempted keys. +- `packages/coding-agent/src/core/agent-session-events.ts:51-78` queues event processing and returns the processing promise to agent-core for protected persistence and for `agent_end` while fallback configuration or an open fallback scope exists. `:80-109` creates the retry wait for any retryable/fallbackable/empty/refusal outcome. `:124-294` performs the classification, retry/fallback handling, compaction, and restore boundary. +- `packages/coding-agent/src/core/agent-session-prompt.ts:121-123` restores before the next idle prompt. `packages/coding-agent/src/core/agent-session-models.ts:90-209` clears the scope when an explicit model, cycle, or thinking-level choice is made. `packages/coding-agent/src/core/agent-session-tool-hooks.ts:96-132` restores before eligible queued follow-up processing. `packages/coding-agent/src/core/agent-session-auto-compaction.ts:266-325` carries the fallback generation through the post-compaction continuation and restores when the same-turn probe finishes or before queued work. +- Current review risks are lifecycle timing and observability: returning `agent_end` processing for every configured fallback can alter settlement timing for ordinary turns and can couple extension work to agent-core; no dedicated slow-listener/deadlock regression was found. The fallback-end event is now tied to an open fallback scope, so an exhausted chain that never selected a candidate may not emit the former unconditional failure event. Restore ordering around nested post-compaction continuation is complex and lacks a dedicated regression. + +### Workflow candidate walk and retry seam + +- `packages/workflows/src/runs/foreground/stage-runner-controller.ts:322-395` is the main candidate walk. It resolves candidates, resumes an existing session when applicable, creates a candidate session, prompts it, scans terminal assistant failures, records model attempts, and calls `handleCandidateFailure` before advancing. +- `createInitialSession` at `:657-684` uses a rejection-only continuation around the first creation attempt to preserve fast attached-stream timing while entering `createInitialSessionWithRetry`. `createSessionWithThrownErrorRetry` at `:707-756` applies the shared retry decision to creation failures and returns a pause result when a retry is paused. +- `promptWithThrownErrorRetry` at `:543-631` applies the same policy to prompt throws. It captures message snapshots/checkpoints, retries via `_runAgentContinue` when the prompt was admitted, restores failed inputs, waits with `sleepOrAbort`, handles pause resume text, and stops on workflow abort/disposal or structured-output capture. After retry exhaustion, control returns to the candidate walk and `handleCandidateFailure` (`:983-1012`) records the failed attempt, disposes the current session, and advances to the next candidate when the error is fallbackable. +- `packages/workflows/src/runs/shared/retry.ts:1-57` is the one workflow retry policy location. `thrownRetryDecision` checks settings, enabled state, maximum retries, and a supplied classifier; `sleepOrAbort` provides abort-aware bounded delay. `packages/workflows/src/durable/tool-primitive.ts:19-30` now imports and re-exports that helper instead of carrying a duplicate implementation. +- `packages/workflows/src/extension/atomic-stage-session.ts:10-52` extends the settings-manager shape with optional retry settings and a callback for observing the manager. `packages/workflows/src/extension/wiring.ts:134-198` captures the manager before SDK session creation and attaches it to a thrown error when creation fails. `stage-runner-controller.ts:98-105` reads that private hint. +- `packages/workflows/src/runs/foreground/stage-runner-context.ts:235-238` routes `ctx.abort()` through the controller so pending retry delays and pause waiters can be cancelled. `stage-runner-controller.ts:427-482` tracks abort generations and rejects pending pause/retry state. `stage-runner-pause.ts` and executor files were inspected during review but are not modified in this worktree. +- Current workflow risk: an eager `ensureSession()` creation failure without an explicit model fallback still uses the initial-session retry path and can throw after exhaustion without entering a candidate walk. The inherited implementation has no test proving that this eager failure advances to a configured fallback. Retry-settings manager precedence (`sessionSettingsManager`, attached session manager, explicit stage options) is also not pinned by a differing-manager test. The new `ctx.abort()` behavior rejects a pending pause and increments the creation generation; existing interrupt/pause callers need regression coverage. + +### Durable rollback seam + +- `packages/coding-agent/src/core/session-manager-types.ts:6-9` defines reserved custom type `atomic:retry-rollback`. +- `packages/coding-agent/src/core/session-manager-core.ts:228-261` adds `rollbackMessageEntriesSince(checkpointLeafId)`. It finds assistant error entries after the checkpoint, records their IDs in an append-only custom entry, and returns the messages for live-state cleanup. +- `packages/coding-agent/src/core/session-manager-history.ts:21-39,129-190` filters rollback marker entries and their target IDs while rebuilding the active branch for context and session settings. `packages/coding-agent/docs/session-format.md:274-278` documents the marker and reopen behavior. +- `agent-session-methods.ts:253-264,394-404` exposes the private rollback and fallback fields to the internal surface. The workflow controller calls rollback only when a retry will actually happen (`stage-runner-controller.ts:554-598`), while lightweight adapters use in-memory message restoration. +- No direct native `SessionManager` regression was found for checkpoint, multiple assistant errors, rollback, successful retry, and reopen. The debugger specifically reported that rollback durability/reopen behavior remains untested. The downstream orchestrator should validate this before treating the persistence change as complete. + +### Tests, docs, and package metadata + +- `test/unit/main-chat-model-fallback.test.ts:36-206` covers structured diagnostics, model-unavailable fallbackability, Codex invalidation split, candidate switching, restore events, and reasoning-level behavior. +- `test/unit/stage-runner-thrown-retry.test.ts:46-504` covers same-session prompt retry, message restoration, concurrent user messages, retry exhaustion and candidate advancement, session creation retry, disabled retry, auth/request-incompatible throws, structured-output suppression, abort, and pause/resume. It contains 14 tests in the recorded run. +- `test/unit/model-fallback-classifier-conformance.test.ts:373-384` asserts companion export identity and fixture parity. +- User-facing docs were updated in `packages/coding-agent/docs/settings.md`, `docs/workflows.md`, `docs/subagents.md`, and `docs/session-format.md`. Changelog entries were added under `[Unreleased]` in coding-agent, workflows, and subagents with issue attribution. +- The new public helper names in `packages/coding-agent/src/index.ts` are broad (`errorMessage`, `modelFailureMessage`) and `docs/sdk.md` does not document them. This is a public-surface decision for downstream validation. + +## Validation recorded before handoff + +The following results are evidence from this research stage; they are not a substitute for a fresh downstream validation run. + +- `npm run check`: passed after formatting. Biome reported no errors, TypeScript typecheck passed, and the coding-agent shrinkwrap check passed. +- Targeted root Vitest: passed. `model-fallback-classifier-conformance.test.ts` (3), `main-chat-model-fallback.test.ts` (9), `stage-runner-thrown-retry.test.ts` (14), `executor-stage-control-registry-1.test.ts` (8), and `executor-stage-control-registry-4.test.ts` (4) passed. A broader stage-runner/executor glob passed 238 tests across 27 files. +- Targeted coding-agent Vitest: passed 27 tests across `agent-session-retry-events.test.ts`, `6904-dns-transport-retry.test.ts`, and `agent-session-retry.test.ts`. +- `npm run test:unit`: failing overall in the recorded review: 5,836 passed, 2 skipped, 16 failed. The reviewer reproduced the same 16 failures on a clean `HEAD` worktree and classified them as pre-existing workflow reload/durable-tool behavior. This still leaves the required check non-green until the downstream orchestrator confirms the baseline and reports it. +- `npm run test --workspace=@bastani/atomic`: not successfully completed as the full workspace command. An attempted command passed an extra `--run` to a package script that already contains `vitest --run`, and Vitest exited with `Expected a single value for option "--run"`. Direct targeted `npx vitest` in `packages/coding-agent` passed 27 tests. A fresh full workspace run remains required. +- A transient intermediate typecheck failure (`isRetryableAssistantError` missing) was later cleared; the recorded final `npm run check` passed. + +## PRE-EXISTING IMPLEMENTATION IN WORKTREE + +These are inherited, uncommitted changes present before the downstream implementation stage. The downstream orchestrator must treat these uncommitted changes as inherited work to validate, complete, test, and commit — not as its own starting point to redo from scratch. + +### Coding-agent package + +| File | Inherited change and direction | State, gaps, and validation need | +|---|---|---| +| `packages/coding-agent/CHANGELOG.md` | Adds an `[Unreleased]` fixed entry for main-chat classifier convergence, Codex fallback, and turn restore. **Classifier; switch restore.** | Entry is present and issue-linked. Verify wording against final shipped behavior. | +| `packages/coding-agent/docs/session-format.md` | Documents reserved `atomic:retry-rollback` custom entries and reopen filtering. **Retry-then-advance support.** | Present. Must remain aligned with any rollback format changes and native-session regression. | +| `packages/coding-agent/docs/settings.md` | Documents fallback restoration before the next turn and explicit `/model` cancellation of restore. **Switch restore.** | Present. Validate event/lifecycle wording against final behavior. | +| `packages/coding-agent/docs/subagents.md` | Notes the shared classifier and consistent auth/availability/request/transport handling. **Classifier.** | Present. Validate public helper/package boundaries. | +| `packages/coding-agent/docs/workflows.md` | Documents same-candidate thrown retry using `settings.retry` and immediate advance when disabled. **Retry-then-advance.** | Present. Confirm eager session-creation semantics are documented if retained. | +| `packages/coding-agent/src/core/agent-session-auto-compaction.ts` | Carries fallback scope generation through post-compaction continuation, restores before queued work, and clears/awaits continuation state. **Switch restore.** | Implemented but nested continuation ordering is untested. Add or inspect a regression for restore during compaction continuation. | +| `packages/coding-agent/src/core/agent-session-events.ts` | Adds fallbackable classification, creates retry waits for fallbackable errors, adjusts event-await behavior for fallback scopes, and restores after compaction/turn completion. **Classifier; switch restore.** | Targeted tests pass. Slow extension listener, ordinary-turn timing, and fallback-end exhaustion event behavior are not directly tested. | +| `packages/coding-agent/src/core/agent-session-methods.ts` | Adds internal methods/fields for fallback scope and durable prompt rollback. **Classifier; switch restore; retry-then-advance.** | Type surface is present and typecheck passed. Validate private-surface compatibility. | +| `packages/coding-agent/src/core/agent-session-models.ts` | Clears a pending fallback restore when explicit model/cycle/thinking-level selection occurs. **Switch restore.** | Targeted restore behavior passes. Explicit `/model` during asynchronous restore needs an integration-level race test. | +| `packages/coding-agent/src/core/agent-session-prompt.ts` | Restores the origin model before validating credentials for a new idle prompt. **Switch restore.** | Present. Verify it does not restore while the same turn is still streaming or while a continuation owns the turn. | +| `packages/coding-agent/src/core/agent-session-retry.ts` | Splits `_isFallbackableError` and `_isRetryableError`, handles Codex invalidation, records per-turn origin state, switches candidates, restores model/thinking level, emits lifecycle/history events, and keeps same-model retry logic. **Classifier; switch restore.** | Core implementation and targeted tests pass. Review shared classifier carve-out, missing fallback-end event when no scope exists, and async restore races. | +| `packages/coding-agent/src/core/agent-session-tool-hooks.ts` | Restores fallback before eligible queued follow-up messages and waits for the event queue. **Switch restore.** | Present. Coupling `prepareNextTurnWithContext` to event listeners lacks a deadlock/timing test. | +| `packages/coding-agent/src/core/agent-session.ts` | Adds origin model/thinking-level, generation, restore-error, and scope state fields. **Switch restore.** | Present and typechecked. Validate initialization and reload/session reuse paths. | +| `packages/coding-agent/src/core/session-manager-core.ts` | Adds append-only rollback marker creation and live failed-message collection. **Retry-then-advance.** | Present. Native durable retry/reopen behavior is not directly tested. | +| `packages/coding-agent/src/core/session-manager-history.ts` | Filters rollback markers and target message IDs from active context reconstruction. **Retry-then-advance.** | Present. It scans the active branch on each context build; test compaction/export/resume interactions. | +| `packages/coding-agent/src/core/session-manager-types.ts` | Defines `RETRY_ROLLBACK_CUSTOM_TYPE = "atomic:retry-rollback"`. **Retry-then-advance.** | Present and documented. Check reserved-type compatibility and malformed marker handling. | +| `packages/coding-agent/src/index.ts` | Exports classifier types and helpers, including `errorMessage`, `modelFailureMessage`, `isSafetyRefusalFailure`, and `normalizeModelFailureSignal`. **Classifier.** | Public exports typecheck. `docs/sdk.md` has no matching API documentation; decide whether this surface is intended. | + +### Subagents and workflows packages + +| File | Inherited change and direction | State, gaps, and validation need | +|---|---|---| +| `packages/subagents/CHANGELOG.md` | Adds an `[Unreleased]` classifier-sharing fix entry. **Classifier.** | Present and issue-linked. Verify it describes shipped behavior only. | +| `packages/subagents/src/runs/shared/model-fallback.ts` | Removes the duplicate classifier and re-exports the coding-agent implementation while retaining subagent candidate formatting. **Classifier.** | Identity test passes. Raw-package loading and published-package dependency boundary need a fresh package-level check. | +| `packages/workflows/CHANGELOG.md` | Adds an `[Unreleased]` thrown-retry/fallback timing entry. **Retry-then-advance.** | Present and issue-linked. Verify final retry/creation behavior. | +| `packages/workflows/src/durable/tool-primitive.ts` | Imports/re-exports shared `sleepOrAbort` and removes the local copy. **Retry-then-advance support.** | Typecheck passed. This changes a helper shared with durable tools; abort reason behavior needs existing durable-tool regression coverage. | +| `packages/workflows/src/extension/atomic-stage-session.ts` | Adds optional retry settings to the SDK manager projection and an `onSettingsManager` preparation callback. **Retry-then-advance.** | Present. Check adapter compatibility when managers omit retry settings. | +| `packages/workflows/src/extension/wiring.ts` | Captures the settings manager before SDK session creation and attaches it to creation errors, wrapping non-extensible/frozen values with a cause. **Retry-then-advance.** | Present and typechecked. Error mutation/private hint behavior should be reviewed for frozen, proxy, primitive, and serialization cases. | +| `packages/workflows/src/runs/foreground/stage-runner-context.ts` | Routes `ctx.abort()` through `StageSessionController.abort()` instead of directly aborting the current session. **Retry-then-advance support.** | Needed to cancel retry waits and pause state, but changes pause/interrupt semantics. Existing pause-then-interrupt callers need regression coverage. | +| `packages/workflows/src/runs/foreground/stage-runner-controller.ts` | Adds shared thrown retry policy to prompt and session creation, pause/resume/abort generation handling, rollback restoration, settings-manager lookup, initial creation fast path, and candidate-walk retry integration. **Retry-then-advance.** | 14 thrown-retry tests pass. Eager creation exhaustion may bypass fallback candidate advancement; manager precedence, terminal rollback, and session replacement races need validation. | +| `packages/workflows/src/runs/foreground/stage-runner-types.ts` | Adds `WorkflowRetrySettings` and optional `getRetrySettings()` to the stage settings-manager projection. **Retry-then-advance.** | Present and typechecked. Validate raw TypeScript consumers with older manager shapes. | +| `packages/workflows/src/runs/shared/model-fallback-failures.ts` | Replaces the workflow classifier copy with compatibility exports from `@bastani/atomic`. **Classifier.** | Present; identity conformance passes. Verify package resolution in raw workflow execution and bundled Atomic use. | +| `packages/workflows/src/runs/shared/retry.ts` | Adds the single workflow thrown-retry decision and abort-aware sleep policy. **Retry-then-advance.** | Present; targeted tests exercise controller use. Add direct policy edge tests for zero/negative settings and abort reasons if needed. | + +### Tests and scratch artifacts + +| File | Inherited change and direction | State, gaps, and validation need | +|---|---|---| +| `test/unit/main-chat-model-fallback.test.ts` | Adds tests for fallback-vs-retry classification, Codex invalidation, candidate switching, turn restore, lifecycle events, and reasoning levels. **Classifier; switch restore.** | 9 tests pass. Does not cover explicit `/model` race, compaction restore, ordinary-turn event settlement, or no-candidate fallback-end behavior. | +| `test/unit/model-fallback-classifier-conformance.test.ts` | Changes conformance from parallel-copy comparison to export identity plus a shared fixture corpus; adds nested transport cancellation/refusal fixtures. **Classifier.** | 3 tests pass. Corpus is finite; shared implementation behavior changes still need review. | +| `test/unit/stage-runner-thrown-retry.test.ts` | Adds 14 workflow prompt/session-creation retry tests, message rollback, fallback advance, disabled retry, structured output, abort, and pause/resume. **Retry-then-advance.** | 14 tests pass. It does not cover eager `ensureSession` creation exhaustion advancing to a fallback, differing retry-manager precedence, native durable rollback/reopen, or all pause/interrupt races. | +| `issues.md` | Untracked implementation notes created during debugging. It records current fallback/retry work and earlier typecheck/test observations. **Supporting artifact; all directions.** | Scratch artifact. It is intentionally preserved under the supervisor amendment; downstream hygiene may remove it only after incorporating its findings. | +| `progress.md` | Untracked empty progress template. **Supporting artifact.** | No implementation content. It is intentionally preserved because this research stage was instructed to leave the worktree unchanged; downstream cleanup can decide its fate. | + +## Open Questions for the downstream orchestrator + +1. Does the intended contract require the shared classifier to preserve exact pre-change behavior, or are the added nested-priority, transport-wrapper, and message-precedence changes accepted as part of convergence? +2. Should `agent_end` await fallback reconciliation only while a fallback scope is active, rather than for every session with configured fallback models? +3. What should happen to `model_fallback_end` when no usable candidate ever starts a scope? +4. Can an eagerly created primary session that exhausts thrown creation retries enter the normal candidate walk and advance to its fallback without losing attached-stream timing? +5. What is the authoritative retry-settings manager when the attached session and explicit stage options differ? +6. What native-session test proves rollback marker durability across retry success and `SessionManager.open()`? +7. Are the new classifier helpers intended as public SDK API, and if so should `packages/coding-agent/docs/sdk.md` document them? +8. Which of the 16 root unit failures are accepted baseline failures, and can the required full commands be rerun without masking regressions? + +## Historical Context + +No prior research documents were found in the requested `research/` location before this artifact. The retained review reports that the earlier parallel pause/resume hang was caused by an external wait around `promptWithPauseResume`; the current worktree does not show an `executor-stage-call.ts` modification, and the targeted pause/control suites passed. The same review reports that the direct creation retry wrapper was shaped as a rejection-only continuation to preserve attached-stream timing. + +## Related Research + +- `packages/coding-agent/docs/settings.md` — user-facing main-chat fallback settings. +- `packages/coding-agent/docs/workflows.md` — workflow model candidates and retry settings. +- `packages/coding-agent/docs/subagents.md` — subagent fallback model behavior. +- `packages/coding-agent/docs/session-format.md` — append-only session entry format and the inherited rollback marker. + +## Handoff + +This document is the research-only handoff. No source, test, documentation, changelog, or scratch file outside `research/` was edited after the supervisor's course correction. The worktree remains intentionally dirty so the downstream orchestrator can validate and complete the inherited implementation rather than silently losing it. From f4f2fe4cc23bf94c287ff3898e7c2c32dda11c3d Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 13:14:38 -0700 Subject: [PATCH 03/13] fix(workflows): only continue a stage retry from a valid transcript tail (#2170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit restoreSessionMessages() retains non-error assistant messages, so an attempt that streamed a completed assistant before throwing left an assistant tail. pi-agent-core's Agent.continue() rejects that with "Cannot continue from message role: assistant", which turned a recoverable retry into a hard stop. Gate the continuation path on the contract continue() actually states: the restored transcript must be non-empty and must not end in an assistant. When the tail is ineligible the retry drops the retained prompt and re-prompts instead, so the input is re-sent exactly once. Cover the four admitted orderings on a real-AgentSession probe — tool-result tail, non-error assistant tail, pause during backoff, and abort during backoff — and add a main-chat case that drives the production compaction path from a failed overflow compaction through to a real fallback switch, rather than injecting the unresolved-overflow flag. Mark the research ticket superseded and correct the claims that no longer describe the repository. Refs: #2170 Assistant-model: Claude Opus 5 --- .../foreground/stage-runner-controller.ts | 21 ++- ...6-08-04-2170-model-fallback-convergence.md | 28 +++- test/unit/main-chat-model-fallback.test.ts | 93 ++++++++++- test/unit/stage-runner-thrown-retry.test.ts | 145 ++++++++++++++++++ 4 files changed, 282 insertions(+), 5 deletions(-) diff --git a/packages/workflows/src/runs/foreground/stage-runner-controller.ts b/packages/workflows/src/runs/foreground/stage-runner-controller.ts index a2f869d5c..acdcfdd86 100644 --- a/packages/workflows/src/runs/foreground/stage-runner-controller.ts +++ b/packages/workflows/src/runs/foreground/stage-runner-controller.ts @@ -116,6 +116,17 @@ function retryableAgentSession(activeSession: StageSessionRuntime): RetryableAge return typeof candidate._runAgentContinue === "function" ? (session as RetryableAgentSession) : undefined; } +/** + * pi-agent-core's `Agent.continue()` rejects an empty transcript and an + * assistant tail. A retry that restored live state may leave either — a + * non-error assistant admitted during the failed attempt is retained — so the + * continuation path is only eligible when the tail satisfies that contract. + */ +function canContinueFromTranscript(activeSession: StageSessionRuntime): boolean { + const last = activeSession.messages[activeSession.messages.length - 1]; + return last !== undefined && last.role !== "assistant"; +} + class ThrownErrorRetryPaused extends Error { constructor(readonly resume: RetryPauseResume) { super("atomic-workflows: thrown-error retry paused"); @@ -635,7 +646,15 @@ export class StageSessionController { if (this.disposed) throw new Error(`atomic-workflows: stage "${this.opts.stageName}" session has been disposed`); if (this.opts.signal?.aborted) throw this.workflowAbortReason(); - retryAdmittedPrompt = willContinue; + // Only resume the existing turn when the restored transcript still + // satisfies pi-agent-core's continuation contract. A retained + // non-error assistant tail would otherwise be rejected, so fall back + // to re-prompting and drop the input that path re-sends. + retryAdmittedPrompt = willContinue && canContinueFromTranscript(activeSession); + if (!retryAdmittedPrompt) { + this.dropRetainedPrompt(activeSession, retainedPrompt); + retainedPrompt = undefined; + } } } } diff --git a/research/tickets/2026-08-04-2170-model-fallback-convergence.md b/research/tickets/2026-08-04-2170-model-fallback-convergence.md index 3530752e4..f2cff48bc 100644 --- a/research/tickets/2026-08-04-2170-model-fallback-convergence.md +++ b/research/tickets/2026-08-04-2170-model-fallback-convergence.md @@ -2,19 +2,41 @@ date: 2026-08-04 09:27:05 PDT researcher: GPT-5.6 Luna git_commit: 226eacfa71793cd561c9e86724f67ecbf00a2fdc -branch: atomic-issue-2170 +branch: fix/2170-model-fallback-convergence repository: atomic-issue-2170 topic: "Issue #2170 model fallback convergence" tags: [research, codebase, model-fallback, main-chat, workflows, retries] -status: complete +status: superseded last_updated: 2026-08-04 -last_updated_by: GPT-5.6 Luna +last_updated_by: Claude Opus 5 breaking_changes_allowed: false compatibility_context: "The repository publishes @bastani/atomic and raw-TypeScript companion packages. Preserve existing public APIs, session history behavior, workflow controls, and provider retry semantics unless the issue explicitly changes them." --- # Research +> **Status: superseded by the delivered change.** This document is the research +> snapshot that preceded implementation, kept for the seams and rationale it +> records. Everything below it describing the work as uncommitted, listing a +> worktree inventory, or reporting validation counts is a snapshot of that +> moment and no longer describes the repository. +> +> What actually shipped, on branch `fix/2170-model-fallback-convergence`: +> +> - The classifier lives in `packages/coding-agent/src/core/model-fallback-failures.ts`; +> workflows and subagents re-export it. +> - The retry decision and backoff live in `packages/coding-agent/src/core/retry-policy.ts` +> as `nextRetryDecision`, exported from the package index and re-exported by +> `packages/workflows/src/runs/shared/retry.ts`. The `thrownRetryDecision` helper +> named below was replaced by it and no longer exists. +> - The durable `atomic:retry-rollback` session entry described below was dropped as +> out of scope; workflow retry restores live state only, matching main-chat retry. +> - Context overflow advances the fallback chain once compaction is disabled, fails, +> or reports it unresolved. +> - All three gates pass: `npm run check`, `npm run test:unit`, and +> `npm run test --workspace=@bastani/atomic` (the last after +> `npm run build --workspace=@bastani/atomic-natives`). + ## Research Question Document the seams and current implementation for issue #2170: diff --git a/test/unit/main-chat-model-fallback.test.ts b/test/unit/main-chat-model-fallback.test.ts index 4aa1f7c51..08b3cfe4d 100644 --- a/test/unit/main-chat-model-fallback.test.ts +++ b/test/unit/main-chat-model-fallback.test.ts @@ -2,7 +2,10 @@ import assert from "node:assert/strict"; import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; import type { Api, AssistantMessage, Model } from "@earendil-works/pi-ai/compat"; import { test } from "vitest"; -import { _checkCompaction } from "../../packages/coding-agent/src/core/agent-session-auto-compaction.js"; +import { + _checkCompaction, + _runAutoCompaction, +} from "../../packages/coding-agent/src/core/agent-session-auto-compaction.js"; import { _createRetryPromiseForAgentEnd, _processAgentEvent, @@ -673,3 +676,91 @@ test("compaction disabled leaves an ordinary provider error resolvable by same-m assert.equal(session._contextOverflowUnresolved, false); }); + +test("a failed overflow compaction advances the real fallback chain end to end", async () => { + // Runs the production compaction path — _checkCompaction into + // _runAutoCompaction — and the production fallback switch, rather than + // injecting the unresolved flag. + const primary = model("openai-codex", "gpt-5.5"); + const fallback = model("anthropic", "claude-opus-4-8"); + const overflow = overflowMessage(); + const events: Array<{ type: string; [key: string]: unknown }> = []; + let continued = 0; + const session: Record = { + model: primary, + thinkingLevel: "high" as ThinkingLevel, + _lastAssistantMessage: overflow, + _protectedStreamingCustomMessages: [], + _postToolCompactionPreflightError: undefined, + _pendingPostToolCompactionGuard: undefined, + _pendingPostCompactionContinuation: undefined, + _contextOverflowUnresolved: false, + _overflowRecoveryAttempted: false, + _fallbackModels: ["anthropic/claude-opus-4-8:high"], + _fallbackAttemptedKeys: new Set(), + _retryAttempt: 0, + settingsManager: { + getCompactionSettings: () => ({ enabled: true }), + getDefaultThinkingLevel: () => "high" as ThinkingLevel, + getDefaultProvider: () => "openai-codex", + }, + _modelRuntime: { + getAvailableSnapshot: () => [primary, fallback], + getModel: (provider: string, id: string) => + provider === fallback.provider && id === fallback.id ? fallback : undefined, + hasConfiguredAuth: () => true, + }, + agent: { + state: { model: primary, thinkingLevel: "high" as ThinkingLevel, messages: [overflow] }, + continue: async () => { + continued += 1; + }, + }, + sessionManager: { + getBranch: () => [], + appendModelChange: (provider: string, id: string) => events.push({ type: "session_model", provider, id }), + appendThinkingLevelChange: () => undefined, + }, + // The compaction planner fails, which is what makes the overflow unresolved. + _applyVerbatimCompaction: async () => { + throw new Error("planner unavailable"); + }, + _getRequiredRequestAuth: async () => ({}), + _dropTrailingAutoCompactionRetryAssistantIfPresent: () => undefined, + _schedulePostAutoCompactionContinuationProbe: () => undefined, + _withContextWindowForModelSwitch: (candidate: Model) => candidate, + _refreshBaseSystemPromptFromActiveTools: () => undefined, + _applyInterruptAbortMessage: () => undefined, + _applyProviderErrorGuidance: () => undefined, + _emitExtensionEvent: async () => undefined, + _emit: (event: { type: string; [key: string]: unknown }) => events.push(event), + _emitModelChanged: () => undefined, + _emitModelSelect: async () => undefined, + _resolveRetry: () => undefined, + _isFallbackableError, + _isRetryableError, + _isEmptyCompletion: () => false, + _isSafetyRefusal: () => false, + _handleRetryableError: async () => false, + _restoreFallbackModel: async () => false, + // Production compaction and fallback implementations. + _checkCompaction, + _runAutoCompaction, + _trySwitchToFallbackModel, + }; + + await _processAgentEvent.call(session as never, { type: "agent_end" } as never); + await new Promise((resolve) => setTimeout(resolve, 5)); + + assert.ok( + events.some((event) => event.type === "compaction_end" && event.unresolvedOverflow === true), + "a failed overflow compaction must report the overflow unresolved", + ); + assert.ok( + events.some((event) => event.type === "model_fallback_start" && event.to === "anthropic/claude-opus-4-8"), + "the unresolved overflow must reach the fallback chain", + ); + assert.equal((session.agent as { state: { model: Model } }).state.model, fallback); + assert.equal(continued, 1); + assert.equal(session._contextOverflowUnresolved, false); +}); diff --git a/test/unit/stage-runner-thrown-retry.test.ts b/test/unit/stage-runner-thrown-retry.test.ts index 34fc69409..0c909d159 100644 --- a/test/unit/stage-runner-thrown-retry.test.ts +++ b/test/unit/stage-runner-thrown-retry.test.ts @@ -587,3 +587,148 @@ describe("shared retry policy", () => { assert.equal(workflowsNextRetryDecision({ enabled: true, maxRetries: 3, baseDelayMs: 250 }, 0, false), undefined); }); }); + +/** + * Real-AgentSession-shaped probe: `asAgentSession()`/`retryableAgentSession()` + * accept it, so tests reach the `_runAgentContinue()` continuation branch that + * lightweight adapters skip. + */ +function realSessionProbe( + settings: ReturnType, + options: { + readonly failCalls: number; + /** Extra messages the failing attempt admits after its user prompt. */ + readonly admit?: (messages: StageSessionRuntime["messages"], call: number) => void; + }, +): { + readonly agentSession: AgentSessionAdapter; + readonly messages: StageSessionRuntime["messages"]; + readonly continuedTranscripts: Array; + readonly promptTexts: string[]; +} { + const messages: StageSessionRuntime["messages"] = []; + const continuedTranscripts: Array = []; + const promptTexts: string[] = []; + let call = 0; + const agentSession: AgentSessionAdapter = { + async create() { + return sessionWithSettings( + settings, + async (text) => { + call += 1; + promptTexts.push(text); + messages.push({ role: "user", content: text, timestamp: Date.now() } as never); + options.admit?.(messages, call); + if (call > options.failCalls) return "ok"; + messages.push({ + role: "assistant", + stopReason: "error", + errorMessage: "503 service unavailable", + content: [], + } as never); + throw new Error("503 service unavailable"); + }, + { + messages, + getLastAssistantText: () => "ok", + state: { messages }, + sessionManager: {}, + modelRuntime: {}, + getContextUsage: () => ({}), + _runAgentContinue: async () => { + continuedTranscripts.push([...messages]); + messages.push({ + role: "assistant", + stopReason: "stop", + content: [{ type: "text", text: "ok" }], + } as never); + }, + } as unknown as Partial, + ); + }, + }; + return { agentSession, messages, continuedTranscripts, promptTexts }; +} + +function userMessagesWithText(messages: StageSessionRuntime["messages"], text: string): number { + return messages.filter((message) => message.role === "user" && message.content === text).length; +} + +describe("createStageContext — continuation eligibility across admitted orderings", () => { + test("a tool-result tail is a valid continuation and keeps the same turn", async () => { + const probe = realSessionProbe(retrySettings(), { + failCalls: 1, + admit: (messages, call) => { + if (call === 1) messages.push({ role: "toolResult", content: "tool output" } as never); + }, + }); + const ctx = createStageContext( + makeOpts({ adapters: { agentSession: probe.agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("do it"), "ok"); + assert.deepEqual(probe.promptTexts, ["do it"], "a valid tail must not re-prompt"); + assert.equal(probe.continuedTranscripts.length, 1); + const observed = probe.continuedTranscripts[0]!; + assert.equal(observed[observed.length - 1]?.role, "toolResult"); + }); + + test("a retained non-error assistant tail re-prompts instead of an invalid continuation", async () => { + // pi-agent-core rejects continue() with "Cannot continue from message + // role: assistant", so the retry must not take the continuation path. + const probe = realSessionProbe(retrySettings(), { + failCalls: 1, + admit: (messages, call) => { + if (call === 1) { + messages.push({ + role: "assistant", + stopReason: "stop", + content: [{ type: "text", text: "partial" }], + } as never); + } + }, + }); + const ctx = createStageContext( + makeOpts({ adapters: { agentSession: probe.agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("do it"), "ok"); + assert.deepEqual(probe.continuedTranscripts, [], "an assistant tail is not a valid continuation"); + assert.deepEqual(probe.promptTexts, ["do it", "do it"], "the retry must re-send the prompt instead"); + assert.equal(userMessagesWithText(probe.messages, "do it"), 1, "the re-prompt must not duplicate the input"); + }); + + test("a pause during backoff drops the retained prompt and uses the resumed text", async () => { + const probe = realSessionProbe(retrySettings({ baseDelayMs: 1000 }), { failCalls: 1 }); + const ctx = createStageContext( + makeOpts({ adapters: { agentSession: probe.agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + + const prompt = ctx.prompt("do it"); + await flushMicrotasks(); + await ctx.__requestPause(); + await new Promise((resolve) => setTimeout(resolve, 15)); + await ctx.__resume("resumed"); + await flushMicrotasks(); + + assert.equal(await prompt, "ok"); + assert.deepEqual(probe.continuedTranscripts, [], "a resumed prompt does not continue the old turn"); + assert.deepEqual(probe.promptTexts, ["do it", "resumed"]); + assert.equal(userMessagesWithText(probe.messages, "do it"), 0, "the abandoned input must not survive the resume"); + }); + + test("ctx.abort during backoff stops before any continuation", async () => { + const probe = realSessionProbe(retrySettings({ baseDelayMs: 1000 }), { failCalls: 1 }); + const ctx = createStageContext( + makeOpts({ adapters: { agentSession: probe.agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + + const prompt = ctx.prompt("do it"); + await flushMicrotasks(); + await ctx.abort(); + + await assert.rejects(prompt, /stage aborted/); + assert.deepEqual(probe.promptTexts, ["do it"]); + assert.deepEqual(probe.continuedTranscripts, [], "an aborted retry must not resume the turn"); + }); +}); From e6af7e84868350f235b82ee1bedc9d64d4e322e0 Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 13:16:31 -0700 Subject: [PATCH 04/13] docs(changelog): note the stage-retry continuation eligibility fix (#2170) Refs: #2170 Assistant-model: Claude Opus 5 --- packages/workflows/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/workflows/CHANGELOG.md b/packages/workflows/CHANGELOG.md index 216c411f2..97da5e95e 100644 --- a/packages/workflows/CHANGELOG.md +++ b/packages/workflows/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Fixed thrown retryable stage failures to retry the current model with bounded exponential backoff from `settings.retry` before advancing through `fallbackModels`; disabling retry keeps immediate candidate advancement ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). - Fixed a same-candidate stage retry losing the stage prompt. The retry resumed the existing session turn with a transcript that no longer ended in the prompt it was resuming, which the agent rejects; the prompt is now retained for that path and still re-sent on the re-prompt path ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed a same-candidate stage retry stopping outright when the failed attempt had already streamed a completed assistant message. Resuming the existing turn is now attempted only when the transcript still ends in a message the agent can continue from; otherwise the retry re-sends the prompt, which recovers instead of failing ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). ## [0.9.12] - 2026-08-04 From 534dc05199781e64e931f46a3e0464b6fc3ed9d1 Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 14:11:57 -0700 Subject: [PATCH 05/13] fix(fallback): keep the turn-scoped restore across reasoning changes (#2170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setThinkingLevel() cleared the fallback scope on every call, before the isChanging check. That cancelled the pending primary-model restore both when a user changed reasoning level during a fallback turn and when the registry refresh re-applied the current level unchanged, leaving the session stranded on the fallback model. The contract's only stated exception is an explicit /model choice. Drop the clear. When the level actually changes during an open fallback scope, carry it into the scope so the restore returns the primary model without overwriting the reasoning level the user picked — the same rule the model already follows. setModel() and both model cycles still clear the scope, so an explicit /model still cancels the restore. Also add ResourceExhausted to the shared retryable pattern list. Upstream pi-ai retries it, but the shared classifier did not, so a transient gRPC provider error stopped a session with no fallback chain instead of spending its same-model retry budget. It goes in the one shared list rather than a main-chat-only pattern, and the shared conformance corpus covers it so both companion packages assert the same answer. Refs: #2170 Assistant-model: Claude Opus 5 --- packages/coding-agent/CHANGELOG.md | 5 +- packages/coding-agent/docs/settings.md | 2 + packages/coding-agent/docs/workflows.md | 2 +- .../src/core/agent-session-models.ts | 7 +- .../src/core/model-fallback-failures.ts | 2 + test/unit/main-chat-model-fallback.test.ts | 94 +++++++++++++++++++ ...el-fallback-classifier-conformance.test.ts | 7 ++ 7 files changed, 114 insertions(+), 5 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 0486a714a..f69810542 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,12 +4,11 @@ ### Fixed -<<<<<<< HEAD - Workflow runs awaiting human input now use the blue `？` indicator in the BACKGROUND panel, `/workflow connect` picker, and `/workflow status` listing, including prompts raised by hidden nested workflow children; the indicator returns to the run's current state when the prompt resolves. -======= - Fixed main-chat model fallback to classify provider failures consistently with workflows, advance rejected credentials and incompatible or unavailable models to the next candidate, and restore the user-selected model at the next turn without overriding an explicit `/model` choice ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). - Fixed a context overflow that compaction cannot resolve to advance the configured `fallbackModels` chain instead of ending the turn, so a larger-context candidate can answer. Compaction still runs first, and a compactable overflow spends no fallback candidate ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). ->>>>>>> d51dd122a (fix(fallback): converge main-chat and workflow model fallback (#2170)) +- Fixed a reasoning-level change during a model fallback stranding the session on the fallback model. Changing reasoning effort is not a model choice, so it no longer cancels the pending restore; the next turn returns to the user-selected primary and keeps the reasoning level that was chosen. Only an explicit `/model` selection or model cycle cancels the restore ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed gRPC `ResourceExhausted` provider errors, seen from providers such as NVIDIA NIM, bypassing the same-model auto-retry budget. They are now retried like other transient provider failures, matching upstream pi-ai, and still advance a configured fallback chain when the retries are exhausted ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). ## [0.9.12] - 2026-08-04 diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index 2edee9f84..050e01063 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -60,6 +60,8 @@ Settings and trust JSON files may start with a UTF-8 BOM, as commonly written by Context overflow keeps its normal recovery order: compaction runs first, and a compactable overflow costs no fallback candidate. Only once compaction is disabled, fails, or reports the overflow unresolved does Atomic advance to the next configured candidate, which is how a larger-context model gets a chance at the turn. +Changing the reasoning level during a fallback turn is not a model choice, so it does not cancel the restore: the next turn still starts on the user-selected primary, carrying the reasoning level you picked. Only an explicit `/model` selection or model cycle cancels it. + The same list is also **borrowed by compaction**. When the compaction range planner cannot produce a usable plan on the current model — a rate limit, quota exhaustion, provider error, context overflow, or an empty plan — Atomic runs one planner request against the next configured candidate, using that candidate's own credentials. **A configured fallback model may therefore receive the compaction transcript.** Borrowing is planner-only: it never changes the session model, thinking level, or model history, it appends no model-change entry, and it emits no fallback status. See [Compaction](/compaction#planning-rungs-and-failure-behavior). Fallback entries should be fully qualified `provider/model` ids. Add a reasoning suffix to a candidate to override the effort for that fallback only; valid suffixes are `:off`, `:minimal`, `:low`, `:medium`, `:high`, `:xhigh`, and `:max`. Atomic clamps or hides levels that the selected model's capability map does not support. diff --git a/packages/coding-agent/docs/workflows.md b/packages/coding-agent/docs/workflows.md index d0bc2c7cd..807d38d4f 100644 --- a/packages/coding-agent/docs/workflows.md +++ b/packages/coding-agent/docs/workflows.md @@ -2295,7 +2295,7 @@ readonly fallbackModels?: readonly string[]; readonly fallbackThinkingLevels?: readonly string[]; ``` -`fallbackModels` tries the primary first, each fallback in order, and then the current Atomic-selected model when available. It advances for rate limits and quota or usage-limit exhaustion, including messages such as `The usage limit has been reached` and codes such as `usage_limit_reached` or `insufficient_quota`. Auth/provider outages, unavailable models, network timeouts, generic transport errors such as `Connection error.` or `fetch failed`, and 5xx responses also advance the chain. A thrown retryable failure is retried on the same candidate with exponential backoff from `settings.retry` before the chain advances; `retry.enabled: false` keeps immediate advancement. +`fallbackModels` tries the primary first, each fallback in order, and then the current Atomic-selected model when available. It advances for rate limits and quota or usage-limit exhaustion, including messages such as `The usage limit has been reached` and codes such as `usage_limit_reached` or `insufficient_quota`. Auth/provider outages, unavailable models, network timeouts, generic transport errors such as `Connection error.` or `fetch failed`, and 5xx responses also advance the chain. A thrown retryable failure is retried on the same candidate with exponential backoff from `settings.retry` before the chain advances; `retry.enabled: false` keeps immediate advancement. A same-candidate retry resumes the existing turn when the stage transcript still ends in a message the agent can continue from, and otherwise re-sends the stage prompt; either way the failed provider error is dropped from the live transcript and the prompt is delivered exactly once. Request/context incompatibility also advances it, including HTTP 400/413/422 bad, unprocessable, or payload-too-large requests; unsupported tools or parameters; context-length or context-window overflow; and `too large`, `invalid_request`, or `bad_request` errors. This lets the chain reach the current selected user model when no configured candidate can serve the request. diff --git a/packages/coding-agent/src/core/agent-session-models.ts b/packages/coding-agent/src/core/agent-session-models.ts index 22a876378..952e91fd3 100644 --- a/packages/coding-agent/src/core/agent-session-models.ts +++ b/packages/coding-agent/src/core/agent-session-models.ts @@ -205,7 +205,6 @@ export async function _cycleAvailableModel( */ export function setThinkingLevel(this: AgentSession, level: ThinkingLevel): void { - if (this._fallbackOriginModel !== undefined) this._clearFallbackModelScope?.(); const availableLevels = this.getAvailableThinkingLevels(); const effectiveLevel = availableLevels.includes(level) ? level : this._clampThinkingLevel(level, availableLevels); @@ -216,6 +215,12 @@ export function setThinkingLevel(this: AgentSession, level: ThinkingLevel): void this.agent.state.thinkingLevel = effectiveLevel; if (isChanging) { + // A reasoning choice is not a model choice, so it must not strand the + // session on a fallback candidate: keep the pending restore. Carry the + // explicit level into the scope so the restore does not overwrite it. + // (A no-op level assignment — a registry refresh re-applying the current + // level — changes nothing here.) + if (this._fallbackOriginModel !== undefined) this._fallbackOriginThinkingLevel = effectiveLevel; this.sessionManager.appendThinkingLevelChange(effectiveLevel); this._refreshBaseSystemPromptFromActiveTools(); if (this.supportsThinking() || effectiveLevel !== "off") { diff --git a/packages/coding-agent/src/core/model-fallback-failures.ts b/packages/coding-agent/src/core/model-fallback-failures.ts index 672434f4d..cad84adf6 100644 --- a/packages/coding-agent/src/core/model-fallback-failures.ts +++ b/packages/coding-agent/src/core/model-fallback-failures.ts @@ -61,6 +61,8 @@ const RETRYABLE_MODEL_FAILURE_PATTERNS: readonly RegExp[] = [ /you can retry your request/i, /try your request again/i, /please retry your request/i, + // gRPC based providers (e.g. NVIDIA NIM); upstream pi-ai retries this too. + /ResourceExhausted/i, ]; const NON_RETRYABLE_FAILURE_PATTERNS: readonly RegExp[] = [ diff --git a/test/unit/main-chat-model-fallback.test.ts b/test/unit/main-chat-model-fallback.test.ts index 08b3cfe4d..ae119e44e 100644 --- a/test/unit/main-chat-model-fallback.test.ts +++ b/test/unit/main-chat-model-fallback.test.ts @@ -10,6 +10,7 @@ import { _createRetryPromiseForAgentEnd, _processAgentEvent, } from "../../packages/coding-agent/src/core/agent-session-events.js"; +import { setThinkingLevel } from "../../packages/coding-agent/src/core/agent-session-models.js"; import { _clearFallbackModelScope, _handleRetryableError, @@ -764,3 +765,96 @@ test("a failed overflow compaction advances the real fallback chain end to end", assert.equal(continued, 1); assert.equal(session._contextOverflowUnresolved, false); }); + +/** Session double sufficient for `_trySwitchToFallbackModel` and `setThinkingLevel`. */ +function thinkingLevelFallbackSession( + primary: Model, + fallback: Model, + events: Array<{ type: string; [key: string]: unknown }>, +): Record { + return { + model: primary, + thinkingLevel: "high" as ThinkingLevel, + _fallbackModels: ["anthropic/claude-opus-4-8:high"], + _fallbackAttemptedKeys: new Set(), + _retryAttempt: 0, + settingsManager: { + getDefaultThinkingLevel: () => "high" as ThinkingLevel, + getDefaultProvider: () => "openai-codex", + setDefaultThinkingLevel: (level: ThinkingLevel) => events.push({ type: "settings_thinking", level }), + }, + _modelRuntime: { + getAvailableSnapshot: () => [primary, fallback], + getModel: (provider: string, id: string) => + provider === fallback.provider && id === fallback.id ? fallback : undefined, + hasConfiguredAuth: () => true, + }, + agent: { + state: { model: primary, thinkingLevel: "high" as ThinkingLevel, messages: [retryableMessage()] }, + continue: async () => undefined, + }, + sessionManager: { + appendModelChange: (provider: string, id: string) => events.push({ type: "session_model", provider, id }), + appendThinkingLevelChange: (level: ThinkingLevel) => events.push({ type: "session_thinking", level }), + }, + getAvailableThinkingLevels: () => ["off", "low", "medium", "high"] as ThinkingLevel[], + _clampThinkingLevel: (level: ThinkingLevel) => level, + supportsThinking: () => true, + _extensionRunner: { emit: async () => undefined }, + _withContextWindowForModelSwitch: (candidate: Model) => candidate, + _refreshBaseSystemPromptFromActiveTools: () => undefined, + _emitModelChanged: (next: Model, previous: Model | undefined, source: string) => + events.push({ type: "model_changed", next: next.id, previous: previous?.id, source }), + _emitModelSelect: async () => undefined, + _emit: (event: { type: string; [key: string]: unknown }) => events.push(event), + _clearFallbackModelScope, + }; +} + +test("a reasoning-level change during a fallback still restores the primary model", async () => { + const primary = model("openai-codex", "gpt-5.5"); + const fallback = model("anthropic", "claude-opus-4-8"); + const events: Array<{ type: string; [key: string]: unknown }> = []; + const session = thinkingLevelFallbackSession(primary, fallback, events); + const state = (session.agent as { state: { model: Model; thinkingLevel: ThinkingLevel } }).state; + + assert.equal(await _trySwitchToFallbackModel.call(session as never, retryableMessage()), true); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(state.model, fallback); + + // A reasoning choice is not a model choice and must not cancel the restore. + setThinkingLevel.call(session as never, "low" as ThinkingLevel); + + assert.equal(await _restoreFallbackModel.call(session as never), true); + assert.equal(state.model, primary, "a reasoning change must not strand the session on the fallback"); + assert.equal(state.thinkingLevel, "low", "the explicit reasoning choice must survive the restore"); +}); + +test("a registry refresh re-applying the current level leaves the pending restore intact", async () => { + const primary = model("openai-codex", "gpt-5.5"); + const fallback = model("anthropic", "claude-opus-4-8"); + const events: Array<{ type: string; [key: string]: unknown }> = []; + const session = thinkingLevelFallbackSession(primary, fallback, events); + const state = (session.agent as { state: { model: Model; thinkingLevel: ThinkingLevel } }).state; + + assert.equal(await _trySwitchToFallbackModel.call(session as never, retryableMessage()), true); + await new Promise((resolve) => setTimeout(resolve, 5)); + + // agent-session-extension-bindings re-applies the current level on registry + // refresh, which must be a no-op for the fallback scope. + setThinkingLevel.call(session as never, state.thinkingLevel); + + assert.equal(await _restoreFallbackModel.call(session as never), true); + assert.equal(state.model, primary); + assert.equal(state.thinkingLevel, "high"); +}); + +test("a gRPC ResourceExhausted failure is both same-model retryable and fallbackable", () => { + // Upstream pi-ai retries this; the shared classifier must agree so a + // session with no fallback chain still spends its same-model retry budget. + const session = { model: model("openai", "gpt-5.5") }; + const message = retryableMessage({ errorMessage: "ResourceExhausted" }); + + assert.equal(_isRetryableError.call(session as never, message), true); + assert.equal(_isFallbackableError.call(session as never, message), true); +}); diff --git a/test/unit/model-fallback-classifier-conformance.test.ts b/test/unit/model-fallback-classifier-conformance.test.ts index 643d64a97..9e38951de 100644 --- a/test/unit/model-fallback-classifier-conformance.test.ts +++ b/test/unit/model-fallback-classifier-conformance.test.ts @@ -370,6 +370,13 @@ const CONFORMANCE_FIXTURES: readonly Fixture[] = [ kind: "unknown", retryable: false, }, + { + // gRPC based providers (e.g. NVIDIA NIM); upstream pi-ai retries this. + label: "grpc ResourceExhausted", + failure: new Error("ResourceExhausted"), + kind: "provider_unavailable", + retryable: true, + }, ]; describe("model fallback classifier conformance (subagents vs workflows)", () => { From 92d3f15a2aab6ed9422038e42774066d448d3f9d Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 15:35:23 -0700 Subject: [PATCH 06/13] fix(workflows): walk the candidate chain from eager session creation (#2170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four defects in the stage fallback path, all reproduced before fixing. createInitialSession() created only candidates[0] and handed failures to a retry helper that rethrew after exhausting that one candidate. A session created before any prompt — an eager stage call, a control attach, or ctx.__ensureSession() — therefore failed the stage with its configured fallbacks untried, and ensureSession() cached the rejected promise so every later caller replayed the same failure. Add a creation-only version of the prompt candidate walk, and clear the cached promise by identity after a terminal rejection. The explicit-candidate branch of promptWithFallback() never consumed pendingCreationResumeMessage, so a paused eager creation resumed with a replacement objective sent the stale original instead. Consume and clear it before tryResumeCurrentSession() and before the loop. canContinueFromTranscript() checked the raw tail role, but pi-agent-core requires the converted tail to be user or toolResult. Atomic's converter drops a custom, bash-execution, or branch-summary message that is excluded from context or empty, exposing the assistant beneath it. Evaluate the same convertToLlm() result Atomic sends. An already-unresolved context overflow was fed to the shared retry policy as an ordinary retryable failure, so it burned the whole settings.retry budget on a model whose compaction had already failed. Gate same-candidate retry on !isUnresolvedContextOverflowFailure while leaving the shared classifier and the wrapper untouched, so the candidate walk still advances. Refs: #2170 Assistant-model: Claude Opus 5 --- packages/coding-agent/docs/workflows.md | 4 + packages/workflows/CHANGELOG.md | 5 +- .../foreground/stage-runner-controller.ts | 98 +++++++-- test/unit/stage-runner-thrown-retry.test.ts | 194 ++++++++++++++++++ 4 files changed, 288 insertions(+), 13 deletions(-) diff --git a/packages/coding-agent/docs/workflows.md b/packages/coding-agent/docs/workflows.md index 807d38d4f..e94fe85f8 100644 --- a/packages/coding-agent/docs/workflows.md +++ b/packages/coding-agent/docs/workflows.md @@ -2299,6 +2299,10 @@ readonly fallbackThinkingLevels?: readonly string[]; Request/context incompatibility also advances it, including HTTP 400/413/422 bad, unprocessable, or payload-too-large requests; unsupported tools or parameters; context-length or context-window overflow; and `too large`, `invalid_request`, or `bad_request` errors. This lets the chain reach the current selected user model when no configured candidate can serve the request. +A context overflow that the stage session's compaction has already failed to resolve is terminal for its candidate: it skips the same-candidate retry, because re-sending an identical request cannot fit a context compaction could not shrink, and advances straight to the next candidate. + +The chain also covers session creation. A stage session created eagerly — by `ctx.__ensureSession()`, an eager stage call, or a control attach — retries its candidate under `settings.retry` and then walks to the next configured candidate, so a provider that cannot even open a session does not strand the stage. A creation failure that exhausts the whole chain is not cached: the next call starts a fresh attempt. + Workflow-code errors, tool failures, validation failures, refusals, content-filter or safety blocks, cancellations, and task failures do not advance the chain. A reattached finished stage starts on the model that last succeeded; if that model fails retryably, the full chain restarts from the primary. ### `thinkingLevel` (deprecated) diff --git a/packages/workflows/CHANGELOG.md b/packages/workflows/CHANGELOG.md index 97da5e95e..2eaa8b07e 100644 --- a/packages/workflows/CHANGELOG.md +++ b/packages/workflows/CHANGELOG.md @@ -17,7 +17,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Fixed thrown retryable stage failures to retry the current model with bounded exponential backoff from `settings.retry` before advancing through `fallbackModels`; disabling retry keeps immediate candidate advancement ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). - Fixed a same-candidate stage retry losing the stage prompt. The retry resumed the existing session turn with a transcript that no longer ended in the prompt it was resuming, which the agent rejects; the prompt is now retained for that path and still re-sent on the re-prompt path ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). -- Fixed a same-candidate stage retry stopping outright when the failed attempt had already streamed a completed assistant message. Resuming the existing turn is now attempted only when the transcript still ends in a message the agent can continue from; otherwise the retry re-sends the prompt, which recovers instead of failing ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed a same-candidate stage retry stopping outright when the failed attempt had already streamed a completed assistant message. Resuming the existing turn is now attempted only when the transcript, as converted for the provider, still ends in a message the agent can continue from; otherwise the retry re-sends the prompt, which recovers instead of failing. Messages that convert away — a note excluded from context, an empty branch summary — no longer look like a valid tail ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed an eagerly created stage session never reaching its `fallbackModels`. A session created before any prompt — by an eager stage call, a control attach, or `ctx.__ensureSession()` — retried only its first candidate and then failed the stage. It now walks the configured chain, and a failure that exhausts the chain is no longer cached and replayed to every later caller ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed a paused eager session creation dropping its replacement objective when the stage had explicit model candidates. The resumed text is now authoritative for the next prompt and is sent exactly once ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed a context overflow that compaction had already failed to resolve being retried on the same candidate for the whole `settings.retry` budget. Re-sending an identical request cannot fit a context compaction could not shrink, so the chain now advances immediately while still treating the failure as fallbackable ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). ## [0.9.12] - 2026-08-04 diff --git a/packages/workflows/src/runs/foreground/stage-runner-controller.ts b/packages/workflows/src/runs/foreground/stage-runner-controller.ts index acdcfdd86..bf133305b 100644 --- a/packages/workflows/src/runs/foreground/stage-runner-controller.ts +++ b/packages/workflows/src/runs/foreground/stage-runner-controller.ts @@ -1,6 +1,7 @@ import { type AgentSession, type CreateAgentSessionOptions, + convertToLlm, type PromptOptions, type StructuredOutputCapture, shouldApplyCodexFastModeForScope, @@ -56,6 +57,7 @@ import type { WorkflowRetrySettings, } from "./stage-runner-types.js"; import { + isUnresolvedContextOverflowFailure, nextResumedContextOverflowFallbackIndex, terminatingToolCallId, unresolvedContextOverflowFailure, @@ -117,14 +119,20 @@ function retryableAgentSession(activeSession: StageSessionRuntime): RetryableAge } /** - * pi-agent-core's `Agent.continue()` rejects an empty transcript and an - * assistant tail. A retry that restored live state may leave either — a - * non-error assistant admitted during the failed attempt is retained — so the - * continuation path is only eligible when the tail satisfies that contract. + * pi-agent-core requires the last message of a continued transcript to convert + * to a `user` or `toolResult` provider message (`agent-loop.js` `agentLoopContinue`); + * anything else is rejected once the request reaches the provider. + * + * The raw tail is not enough to answer that. A retry that restored live state + * can leave a retained non-error assistant, or a `custom`/`bashExecution`/ + * `branchSummary` message that Atomic's converter drops when it is excluded + * from context or empty — leaving an assistant as the converted tail. Evaluate + * the same `convertToLlm()` result Atomic sends. */ function canContinueFromTranscript(activeSession: StageSessionRuntime): boolean { - const last = activeSession.messages[activeSession.messages.length - 1]; - return last !== undefined && last.role !== "assistant"; + const converted = convertToLlm([...activeSession.messages]); + const last = converted[converted.length - 1]; + return last !== undefined && (last.role === "user" || last.role === "toolResult"); } class ThrownErrorRetryPaused extends Error { @@ -235,7 +243,16 @@ export class StageSessionController { async ensureSession(consumer: AgentSessionConsumer = "prompt"): Promise { if (this.disposed) throw new Error(`atomic-workflows: stage "${this.opts.stageName}" session has been disposed`); if (this.session !== undefined) return this.session; - if (!this.sessionPromise) this.sessionPromise = this.createInitialSession(consumer); + if (!this.sessionPromise) { + const pending = this.createInitialSession(consumer); + this.sessionPromise = pending; + // A terminal creation failure must not be replayed to every later + // caller. Clear by identity so a walk that replaced the promise, or a + // cancellation that already cleared it, is left alone. + pending.catch(() => { + if (this.sessionPromise === pending) this.sessionPromise = undefined; + }); + } return this.sessionPromise; } @@ -349,9 +366,13 @@ export class StageSessionController { return; } - if (await this.tryResumeCurrentSession(text, sdkOptions, candidates)) return; + // A paused eager creation resumed with a replacement objective: that text + // is authoritative for this prompt and must not be dropped here. + const resumedText = this.pendingCreationResumeMessage; + this.pendingCreationResumeMessage = undefined; + let promptText = resumedText ?? text; + if (await this.tryResumeCurrentSession(promptText, sdkOptions, candidates)) return; let index = this.activeCandidateIndex ?? 0; - let promptText = text; while (index < candidates.length) { const candidate = candidates[index]!; try { @@ -595,10 +616,14 @@ export class StageSessionController { } catch (error) { const errorSettingsManager = retrySettingsManagerFromError(error); if (errorSettingsManager !== undefined) this.sessionSettingsManager = errorSettingsManager; - const decision = nextRetryDecision(this.retrySettings(), retryAttempt, isRetryableModelFailure(error)); + // An already-unresolved context overflow is terminal for this model: + // compaction has run and failed, so another identical request cannot + // help. It stays fallbackable so `handleCandidateFailure()` advances. + const retryableFailure = isRetryableModelFailure(error); + const sameCandidateRetryable = retryableFailure && !isUnresolvedContextOverflowFailure(error); + const decision = nextRetryDecision(this.retrySettings(), retryAttempt, sameCandidateRetryable); const continuationSession = retryableAgentSession(activeSession); const admittedMessages = activeSession.messages.length > messagesBeforeAttempt.length; - const retryableFailure = isRetryableModelFailure(error); const willRetry = decision !== undefined && !this.disposed && @@ -708,10 +733,59 @@ export class StageSessionController { this.activeCandidateIndex = 0; this.selectedModel = first.id; return this.createSession(first, consumer).catch((error) => - this.createInitialSessionWithRetry(first, consumer, { error }), + this.createInitialSessionCandidateWalk(candidates, consumer, 0, { error }), ); } + /** + * Creation-only version of the prompt candidate walk. + * + * An eagerly created session (`ctx.__ensureSession()`, an eager stage call, a + * control attach) has no prompt to drive `promptWithFallback()`, so without + * this a candidate whose creation keeps failing would exhaust its retries and + * throw while the configured fallbacks were never tried. + */ + private async createInitialSessionCandidateWalk( + candidates: readonly WorkflowResolvedModelCandidate[], + consumer: AgentSessionConsumer, + startIndex: number, + initialFailure: { readonly error: unknown } | undefined, + ): Promise { + let index = startIndex; + let pendingFailure = initialFailure; + let lastError: unknown = initialFailure?.error; + while (index < candidates.length) { + const candidate = candidates[index]!; + this.activeCandidateIndex = index; + this.selectedModel = candidate.id; + try { + const created = await this.createSessionWithThrownErrorRetry(candidate, consumer, pendingFailure); + pendingFailure = undefined; + if (!isSessionCreationPauseResult(created)) { + this.notifyModelFallbackMetaChange(); + return created; + } + if (created.resumeMessage === undefined) { + // A pause without a replacement objective cancels this pending + // creation. Let the next prompt start a fresh creation attempt. + this.pendingCreationResumeMessage = undefined; + this.sessionPromise = undefined; + throw new StageSessionCreationCancelled(); + } + // The replacement objective belongs to the next prompt; this + // candidate is still the one being created. + this.pendingCreationResumeMessage = created.resumeMessage; + } catch (error) { + if (error instanceof StageSessionCreationCancelled) throw error; + pendingFailure = undefined; + lastError = error; + if ((await this.handleCandidateFailure(error, candidate, candidates, index)) !== "retry") throw error; + index += 1; + } + } + throw lastError ?? new Error(`atomic-workflows: stage "${this.opts.stageName}" has no usable model candidate`); + } + private async createInitialSessionWithRetry( candidate: WorkflowResolvedModelCandidate | undefined, consumer: AgentSessionConsumer, diff --git a/test/unit/stage-runner-thrown-retry.test.ts b/test/unit/stage-runner-thrown-retry.test.ts index 0c909d159..1ed5daa54 100644 --- a/test/unit/stage-runner-thrown-retry.test.ts +++ b/test/unit/stage-runner-thrown-retry.test.ts @@ -731,4 +731,198 @@ describe("createStageContext — continuation eligibility across admitted orderi assert.deepEqual(probe.promptTexts, ["do it"]); assert.deepEqual(probe.continuedTranscripts, [], "an aborted retry must not resume the turn"); }); + + test("a tail that converts away is not a valid continuation", async () => { + // pi-agent-core requires the CONVERTED tail to be user/toolResult. A custom + // message excluded from context is dropped by convertToLlm(), exposing the + // retained assistant beneath it, so the raw role alone cannot decide. + const probe = realSessionProbe(retrySettings(), { + failCalls: 1, + admit: (messages, call) => { + if (call !== 1) return; + messages.push({ + role: "assistant", + stopReason: "stop", + content: [{ type: "text", text: "partial" }], + } as never); + messages.push({ + role: "custom", + customType: "workflow:note", + content: "hidden note", + excludeFromContext: true, + } as never); + }, + }); + const ctx = createStageContext( + makeOpts({ adapters: { agentSession: probe.agentSession }, stageOptions: { model: "anthropic/primary" } }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("do it"), "ok"); + assert.deepEqual(probe.continuedTranscripts, [], "a converted assistant tail is not a valid continuation"); + assert.deepEqual(probe.promptTexts, ["do it", "do it"], "the retry must re-send the prompt instead"); + assert.equal(userMessagesWithText(probe.messages, "do it"), 1, "the re-prompt must not duplicate the input"); + }); +}); + +describe("createStageContext — eager session creation walks the candidate chain", () => { + const eagerSettingsManager = (settings: ReturnType): WorkflowFastModeSettingsManager => ({ + getCodexFastModeSettings: () => ({ chat: false, workflow: false }), + getRetrySettings: () => settings, + }); + + test("an eagerly created session advances to a fallback candidate", async () => { + // ctx.__ensureSession() has no prompt to drive promptWithFallback(), so the + // creation path must walk the chain itself. + const created: string[] = []; + const settings = retrySettings(); + const agentSession: AgentSessionAdapter = { + async create(options) { + const model = modelFor(options); + created.push(model); + if (model === "anthropic/primary") throw new Error("503 service unavailable during create"); + return sessionWithSettings(settings, async () => "fallback answer", { + getLastAssistantText: () => "fallback answer", + }); + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary", + fallbackModels: ["openai/fallback"], + settingsManager: eagerSettingsManager(settings) as never, + }, + }), + ) as InternalStageContext; + + await ctx.__ensureSession(); + + assert.deepEqual(created, ["anthropic/primary", "anthropic/primary", "anthropic/primary", "openai/fallback"]); + assert.equal(await ctx.prompt("go"), "fallback answer"); + }); + + test("a terminal creation failure is not replayed to a later ensureSession", async () => { + let creates = 0; + const settings = retrySettings(); + const agentSession: AgentSessionAdapter = { + async create(options) { + creates += 1; + // Three attempts on the primary plus three on the fallback exhaust + // the first walk; anything after that succeeds. + if (creates <= 6) throw new Error(`503 service unavailable during create ${modelFor(options)}`); + return sessionWithSettings(settings, async () => "late answer", { + getLastAssistantText: () => "late answer", + }); + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary", + fallbackModels: ["openai/fallback"], + settingsManager: eagerSettingsManager(settings) as never, + }, + }), + ) as InternalStageContext; + + await assert.rejects(ctx.__ensureSession(), /503 service unavailable during create/); + assert.equal(creates, 6); + + // The cached rejection must not be handed to the next caller. + await ctx.__ensureSession(); + assert.equal(creates, 7); + }); + + test("a paused eager creation resumes with the replacement objective", async () => { + const promptTexts: string[] = []; + let creates = 0; + const settings = retrySettings({ baseDelayMs: 1000 }); + const agentSession: AgentSessionAdapter = { + async create() { + creates += 1; + if (creates === 1) throw new Error("503 service unavailable during create"); + return sessionWithSettings( + settings, + async (text) => { + promptTexts.push(text); + return "resumed answer"; + }, + { getLastAssistantText: () => "resumed answer" }, + ); + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary", + fallbackModels: ["openai/fallback"], + settingsManager: eagerSettingsManager(settings) as never, + }, + }), + ) as InternalStageContext; + + const eager = ctx.__ensureSession(); + await flushMicrotasks(); + await ctx.__requestPause(); + await ctx.__resume("replacement objective"); + await eager; + + assert.equal(await ctx.prompt("stale original objective"), "resumed answer"); + assert.deepEqual(promptTexts, ["replacement objective"], "the replacement objective is authoritative"); + }); +}); + +describe("createStageContext — unresolved overflow is terminal for its candidate", () => { + test("an already-unresolved overflow advances without retrying the same model", async () => { + const prompts: string[] = []; + const settings = retrySettings({ maxRetries: 2 }); + const settingsManager: WorkflowFastModeSettingsManager = { + getCodexFastModeSettings: () => ({ chat: false, workflow: false }), + getRetrySettings: () => settings, + }; + const agentSession: AgentSessionAdapter = { + async create(options) { + const model = String(options.model); + const mock = makeMockSession({ + async prompt() { + prompts.push(model); + if (model === "anthropic/primary") { + mock.emit({ + type: "compaction_end", + reason: "overflow", + result: undefined, + aborted: false, + willRetry: false, + unresolvedOverflow: true, + errorMessage: "Context overflow recovery failed after one compact-and-retry attempt.", + }); + } + return undefined; + }, + getLastAssistantText() { + return model === "openai/fallback" ? "fallback answer" : undefined; + }, + }); + return mock.session; + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary", + fallbackModels: ["openai/fallback"], + settingsManager: settingsManager as never, + }, + }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("go"), "fallback answer"); + // Compaction already ran and failed, so re-sending the same request cannot + // help: one attempt per candidate, not one per retry budget. + assert.deepEqual(prompts, ["anthropic/primary", "openai/fallback"]); + }); }); From e45a42304daae6674c4a3fe037198c2ab0d3130d Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 16:38:02 -0700 Subject: [PATCH 07/13] fix(fallback): skip reasoning-only variants of a condemned model (#2170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _trySwitchToFallbackModel() skipped a candidate only when it matched the failed model AND its reasoning level, so openai-codex/gpt-5.5:high failing with an invalidated OAuth token fell back to openai-codex/gpt-5.5:low — the same dead credential, with model_fallback_start reporting an identical from and to. The boundary is not "auth"; it is the distinction the shared classifier already draws. A failure that may spend a candidate but cannot be repaired by another request to the same model — rejected credential, unavailable model, request the model cannot serve — is unaffected by reasoning level, so every reasoning variant of that model is out for the rest of the turn. Track those models in a turn-scoped blocked list and skip matching candidates before the fallback scope opens, so a skipped variant emits no event and starts no lifecycle. The list clears with the existing fallback-attempt state on a new user turn, a successful turn, and scope close. Transient rate-limit and transport failures stay same-model retryable and keep their reasoning variants. The pre-existing same-provider reasoning test used the fixture's default "Not Found" message, which is fallbackable but not same-model retryable, so it was asserting the defect. It now uses a real transient failure and covers the behavior that must survive. Refs: #2170 Assistant-model: Claude Opus 5 --- packages/coding-agent/CHANGELOG.md | 2 +- packages/coding-agent/docs/settings.md | 2 + .../src/core/agent-session-events.ts | 2 + .../src/core/agent-session-methods.ts | 1 + .../src/core/agent-session-retry.ts | 16 ++ .../coding-agent/src/core/agent-session.ts | 2 + test/unit/main-chat-model-fallback.test.ts | 141 +++++++++++++++++- 7 files changed, 162 insertions(+), 4 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index f69810542..74127e167 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -5,7 +5,7 @@ ### Fixed - Workflow runs awaiting human input now use the blue `？` indicator in the BACKGROUND panel, `/workflow connect` picker, and `/workflow status` listing, including prompts raised by hidden nested workflow children; the indicator returns to the run's current state when the prompt resolves. -- Fixed main-chat model fallback to classify provider failures consistently with workflows, advance rejected credentials and incompatible or unavailable models to the next candidate, and restore the user-selected model at the next turn without overriding an explicit `/model` choice ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed main-chat model fallback to classify provider failures consistently with workflows, advance rejected credentials and incompatible or unavailable models to the next candidate, and restore the user-selected model at the next turn without overriding an explicit `/model` choice. A failure that requesting the same model again cannot repair now takes that model out of the chain for the rest of the turn at every reasoning level, so a fallback entry differing only by its `:low`/`:high` suffix is skipped instead of retrying the same dead credential; transient rate-limit and transport failures keep those reasoning variants ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). - Fixed a context overflow that compaction cannot resolve to advance the configured `fallbackModels` chain instead of ending the turn, so a larger-context candidate can answer. Compaction still runs first, and a compactable overflow spends no fallback candidate ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). - Fixed a reasoning-level change during a model fallback stranding the session on the fallback model. Changing reasoning effort is not a model choice, so it no longer cancels the pending restore; the next turn returns to the user-selected primary and keeps the reasoning level that was chosen. Only an explicit `/model` selection or model cycle cancels the restore ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). - Fixed gRPC `ResourceExhausted` provider errors, seen from providers such as NVIDIA NIM, bypassing the same-model auto-retry budget. They are now retried like other transient provider failures, matching upstream pi-ai, and still advance a configured fallback chain when the retries are exhausted ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index 050e01063..94381ba69 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -58,6 +58,8 @@ Settings and trust JSON files may start with a UTF-8 BOM, as commonly written by `fallbackModels` gives ordinary main-chat turns an ordered model fallback chain. Atomic starts with the selected/default model. If that model exhausts the normal same-model auto-retry loop for a retryable provider/model failure — including rate limits and quota/usage-limit exhaustion such as a provider reporting `The usage limit has been reached` — Atomic switches to the next configured fallback model and continues the same turn. If `retry.enabled` is `false`, Atomic skips same-model retries and moves directly to the next fallback for retryable failures. Non-retryable task failures and cancellations do not trigger model fallback. After a successful or exhausted fallback turn, Atomic restores the user-selected primary before the next user turn; an explicit `/model` choice during fallback cancels that restore. +A failure that another request to the same model cannot repair — a rejected credential, an unavailable model, a request that model cannot serve — takes that model out of the chain for the rest of the turn at **every** reasoning level, so a candidate that differs only by its `:low`/`:high` suffix is skipped rather than spent. Transient rate-limit and transport failures keep those reasoning variants, because retrying them can succeed. + Context overflow keeps its normal recovery order: compaction runs first, and a compactable overflow costs no fallback candidate. Only once compaction is disabled, fails, or reports the overflow unresolved does Atomic advance to the next configured candidate, which is how a larger-context model gets a chance at the turn. Changing the reasoning level during a fallback turn is not a model choice, so it does not cancel the restore: the next turn still starts on the user-selected primary, carrying the reasoning level you picked. Only an explicit `/model` selection or model cycle cancels it. diff --git a/packages/coding-agent/src/core/agent-session-events.ts b/packages/coding-agent/src/core/agent-session-events.ts index ab5648aa5..aa2371285 100644 --- a/packages/coding-agent/src/core/agent-session-events.ts +++ b/packages/coding-agent/src/core/agent-session-events.ts @@ -133,6 +133,7 @@ export async function _processAgentEvent(this: AgentSession, event: AgentEvent): if (event.type === "message_start" && event.message.role === "user") { this._overflowRecoveryAttempted = false; this._fallbackAttemptedKeys.clear(); + this._fallbackBlockedModels.length = 0; const messageText = this._getUserMessageText(event.message); if (messageText) { // Check steering queue first @@ -218,6 +219,7 @@ export async function _processAgentEvent(this: AgentSession, event: AgentEvent): } if (!assistantFailed && assistantMsg.stopReason === "stop") { this._fallbackAttemptedKeys.clear(); + this._fallbackBlockedModels.length = 0; } // A non-truncated assistant response means the length-continuation loop diff --git a/packages/coding-agent/src/core/agent-session-methods.ts b/packages/coding-agent/src/core/agent-session-methods.ts index 42d4b4910..192ad2dd7 100644 --- a/packages/coding-agent/src/core/agent-session-methods.ts +++ b/packages/coding-agent/src/core/agent-session-methods.ts @@ -393,6 +393,7 @@ export interface AgentSessionInternalSurface extends AgentSessionMethodSurface, _scopedModels: Array<{ model: Model; thinkingLevel?: ThinkingLevel }>; _fallbackModels: string[]; _fallbackAttemptedKeys: Set; + _fallbackBlockedModels: Array>; _fallbackOriginModel: Model | undefined; _fallbackOriginThinkingLevel: ThinkingLevel | undefined; _fallbackScopeGeneration: number; diff --git a/packages/coding-agent/src/core/agent-session-retry.ts b/packages/coding-agent/src/core/agent-session-retry.ts index 0fb55cf6b..5ce814459 100644 --- a/packages/coding-agent/src/core/agent-session-retry.ts +++ b/packages/coding-agent/src/core/agent-session-retry.ts @@ -212,6 +212,7 @@ function clearFallbackModelScopeState(this: AgentSession): void { this._fallbackOriginThinkingLevel = undefined; this._fallbackRestoreError = undefined; this._fallbackAttemptedKeys.clear(); + this._fallbackBlockedModels.length = 0; } function finishFallbackModelScope( @@ -307,6 +308,18 @@ export async function _trySwitchToFallbackModel(this: AgentSession, message: Ass const currentModel = this.agent.state.model ?? this.model; if (this._fallbackModels.length === 0 || currentModel === undefined) return false; + // A failure the chain may spend a candidate on, but that requesting the same + // model again cannot repair — a rejected credential, an unavailable model, a + // request this model cannot serve — condemns that model for the rest of the + // turn. Reasoning level does not change any of those answers, so every + // reasoning variant of it is blocked too. Transient rate-limit and transport + // failures stay same-model retryable and keep their reasoning variants. + if (isRetryableModelFailure(message) && !isRetryableSameModelFailure(message)) { + if (!this._fallbackBlockedModels.some((blocked) => modelsAreEqual(blocked, currentModel))) { + this._fallbackBlockedModels.push(currentModel); + } + } + const currentThinkingLevel = this.agent.state.thinkingLevel ?? this.thinkingLevel; const fromModel = currentModel; for (const rawCandidate of this._fallbackModels) { @@ -323,6 +336,9 @@ export async function _trySwitchToFallbackModel(this: AgentSession, message: Ass const key = fallbackKey(nextModel, nextLevel); if (this._fallbackAttemptedKeys.has(key)) continue; if (modelsAreEqual(nextModel, fromModel) && nextLevel === currentThinkingLevel) continue; + // Skipped before any lifecycle starts: a blocked variant emits no fallback + // event and opens no scope. + if (this._fallbackBlockedModels.some((blocked) => modelsAreEqual(blocked, nextModel))) continue; // Do not create a fallback lifecycle until a candidate can actually be // selected. An exhausted or unresolvable chain has no start to close. diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 23df42f87..4ff5193ed 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -75,6 +75,8 @@ class AgentSessionBase { protected _scopedModels: Array<{ model: Model; thinkingLevel?: ThinkingLevel }>; protected _fallbackModels: string[]; protected _fallbackAttemptedKeys: Set = new Set(); + /** Models condemned for this turn by a failure retrying them cannot repair. */ + protected _fallbackBlockedModels: Array> = []; protected _fallbackOriginModel: Model | undefined; protected _fallbackOriginThinkingLevel: ThinkingLevel | undefined; protected _fallbackScopeGeneration = 0; diff --git a/test/unit/main-chat-model-fallback.test.ts b/test/unit/main-chat-model-fallback.test.ts index ae119e44e..bd83e1cab 100644 --- a/test/unit/main-chat-model-fallback.test.ts +++ b/test/unit/main-chat-model-fallback.test.ts @@ -108,6 +108,7 @@ test("main-chat fallback switches models after same-model retry exhaustion", asy thinkingLevel: "high" as ThinkingLevel, _fallbackModels: ["anthropic/claude-opus-4-8:high"], _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], _retryAttempt: 0, settingsManager: { getRetrySettings: () => ({ enabled: true, maxRetries: 0, baseDelayMs: 1 }), @@ -175,6 +176,7 @@ test("main-chat fallback restores the primary model at the next turn boundary", thinkingLevel: "high" as ThinkingLevel, _fallbackModels: ["anthropic/claude-opus-4-8:high"], _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], _retryAttempt: 0, settingsManager: { getDefaultThinkingLevel: () => "high" as ThinkingLevel, @@ -214,7 +216,10 @@ test("main-chat fallback restores the primary model at the next turn boundary", assert.equal(await _restoreFallbackModel.call(session as never), false); }); -test("main-chat fallback can change reasoning on the same provider/model", async () => { +test("a transient failure can still change reasoning on the same provider/model", async () => { + // Rate limits and transport blips clear on their own, so the same model at + // another reasoning level stays a legitimate candidate. + const transient = retryableMessage({ errorMessage: "rate limit exceeded" }); const primary = model("openai", "gpt-5-mini"); const events: Array<{ type: string; [key: string]: unknown }> = []; const session = { @@ -222,6 +227,7 @@ test("main-chat fallback can change reasoning on the same provider/model", async thinkingLevel: "high" as ThinkingLevel, _fallbackModels: ["openai/gpt-5-mini:low"], _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], _retryAttempt: 0, settingsManager: { getDefaultThinkingLevel: () => "high" as ThinkingLevel, @@ -234,7 +240,7 @@ test("main-chat fallback can change reasoning on the same provider/model", async hasConfiguredAuth: () => true, }, agent: { - state: { model: primary, thinkingLevel: "high" as ThinkingLevel, messages: [retryableMessage()] }, + state: { model: primary, thinkingLevel: "high" as ThinkingLevel, messages: [transient] }, continue: async () => undefined, }, sessionManager: { @@ -248,7 +254,7 @@ test("main-chat fallback can change reasoning on the same provider/model", async _emit: (event: { type: string; [key: string]: unknown }) => events.push(event), }; - const handled = await _trySwitchToFallbackModel.call(session as never, retryableMessage()); + const handled = await _trySwitchToFallbackModel.call(session as never, transient); assert.equal(handled, true); assert.equal(session.agent.state.model, primary); @@ -274,6 +280,7 @@ test("main-chat retry-disabled fallback keeps prompt waiting for fallback comple thinkingLevel: "high" as ThinkingLevel, _fallbackModels: ["anthropic/claude-opus-4-8:high"], _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], _retryAttempt: 0, _retryPromise: undefined as Promise | undefined, _retryResolve: undefined as (() => void) | undefined, @@ -346,6 +353,7 @@ test("main-chat fallback rejection settles the retry wait", async () => { thinkingLevel: "high" as ThinkingLevel, _fallbackModels: ["anthropic/claude-opus-4-8:high"], _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], _retryAttempt: 1, _retryPromise: retryPromise as Promise | undefined, _retryResolve: resolveRetry as (() => void) | undefined, @@ -411,6 +419,7 @@ test("main-chat fallback continuation resolution does not mark assistant errors thinkingLevel: "high" as ThinkingLevel, _fallbackModels: ["anthropic/claude-opus-4-8:high"], _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], _retryAttempt: 0, settingsManager: { getDefaultThinkingLevel: () => "high" as ThinkingLevel, @@ -461,6 +470,7 @@ test("a rejected codex credential advances to the next candidate without re-requ thinkingLevel: "high" as ThinkingLevel, _fallbackModels: ["anthropic/claude-opus-4-8:high"], _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], _retryAttempt: 0, settingsManager: { // Same-model retry is enabled and generously budgeted; a dead credential @@ -520,6 +530,7 @@ test("an explicit /model choice during a fallback is not overwritten by the rest thinkingLevel: "high" as ThinkingLevel, _fallbackModels: ["anthropic/claude-opus-4-8:high"], _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], _retryAttempt: 0, settingsManager: { getDefaultThinkingLevel: () => "high" as ThinkingLevel, @@ -586,6 +597,8 @@ function overflowTurnSession(checkCompaction: (session: Record) model: model("openai-codex", "gpt-5.5"), _lastAssistantMessage: overflowMessage(), _protectedStreamingCustomMessages: [], + _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], _postToolCompactionPreflightError: undefined, _pendingPostCompactionContinuation: undefined, _contextOverflowUnresolved: false, @@ -699,6 +712,7 @@ test("a failed overflow compaction advances the real fallback chain end to end", _overflowRecoveryAttempted: false, _fallbackModels: ["anthropic/claude-opus-4-8:high"], _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], _retryAttempt: 0, settingsManager: { getCompactionSettings: () => ({ enabled: true }), @@ -777,6 +791,7 @@ function thinkingLevelFallbackSession( thinkingLevel: "high" as ThinkingLevel, _fallbackModels: ["anthropic/claude-opus-4-8:high"], _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], _retryAttempt: 0, settingsManager: { getDefaultThinkingLevel: () => "high" as ThinkingLevel, @@ -858,3 +873,123 @@ test("a gRPC ResourceExhausted failure is both same-model retryable and fallback assert.equal(_isRetryableError.call(session as never, message), true); assert.equal(_isFallbackableError.call(session as never, message), true); }); + +test("a terminal provider failure skips reasoning-only variants of the failed model", async () => { + // The credential, not the reasoning effort, is what failed. Re-requesting the + // same provider/model at another level would spend a candidate on the same + // dead credential, so the chain must reach a different provider. + const primary = model("openai-codex", "gpt-5.5"); + const anthropic = model("anthropic", "claude-opus-4-8"); + const invalidated = retryableMessage({ errorMessage: "OAuth token invalidated" }); + const events: Array<{ type: string; [key: string]: unknown }> = []; + const continuedOn: Array = []; + const session = { + model: primary, + thinkingLevel: "high" as ThinkingLevel, + _fallbackModels: ["openai-codex/gpt-5.5:low", "anthropic/claude-opus-4-8:high"], + _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], + _retryAttempt: 0, + settingsManager: { + getRetrySettings: () => ({ enabled: true, maxRetries: 3, baseDelayMs: 1000 }), + getDefaultThinkingLevel: () => "high" as ThinkingLevel, + getDefaultProvider: () => "openai-codex", + }, + _modelRuntime: { + getAvailableSnapshot: () => [primary, anthropic], + getModel: (provider: string, id: string) => { + if (provider === primary.provider && id === primary.id) return primary; + return provider === anthropic.provider && id === anthropic.id ? anthropic : undefined; + }, + hasConfiguredAuth: () => true, + }, + agent: { + state: { model: primary, thinkingLevel: "high" as ThinkingLevel, messages: [invalidated] }, + continue: async () => { + continuedOn.push(session.agent.state.model.id === primary.id ? "openai-codex" : "anthropic"); + }, + }, + sessionManager: { + appendModelChange: (provider: string, id: string) => events.push({ type: "session_model", provider, id }), + appendThinkingLevelChange: () => undefined, + }, + _withContextWindowForModelSwitch: (candidate: Model) => candidate, + _refreshBaseSystemPromptFromActiveTools: () => undefined, + _emitModelChanged: () => undefined, + _emitModelSelect: async () => undefined, + _emit: (event: { type: string; [key: string]: unknown }) => events.push(event), + _resolveRetry: () => undefined, + _isRetryableError, + _isFallbackableError, + _trySwitchToFallbackModel, + }; + + assert.equal(await _handleRetryableError.call(session as never, invalidated), true); + await new Promise((resolve) => setTimeout(resolve, 5)); + + assert.equal( + events.some((event) => event.type === "auto_retry_start"), + false, + "a rejected credential must not be re-requested on the same model", + ); + const starts = events.filter((event) => event.type === "model_fallback_start"); + assert.equal(starts.length, 1); + assert.equal(starts[0]?.from, "openai-codex/gpt-5.5"); + assert.equal(starts[0]?.to, "anthropic/claude-opus-4-8", "the reasoning-only variant must be skipped"); + assert.notEqual(starts[0]?.from, starts[0]?.to); + assert.equal(session.agent.state.model, anthropic); + assert.deepEqual(continuedOn, ["anthropic"], "exactly one continuation, on a different provider"); +}); + +test("a blocked model stays blocked for the rest of the turn", async () => { + // After moving to another provider, a later failure must not walk back into a + // reasoning variant of the model whose credential is already known dead. + const primary = model("openai-codex", "gpt-5.5"); + const anthropic = model("anthropic", "claude-opus-4-8"); + const invalidated = retryableMessage({ errorMessage: "OAuth token invalidated" }); + const events: Array<{ type: string; [key: string]: unknown }> = []; + const session = { + model: primary, + thinkingLevel: "high" as ThinkingLevel, + _fallbackModels: ["anthropic/claude-opus-4-8:high", "openai-codex/gpt-5.5:low"], + _fallbackAttemptedKeys: new Set(), + _fallbackBlockedModels: [] as Model[], + _retryAttempt: 0, + settingsManager: { + getDefaultThinkingLevel: () => "high" as ThinkingLevel, + getDefaultProvider: () => "openai-codex", + }, + _modelRuntime: { + getAvailableSnapshot: () => [primary, anthropic], + getModel: (provider: string, id: string) => { + if (provider === primary.provider && id === primary.id) return primary; + return provider === anthropic.provider && id === anthropic.id ? anthropic : undefined; + }, + hasConfiguredAuth: () => true, + }, + agent: { + state: { model: primary, thinkingLevel: "high" as ThinkingLevel, messages: [invalidated] }, + continue: async () => undefined, + }, + sessionManager: { + appendModelChange: () => undefined, + appendThinkingLevelChange: () => undefined, + }, + _withContextWindowForModelSwitch: (candidate: Model) => candidate, + _refreshBaseSystemPromptFromActiveTools: () => undefined, + _emitModelChanged: () => undefined, + _emitModelSelect: async () => undefined, + _emit: (event: { type: string; [key: string]: unknown }) => events.push(event), + }; + + // The dead credential moves the turn to Anthropic. + assert.equal(await _trySwitchToFallbackModel.call(session as never, invalidated), true); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(session.agent.state.model, anthropic); + + // A transient Anthropic failure must not return to the blocked Codex model. + const transient = retryableMessage({ errorMessage: "rate limit exceeded" }); + assert.equal(await _trySwitchToFallbackModel.call(session as never, transient), false); + assert.equal(session.agent.state.model, anthropic); + assert.equal(events.filter((event) => event.type === "model_fallback_start").length, 1); +}); From 9c085f1619622ee5ce4d993cb65eb7200dcd12c2 Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 17:54:56 -0700 Subject: [PATCH 08/13] fix(workflows): one creation gate and pause-aware continuation (#2170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three races in the stage runner, each reproduced before fixing. disposeCurrentSession() cleared the shared creation promise unconditionally, including while a candidate walk was still creating the next session. A concurrent ctx.__ensureSession(), or a first ctx.prompt() whose explicit-candidate branch bypassed ensureSession() entirely, could then start a second walk: duplicate provider work, with the losing session left live and undisposed. Mark the promise a walk owns, clear it on dispose only when it is not that promise, and let the explicit-candidate prompt join a creation already in flight. createSession() was awaited with no pause observer, so a controlled pause that both started and finished while the adapter was in flight was invisible; completing a resume clears the pause request, so a post-await currentResume() check sees nothing and the stale objective is prompted. Register an observer before creating and latch any replacement objective onto the resume itself. The observer set is separate from the thrown-retry states and the creation promise is returned untouched, so abort semantics, backoff timing, and attached-stream observability are all unchanged — an earlier attempt that added one microtask hop broke the executor pause and abort suites, which is what caught it. _runAgentContinue() was called directly, so the abort a controlled pause causes was classified as a terminal model failure and the stage became unrecoverable by resume. Apply the prompt's pause rules to the continuation: settle the pause, await its delivery settlement, then re-prompt with the replacement objective instead of spending a fallback candidate. Refs: #2170 Assistant-model: Claude Opus 5 --- packages/coding-agent/docs/workflows.md | 4 + packages/workflows/CHANGELOG.md | 3 + .../foreground/stage-runner-controller.ts | 163 +++++++++++-- test/unit/stage-runner-thrown-retry.test.ts | 220 ++++++++++++++++++ 4 files changed, 375 insertions(+), 15 deletions(-) diff --git a/packages/coding-agent/docs/workflows.md b/packages/coding-agent/docs/workflows.md index e94fe85f8..95958fa67 100644 --- a/packages/coding-agent/docs/workflows.md +++ b/packages/coding-agent/docs/workflows.md @@ -2303,6 +2303,10 @@ A context overflow that the stage session's compaction has already failed to res The chain also covers session creation. A stage session created eagerly — by `ctx.__ensureSession()`, an eager stage call, or a control attach — retries its candidate under `settings.retry` and then walks to the next configured candidate, so a provider that cannot even open a session does not strand the stage. A creation failure that exhausts the whole chain is not cached: the next call starts a fresh attempt. +That walk runs behind a single creation gate. A concurrent `ctx.__ensureSession()` or a first `ctx.prompt()` joins the creation already in flight rather than starting a second walk, so the stage never has two live sessions competing for the same generation. + +Controlled pauses are honored throughout. A pause that starts and finishes while a session is still being created keeps its replacement objective, which the next prompt sends exactly once; a pause during a same-candidate continuation is settled as a pause rather than a model failure, so resuming recovers the stage instead of spending a fallback candidate. + Workflow-code errors, tool failures, validation failures, refusals, content-filter or safety blocks, cancellations, and task failures do not advance the chain. A reattached finished stage starts on the model that last succeeded; if that model fails retryably, the full chain restarts from the primary. ### `thinkingLevel` (deprecated) diff --git a/packages/workflows/CHANGELOG.md b/packages/workflows/CHANGELOG.md index 2eaa8b07e..09b367589 100644 --- a/packages/workflows/CHANGELOG.md +++ b/packages/workflows/CHANGELOG.md @@ -21,6 +21,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Fixed an eagerly created stage session never reaching its `fallbackModels`. A session created before any prompt — by an eager stage call, a control attach, or `ctx.__ensureSession()` — retried only its first candidate and then failed the stage. It now walks the configured chain, and a failure that exhausts the chain is no longer cached and replayed to every later caller ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). - Fixed a paused eager session creation dropping its replacement objective when the stage had explicit model candidates. The resumed text is now authoritative for the next prompt and is sent exactly once ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). - Fixed a context overflow that compaction had already failed to resolve being retried on the same candidate for the whole `settings.retry` budget. Re-sending an identical request cannot fit a context compaction could not shrink, so the chain now advances immediately while still treating the failure as fallbackable ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed a stage running two candidate walks at once. Advancing a candidate released the shared creation gate while the walk was still creating the next session, so a concurrent `ctx.__ensureSession()` or a first `ctx.prompt()` could start a second walk, duplicate provider work, and leave the losing session live and undisposed. Concurrent callers now join the creation already in flight ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed a controlled pause that both started and finished while a stage session was still being created losing its replacement objective, so the stale objective was prompted instead ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed a controlled pause during a same-candidate continuation aborting the stage unrecoverably. The abort a pause causes is no longer classified as a terminal model failure: the pause settles, and resuming with a replacement objective re-prompts on the same candidate instead of spending a fallback ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). ## [0.9.12] - 2026-08-04 diff --git a/packages/workflows/src/runs/foreground/stage-runner-controller.ts b/packages/workflows/src/runs/foreground/stage-runner-controller.ts index bf133305b..2524cd56e 100644 --- a/packages/workflows/src/runs/foreground/stage-runner-controller.ts +++ b/packages/workflows/src/runs/foreground/stage-runner-controller.ts @@ -64,13 +64,18 @@ import { unresolvedContextOverflowMessage, } from "./stage-runner-unresolved-overflow.js"; -type RetryPauseResume = Promise<{ readonly message?: string }>; +type RetryPauseResume = NonNullable>; interface ThrownErrorRetryState { readonly controller: AbortController; pauseResume?: RetryPauseResume; } +/** Observes a controlled pause raised while a session creation is in flight. */ +interface CreationPauseObserver { + pauseResume?: RetryPauseResume; +} + interface SessionCreationPauseResult { readonly kind: "paused"; readonly resumeMessage?: string; @@ -145,6 +150,8 @@ class ThrownErrorRetryPaused extends Error { export class StageSessionController { private session: StageSessionRuntime | undefined; private activeCreation: Promise | undefined; + /** The creation promise a candidate walk still owns, if one is advancing. */ + private ownedCreationPromise: Promise | undefined; private abortGeneration = 0; private abortReason: Error | DOMException | string | undefined; private abortReasonGeneration = 0; @@ -174,6 +181,7 @@ export class StageSessionController { private readonly modelCatalog: WorkflowModelCatalogPort | undefined; private sessionSettingsManager: WorkflowFastModeSettingsManager | undefined; private readonly thrownErrorRetryStates = new Set(); + private readonly creationPauseObservers = new Set(); private readonly replacement = new StageSessionReplacement(); private pendingCreationResumeMessage: string | undefined; private readonly messageAdmission = new StageMessageAdmission(); @@ -246,10 +254,17 @@ export class StageSessionController { if (!this.sessionPromise) { const pending = this.createInitialSession(consumer); this.sessionPromise = pending; + // One creation gate: the walk owns this promise while it advances + // candidates, so a concurrent caller joins it instead of racing it. + this.ownedCreationPromise = pending; + const release = (): void => { + if (this.ownedCreationPromise === pending) this.ownedCreationPromise = undefined; + }; // A terminal creation failure must not be replayed to every later // caller. Clear by identity so a walk that replaced the promise, or a // cancellation that already cleared it, is left alone. - pending.catch(() => { + pending.then(release, () => { + release(); if (this.sessionPromise === pending) this.sessionPromise = undefined; }); } @@ -366,8 +381,21 @@ export class StageSessionController { return; } - // A paused eager creation resumed with a replacement objective: that text - // is authoritative for this prompt and must not be dropped here. + // A creation already in flight — an eager walk, or a concurrent caller — + // owns the candidate chain. Join it rather than starting a second walk that + // would duplicate provider work and leak whichever session lost the race. + if (this.session === undefined && this.sessionPromise !== undefined) { + try { + await this.sessionPromise; + } catch (error) { + if (error instanceof StageSessionCreationCancelled) return; + // The exhausted walk cleared its own promise by identity; fall through + // and let the walk below report the failure for this prompt. + } + } + + // A paused creation resumed with a replacement objective: that text is + // authoritative for this prompt and must not be dropped here. const resumedText = this.pendingCreationResumeMessage; this.pendingCreationResumeMessage = undefined; let promptText = resumedText ?? text; @@ -452,7 +480,16 @@ export class StageSessionController { requestPause(): Promise { const pause = this.pauseControl.requestPause(); const resume = this.pauseControl.currentResume(); - if (resume !== undefined) this.pauseThrownErrorRetries(resume); + if (resume !== undefined) { + this.pauseThrownErrorRetries(resume); + // A creation in flight cannot see the pause request itself, because + // completing the resume clears it before creation settles. + for (const observer of this.creationPauseObservers) { + if (observer.pauseResume !== undefined) continue; + observer.pauseResume = resume; + this.latchCreationResume(resume); + } + } return pause; } resume( @@ -581,6 +618,40 @@ export class StageSessionController { } } + /** + * Resume the existing turn under the same pause rules as a prompt. + * + * A controlled pause aborts the in-flight request, which must not be mistaken + * for a terminal model failure: the stage would advance a candidate and become + * unrecoverable by resume. + */ + private async continueWithPauseResume( + continuationSession: RetryableAgentSession, + ): Promise<{ readonly kind: "continued" } | { readonly kind: "paused"; readonly message?: string }> { + const settlePause = async (resume: RetryPauseResume): Promise<{ kind: "paused"; message?: string }> => { + const resumed = await resume; + await resumed.runnerOwnedDeliverySettlement; + return { kind: "paused", ...(resumed.message === undefined ? {} : { message: resumed.message }) }; + }; + const pauseBeforeContinue = this.pauseControl.currentResume(); + if (pauseBeforeContinue !== undefined) return settlePause(pauseBeforeContinue); + + const state: ThrownErrorRetryState = { controller: new AbortController() }; + this.thrownErrorRetryStates.add(state); + try { + await continuationSession._runAgentContinue(); + } catch (error) { + const resume = state.pauseResume ?? this.pauseControl.currentResume(); + if (resume === undefined) throw error; + return settlePause(resume); + } finally { + this.thrownErrorRetryStates.delete(state); + } + const pauseAfterContinue = state.pauseResume ?? this.pauseControl.currentResume(); + if (pauseAfterContinue !== undefined) return settlePause(pauseAfterContinue); + return { kind: "continued" }; + } + private async promptWithThrownErrorRetry( activeSession: StageSessionRuntime, text: string, @@ -597,11 +668,26 @@ export class StageSessionController { if (retryAdmittedPrompt) { const continuationSession = retryableAgentSession(activeSession); if (continuationSession !== undefined) { - await continuationSession._runAgentContinue(); - return { - terminalScanStartIndex: - terminalScanStartIndex ?? this.lastPromptStartIndex ?? messagesBeforeAttempt.length, - }; + const outcome = await this.continueWithPauseResume(continuationSession); + if (outcome.kind === "continued") { + return { + terminalScanStartIndex: + terminalScanStartIndex ?? this.lastPromptStartIndex ?? messagesBeforeAttempt.length, + }; + } + // A controlled pause interrupted the continuation. The resumed + // objective owns the turn from here, so re-prompt with it rather + // than resuming a turn the operator replaced. + this.dropRetainedPrompt(activeSession, retainedPrompt); + retainedPrompt = undefined; + retryAdmittedPrompt = false; + retryAttempt = 0; + terminalScanStartIndex = undefined; + if (outcome.message === undefined) { + return { terminalScanStartIndex: activeSession.messages.length }; + } + nextText = outcome.message; + continue; } // No continuation is possible after all, so the retained prompt // must not survive into the re-prompt below. @@ -710,14 +796,14 @@ export class StageSessionController { private async createInitialSession(consumer: AgentSessionConsumer): Promise { if (!this.hasExplicitModelFallbackConfig) { - return this.createSession(undefined, consumer).catch((error) => + return this.createSessionObservingPause(undefined, consumer).catch((error) => this.createInitialSessionWithRetry(undefined, consumer, { error }), ); } const candidates = await this.modelCandidates(); const first = candidates[0]; if (first === undefined) { - return this.createSession(undefined, consumer).catch((error) => + return this.createSessionObservingPause(undefined, consumer).catch((error) => this.createInitialSessionWithRetry(undefined, consumer, { error }), ); } @@ -732,7 +818,7 @@ export class StageSessionController { } this.activeCandidateIndex = 0; this.selectedModel = first.id; - return this.createSession(first, consumer).catch((error) => + return this.createSessionObservingPause(first, consumer).catch((error) => this.createInitialSessionCandidateWalk(candidates, consumer, 0, { error }), ); } @@ -807,6 +893,50 @@ export class StageSessionController { } } + /** + * Latch a paused creation's replacement objective for the next prompt. + * + * The created session is already attached, so the pause cannot cancel it + * without leaking it; the replacement text becomes the next prompt's input. + */ + private latchCreationResume(resume: RetryPauseResume): void { + void resume + .then(async (resolved) => { + await resolved.runnerOwnedDeliverySettlement; + if (resolved.message !== undefined) this.pendingCreationResumeMessage = resolved.message; + }) + .catch(() => {}); + } + + /** + * Create while a controlled pause can still be observed. + * + * `requestPause()` latches its resume into every registered creation observer, + * so a pause that both starts and finishes while the adapter is still in + * flight is caught. Checking `currentResume()` after the await would miss it, + * because completing a resume clears the pause request. + * + * The creation promise is returned untouched, and observers are kept out of + * the thrown-retry set: this must add no microtask hop and must not alter + * abort or backoff semantics. An attached stream stays observable in the + * caller's own turn. + */ + private createSessionObservingPause( + candidate: WorkflowResolvedModelCandidate | undefined, + consumer: AgentSessionConsumer, + ): Promise { + const activePause = this.pauseControl.currentResume(); + const observer: CreationPauseObserver = { pauseResume: activePause }; + if (activePause !== undefined) this.latchCreationResume(activePause); + this.creationPauseObservers.add(observer); + const creation = this.createSession(candidate, consumer); + const settle = (): void => { + this.creationPauseObservers.delete(observer); + }; + creation.then(settle, settle); + return creation; + } + private async createSessionWithThrownErrorRetry( candidate: WorkflowResolvedModelCandidate | undefined, consumer: AgentSessionConsumer, @@ -821,7 +951,7 @@ export class StageSessionController { pendingFailure = undefined; throw failure.error; } - return await this.createSession(candidate, consumer); + return await this.createSessionObservingPause(candidate, consumer); } catch (error) { const errorSettingsManager = retrySettingsManagerFromError(error); if (errorSettingsManager !== undefined) this.sessionSettingsManager = errorSettingsManager; @@ -968,7 +1098,10 @@ export class StageSessionController { this.messageAdmission.reset(); this.replacement.retire(current); this.session = undefined; - this.sessionPromise = undefined; + // A candidate walk still advancing owns the shared creation promise: clearing + // it here would let a concurrent caller start a second walk, duplicating + // provider work and leaking whichever session lost the race. + if (this.sessionPromise !== this.ownedCreationPromise) this.sessionPromise = undefined; this.sessionSettingsManager = undefined; this.resumeCurrentSession = false; for (const unsubscribe of this.listenerUnsubscribes.values()) unsubscribe(); diff --git a/test/unit/stage-runner-thrown-retry.test.ts b/test/unit/stage-runner-thrown-retry.test.ts index 1ed5daa54..3645560a9 100644 --- a/test/unit/stage-runner-thrown-retry.test.ts +++ b/test/unit/stage-runner-thrown-retry.test.ts @@ -926,3 +926,223 @@ describe("createStageContext — unresolved overflow is terminal for its candida assert.deepEqual(prompts, ["anthropic/primary", "openai/fallback"]); }); }); + +describe("createStageContext — one creation gate across concurrent callers", () => { + const noRetry = retrySettings({ enabled: false }); + const gateSettingsManager: WorkflowFastModeSettingsManager = { + getCodexFastModeSettings: () => ({ chat: false, workflow: false }), + getRetrySettings: () => noRetry, + }; + + test("a second ensureSession joins the walk in flight instead of starting another", async () => { + const created: string[] = []; + const disposed: string[] = []; + let releaseFallback: (() => void) | undefined; + const fallbackReady = new Promise((resolve) => { + releaseFallback = resolve; + }); + const agentSession: AgentSessionAdapter = { + async create(options) { + const model = modelFor(options); + created.push(model); + if (model === "anthropic/primary") throw new Error("503 service unavailable during create"); + await fallbackReady; + const { session } = makeMockSession({ + async prompt() { + return undefined; + }, + dispose() { + disposed.push(model); + }, + getLastAssistantText: () => "fallback answer", + }); + return { session, settingsManager: gateSettingsManager }; + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary", + fallbackModels: ["openai/fallback"], + settingsManager: gateSettingsManager as never, + }, + }), + ) as InternalStageContext; + + const first = ctx.__ensureSession(); + // The walk has failed the primary, disposed it, and is now awaiting the + // fallback adapter — exactly the window where the promise used to be lost. + await flushMicrotasks(); + await flushMicrotasks(); + const second = ctx.__ensureSession(); + releaseFallback?.(); + await first; + await second; + + assert.deepEqual(created, ["anthropic/primary", "openai/fallback"], "only one candidate walk may run"); + assert.equal(created.filter((model) => model === "openai/fallback").length, 1, "no duplicate live session"); + assert.deepEqual(disposed, [], "the single attached session is still live"); + }); + + test("a first prompt joins a creation already in flight", async () => { + const created: string[] = []; + let releaseFallback: (() => void) | undefined; + const fallbackReady = new Promise((resolve) => { + releaseFallback = resolve; + }); + const agentSession: AgentSessionAdapter = { + async create(options) { + const model = modelFor(options); + created.push(model); + if (model === "anthropic/primary") throw new Error("503 service unavailable during create"); + await fallbackReady; + return sessionWithSettings(noRetry, async () => "fallback answer", { + getLastAssistantText: () => "fallback answer", + }); + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary", + fallbackModels: ["openai/fallback"], + settingsManager: gateSettingsManager as never, + }, + }), + ) as InternalStageContext; + + const eager = ctx.__ensureSession(); + await flushMicrotasks(); + await flushMicrotasks(); + const prompt = ctx.prompt("go"); + releaseFallback?.(); + await eager; + + assert.equal(await prompt, "fallback answer"); + assert.deepEqual(created, ["anthropic/primary", "openai/fallback"], "the prompt must not start a second walk"); + }); +}); + +describe("createStageContext — pauses observed while a creation is in flight", () => { + test("a pause that resolves during creation still delivers its replacement objective", async () => { + const promptTexts: string[] = []; + const created: string[] = []; + let releaseCreate: (() => void) | undefined; + const createReady = new Promise((resolve) => { + releaseCreate = resolve; + }); + const settings = retrySettings(); + const settingsManager: WorkflowFastModeSettingsManager = { + getCodexFastModeSettings: () => ({ chat: false, workflow: false }), + getRetrySettings: () => settings, + }; + const agentSession: AgentSessionAdapter = { + async create(options) { + created.push(modelFor(options)); + await createReady; + return sessionWithSettings( + settings, + async (text) => { + promptTexts.push(text); + return "resumed answer"; + }, + { getLastAssistantText: () => "resumed answer" }, + ); + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { model: "anthropic/primary", settingsManager: settingsManager as never }, + }), + ) as InternalStageContext; + + const eager = ctx.__ensureSession(); + await flushMicrotasks(); + // The pause both starts and finishes while the adapter is still in flight, + // so a post-await currentResume() check would miss it. + await ctx.__requestPause(); + await ctx.__resume("replacement objective"); + releaseCreate?.(); + await eager; + + assert.equal(await ctx.prompt("stale objective"), "resumed answer"); + assert.deepEqual(promptTexts, ["replacement objective"], "the replacement objective is authoritative"); + assert.deepEqual(created, ["anthropic/primary"], "the pause must not create a second session"); + }); +}); + +describe("createStageContext — pauses around a same-turn continuation", () => { + test("a pause during the continuation recovers with the replacement objective", async () => { + const messages: StageSessionRuntime["messages"] = []; + const promptTexts: string[] = []; + const created: string[] = []; + let continueCalls = 0; + let releaseContinue: (() => void) | undefined; + const continueReady = new Promise((resolve) => { + releaseContinue = resolve; + }); + const settings = retrySettings(); + let sessionRef: { abort: () => Promise } | undefined; + const agentSession: AgentSessionAdapter = { + async create(options) { + created.push(modelFor(options)); + const result = sessionWithSettings( + settings, + async (text) => { + promptTexts.push(text); + messages.push({ role: "user", content: text, timestamp: Date.now() } as never); + if (promptTexts.length === 1) { + messages.push({ + role: "assistant", + stopReason: "error", + errorMessage: "503 service unavailable", + content: [], + } as never); + throw new Error("503 service unavailable"); + } + return "resumed answer"; + }, + { + messages, + getLastAssistantText: () => "resumed answer", + state: { messages }, + sessionManager: {}, + modelRuntime: {}, + getContextUsage: () => ({}), + _runAgentContinue: async () => { + continueCalls += 1; + await continueReady; + // A controlled pause aborts the in-flight continuation. + throw new Error("Request aborted"); + }, + } as unknown as Partial, + ); + sessionRef = result.session; + return result; + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { model: "anthropic/primary", fallbackModels: ["openai/fallback"] }, + }), + ) as InternalStageContext; + + const prompt = ctx.prompt("do it"); + // Let the first attempt fail, the backoff elapse, and the continuation start. + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(continueCalls, 1, "the retry must reach the continuation path"); + const pause = ctx.__requestPause(); + releaseContinue?.(); + await pause; + await ctx.__resume("replacement objective"); + + assert.equal(await prompt, "resumed answer"); + assert.deepEqual(promptTexts, ["do it", "replacement objective"]); + assert.deepEqual(created, ["anthropic/primary"], "a controlled pause must not spend a fallback candidate"); + assert.equal(typeof sessionRef?.abort, "function"); + }); +}); From 8bf07a82a52923ae6b8538cb32ab5e5fd39c1d8f Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 18:51:59 -0700 Subject: [PATCH 09/13] fix(fallback): prefer Codex invalidation over abort wrappers (#2170) Assistant-model: GPT-5.4 --- packages/coding-agent/CHANGELOG.md | 1 + .../src/core/agent-session-retry.ts | 4 +-- .../test/agent-session-retry.test.ts | 26 +++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 74127e167..5901f000e 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -34,6 +34,7 @@ Cumulative release of the `0.9.12-alpha.1` prerelease. The summary below covers - The compaction relevance query is no longer truncated to 1,000 characters. Truncation made prompt section order the retention policy: for a long structured prompt only the leading section reached the planner, so a constraint stated later could not influence what was kept, while the objective it qualified survived and was acted on. Long queries are safe — an oversized planner request surfaces as an explicit provider-overflow failure rather than silent truncation ([#2172](https://github.com/bastani-inc/atomic/issues/2172)). - Cleared three advisories in the shipped dependency tree. `undici` moves 8.5.0 → 8.9.0, which covers five advisories against 8.0.0–8.8.0 — response desynchronization via the retry interceptor, two cross-user cache-directive disclosures, CRLF injection through a blob body `type`, and cookie-attribute injection — and it is the dispatcher behind `fetch_url` and every agent HTTP request. Transitively under `@modelcontextprotocol/sdk`, `ip-address` is pinned to 10.3.1 for three SSRF and trust-boundary bypasses (octal-decoded leading-zero octets, CIDR suffixes suppressing special-use classification, and misclassified IPv4-mapped/NAT64 addresses), and `hono` to 4.12.34 for a CORS-middleware ReDoS. `npm audit` is clean. + - Fixed bundled builtin extensions failing to load on Windows when shared modules were evaluated twice, so bundled workflows and tools initialize correctly. ## [0.9.11] - 2026-08-03 diff --git a/packages/coding-agent/src/core/agent-session-retry.ts b/packages/coding-agent/src/core/agent-session-retry.ts index 5ce814459..f225e98e9 100644 --- a/packages/coding-agent/src/core/agent-session-retry.ts +++ b/packages/coding-agent/src/core/agent-session-retry.ts @@ -53,10 +53,10 @@ export function _isFallbackableError(this: AgentSession, message: AssistantMessa if (message.stopReason !== "error") return false; const contextWindow = this.model?.contextWindow ?? 0; if (isContextOverflow(message, contextWindow)) return false; - const signal = normalizeModelFailureSignal(message); - if (signal.kind === "cancelled" || signal.kind === "task_failure") return false; const provider = message.provider || this.model?.provider; if (provider && containsCodexTokenInvalidation(provider, message)) return true; + const signal = normalizeModelFailureSignal(message); + if (signal.kind === "cancelled" || signal.kind === "task_failure") return false; return isRetryableModelFailure(message); } diff --git a/packages/coding-agent/test/agent-session-retry.test.ts b/packages/coding-agent/test/agent-session-retry.test.ts index 6404e3d56..630f55145 100644 --- a/packages/coding-agent/test/agent-session-retry.test.ts +++ b/packages/coding-agent/test/agent-session-retry.test.ts @@ -438,6 +438,32 @@ describe("AgentSession retry", () => { ).toBe(false); }); + it("treats Codex token invalidation wrapped in abort diagnostics as fallbackable only", async () => { + const created = await createSession({ failCount: 0 }); + const probe = created.session as unknown as { + _isFallbackableError(message: AssistantMessage): boolean; + _isRetryableError(message: AssistantMessage): boolean; + }; + const wrappedInvalidation = { + ...createAssistantMessage("", { + stopReason: "error", + errorMessage: "invalidated oauth token", + provider: "openai-codex", + api: "openai-codex-responses", + model: "gpt-5.5", + }), + diagnostics: [ + { + type: "provider_transport_failure", + error: { name: "AbortError", message: "The operation was aborted" }, + }, + ], + } as AssistantMessage; + + expect(probe._isFallbackableError(wrappedInvalidation)).toBe(true); + expect(probe._isRetryableError(wrappedInvalidation)).toBe(false); + }); + it("does not classify a reasoning-only turn with output tokens as empty", async () => { const created = await createSession({ failCount: 0 }); const probe = created.session as unknown as { _isEmptyCompletion(message: AssistantMessage): boolean }; From 83bcaf7aa007fe47969dc7136459d12a43f8ffd3 Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 19:27:36 -0700 Subject: [PATCH 10/13] fix(fallback): avoid CodeQL regex backtracking in classifier (#2170) Assistant-model: GPT-5.4 --- .../src/core/model-fallback-failures.ts | 52 ++++++++++++------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/packages/coding-agent/src/core/model-fallback-failures.ts b/packages/coding-agent/src/core/model-fallback-failures.ts index cad84adf6..bde989489 100644 --- a/packages/coding-agent/src/core/model-fallback-failures.ts +++ b/packages/coding-agent/src/core/model-fallback-failures.ts @@ -24,8 +24,6 @@ const RETRYABLE_MODEL_FAILURE_PATTERNS: readonly RegExp[] = [ /invalidated[_\s-]+(?:oauth|auth)[_\s-]+token/i, /forbidden/i, /invalid\s*key/i, - /model.*(?:unavailable|disabled|not\s*found|unknown)/i, - /(?:unavailable|disabled|not\s*found|unknown).*model/i, /overloaded/i, /temporarily\s*unavailable/i, /service\s*unavailable/i, @@ -180,9 +178,7 @@ function diagnosticErrors(value: unknown): readonly unknown[] { const diagnosticType = stringField(diagnostic, "type"); const diagnosticError = field(diagnostic, "error"); errors.push( - diagnosticType !== undefined && /provider[_\s-]?transport[_\s-]?failure/i.test(diagnosticType) - ? diagnostic - : (diagnosticError ?? diagnostic), + normalizeCode(diagnosticType) === "provider_transport_failure" ? diagnostic : (diagnosticError ?? diagnostic), ); } return errors; @@ -190,14 +186,36 @@ function diagnosticErrors(value: unknown): readonly unknown[] { function normalizeCode(value: string | number | undefined): string | undefined { if (value === undefined) return undefined; - const normalized = String(value) - .trim() - .toLowerCase() - .replace(/[^a-z0-9]+/g, "_") - .replace(/^_+|_+$/g, ""); + const raw = String(value).trim().toLowerCase(); + if (!raw) return undefined; + let normalized = ""; + let needsSeparator = false; + for (const char of raw) { + const code = char.charCodeAt(0); + const isLowerAlpha = code >= 97 && code <= 122; + const isDigit = code >= 48 && code <= 57; + if (isLowerAlpha || isDigit) { + normalized += char; + needsSeparator = true; + continue; + } + if (needsSeparator && normalized[normalized.length - 1] !== "_") normalized += "_"; + } + if (normalized.endsWith("_")) normalized = normalized.slice(0, -1); return normalized.length > 0 ? normalized : undefined; } +function messageLooksLikeModelUnavailable(message: string): boolean { + const lower = message.toLowerCase(); + return ( + lower.includes("model") && + (lower.includes("unavailable") || + lower.includes("disabled") || + lower.includes("not found") || + lower.includes("unknown")) + ); +} + function kindFromStatus(status: number | undefined): ModelFallbackFailureKind | undefined { switch (status) { case 400: @@ -373,7 +391,10 @@ function fallbackKindFromMessage(message: string, name: string | undefined): Mod if (REQUEST_INCOMPATIBLE_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "request_incompatible"; const nameKind = kindFromCode(name); if (nameKind !== undefined) return nameKind; - if (!RETRYABLE_MODEL_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return undefined; + const hasRetryablePattern = + RETRYABLE_MODEL_FAILURE_PATTERNS.some((pattern) => pattern.test(message)) || + messageLooksLikeModelUnavailable(message); + if (!hasRetryablePattern) return undefined; if (/rate\s*limit|too\s*many\s*requests|\b429\b|quota|usage[\s_-]*limit|billing|credit/i.test(message)) return "rate_limit"; if ( @@ -382,12 +403,7 @@ function fallbackKindFromMessage(message: string, name: string | undefined): Mod ) ) return "auth_on_candidate_provider"; - if ( - /model.*(?:unavailable|disabled|not\s*found|unknown)|(?:unavailable|disabled|not\s*found|unknown).*model/i.test( - message, - ) - ) - return "model_unavailable"; + if (messageLooksLikeModelUnavailable(message)) return "model_unavailable"; if (/network|fetch|socket|connection\s*refused|getaddrinfo|ENOTFOUND|EAI_AGAIN|timeout|timed\s*out/i.test(message)) return "network_timeout"; return "provider_unavailable"; @@ -492,7 +508,7 @@ function structuredSignal( const stopReason = stopReasonFrom(value)?.toLowerCase(); const diagnosticType = stringField(value, "type"); const providerTransportSignal = - diagnosticType !== undefined && /provider[_\s-]?transport[_\s-]?failure/i.test(diagnosticType) + diagnosticType !== undefined && normalizeCode(diagnosticType) === "provider_transport_failure" ? makeSignal("transport_error", value, source) : undefined; if (stopReason === "aborted") return makeSignal("cancelled", value, source); From 19bf13e45a54ce083a1ca705bca91684576cb037 Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 20:09:07 -0700 Subject: [PATCH 11/13] test(interactive-engine): widen subprocess startup waits Assistant-model: GPT-5.4 --- test/unit/interactive-engine-cycle-fallback.test.ts | 7 ++++++- test/unit/interactive-engine-inherited-discovery.test.ts | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/unit/interactive-engine-cycle-fallback.test.ts b/test/unit/interactive-engine-cycle-fallback.test.ts index 687f6a1bd..a28104efe 100644 --- a/test/unit/interactive-engine-cycle-fallback.test.ts +++ b/test/unit/interactive-engine-cycle-fallback.test.ts @@ -20,6 +20,7 @@ const serialTest = process.platform === "win32" ? test.sequential.skip : test.se const prefix = "@@ATOMIC_TEST@@"; const warning = "Configured default model is unavailable or unsupported. Update defaultProvider/defaultModel or use /model."; +const INTERACTIVE_STARTUP_TIMEOUT_MS = 20_000; interface Report { type?: string; @@ -292,7 +293,11 @@ serialTest( cwd, ); try { - const ready = await driver.waitFor((report) => report.type === "input_loop_ready"); + const ready = await driver.waitFor( + (report) => report.type === "input_loop_ready", + 0, + INTERACTIVE_STARTUP_TIMEOUT_MS, + ); const readyIndex = driver.reports.indexOf(ready); const locked = await driver.waitForState( (state) => state.modelFallbackReason === "configured-provider-unsupported", diff --git a/test/unit/interactive-engine-inherited-discovery.test.ts b/test/unit/interactive-engine-inherited-discovery.test.ts index 6e91869e4..7daf5ba85 100644 --- a/test/unit/interactive-engine-inherited-discovery.test.ts +++ b/test/unit/interactive-engine-inherited-discovery.test.ts @@ -15,6 +15,7 @@ import { const serialTest = process.platform === "win32" ? test.sequential.skip : test.sequential; const PREFIX = "@@ATOMIC_TEST@@"; +const INHERITED_DISCOVERY_TIMEOUT_MS = 20_000; interface HarnessReport { type?: string; @@ -165,7 +166,7 @@ function args(): string[] { } async function waitForCommand(driver: InteractiveDriver): Promise> { - const deadline = performance.now() + 10_000; + const deadline = performance.now() + INHERITED_DISCOVERY_TIMEOUT_MS; let names = new Set(); while (performance.now() < deadline) { names = await driver.autocomplete("/legacy-compatible"); From cfe8dad0350c8f4e9a146fa5affb81e611e6128c Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 20:48:33 -0700 Subject: [PATCH 12/13] fix(fallback): use same-model retry eligibility in workflow candidate walk (#2170) Prompt delivery and session creation authorized same-model retries with the broad fallback-eligibility predicate, so a 401/400/model-unavailable failure burned the whole settings.retry budget on a candidate that had already definitively rejected the request before advancing. Both call sites now use the shared isRetryableSameModelFailure() classifier, keeping the broad predicate for fallback advancement and the unresolved-context-overflow exclusion. Addresses Greptile review on #2201. Assistant-model: Claude Fable 5 --- packages/coding-agent/docs/workflows.md | 4 +- packages/coding-agent/src/index.ts | 1 + packages/workflows/CHANGELOG.md | 2 +- .../foreground/stage-runner-controller.ts | 10 ++++- .../runs/shared/model-fallback-failures.ts | 1 + test/unit/stage-runner-thrown-retry.test.ts | 38 ++++++++++++++++++- 6 files changed, 49 insertions(+), 7 deletions(-) diff --git a/packages/coding-agent/docs/workflows.md b/packages/coding-agent/docs/workflows.md index 95958fa67..69e09c512 100644 --- a/packages/coding-agent/docs/workflows.md +++ b/packages/coding-agent/docs/workflows.md @@ -2295,13 +2295,13 @@ readonly fallbackModels?: readonly string[]; readonly fallbackThinkingLevels?: readonly string[]; ``` -`fallbackModels` tries the primary first, each fallback in order, and then the current Atomic-selected model when available. It advances for rate limits and quota or usage-limit exhaustion, including messages such as `The usage limit has been reached` and codes such as `usage_limit_reached` or `insufficient_quota`. Auth/provider outages, unavailable models, network timeouts, generic transport errors such as `Connection error.` or `fetch failed`, and 5xx responses also advance the chain. A thrown retryable failure is retried on the same candidate with exponential backoff from `settings.retry` before the chain advances; `retry.enabled: false` keeps immediate advancement. A same-candidate retry resumes the existing turn when the stage transcript still ends in a message the agent can continue from, and otherwise re-sends the stage prompt; either way the failed provider error is dropped from the live transcript and the prompt is delivered exactly once. +`fallbackModels` tries the primary first, each fallback in order, and then the current Atomic-selected model when available. It advances for rate limits and quota or usage-limit exhaustion, including messages such as `The usage limit has been reached` and codes such as `usage_limit_reached` or `insufficient_quota`. Auth/provider outages, unavailable models, network timeouts, generic transport errors such as `Connection error.` or `fetch failed`, and 5xx responses also advance the chain. A thrown failure that another request to the same candidate can plausibly repair — a rate limit, provider outage, network timeout, or transport error — is retried on that candidate with exponential backoff from `settings.retry` before the chain advances; `retry.enabled: false` keeps immediate advancement. A failure the same candidate has already definitively rejected — a rejected credential, an unavailable model, or an incompatible request — skips the same-candidate retry and advances immediately, exactly as in main chat. A same-candidate retry resumes the existing turn when the stage transcript still ends in a message the agent can continue from, and otherwise re-sends the stage prompt; either way the failed provider error is dropped from the live transcript and the prompt is delivered exactly once. Request/context incompatibility also advances it, including HTTP 400/413/422 bad, unprocessable, or payload-too-large requests; unsupported tools or parameters; context-length or context-window overflow; and `too large`, `invalid_request`, or `bad_request` errors. This lets the chain reach the current selected user model when no configured candidate can serve the request. A context overflow that the stage session's compaction has already failed to resolve is terminal for its candidate: it skips the same-candidate retry, because re-sending an identical request cannot fit a context compaction could not shrink, and advances straight to the next candidate. -The chain also covers session creation. A stage session created eagerly — by `ctx.__ensureSession()`, an eager stage call, or a control attach — retries its candidate under `settings.retry` and then walks to the next configured candidate, so a provider that cannot even open a session does not strand the stage. A creation failure that exhausts the whole chain is not cached: the next call starts a fresh attempt. +The chain also covers session creation. A stage session created eagerly — by `ctx.__ensureSession()`, an eager stage call, or a control attach — retries transient creation failures on its candidate under `settings.retry` and then walks to the next configured candidate, so a provider that cannot even open a session does not strand the stage. Creation failures that same-candidate retry cannot repair — auth, unavailable model, incompatible request — advance immediately. A creation failure that exhausts the whole chain is not cached: the next call starts a fresh attempt. That walk runs behind a single creation gate. A concurrent `ctx.__ensureSession()` or a first `ctx.prompt()` joins the creation already in flight rather than starting a second walk, so the stage never has two live sessions competing for the same generation. diff --git a/packages/coding-agent/src/index.ts b/packages/coding-agent/src/index.ts index a2ce93fd5..697385068 100644 --- a/packages/coding-agent/src/index.ts +++ b/packages/coding-agent/src/index.ts @@ -133,6 +133,7 @@ export type { export { errorMessage, isRetryableModelFailure, + isRetryableSameModelFailure, modelFailureMessage, normalizeModelFailureSignal, } from "./core/model-fallback-failures.ts"; diff --git a/packages/workflows/CHANGELOG.md b/packages/workflows/CHANGELOG.md index 09b367589..465d13270 100644 --- a/packages/workflows/CHANGELOG.md +++ b/packages/workflows/CHANGELOG.md @@ -15,7 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed -- Fixed thrown retryable stage failures to retry the current model with bounded exponential backoff from `settings.retry` before advancing through `fallbackModels`; disabling retry keeps immediate candidate advancement ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). +- Fixed thrown transient stage failures — rate limits, provider outages, network timeouts, transport errors — to retry the current model with bounded exponential backoff from `settings.retry` before advancing through `fallbackModels`; disabling retry keeps immediate candidate advancement. Failures another request to the same model cannot repair — a rejected credential, an unavailable model, an incompatible request — skip the same-model retry and advance to the next candidate immediately, for prompt delivery and session creation alike ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). - Fixed a same-candidate stage retry losing the stage prompt. The retry resumed the existing session turn with a transcript that no longer ended in the prompt it was resuming, which the agent rejects; the prompt is now retained for that path and still re-sent on the re-prompt path ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). - Fixed a same-candidate stage retry stopping outright when the failed attempt had already streamed a completed assistant message. Resuming the existing turn is now attempted only when the transcript, as converted for the provider, still ends in a message the agent can continue from; otherwise the retry re-sends the prompt, which recovers instead of failing. Messages that convert away — a note excluded from context, an empty branch summary — no longer look like a valid tail ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). - Fixed an eagerly created stage session never reaching its `fallbackModels`. A session created before any prompt — by an eager stage call, a control attach, or `ctx.__ensureSession()` — retried only its first candidate and then failed the stage. It now walks the configured chain, and a failure that exhausts the chain is no longer cached and replayed to every later caller ([#2170](https://github.com/bastani-inc/atomic/issues/2170)). diff --git a/packages/workflows/src/runs/foreground/stage-runner-controller.ts b/packages/workflows/src/runs/foreground/stage-runner-controller.ts index 2524cd56e..269918164 100644 --- a/packages/workflows/src/runs/foreground/stage-runner-controller.ts +++ b/packages/workflows/src/runs/foreground/stage-runner-controller.ts @@ -19,6 +19,7 @@ import { buildModelCandidatesFromCatalog, errorMessage, isRetryableModelFailure, + isRetryableSameModelFailure, type WorkflowResolvedModelCandidate, workflowModelId, } from "../shared/model-fallback.js"; @@ -705,8 +706,13 @@ export class StageSessionController { // An already-unresolved context overflow is terminal for this model: // compaction has run and failed, so another identical request cannot // help. It stays fallbackable so `handleCandidateFailure()` advances. + // Same-model retry eligibility is narrower than fallback eligibility: + // auth, model-unavailable, and request-incompatible failures advance + // to the next candidate immediately instead of burning the retry + // budget on a request the same model has already rejected. const retryableFailure = isRetryableModelFailure(error); - const sameCandidateRetryable = retryableFailure && !isUnresolvedContextOverflowFailure(error); + const sameCandidateRetryable = + isRetryableSameModelFailure(error) && !isUnresolvedContextOverflowFailure(error); const decision = nextRetryDecision(this.retrySettings(), retryAttempt, sameCandidateRetryable); const continuationSession = retryableAgentSession(activeSession); const admittedMessages = activeSession.messages.length > messagesBeforeAttempt.length; @@ -955,7 +961,7 @@ export class StageSessionController { } catch (error) { const errorSettingsManager = retrySettingsManagerFromError(error); if (errorSettingsManager !== undefined) this.sessionSettingsManager = errorSettingsManager; - const decision = nextRetryDecision(this.retrySettings(), retryAttempt, isRetryableModelFailure(error)); + const decision = nextRetryDecision(this.retrySettings(), retryAttempt, isRetryableSameModelFailure(error)); if ( decision === undefined || this.disposed || diff --git a/packages/workflows/src/runs/shared/model-fallback-failures.ts b/packages/workflows/src/runs/shared/model-fallback-failures.ts index dd2ae1994..a504852de 100644 --- a/packages/workflows/src/runs/shared/model-fallback-failures.ts +++ b/packages/workflows/src/runs/shared/model-fallback-failures.ts @@ -13,6 +13,7 @@ export type { export { errorMessage, isRetryableModelFailure, + isRetryableSameModelFailure, modelFailureMessage, normalizeModelFailureSignal, } from "@bastani/atomic"; diff --git a/test/unit/stage-runner-thrown-retry.test.ts b/test/unit/stage-runner-thrown-retry.test.ts index 3645560a9..769e4aa97 100644 --- a/test/unit/stage-runner-thrown-retry.test.ts +++ b/test/unit/stage-runner-thrown-retry.test.ts @@ -308,6 +308,40 @@ describe("createStageContext — thrown model failure retry", () => { ); }); + test("session creation advances immediately on auth and request-incompatible failures", async () => { + for (const failure of ["401 unauthorized during create", "400 bad request during create"]) { + const created: string[] = []; + const settings = retrySettings(); + const settingsManager = { + getCodexFastModeSettings: () => ({ chat: false, workflow: false }), + getRetrySettings: () => settings, + }; + const agentSession: AgentSessionAdapter = { + async create(options) { + const model = modelFor(options); + created.push(model); + if (model === "anthropic/primary") throw new Error(failure); + return sessionWithSettings(settings, async () => "fallback answer", { + getLastAssistantText: () => "fallback answer", + }); + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary", + fallbackModels: ["openai/fallback"], + settingsManager: settingsManager as never, + }, + }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("go"), "fallback answer"); + assert.deepEqual(created, ["anthropic/primary", "openai/fallback"]); + } + }); + test("disabled retry advances immediately without another prompt", async () => { const calls: string[] = []; const settings = retrySettings({ enabled: false }); @@ -337,7 +371,7 @@ describe("createStageContext — thrown model failure retry", () => { assert.deepEqual(calls, ["anthropic/primary", "openai/fallback"]); }); - test("retries auth and request-incompatible thrown failures before advancing", async () => { + test("auth and request-incompatible thrown failures advance immediately without same-model retry", async () => { for (const failure of ["401 unauthorized", "400 bad request"]) { const calls: string[] = []; const settings = retrySettings(); @@ -363,7 +397,7 @@ describe("createStageContext — thrown model failure retry", () => { ) as InternalStageContext; assert.equal(await ctx.prompt("go"), "fallback answer"); - assert.deepEqual(calls, ["anthropic/primary", "anthropic/primary", "anthropic/primary", "openai/fallback"]); + assert.deepEqual(calls, ["anthropic/primary", "openai/fallback"]); } }); From 71728a5dd524226dd2a76011e2eed9d27bda6a1a Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Tue, 4 Aug 2026 21:13:59 -0700 Subject: [PATCH 13/13] chore: re-trigger CI for 7500aa72b (Tests workflow never started)