Skip to content

Add exponential backoff for mid-stream retry failures #8888

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 28, 2025
Merged

Add exponential backoff for mid-stream retry failures #8888

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c135224
Add exponential backoff for mid-stream retry failures
daniel-lxs Oct 28, 2025
45ed85d
Add abort check in retry countdown loop
daniel-lxs Oct 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 89 additions & 41 deletions src/core/task/Task.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1749,9 +1749,10 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
interface StackItem {
userContent: Anthropic.Messages.ContentBlockParam[]
includeFileDetails: boolean
retryAttempt?: number
}

const stack: StackItem[] = [{ userContent, includeFileDetails }]
const stack: StackItem[] = [{ userContent, includeFileDetails, retryAttempt: 0 }]

while (stack.length > 0) {
const currentItem = stack.pop()!
Expand Down Expand Up @@ -2231,10 +2232,21 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
`[Task#${this.taskId}.${this.instanceId}] Stream failed, will retry: ${streamingFailedMessage}`,
)

// Push the same content back onto the stack to retry
// Apply exponential backoff similar to first-chunk errors when auto-resubmit is enabled
const stateForBackoff = await this.providerRef.deref()?.getState()
if (stateForBackoff?.autoApprovalEnabled && stateForBackoff?.alwaysApproveResubmit) {
await this.backoffAndAnnounce(
currentItem.retryAttempt ?? 0,
error,
streamingFailedMessage,
)
}

// Push the same content back onto the stack to retry, incrementing the retry attempt counter
stack.push({
userContent: currentUserContent,
includeFileDetails: false,
retryAttempt: (currentItem.retryAttempt ?? 0) + 1,
})

// Continue to retry the request
Expand Down Expand Up @@ -2775,45 +2787,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
errorMsg = "Unknown error"
}

const baseDelay = requestDelaySeconds || 5
let exponentialDelay = Math.min(
Math.ceil(baseDelay * Math.pow(2, retryAttempt)),
MAX_EXPONENTIAL_BACKOFF_SECONDS,
)

// If the error is a 429, and the error details contain a retry delay, use that delay instead of exponential backoff
if (error.status === 429) {
const geminiRetryDetails = error.errorDetails?.find(
(detail: any) => detail["@type"] === "type.googleapis.com/google.rpc.RetryInfo",
)
if (geminiRetryDetails) {
const match = geminiRetryDetails?.retryDelay?.match(/^(\d+)s$/)
if (match) {
exponentialDelay = Number(match[1]) + 1
}
}
}

// Wait for the greater of the exponential delay or the rate limit delay
const finalDelay = Math.max(exponentialDelay, rateLimitDelay)

// Show countdown timer with exponential backoff
for (let i = finalDelay; i > 0; i--) {
await this.say(
"api_req_retry_delayed",
`${errorMsg}\n\nRetry attempt ${retryAttempt + 1}\nRetrying in ${i} seconds...`,
undefined,
true,
)
await delay(1000)
}

await this.say(
"api_req_retry_delayed",
`${errorMsg}\n\nRetry attempt ${retryAttempt + 1}\nRetrying now...`,
undefined,
false,
)
// Apply shared exponential backoff and countdown UX
await this.backoffAndAnnounce(retryAttempt, error, errorMsg)

// Delegate generator output from the recursive call with
// incremented retry count.
Expand Down Expand Up @@ -2851,6 +2826,79 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
yield* iterator
}

// Shared exponential backoff for retries (first-chunk and mid-stream)
private async backoffAndAnnounce(retryAttempt: number, error: any, header?: string): Promise<void> {
try {
const state = await this.providerRef.deref()?.getState()
const baseDelay = state?.requestDelaySeconds || 5

let exponentialDelay = Math.min(
Math.ceil(baseDelay * Math.pow(2, retryAttempt)),
MAX_EXPONENTIAL_BACKOFF_SECONDS,
)

// Respect provider rate limit window
let rateLimitDelay = 0
const rateLimit = state?.apiConfiguration?.rateLimitSeconds || 0
if (Task.lastGlobalApiRequestTime && rateLimit > 0) {
const elapsed = performance.now() - Task.lastGlobalApiRequestTime
rateLimitDelay = Math.ceil(Math.min(rateLimit, Math.max(0, rateLimit * 1000 - elapsed) / 1000))
}

// Prefer RetryInfo on 429 if present
if (error?.status === 429) {
const retryInfo = error?.errorDetails?.find(
(d: any) => d["@type"] === "type.googleapis.com/google.rpc.RetryInfo",
)
const match = retryInfo?.retryDelay?.match?.(/^(\d+)s$/)
if (match) {
exponentialDelay = Number(match[1]) + 1
}
}

const finalDelay = Math.max(exponentialDelay, rateLimitDelay)
if (finalDelay <= 0) return

// Build header text; fall back to error message if none provided
let headerText = header
if (!headerText) {
if (error?.error?.metadata?.raw) {
headerText = JSON.stringify(error.error.metadata.raw, null, 2)
} else if (error?.message) {
headerText = error.message
} else {
headerText = "Unknown error"
}
}
headerText = headerText ? `${headerText}\n\n` : ""

// Show countdown timer with exponential backoff
for (let i = finalDelay; i > 0; i--) {
// Check abort flag during countdown to allow early exit
if (this.abort) {
throw new Error(`[Task#${this.taskId}] Aborted during retry countdown`)
}

await this.say(
"api_req_retry_delayed",
`${headerText}Retry attempt ${retryAttempt + 1}\nRetrying in ${i} seconds...`,
undefined,
true,
)
await delay(1000)
}

await this.say(
"api_req_retry_delayed",
`${headerText}Retry attempt ${retryAttempt + 1}\nRetrying now...`,
undefined,
false,
)
} catch (err) {
console.error("Exponential backoff failed:", err)
}
}

// Checkpoints

public async checkpointSave(force: boolean = false, suppressMessage: boolean = false) {
Expand Down