-
Notifications
You must be signed in to change notification settings - Fork 2.8k
feat(retry): add configurable delay and retry limits #4704
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
30bd8e4
c2214db
44b40d5
7747d61
0c8e02d
389a4ec
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| "kilo-code": minor | ||
| --- | ||
|
|
||
| feat(retry): implement configurable delay and max retries |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -162,7 +162,6 @@ import { MessageManager } from "../message-manager" | |
| import { validateAndFixToolResultIds } from "./validateToolResultIds" | ||
| import { deduplicateToolUseBlocks } from "./deduplicateToolUseBlocks" | ||
|
|
||
| const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes | ||
| const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds | ||
| const FORCED_CONTEXT_REDUCTION_PERCENT = 75 // Keep 75% of context (remove 25%) on context window errors | ||
| const MAX_CONTEXT_WINDOW_RETRIES = 3 // Maximum retries for context window errors | ||
|
|
@@ -3595,9 +3594,14 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike { | |
| `[Task#${this.taskId}.${this.instanceId}] Stream failed, will retry: ${streamingFailedMessage}`, | ||
| ) | ||
|
|
||
| // Apply exponential backoff similar to first-chunk errors when auto-resubmit is enabled | ||
| // Apply backoff similar to first-chunk errors when auto-resubmit is enabled | ||
| const stateForBackoff = await this.providerRef.deref()?.getState() | ||
| if (stateForBackoff?.autoApprovalEnabled) { | ||
| const retryMax = stateForBackoff?.requestRetryMax ?? 0 | ||
| if ( | ||
| stateForBackoff?.autoApprovalEnabled && | ||
| stateForBackoff?.alwaysApproveResubmit && // kilocode_change | ||
| (retryMax === 0 || (currentItem.retryAttempt ?? 0) < retryMax) | ||
| ) { | ||
| await this.backoffAndAnnounce(currentItem.retryAttempt ?? 0, error) | ||
|
|
||
| // Check if task was aborted during the backoff | ||
|
|
@@ -3925,7 +3929,12 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike { | |
|
|
||
| // Check if we should auto-retry or prompt the user | ||
| // Reuse the state variable from above | ||
| if (state?.autoApprovalEnabled) { | ||
| const retryMax = state?.requestRetryMax ?? 0 | ||
| if ( | ||
| state?.autoApprovalEnabled && | ||
| state?.alwaysApproveResubmit && // kilocode_change | ||
| (retryMax === 0 || (currentItem.retryAttempt ?? 0) < retryMax) | ||
| ) { | ||
| // Auto-retry with backoff - don't persist failure message when retrying | ||
| await this.backoffAndAnnounce( | ||
| currentItem.retryAttempt ?? 0, | ||
|
|
@@ -4801,8 +4810,9 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike { | |
| } | ||
| // kilocode_change end | ||
| // note that this api_req_failed ask is unique in that we only present this option if the api hasn't streamed any content yet (ie it fails on the first chunk due), as it would allow them to hit a retry button. However if the api failed mid-stream, it could be in any arbitrary state where some tools may have executed, so that error is handled differently and requires cancelling the task entirely. | ||
| if (autoApprovalEnabled) { | ||
| // Apply shared exponential backoff and countdown UX | ||
| const retryMax = state?.requestRetryMax ?? 0 // kilocode_change | ||
| if (autoApprovalEnabled && state?.alwaysApproveResubmit && (retryMax === 0 || retryAttempt < retryMax)) { | ||
| // Apply shared backoff and countdown UX | ||
| await this.backoffAndAnnounce(retryAttempt, error) | ||
|
|
||
| // CRITICAL: Check if task was aborted during the backoff countdown | ||
|
|
@@ -4866,16 +4876,11 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike { | |
| // kilocode_change end | ||
| } | ||
|
|
||
| // Shared exponential backoff for retries (first-chunk and mid-stream) | ||
| // Shared backoff for retries (first-chunk and mid-stream) | ||
| private async backoffAndAnnounce(retryAttempt: number, error: any): Promise<void> { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: The Also, the comment on line 4925 ("Show countdown timer with exponential backoff") and the error message on line 4938 ("Exponential backoff failed") are stale — the method no longer uses exponential backoff. |
||
| try { | ||
| const state = await this.providerRef.deref()?.getState() | ||
| const baseDelay = state?.requestDelaySeconds || 5 | ||
|
|
||
| let exponentialDelay = Math.min( | ||
| Math.ceil(baseDelay * Math.pow(2, retryAttempt)), | ||
| MAX_EXPONENTIAL_BACKOFF_SECONDS, | ||
| ) | ||
| let requestDelaySeconds = state?.requestDelaySeconds ?? 10 | ||
|
|
||
| // Respect provider rate limit window | ||
| let rateLimitDelay = 0 | ||
|
|
@@ -4892,11 +4897,11 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike { | |
| ) | ||
| const match = retryInfo?.retryDelay?.match?.(/^(\d+)s$/) | ||
| if (match) { | ||
| exponentialDelay = Number(match[1]) + 1 | ||
| requestDelaySeconds = Number(match[1]) + 1 | ||
| } | ||
| } | ||
|
|
||
| const finalDelay = Math.max(exponentialDelay, rateLimitDelay) | ||
| const finalDelay = Math.max(requestDelaySeconds, rateLimitDelay) | ||
| if (finalDelay <= 0) { | ||
| return | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,125 @@ | ||
| import * as os from "os" | ||
| import * as path from "path" | ||
| import * as vscode from "vscode" | ||
| import { Task } from "../Task" | ||
|
|
||
| // Mock dependencies | ||
| vi.mock("delay", () => ({ | ||
| __esModule: true, | ||
| default: vi.fn().mockResolvedValue(undefined), | ||
| })) | ||
|
|
||
| vi.mock("p-wait-for", () => ({ | ||
| default: vi.fn().mockImplementation(async () => Promise.resolve()), | ||
| })) | ||
|
|
||
| vi.mock("vscode", () => { | ||
| return { | ||
| workspace: { | ||
| getConfiguration: vi.fn(() => ({ get: vi.fn() })), | ||
| }, | ||
| env: { | ||
| uriScheme: "vscode", | ||
| language: "en", | ||
| }, | ||
| EventEmitter: vi.fn().mockImplementation(() => ({ | ||
| event: vi.fn(), | ||
| fire: vi.fn(), | ||
| })), | ||
| } | ||
| }) | ||
|
|
||
| describe("Auto-Retry Logic", () => { | ||
| let mockProvider: any | ||
| let mockApiConfig: any | ||
| let mockExtensionContext: any | ||
|
|
||
| beforeEach(() => { | ||
| mockExtensionContext = { | ||
| globalState: { | ||
| get: vi.fn(), | ||
| update: vi.fn(), | ||
| keys: vi.fn().mockReturnValue([]), | ||
| }, | ||
| globalStorageUri: { fsPath: path.join(os.tmpdir(), "test-storage") }, | ||
| secrets: { | ||
| get: vi.fn().mockResolvedValue(undefined), | ||
| store: vi.fn().mockResolvedValue(undefined), | ||
| }, | ||
| extensionUri: { fsPath: "/mock/path" }, | ||
| extension: { packageJSON: { version: "1.0.0" } }, | ||
| } | ||
|
|
||
| mockProvider = { | ||
| getState: vi.fn().mockResolvedValue({ | ||
| autoApprovalEnabled: true, | ||
| requestDelaySeconds: 1, | ||
| requestRetryMax: 3, | ||
| }), | ||
| postMessageToWebview: vi.fn().mockResolvedValue(undefined), | ||
| postStateToWebview: vi.fn().mockResolvedValue(undefined), | ||
| } | ||
|
|
||
| mockApiConfig = { | ||
| apiProvider: "anthropic", | ||
| apiModelId: "claude-3-5-sonnet-20241022", | ||
| } | ||
| }) | ||
|
|
||
| it("should calculate correct delay", async () => { | ||
| const task = new Task({ | ||
| context: mockExtensionContext, | ||
| provider: mockProvider, | ||
| apiConfiguration: mockApiConfig, | ||
| task: "test", | ||
| startTask: false, | ||
| }) | ||
|
|
||
| const delay = (task as any).backoffAndAnnounce(1, new Error("test")) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: Test has no assertions — this test calls The comment on line 79 acknowledges this limitation. Consider either:
|
||
| // We can't easily await this because it has a loop with delay() | ||
| // but we can check the internal logic if we expose it or mock delay better | ||
| }) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: No-op test — this test creates a Consider either:
|
||
|
|
||
| it("should respect requestRetryMax and alwaysApproveResubmit", async () => { | ||
| const state = { | ||
| autoApprovalEnabled: true, | ||
| alwaysApproveResubmit: true, | ||
| requestRetryMax: 2 | ||
| } | ||
|
|
||
| const shouldRetry = (attempt: number) => | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: Tests re-implement retry logic locally instead of testing the actual Consider testing the real |
||
| state.autoApprovalEnabled && | ||
| state.alwaysApproveResubmit && | ||
| (state.requestRetryMax === 0 || attempt < state.requestRetryMax) | ||
|
|
||
| // retryAttempt 0 < 2 -> should retry | ||
| expect(shouldRetry(0)).toBe(true) | ||
| // retryAttempt 1 < 2 -> should retry | ||
| expect(shouldRetry(1)).toBe(true) | ||
| // retryAttempt 2 == 2 -> should NOT retry | ||
| expect(shouldRetry(2)).toBe(false) | ||
|
|
||
| // If alwaysApproveResubmit is false, should NOT retry | ||
| state.alwaysApproveResubmit = false | ||
| expect(shouldRetry(0)).toBe(false) | ||
| }) | ||
|
|
||
| it("should handle unlimited retries when requestRetryMax is 0", async () => { | ||
| const state = { | ||
| autoApprovalEnabled: true, | ||
| alwaysApproveResubmit: true, | ||
| requestRetryMax: 0 | ||
| } | ||
|
|
||
| const shouldRetry = (attempt: number) => | ||
| state.autoApprovalEnabled && | ||
| state.alwaysApproveResubmit && | ||
| (state.requestRetryMax === 0 || attempt < state.requestRetryMax) | ||
|
|
||
| expect(shouldRetry(100)).toBe(true) | ||
|
|
||
| // If alwaysApproveResubmit is false, should NOT retry even with unlimited retries | ||
| state.alwaysApproveResubmit = false | ||
| expect(shouldRetry(100)).toBe(false) | ||
| }) | ||
| }) | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
CRITICAL: The
retryMaxcheck here only gates the backoff delay — it does NOT gate the actual retry. Lines 3619–3627 (stack.push+continue) execute unconditionally outside thisifblock, meaning whenretryMaxis reached, the backoff is skipped but the retry still happens. This creates an infinite retry loop without any delay.The
stack.push+continueblock should be moved inside thisif, with anelsebranch that either breaks or prompts the user (similar to the empty-assistant-response path at line 3965 which correctly usesif/else).