diff --git a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts index 273474cfcb..cbe3b4a678 100644 --- a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts +++ b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts @@ -368,9 +368,13 @@ function mockCreatedInfraRetryFlow( // --- Tests --- -import type { POST as POSTType } from './route'; +import type { + POST as POSTType, + isWorkspaceCapacityFailure as isWorkspaceCapacityFailureType, +} from './route'; let POST: typeof POSTType; +let isWorkspaceCapacityFailure: typeof isWorkspaceCapacityFailureType; beforeEach(async () => { jest.clearAllMocks(); @@ -431,7 +435,27 @@ beforeEach(async () => { ); mockDisableCodeReviewForActionRequiredFailure.mockResolvedValue(undefined); mockDisableCodeReviewForRepeatedCloneTimeoutsToday.mockResolvedValue(null); - ({ POST } = await import('./route')); + ({ POST, isWorkspaceCapacityFailure } = await import('./route')); +}); + +describe('isWorkspaceCapacityFailure', () => { + it('detects a full-disk failure from the structured subtype or the message', () => { + expect( + isWorkspaceCapacityFailure(undefined, 'Workspace setup failed: sandbox storage full') + ).toBe(true); + // Orchestrator session-start path reports it inside a "(500)" message. + expect( + isWorkspaceCapacityFailure( + undefined, + 'initiate failed (500): Workspace admission rejected: 1036 MB available below 2048 MB threshold after cleanup' + ) + ).toBe(true); + }); + + it('ignores unrelated failures', () => { + expect(isWorkspaceCapacityFailure(undefined, 'The message could not be delivered')).toBe(false); + expect(isWorkspaceCapacityFailure(undefined, undefined)).toBe(false); + }); }); describe('POST /api/internal/code-review-status/[reviewId]', () => { diff --git a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts index bc2638eadf..3759b99504 100644 --- a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts +++ b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts @@ -348,6 +348,19 @@ function normalizePayload(raw: StatusUpdatePayload): { terminalReason = 'billing'; } + // Infer workspace capacity (transient sandbox disk pressure) so it is + // classified distinctly instead of counted as an unknown delivery failure. + // Capacity arrives either with no terminal reason (delivery path) or with a + // generic 'sandbox_error' (orchestrator session-start path), so override that + // one generic value too — otherwise the column stays 'sandbox_error' and only + // the admin router's message match recovers the category. + if ( + (!terminalReason || terminalReason === 'sandbox_error') && + isWorkspaceCapacityFailure(failure, raw.errorMessage) + ) { + terminalReason = 'workspace_capacity'; + } + if ( (raw.status === 'failed' || raw.status === 'interrupted') && isModelNotFoundCodeReviewTerminalReason(terminalReason, raw.errorMessage) @@ -371,6 +384,29 @@ function normalizePayload(raw: StatusUpdatePayload): { }; } +/** + * Detects a workspace disk-capacity failure so it is recorded with a distinct + * `workspace_capacity` terminal reason. Prefers the structured failure subtype + * from cloud-agent-next and falls back to the safe error message for older + * payloads. + * + * The message fallback is intentionally broad ('admission rejected'); a stricter, + * phrasing-anchored variant lives in + * services/code-review-infra/src/code-review-orchestrator.ts + * (isWorkspaceAdmissionCapacityFailure). Keep the two in mind together if the + * admission-rejection message format ever changes. + */ +export function isWorkspaceCapacityFailure( + failure: CloudAgentSafeFailure | undefined, + errorMessage: string | undefined +): boolean { + if (failure?.code === 'workspace_setup_failed' && failure.subtype === 'sandbox_storage_full') { + return true; + } + const message = (errorMessage ?? '').toLowerCase(); + return message.includes('sandbox storage full') || message.includes('admission rejected'); +} + function isBillingCodeReviewTerminalReason( terminalReason?: CodeReviewTerminalReason, errorMessage?: string | null diff --git a/apps/web/src/routers/admin-code-reviews-router.test.ts b/apps/web/src/routers/admin-code-reviews-router.test.ts index 1720b78279..c138c8ee96 100644 --- a/apps/web/src/routers/admin-code-reviews-router.test.ts +++ b/apps/web/src/routers/admin-code-reviews-router.test.ts @@ -341,6 +341,60 @@ describe('adminCodeReviewsRouter', () => { expect(exportRows[0]).toHaveProperty('attempt_status'); }); + it('buckets sandbox capacity and delivery failures instead of Other', async () => { + const owner = { type: 'user', id: adminUser.id } satisfies ReviewOwner; + + await db.insert(cloud_agent_code_reviews).values([ + reviewValues({ + owner, + status: 'failed', + createdAt: timestamp(700), + terminalReason: 'workspace_capacity', + errorMessage: 'Workspace setup failed: sandbox storage full', + }), + // terminal_reason is authoritative and must win over the generic '%404%' + // branch that this admission-rejected message would otherwise match. + reviewValues({ + owner, + status: 'failed', + createdAt: timestamp(705), + terminalReason: 'workspace_capacity', + errorMessage: 'Workspace admission rejected: 404 MB available below 2048 MB threshold', + }), + // Orchestrator session-start path: reports capacity as terminal_reason + // 'sandbox_error' with a "(500)" message. The capacity message match must + // win over the generic '%500%' Upstream Server Error branch. + reviewValues({ + owner, + status: 'failed', + createdAt: timestamp(707), + terminalReason: 'sandbox_error', + errorMessage: + 'initiate failed (500): Workspace admission rejected: 1036 MB available below 2048 MB threshold after cleanup', + }), + reviewValues({ + owner, + status: 'failed', + createdAt: timestamp(710), + errorMessage: 'The message could not be delivered', + }), + ]); + + const caller = await createCallerForUser(adminUser.id); + const errors = await caller.admin.codeReviews.getErrorAnalysis(filterInput()); + + expect(errors.categories).toEqual( + expect.arrayContaining([ + expect.objectContaining({ category: 'Sandbox Capacity', count: 3 }), + expect.objectContaining({ category: 'Delivery Failure', count: 1 }), + ]) + ); + const categoryNames = errors.categories.map(category => category.category); + expect(categoryNames).not.toContain('Other'); + expect(categoryNames).not.toContain('Not Found'); + expect(categoryNames).not.toContain('Upstream Server Error'); + }); + it('classifies final model-not-found outcomes as cancellations instead of failures', async () => { const owner = { type: 'user', id: adminUser.id } satisfies ReviewOwner; diff --git a/apps/web/src/routers/admin-code-reviews-router.ts b/apps/web/src/routers/admin-code-reviews-router.ts index 9551a032bb..01d5b4f8a5 100644 --- a/apps/web/src/routers/admin-code-reviews-router.ts +++ b/apps/web/src/routers/admin-code-reviews-router.ts @@ -83,6 +83,9 @@ const excludeModelNotFoundAttempt = sql`COALESCE(${cloud_agent_code_review_attem */ const errorCategoryExpr = sql`CASE WHEN ${cloud_agent_code_reviews.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'gitlab_project_access_required', 'byok_invalid_key', 'selected_model_unavailable') THEN 'Action Required' + WHEN ${cloud_agent_code_reviews.terminal_reason} = 'workspace_capacity' THEN 'Sandbox Capacity' + WHEN ${cloud_agent_code_reviews.error_message} LIKE '%sandbox storage full%' OR ${cloud_agent_code_reviews.error_message} LIKE '%admission rejected%' OR ${cloud_agent_code_reviews.error_message} LIKE '%storage full%' THEN 'Sandbox Capacity' + WHEN ${cloud_agent_code_reviews.error_message} LIKE '%connect to the sandbox%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Sandbox connection failed%' OR ${cloud_agent_code_reviews.error_message} LIKE '%container shut down%' THEN 'Sandbox Connection' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%rate limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Rate limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%429%' THEN 'Rate Limited' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%timeout%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Timeout%' OR ${cloud_agent_code_reviews.error_message} LIKE '%ETIMEDOUT%' OR ${cloud_agent_code_reviews.error_message} LIKE '%timed out%' THEN 'Timeout' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%context window%' OR ${cloud_agent_code_reviews.error_message} LIKE '%token limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%too large%' OR ${cloud_agent_code_reviews.error_message} LIKE '%maximum context length%' THEN 'Context Window Exceeded' @@ -91,12 +94,17 @@ const errorCategoryExpr = sql`CASE WHEN ${cloud_agent_code_reviews.error_message} LIKE '%500%' OR ${cloud_agent_code_reviews.error_message} LIKE '%502%' OR ${cloud_agent_code_reviews.error_message} LIKE '%503%' OR ${cloud_agent_code_reviews.error_message} LIKE '%internal server%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Internal Server%' THEN 'Upstream Server Error' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%ECONNREFUSED%' OR ${cloud_agent_code_reviews.error_message} LIKE '%ECONNRESET%' OR ${cloud_agent_code_reviews.error_message} LIKE '%socket hang up%' OR ${cloud_agent_code_reviews.error_message} LIKE '%network%' THEN 'Network Error' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%parse%' OR ${cloud_agent_code_reviews.error_message} LIKE '%JSON%' OR ${cloud_agent_code_reviews.error_message} LIKE '%unexpected token%' THEN 'Parse Error' + WHEN ${cloud_agent_code_reviews.error_message} LIKE '%could not be delivered%' THEN 'Delivery Failure' + WHEN ${cloud_agent_code_reviews.error_message} LIKE '%repository_not_installed%' OR ${cloud_agent_code_reviews.error_message} LIKE '%app installation required%' THEN 'Action Required' WHEN ${cloud_agent_code_reviews.error_message} IS NULL THEN 'Unknown Error' ELSE 'Other' END`; const attemptErrorCategoryExpr = sql`CASE WHEN ${cloud_agent_code_review_attempts.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'gitlab_project_access_required', 'byok_invalid_key', 'selected_model_unavailable') THEN 'Action Required' + WHEN ${cloud_agent_code_review_attempts.terminal_reason} = 'workspace_capacity' THEN 'Sandbox Capacity' + WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%sandbox storage full%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%admission rejected%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%storage full%' THEN 'Sandbox Capacity' + WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%connect to the sandbox%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Sandbox connection failed%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%container shut down%' THEN 'Sandbox Connection' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%rate limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Rate limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%429%' THEN 'Rate Limited' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%timeout%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Timeout%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%ETIMEDOUT%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%timed out%' THEN 'Timeout' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%context window%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%token limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%too large%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%maximum context length%' THEN 'Context Window Exceeded' @@ -105,6 +113,8 @@ const attemptErrorCategoryExpr = sql`CASE WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%500%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%502%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%503%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%internal server%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Internal Server%' THEN 'Upstream Server Error' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%ECONNREFUSED%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%ECONNRESET%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%socket hang up%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%network%' THEN 'Network Error' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%parse%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%JSON%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%unexpected token%' THEN 'Parse Error' + WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%could not be delivered%' THEN 'Delivery Failure' + WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%repository_not_installed%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%app installation required%' THEN 'Action Required' WHEN ${cloud_agent_code_review_attempts.error_message} IS NULL THEN 'Unknown Error' ELSE 'Other' END`; diff --git a/packages/db/src/schema-types.ts b/packages/db/src/schema-types.ts index 245e8a3fea..afe1eb3a20 100644 --- a/packages/db/src/schema-types.ts +++ b/packages/db/src/schema-types.ts @@ -2000,6 +2000,7 @@ export const CODE_REVIEW_TERMINAL_REASONS = [ 'timeout', 'upstream_error', 'sandbox_error', + 'workspace_capacity', 'unknown', ] as const; diff --git a/packages/worker-utils/src/cloud-agent-next-client.ts b/packages/worker-utils/src/cloud-agent-next-client.ts index 1ac032c5e0..a53ccb2b30 100644 --- a/packages/worker-utils/src/cloud-agent-next-client.ts +++ b/packages/worker-utils/src/cloud-agent-next-client.ts @@ -164,6 +164,7 @@ export type CloudAgentTerminalReason = | 'timeout' | 'upstream_error' | 'sandbox_error' + | 'workspace_capacity' | 'unknown'; export class CloudAgentNextError extends Error { diff --git a/services/cloud-agent-next/src/execution/orchestrator.test.ts b/services/cloud-agent-next/src/execution/orchestrator.test.ts index e8679e65a3..11f469db33 100644 --- a/services/cloud-agent-next/src/execution/orchestrator.test.ts +++ b/services/cloud-agent-next/src/execution/orchestrator.test.ts @@ -360,7 +360,7 @@ describe('ExecutionOrchestrator AgentSandbox delivery', () => { await expect(orchestrator.execute(basePlan)).rejects.toBe(finalizingError); }); - it('does not recover the shared sandbox for plain capacity admission rejection', async () => { + it('translates a capacity admission rejection to a retryable sandbox_storage_full failure without recovering the shared sandbox', async () => { const { orchestrator, ensureWrapper, deleteSandbox } = createOrchestrator(); ensureWrapper.mockRejectedValueOnce( new WorkspaceCapacityAdmissionRejectedError({ @@ -371,7 +371,13 @@ describe('ExecutionOrchestrator AgentSandbox delivery', () => { }) ); - await expect(orchestrator.execute(basePlan)).rejects.toThrow('Failed to start wrapper'); + // Must not degrade to a generic WRAPPER_START_FAILED: the subtype is what + // routes the pending flush to the capacity-specific retry backoff. + await expect(orchestrator.execute(basePlan)).rejects.toMatchObject({ + code: 'WORKSPACE_SETUP_FAILED', + retryable: true, + workspaceFailureSubtype: 'sandbox_storage_full', + } satisfies Partial); expect(deleteSandbox).not.toHaveBeenCalled(); }); diff --git a/services/cloud-agent-next/src/execution/orchestrator.ts b/services/cloud-agent-next/src/execution/orchestrator.ts index b616880d9f..578229a71d 100644 --- a/services/cloud-agent-next/src/execution/orchestrator.ts +++ b/services/cloud-agent-next/src/execution/orchestrator.ts @@ -18,6 +18,7 @@ import { ExecutionError } from './errors.js'; import { SessionService } from '../session-service.js'; import { logger } from '../logger.js'; import { WrapperError } from '../kilo/wrapper-client.js'; +import { WorkspaceCapacityAdmissionRejectedError } from '../workspace-errors.js'; import { withDORetry } from '../utils/do-retry.js'; import { withTimeout } from '@kilocode/worker-utils'; import { logSandboxOperationTimeout } from '../sandbox-timeout-logging.js'; @@ -57,6 +58,17 @@ function withWorkspacePreparationTimeout(operation: Promise, step: string) function translateKnownWrapperFailure(error: unknown): Error | undefined { if (error instanceof ExecutionError) return error; + // A full workspace disk surfaces here as a raw WorkspaceCapacityAdmissionRejectedError. + // Translate it into a workspace_setup_failed / sandbox_storage_full ExecutionError + // so it flows through the shared subtype-based classification and gets the + // capacity-specific retry backoff instead of being wrapped as a generic + // wrapperStartFailed (which would lose its identity and the longer budget). + if (error instanceof WorkspaceCapacityAdmissionRejectedError) { + return ExecutionError.workspaceSetupFailed(error.message, error, { + subtype: 'sandbox_storage_full', + retryable: true, + }); + } if (!(error instanceof WrapperError)) return undefined; if (error.code === 'WORKSPACE_SETUP_FAILED') { diff --git a/services/cloud-agent-next/src/session/pending-messages.test.ts b/services/cloud-agent-next/src/session/pending-messages.test.ts index e4a9b1955b..496d024c76 100644 --- a/services/cloud-agent-next/src/session/pending-messages.test.ts +++ b/services/cloud-agent-next/src/session/pending-messages.test.ts @@ -371,6 +371,101 @@ describe('recordPendingFlushFailure', () => { expect(delays).toEqual([2_000, undefined]); }); + it('gives a full-disk workspace failure a longer backed-off retry budget', async () => { + const storage = createMemoryStorage(); + let message = makeMessage(); + await storePendingSessionMessage(storage, message); + + const delays: (number | undefined)[] = []; + const now = 100_000; + + // A workspace_setup_failed with the sandbox_storage_full subtype is transient + // disk backpressure and should retry several times with growing delays before + // exhausting, unlike a plain workspace failure (one warm-followup retry above). + for (let i = 0; i < 4; i++) { + const result = await recordPendingFlushFailure( + storage, + message, + 'sandbox storage full', + now, + { + policy: 'warm-followup', + code: 'WORKSPACE_SETUP_FAILED', + subtype: 'sandbox_storage_full', + } + ); + delays.push( + result.nextFlushAttemptAt !== undefined ? result.nextFlushAttemptAt - now : undefined + ); + message = result.message; + } + + expect(delays).toEqual([10_000, 30_000, 60_000, undefined]); + }); + + it('starts the capacity retry budget fresh after a different earlier failure', async () => { + const storage = createMemoryStorage(); + const message = makeMessage({ + createdAt: 1, + flushAttempts: 2, + lastFlushFailureCode: 'WRAPPER_START_FAILED', + }); + await storePendingSessionMessage(storage, message); + + const result = await recordPendingFlushFailure( + storage, + message, + 'sandbox storage full', + 100_000, + { + policy: 'warm-followup', + code: 'WORKSPACE_SETUP_FAILED', + subtype: 'sandbox_storage_full', + } + ); + + // Reset to attempt 1 so the full 10s/30s/60s budget applies, not truncated + // by the two earlier non-capacity attempts. + expect(result.attempts).toBe(1); + expect(result.exhausted).toBe(false); + expect(result.nextFlushAttemptAt).toBe(110_000); + }); + + it('bounds retries when failures alternate between reset-eligible modes', async () => { + const storage = createMemoryStorage(); + let message = makeMessage(); + await storePendingSessionMessage(storage, message); + const now = 100_000; + + // Fresh sandbox-connect failure resets to attempt 1. + let result = await recordPendingFlushFailure(storage, message, 'connect failed', now, { + policy: 'warm-followup', + code: 'SANDBOX_CONNECT_FAILED', + }); + expect(result.attempts).toBe(1); + expect(result.exhausted).toBe(false); + message = result.message; + + // Switching to capacity (both modes reset-eligible) does NOT reset; attempts accumulate. + result = await recordPendingFlushFailure(storage, message, 'sandbox storage full', now, { + policy: 'warm-followup', + code: 'WORKSPACE_SETUP_FAILED', + subtype: 'sandbox_storage_full', + }); + expect(result.attempts).toBe(2); + message = result.message; + + // Switching back to connect keeps accumulating and exceeds the connect budget, + // so the flapping message exhausts instead of resetting forever. + result = await recordPendingFlushFailure(storage, message, 'connect failed', now, { + policy: 'warm-followup', + code: 'SANDBOX_CONNECT_FAILED', + }); + expect(result.attempts).toBe(3); + expect(result.exhausted).toBe(true); + expect(result.nextFlushAttemptAt).toBeUndefined(); + }); + it('retries a sandbox connection failure once after exactly five seconds', async () => { const storage = createMemoryStorage(); let message = makeMessage({ createdAt: 100_000 }); diff --git a/services/cloud-agent-next/src/session/pending-messages.ts b/services/cloud-agent-next/src/session/pending-messages.ts index ad2293f36e..f43717a2b7 100644 --- a/services/cloud-agent-next/src/session/pending-messages.ts +++ b/services/cloud-agent-next/src/session/pending-messages.ts @@ -19,6 +19,11 @@ import { export const PENDING_SESSION_MESSAGE_LIMIT = 10; export const PENDING_FLUSH_RETRY_BASE_DELAY_MS = 2_000; const SANDBOX_CONNECT_RETRY_DELAYS_MS = [5_000] as const; +// A full workspace disk is transient backpressure: stale-workspace cleanup and +// other reviews finishing free space within seconds to minutes. Give it a +// longer, backed-off retry budget instead of failing the delivery after a +// single redelivery. +const WORKSPACE_CAPACITY_RETRY_DELAYS_MS = [10_000, 30_000, 60_000] as const; // Other pending delivery failures currently get one redelivery after the initial failed attempt. const WARM_FOLLOWUP_RETRY_DELAYS_MS = [PENDING_FLUSH_RETRY_BASE_DELAY_MS] as const; const COLD_INIT_RETRY_DELAYS_MS = [PENDING_FLUSH_RETRY_BASE_DELAY_MS] as const; @@ -485,6 +490,22 @@ export function shouldSkipPendingFlush(message: PendingSessionMessage, now: numb return message.nextFlushAttemptAt !== undefined && message.nextFlushAttemptAt > now; } +/** + * Reset-eligible modes each start a fresh retry budget on entry (sandbox-connect + * has a short reconnect budget; sandbox-capacity has a longer backed-off budget + * for transient disk pressure). Alternating between them must NOT keep resetting, + * so callers only reset when entering one of these from a non-reset-eligible state. + */ +function isResetEligibleFailure( + code: PendingFlushFailureCode | undefined, + subtype: WorkspaceFailureSubtype | undefined +): boolean { + return ( + code === 'SANDBOX_CONNECT_FAILED' || + (code === 'WORKSPACE_SETUP_FAILED' && subtype === 'sandbox_storage_full') + ); +} + export async function recordPendingFlushFailure( storage: SessionQueueStorage, message: PendingSessionMessage, @@ -535,17 +556,25 @@ export async function recordPendingFlushFailure( : options.code === 'WORKSPACE_SETUP_FAILED' ? options.safeFailureMessage : undefined; + // Reset the attempt counter only when a message ENTERS a reset-eligible + // transient mode (sandbox-connect or sandbox-capacity) from a state that is + // not itself reset-eligible, so each fresh sequence gets its full backoff + // budget. When failures alternate between the two reset-eligible modes the + // counter is NOT reset, so attempts accumulate and the message still exhausts + // instead of flapping between modes forever. const attempts = - flushFailureCode === 'SANDBOX_CONNECT_FAILED' && - message.lastFlushFailureCode !== 'SANDBOX_CONNECT_FAILED' + isResetEligibleFailure(flushFailureCode, failureSubtype) && + !isResetEligibleFailure(message.lastFlushFailureCode, message.lastFlushFailureSubtype) ? 1 : (message.flushAttempts ?? 0) + 1; const retryDelays = flushFailureCode === 'SANDBOX_CONNECT_FAILED' ? SANDBOX_CONNECT_RETRY_DELAYS_MS - : options.policy === 'cold-init' - ? COLD_INIT_RETRY_DELAYS_MS - : WARM_FOLLOWUP_RETRY_DELAYS_MS; + : flushFailureCode === 'WORKSPACE_SETUP_FAILED' && failureSubtype === 'sandbox_storage_full' + ? WORKSPACE_CAPACITY_RETRY_DELAYS_MS + : options.policy === 'cold-init' + ? COLD_INIT_RETRY_DELAYS_MS + : WARM_FOLLOWUP_RETRY_DELAYS_MS; const retryable = options.retryable ?? isRetryableFlushCode(flushFailureCode); const exhausted = !retryable || attempts > retryDelays.length; const retryDelay = retryDelays[attempts - 1]; diff --git a/services/cloud-agent-next/src/session/session-message-queue.test.ts b/services/cloud-agent-next/src/session/session-message-queue.test.ts index 7309f1d234..b207723552 100644 --- a/services/cloud-agent-next/src/session/session-message-queue.test.ts +++ b/services/cloud-agent-next/src/session/session-message-queue.test.ts @@ -1671,7 +1671,7 @@ describe('SessionMessageQueue', () => { }); await harness.queue.drainNextPendingMessage(); - const [pending] = await listPendingSessionMessages(harness.storage); + let [pending] = await listPendingSessionMessages(harness.storage); expect(pending?.lastFlushFailureCode).toBe('WORKSPACE_SETUP_FAILED'); expect(pending?.lastFlushError).toBe(error); expect(pending?.lastFlushFailureSubtype).toBe('sandbox_storage_full'); @@ -1680,9 +1680,14 @@ describe('SessionMessageQueue', () => { throw new Error('Expected workspace setup failure to be retried before terminalization'); } - vi.spyOn(Date, 'now').mockReturnValueOnce(pending.nextFlushAttemptAt); - await harness.queue.drainNextPendingMessage(); - vi.restoreAllMocks(); + // A full disk is transient backpressure, so it now retries on a longer + // backoff (10s, 30s, 60s). Drain through the whole budget until exhaustion. + for (let attempt = 0; attempt < 5 && pending?.nextFlushAttemptAt !== undefined; attempt++) { + vi.spyOn(Date, 'now').mockReturnValueOnce(pending.nextFlushAttemptAt); + await harness.queue.drainNextPendingMessage(); + vi.restoreAllMocks(); + [pending] = await listPendingSessionMessages(harness.storage); + } expect(harness.terminalizations.at(-1)?.params).toMatchObject({ kind: 'failed', diff --git a/services/code-review-infra/src/types.ts b/services/code-review-infra/src/types.ts index e3a1f84cf4..9f46088efe 100644 --- a/services/code-review-infra/src/types.ts +++ b/services/code-review-infra/src/types.ts @@ -111,6 +111,9 @@ export interface CodeReviewStatusResponse { export type CodeReviewStatusResult = CodeReviewStatusResponse | null; +// KEEP IN SYNC with CODE_REVIEW_TERMINAL_REASONS (packages/db/src/schema-types.ts) +// and CloudAgentTerminalReason (packages/worker-utils/src/cloud-agent-next-client.ts). +// A value missing here is silently coerced to undefined by `.catch(undefined)`. const InternalStatusTerminalReasonSchema = z .enum([ 'billing', @@ -127,6 +130,7 @@ const InternalStatusTerminalReasonSchema = z 'timeout', 'upstream_error', 'sandbox_error', + 'workspace_capacity', 'unknown', ]) .nullable()