From cae043e65b42566484f9b2ef1db74705d70dcaeb Mon Sep 17 00:00:00 2001 From: St0rmz1 Date: Fri, 24 Jul 2026 16:22:51 -0700 Subject: [PATCH 1/5] WIP(code reviewer) better handle disk space pressure --- apps/web/src/routers/admin-code-reviews-router.ts | 8 ++++++++ packages/db/src/schema-types.ts | 1 + packages/worker-utils/src/cloud-agent-next-client.ts | 1 + 3 files changed, 10 insertions(+) diff --git a/apps/web/src/routers/admin-code-reviews-router.ts b/apps/web/src/routers/admin-code-reviews-router.ts index 9551a032bb..4c3a7342cf 100644 --- a/apps/web/src/routers/admin-code-reviews-router.ts +++ b/apps/web/src/routers/admin-code-reviews-router.ts @@ -91,6 +91,10 @@ const errorCategoryExpr = sql`CASE WHEN ${cloud_agent_code_reviews.error_message} LIKE '%500%' OR ${cloud_agent_code_reviews.error_message} LIKE '%502%' OR ${cloud_agent_code_reviews.error_message} LIKE '%503%' OR ${cloud_agent_code_reviews.error_message} LIKE '%internal server%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Internal Server%' THEN 'Upstream Server Error' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%ECONNREFUSED%' OR ${cloud_agent_code_reviews.error_message} LIKE '%ECONNRESET%' OR ${cloud_agent_code_reviews.error_message} LIKE '%socket hang up%' OR ${cloud_agent_code_reviews.error_message} LIKE '%network%' THEN 'Network Error' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%parse%' OR ${cloud_agent_code_reviews.error_message} LIKE '%JSON%' OR ${cloud_agent_code_reviews.error_message} LIKE '%unexpected token%' THEN 'Parse Error' + WHEN ${cloud_agent_code_reviews.terminal_reason} = 'workspace_capacity' OR ${cloud_agent_code_reviews.error_message} LIKE '%sandbox storage full%' OR ${cloud_agent_code_reviews.error_message} LIKE '%admission rejected%' OR ${cloud_agent_code_reviews.error_message} LIKE '%storage full%' THEN 'Sandbox Capacity' + WHEN ${cloud_agent_code_reviews.error_message} LIKE '%connect to the sandbox%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Sandbox connection failed%' OR ${cloud_agent_code_reviews.error_message} LIKE '%container shut down%' THEN 'Sandbox Connection' + WHEN ${cloud_agent_code_reviews.error_message} LIKE '%could not be delivered%' THEN 'Delivery Failure' + WHEN ${cloud_agent_code_reviews.error_message} LIKE '%repository_not_installed%' OR ${cloud_agent_code_reviews.error_message} LIKE '%app installation required%' THEN 'Action Required' WHEN ${cloud_agent_code_reviews.error_message} IS NULL THEN 'Unknown Error' ELSE 'Other' END`; @@ -105,6 +109,10 @@ const attemptErrorCategoryExpr = sql`CASE WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%500%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%502%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%503%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%internal server%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Internal Server%' THEN 'Upstream Server Error' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%ECONNREFUSED%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%ECONNRESET%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%socket hang up%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%network%' THEN 'Network Error' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%parse%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%JSON%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%unexpected token%' THEN 'Parse Error' + WHEN ${cloud_agent_code_review_attempts.terminal_reason} = 'workspace_capacity' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%sandbox storage full%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%admission rejected%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%storage full%' THEN 'Sandbox Capacity' + WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%connect to the sandbox%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Sandbox connection failed%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%container shut down%' THEN 'Sandbox Connection' + WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%could not be delivered%' THEN 'Delivery Failure' + WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%repository_not_installed%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%app installation required%' THEN 'Action Required' WHEN ${cloud_agent_code_review_attempts.error_message} IS NULL THEN 'Unknown Error' ELSE 'Other' END`; diff --git a/packages/db/src/schema-types.ts b/packages/db/src/schema-types.ts index 245e8a3fea..afe1eb3a20 100644 --- a/packages/db/src/schema-types.ts +++ b/packages/db/src/schema-types.ts @@ -2000,6 +2000,7 @@ export const CODE_REVIEW_TERMINAL_REASONS = [ 'timeout', 'upstream_error', 'sandbox_error', + 'workspace_capacity', 'unknown', ] as const; diff --git a/packages/worker-utils/src/cloud-agent-next-client.ts b/packages/worker-utils/src/cloud-agent-next-client.ts index 1ac032c5e0..a53ccb2b30 100644 --- a/packages/worker-utils/src/cloud-agent-next-client.ts +++ b/packages/worker-utils/src/cloud-agent-next-client.ts @@ -164,6 +164,7 @@ export type CloudAgentTerminalReason = | 'timeout' | 'upstream_error' | 'sandbox_error' + | 'workspace_capacity' | 'unknown'; export class CloudAgentNextError extends Error { From 7169821ca63a0eb8860280f80646e2fe012ecbac Mon Sep 17 00:00:00 2001 From: St0rmz1 Date: Fri, 24 Jul 2026 16:42:33 -0700 Subject: [PATCH 2/5] feat(code-review): retry and classify workspace disk capacity failures --- .../[reviewId]/route.test.ts | 22 +++++++++++++ .../code-review-status/[reviewId]/route.ts | 27 ++++++++++++++++ .../routers/admin-code-reviews-router.test.ts | 31 ++++++++++++++++++ .../src/session/pending-messages.test.ts | 32 +++++++++++++++++++ .../src/session/pending-messages.ts | 13 ++++++-- .../src/session/session-message-queue.test.ts | 13 +++++--- .../src/session/session-message-queue.ts | 21 ++++++++++++ 7 files changed, 152 insertions(+), 7 deletions(-) diff --git a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts index 273474cfcb..0f537e6023 100644 --- a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts +++ b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts @@ -705,6 +705,28 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => { ); }); + it('infers workspace_capacity terminalReason for a sandbox storage full failure', async () => { + mockGetCodeReviewById.mockResolvedValue(makeReview()); + + const response = await POST( + makeRequest({ + status: 'failed', + errorMessage: 'Workspace setup failed: sandbox storage full', + }), + makeParams(REVIEW_ID) + ); + + expect(response.status).toBe(200); + expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith( + REVIEW_ID, + 'failed', + expect.objectContaining({ + errorMessage: 'Workspace setup failed: sandbox storage full', + terminalReason: 'workspace_capacity', + }) + ); + }); + it.each([ 'Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.', 'Payment Required: [BYOK] Your API account has insufficient funds. Please check your billing details with your API provider.', diff --git a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts index bc2638eadf..4bc4748526 100644 --- a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts +++ b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts @@ -348,6 +348,12 @@ function normalizePayload(raw: StatusUpdatePayload): { terminalReason = 'billing'; } + // Infer workspace capacity (transient sandbox disk pressure) so it is + // classified distinctly instead of counted as an unknown delivery failure. + if (!terminalReason && isWorkspaceCapacityFailure(failure, raw.errorMessage)) { + terminalReason = 'workspace_capacity'; + } + if ( (raw.status === 'failed' || raw.status === 'interrupted') && isModelNotFoundCodeReviewTerminalReason(terminalReason, raw.errorMessage) @@ -371,6 +377,23 @@ function normalizePayload(raw: StatusUpdatePayload): { }; } +/** + * Detects a workspace disk-capacity failure so it is recorded with a distinct + * `workspace_capacity` terminal reason. Prefers the structured failure subtype + * from cloud-agent-next and falls back to the safe error message for older + * payloads. + */ +function isWorkspaceCapacityFailure( + failure: CloudAgentSafeFailure | undefined, + errorMessage: string | undefined +): boolean { + if (failure?.code === 'workspace_setup_failed' && failure.subtype === 'sandbox_storage_full') { + return true; + } + const message = (errorMessage ?? '').toLowerCase(); + return message.includes('sandbox storage full') || message.includes('admission rejected'); +} + function isBillingCodeReviewTerminalReason( terminalReason?: CodeReviewTerminalReason, errorMessage?: string | null @@ -418,6 +441,10 @@ function hasKnownUnretryableTerminalReason(terminalReason?: CodeReviewTerminalRe terminalReason === 'user_cancelled' || terminalReason === 'superseded' || terminalReason === 'interrupted' || + // Capacity failures are already retried at the delivery layer in + // cloud-agent-next. A fresh web-level retry would re-dispatch onto the same + // full shared sandbox and restart the storm, so terminalize instead. + terminalReason === 'workspace_capacity' || isCodeReviewActionRequiredReason(terminalReason) ); } diff --git a/apps/web/src/routers/admin-code-reviews-router.test.ts b/apps/web/src/routers/admin-code-reviews-router.test.ts index 1720b78279..2c68d62fc8 100644 --- a/apps/web/src/routers/admin-code-reviews-router.test.ts +++ b/apps/web/src/routers/admin-code-reviews-router.test.ts @@ -341,6 +341,37 @@ describe('adminCodeReviewsRouter', () => { expect(exportRows[0]).toHaveProperty('attempt_status'); }); + it('buckets sandbox capacity and delivery failures instead of Other', async () => { + const owner = { type: 'user', id: adminUser.id } satisfies ReviewOwner; + + await db.insert(cloud_agent_code_reviews).values([ + reviewValues({ + owner, + status: 'failed', + createdAt: timestamp(700), + terminalReason: 'workspace_capacity', + errorMessage: 'Workspace setup failed: sandbox storage full', + }), + reviewValues({ + owner, + status: 'failed', + createdAt: timestamp(710), + errorMessage: 'The message could not be delivered', + }), + ]); + + const caller = await createCallerForUser(adminUser.id); + const errors = await caller.admin.codeReviews.getErrorAnalysis(filterInput()); + + expect(errors.categories).toEqual( + expect.arrayContaining([ + expect.objectContaining({ category: 'Sandbox Capacity', count: 1 }), + expect.objectContaining({ category: 'Delivery Failure', count: 1 }), + ]) + ); + expect(errors.categories.map(category => category.category)).not.toContain('Other'); + }); + it('classifies final model-not-found outcomes as cancellations instead of failures', async () => { const owner = { type: 'user', id: adminUser.id } satisfies ReviewOwner; diff --git a/services/cloud-agent-next/src/session/pending-messages.test.ts b/services/cloud-agent-next/src/session/pending-messages.test.ts index e4a9b1955b..9331fef477 100644 --- a/services/cloud-agent-next/src/session/pending-messages.test.ts +++ b/services/cloud-agent-next/src/session/pending-messages.test.ts @@ -371,6 +371,38 @@ describe('recordPendingFlushFailure', () => { expect(delays).toEqual([2_000, undefined]); }); + it('gives a full-disk workspace failure a longer backed-off retry budget', async () => { + const storage = createMemoryStorage(); + let message = makeMessage(); + await storePendingSessionMessage(storage, message); + + const delays: (number | undefined)[] = []; + const now = 100_000; + + // A workspace_setup_failed with the sandbox_storage_full subtype is transient + // disk backpressure and should retry several times with growing delays before + // exhausting, unlike a plain workspace failure (one warm-followup retry above). + for (let i = 0; i < 4; i++) { + const result = await recordPendingFlushFailure( + storage, + message, + 'sandbox storage full', + now, + { + policy: 'warm-followup', + code: 'WORKSPACE_SETUP_FAILED', + subtype: 'sandbox_storage_full', + } + ); + delays.push( + result.nextFlushAttemptAt !== undefined ? result.nextFlushAttemptAt - now : undefined + ); + message = result.message; + } + + expect(delays).toEqual([10_000, 30_000, 60_000, undefined]); + }); + it('retries a sandbox connection failure once after exactly five seconds', async () => { const storage = createMemoryStorage(); let message = makeMessage({ createdAt: 100_000 }); diff --git a/services/cloud-agent-next/src/session/pending-messages.ts b/services/cloud-agent-next/src/session/pending-messages.ts index ad2293f36e..d5c708942e 100644 --- a/services/cloud-agent-next/src/session/pending-messages.ts +++ b/services/cloud-agent-next/src/session/pending-messages.ts @@ -19,6 +19,11 @@ import { export const PENDING_SESSION_MESSAGE_LIMIT = 10; export const PENDING_FLUSH_RETRY_BASE_DELAY_MS = 2_000; const SANDBOX_CONNECT_RETRY_DELAYS_MS = [5_000] as const; +// A full workspace disk is transient backpressure: stale-workspace cleanup and +// other reviews finishing free space within seconds to minutes. Give it a +// longer, backed-off retry budget instead of failing the delivery after a +// single redelivery. +const WORKSPACE_CAPACITY_RETRY_DELAYS_MS = [10_000, 30_000, 60_000] as const; // Other pending delivery failures currently get one redelivery after the initial failed attempt. const WARM_FOLLOWUP_RETRY_DELAYS_MS = [PENDING_FLUSH_RETRY_BASE_DELAY_MS] as const; const COLD_INIT_RETRY_DELAYS_MS = [PENDING_FLUSH_RETRY_BASE_DELAY_MS] as const; @@ -543,9 +548,11 @@ export async function recordPendingFlushFailure( const retryDelays = flushFailureCode === 'SANDBOX_CONNECT_FAILED' ? SANDBOX_CONNECT_RETRY_DELAYS_MS - : options.policy === 'cold-init' - ? COLD_INIT_RETRY_DELAYS_MS - : WARM_FOLLOWUP_RETRY_DELAYS_MS; + : flushFailureCode === 'WORKSPACE_SETUP_FAILED' && failureSubtype === 'sandbox_storage_full' + ? WORKSPACE_CAPACITY_RETRY_DELAYS_MS + : options.policy === 'cold-init' + ? COLD_INIT_RETRY_DELAYS_MS + : WARM_FOLLOWUP_RETRY_DELAYS_MS; const retryable = options.retryable ?? isRetryableFlushCode(flushFailureCode); const exhausted = !retryable || attempts > retryDelays.length; const retryDelay = retryDelays[attempts - 1]; diff --git a/services/cloud-agent-next/src/session/session-message-queue.test.ts b/services/cloud-agent-next/src/session/session-message-queue.test.ts index 7309f1d234..b207723552 100644 --- a/services/cloud-agent-next/src/session/session-message-queue.test.ts +++ b/services/cloud-agent-next/src/session/session-message-queue.test.ts @@ -1671,7 +1671,7 @@ describe('SessionMessageQueue', () => { }); await harness.queue.drainNextPendingMessage(); - const [pending] = await listPendingSessionMessages(harness.storage); + let [pending] = await listPendingSessionMessages(harness.storage); expect(pending?.lastFlushFailureCode).toBe('WORKSPACE_SETUP_FAILED'); expect(pending?.lastFlushError).toBe(error); expect(pending?.lastFlushFailureSubtype).toBe('sandbox_storage_full'); @@ -1680,9 +1680,14 @@ describe('SessionMessageQueue', () => { throw new Error('Expected workspace setup failure to be retried before terminalization'); } - vi.spyOn(Date, 'now').mockReturnValueOnce(pending.nextFlushAttemptAt); - await harness.queue.drainNextPendingMessage(); - vi.restoreAllMocks(); + // A full disk is transient backpressure, so it now retries on a longer + // backoff (10s, 30s, 60s). Drain through the whole budget until exhaustion. + for (let attempt = 0; attempt < 5 && pending?.nextFlushAttemptAt !== undefined; attempt++) { + vi.spyOn(Date, 'now').mockReturnValueOnce(pending.nextFlushAttemptAt); + await harness.queue.drainNextPendingMessage(); + vi.restoreAllMocks(); + [pending] = await listPendingSessionMessages(harness.storage); + } expect(harness.terminalizations.at(-1)?.params).toMatchObject({ kind: 'failed', diff --git a/services/cloud-agent-next/src/session/session-message-queue.ts b/services/cloud-agent-next/src/session/session-message-queue.ts index c91daa9a86..84df4e94fa 100644 --- a/services/cloud-agent-next/src/session/session-message-queue.ts +++ b/services/cloud-agent-next/src/session/session-message-queue.ts @@ -15,6 +15,7 @@ import { logger } from '../logger.js'; import { dispatchedKilocodeModelId } from '../persistence/model-utils.js'; import type { SessionMetadata } from '../persistence/session-metadata.js'; import { isSandboxWorkspaceProbeTimeoutError } from '../sandbox-recovery.js'; +import { WorkspaceCapacityAdmissionRejectedError } from '../workspace-errors.js'; import { WrapperCleanupBlockedError, type WrapperCleanupBlock, @@ -466,6 +467,26 @@ export async function flushNextPendingSessionMessage(params: { remainingCount: totalCount, }; } + if (error instanceof WorkspaceCapacityAdmissionRejectedError) { + // A full workspace disk is transient backpressure, not a review failure. + // Route it through the sandbox_storage_full path so it is classified as a + // capacity failure and retried with a longer backoff (cleanup and other + // reviews finishing free space) instead of dying after one attempt. + const failure = await recordPendingFlushFailure( + params.storage, + message, + error.message, + Date.now(), + { + policy, + code: 'WORKSPACE_SETUP_FAILED', + subtype: 'sandbox_storage_full', + retryable: true, + scheduleTerminalizationRepair: params.scheduleTerminalizationRepair, + } + ); + return toFailureResult(failure, totalCount); + } const code = error instanceof MessageDeliveryRequestValidationError ? error.code From dae2d74f11732dc1caafbdf76fb2186e9832a5cd Mon Sep 17 00:00:00 2001 From: St0rmz1 Date: Sat, 25 Jul 2026 07:34:02 -0700 Subject: [PATCH 3/5] fix(code-review): reset retry budget when entering sandbox-capacity failure --- .../src/session/pending-messages.test.ts | 28 +++++++++++++++++++ .../src/session/pending-messages.ts | 12 ++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/services/cloud-agent-next/src/session/pending-messages.test.ts b/services/cloud-agent-next/src/session/pending-messages.test.ts index 9331fef477..a94a75f5aa 100644 --- a/services/cloud-agent-next/src/session/pending-messages.test.ts +++ b/services/cloud-agent-next/src/session/pending-messages.test.ts @@ -403,6 +403,34 @@ describe('recordPendingFlushFailure', () => { expect(delays).toEqual([10_000, 30_000, 60_000, undefined]); }); + it('starts the capacity retry budget fresh after a different earlier failure', async () => { + const storage = createMemoryStorage(); + const message = makeMessage({ + createdAt: 1, + flushAttempts: 2, + lastFlushFailureCode: 'WRAPPER_START_FAILED', + }); + await storePendingSessionMessage(storage, message); + + const result = await recordPendingFlushFailure( + storage, + message, + 'sandbox storage full', + 100_000, + { + policy: 'warm-followup', + code: 'WORKSPACE_SETUP_FAILED', + subtype: 'sandbox_storage_full', + } + ); + + // Reset to attempt 1 so the full 10s/30s/60s budget applies, not truncated + // by the two earlier non-capacity attempts. + expect(result.attempts).toBe(1); + expect(result.exhausted).toBe(false); + expect(result.nextFlushAttemptAt).toBe(110_000); + }); + it('retries a sandbox connection failure once after exactly five seconds', async () => { const storage = createMemoryStorage(); let message = makeMessage({ createdAt: 100_000 }); diff --git a/services/cloud-agent-next/src/session/pending-messages.ts b/services/cloud-agent-next/src/session/pending-messages.ts index d5c708942e..e905224d73 100644 --- a/services/cloud-agent-next/src/session/pending-messages.ts +++ b/services/cloud-agent-next/src/session/pending-messages.ts @@ -540,9 +540,17 @@ export async function recordPendingFlushFailure( : options.code === 'WORKSPACE_SETUP_FAILED' ? options.safeFailureMessage : undefined; + // A newly-observed sandbox-capacity failure starts its own retry budget, the + // same way a sandbox-connection failure does, so the full 10s/30s/60s backoff + // applies even if the message already failed under a different code first. + const enteringSandboxCapacityRetry = + flushFailureCode === 'WORKSPACE_SETUP_FAILED' && + failureSubtype === 'sandbox_storage_full' && + message.lastFlushFailureSubtype !== 'sandbox_storage_full'; const attempts = - flushFailureCode === 'SANDBOX_CONNECT_FAILED' && - message.lastFlushFailureCode !== 'SANDBOX_CONNECT_FAILED' + (flushFailureCode === 'SANDBOX_CONNECT_FAILED' && + message.lastFlushFailureCode !== 'SANDBOX_CONNECT_FAILED') || + enteringSandboxCapacityRetry ? 1 : (message.flushAttempts ?? 0) + 1; const retryDelays = From 56db212ee1a827bc9c47b6021649756601945c75 Mon Sep 17 00:00:00 2001 From: St0rmz1 Date: Sat, 25 Jul 2026 18:36:39 -0700 Subject: [PATCH 4/5] fix(code-review): bound reset-eligible flush retries by attempt count --- .../[reviewId]/route.test.ts | 50 ++++++++++--------- .../code-review-status/[reviewId]/route.ts | 6 +-- .../routers/admin-code-reviews-router.test.ts | 27 +++++++++- .../src/routers/admin-code-reviews-router.ts | 10 ++-- .../src/execution/orchestrator.test.ts | 10 +++- .../src/execution/orchestrator.ts | 12 +++++ .../src/session/pending-messages.test.ts | 35 +++++++++++++ .../src/session/pending-messages.ts | 34 +++++++++---- .../src/session/session-message-queue.ts | 21 -------- services/code-review-infra/src/types.ts | 4 ++ 10 files changed, 141 insertions(+), 68 deletions(-) diff --git a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts index 0f537e6023..cbe3b4a678 100644 --- a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts +++ b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts @@ -368,9 +368,13 @@ function mockCreatedInfraRetryFlow( // --- Tests --- -import type { POST as POSTType } from './route'; +import type { + POST as POSTType, + isWorkspaceCapacityFailure as isWorkspaceCapacityFailureType, +} from './route'; let POST: typeof POSTType; +let isWorkspaceCapacityFailure: typeof isWorkspaceCapacityFailureType; beforeEach(async () => { jest.clearAllMocks(); @@ -431,7 +435,27 @@ beforeEach(async () => { ); mockDisableCodeReviewForActionRequiredFailure.mockResolvedValue(undefined); mockDisableCodeReviewForRepeatedCloneTimeoutsToday.mockResolvedValue(null); - ({ POST } = await import('./route')); + ({ POST, isWorkspaceCapacityFailure } = await import('./route')); +}); + +describe('isWorkspaceCapacityFailure', () => { + it('detects a full-disk failure from the structured subtype or the message', () => { + expect( + isWorkspaceCapacityFailure(undefined, 'Workspace setup failed: sandbox storage full') + ).toBe(true); + // Orchestrator session-start path reports it inside a "(500)" message. + expect( + isWorkspaceCapacityFailure( + undefined, + 'initiate failed (500): Workspace admission rejected: 1036 MB available below 2048 MB threshold after cleanup' + ) + ).toBe(true); + }); + + it('ignores unrelated failures', () => { + expect(isWorkspaceCapacityFailure(undefined, 'The message could not be delivered')).toBe(false); + expect(isWorkspaceCapacityFailure(undefined, undefined)).toBe(false); + }); }); describe('POST /api/internal/code-review-status/[reviewId]', () => { @@ -705,28 +729,6 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => { ); }); - it('infers workspace_capacity terminalReason for a sandbox storage full failure', async () => { - mockGetCodeReviewById.mockResolvedValue(makeReview()); - - const response = await POST( - makeRequest({ - status: 'failed', - errorMessage: 'Workspace setup failed: sandbox storage full', - }), - makeParams(REVIEW_ID) - ); - - expect(response.status).toBe(200); - expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith( - REVIEW_ID, - 'failed', - expect.objectContaining({ - errorMessage: 'Workspace setup failed: sandbox storage full', - terminalReason: 'workspace_capacity', - }) - ); - }); - it.each([ 'Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.', 'Payment Required: [BYOK] Your API account has insufficient funds. Please check your billing details with your API provider.', diff --git a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts index 4bc4748526..cbb68b8108 100644 --- a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts +++ b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts @@ -383,7 +383,7 @@ function normalizePayload(raw: StatusUpdatePayload): { * from cloud-agent-next and falls back to the safe error message for older * payloads. */ -function isWorkspaceCapacityFailure( +export function isWorkspaceCapacityFailure( failure: CloudAgentSafeFailure | undefined, errorMessage: string | undefined ): boolean { @@ -441,10 +441,6 @@ function hasKnownUnretryableTerminalReason(terminalReason?: CodeReviewTerminalRe terminalReason === 'user_cancelled' || terminalReason === 'superseded' || terminalReason === 'interrupted' || - // Capacity failures are already retried at the delivery layer in - // cloud-agent-next. A fresh web-level retry would re-dispatch onto the same - // full shared sandbox and restart the storm, so terminalize instead. - terminalReason === 'workspace_capacity' || isCodeReviewActionRequiredReason(terminalReason) ); } diff --git a/apps/web/src/routers/admin-code-reviews-router.test.ts b/apps/web/src/routers/admin-code-reviews-router.test.ts index 2c68d62fc8..c138c8ee96 100644 --- a/apps/web/src/routers/admin-code-reviews-router.test.ts +++ b/apps/web/src/routers/admin-code-reviews-router.test.ts @@ -352,6 +352,26 @@ describe('adminCodeReviewsRouter', () => { terminalReason: 'workspace_capacity', errorMessage: 'Workspace setup failed: sandbox storage full', }), + // terminal_reason is authoritative and must win over the generic '%404%' + // branch that this admission-rejected message would otherwise match. + reviewValues({ + owner, + status: 'failed', + createdAt: timestamp(705), + terminalReason: 'workspace_capacity', + errorMessage: 'Workspace admission rejected: 404 MB available below 2048 MB threshold', + }), + // Orchestrator session-start path: reports capacity as terminal_reason + // 'sandbox_error' with a "(500)" message. The capacity message match must + // win over the generic '%500%' Upstream Server Error branch. + reviewValues({ + owner, + status: 'failed', + createdAt: timestamp(707), + terminalReason: 'sandbox_error', + errorMessage: + 'initiate failed (500): Workspace admission rejected: 1036 MB available below 2048 MB threshold after cleanup', + }), reviewValues({ owner, status: 'failed', @@ -365,11 +385,14 @@ describe('adminCodeReviewsRouter', () => { expect(errors.categories).toEqual( expect.arrayContaining([ - expect.objectContaining({ category: 'Sandbox Capacity', count: 1 }), + expect.objectContaining({ category: 'Sandbox Capacity', count: 3 }), expect.objectContaining({ category: 'Delivery Failure', count: 1 }), ]) ); - expect(errors.categories.map(category => category.category)).not.toContain('Other'); + const categoryNames = errors.categories.map(category => category.category); + expect(categoryNames).not.toContain('Other'); + expect(categoryNames).not.toContain('Not Found'); + expect(categoryNames).not.toContain('Upstream Server Error'); }); it('classifies final model-not-found outcomes as cancellations instead of failures', async () => { diff --git a/apps/web/src/routers/admin-code-reviews-router.ts b/apps/web/src/routers/admin-code-reviews-router.ts index 4c3a7342cf..01d5b4f8a5 100644 --- a/apps/web/src/routers/admin-code-reviews-router.ts +++ b/apps/web/src/routers/admin-code-reviews-router.ts @@ -83,6 +83,9 @@ const excludeModelNotFoundAttempt = sql`COALESCE(${cloud_agent_code_review_attem */ const errorCategoryExpr = sql`CASE WHEN ${cloud_agent_code_reviews.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'gitlab_project_access_required', 'byok_invalid_key', 'selected_model_unavailable') THEN 'Action Required' + WHEN ${cloud_agent_code_reviews.terminal_reason} = 'workspace_capacity' THEN 'Sandbox Capacity' + WHEN ${cloud_agent_code_reviews.error_message} LIKE '%sandbox storage full%' OR ${cloud_agent_code_reviews.error_message} LIKE '%admission rejected%' OR ${cloud_agent_code_reviews.error_message} LIKE '%storage full%' THEN 'Sandbox Capacity' + WHEN ${cloud_agent_code_reviews.error_message} LIKE '%connect to the sandbox%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Sandbox connection failed%' OR ${cloud_agent_code_reviews.error_message} LIKE '%container shut down%' THEN 'Sandbox Connection' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%rate limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Rate limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%429%' THEN 'Rate Limited' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%timeout%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Timeout%' OR ${cloud_agent_code_reviews.error_message} LIKE '%ETIMEDOUT%' OR ${cloud_agent_code_reviews.error_message} LIKE '%timed out%' THEN 'Timeout' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%context window%' OR ${cloud_agent_code_reviews.error_message} LIKE '%token limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%too large%' OR ${cloud_agent_code_reviews.error_message} LIKE '%maximum context length%' THEN 'Context Window Exceeded' @@ -91,8 +94,6 @@ const errorCategoryExpr = sql`CASE WHEN ${cloud_agent_code_reviews.error_message} LIKE '%500%' OR ${cloud_agent_code_reviews.error_message} LIKE '%502%' OR ${cloud_agent_code_reviews.error_message} LIKE '%503%' OR ${cloud_agent_code_reviews.error_message} LIKE '%internal server%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Internal Server%' THEN 'Upstream Server Error' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%ECONNREFUSED%' OR ${cloud_agent_code_reviews.error_message} LIKE '%ECONNRESET%' OR ${cloud_agent_code_reviews.error_message} LIKE '%socket hang up%' OR ${cloud_agent_code_reviews.error_message} LIKE '%network%' THEN 'Network Error' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%parse%' OR ${cloud_agent_code_reviews.error_message} LIKE '%JSON%' OR ${cloud_agent_code_reviews.error_message} LIKE '%unexpected token%' THEN 'Parse Error' - WHEN ${cloud_agent_code_reviews.terminal_reason} = 'workspace_capacity' OR ${cloud_agent_code_reviews.error_message} LIKE '%sandbox storage full%' OR ${cloud_agent_code_reviews.error_message} LIKE '%admission rejected%' OR ${cloud_agent_code_reviews.error_message} LIKE '%storage full%' THEN 'Sandbox Capacity' - WHEN ${cloud_agent_code_reviews.error_message} LIKE '%connect to the sandbox%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Sandbox connection failed%' OR ${cloud_agent_code_reviews.error_message} LIKE '%container shut down%' THEN 'Sandbox Connection' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%could not be delivered%' THEN 'Delivery Failure' WHEN ${cloud_agent_code_reviews.error_message} LIKE '%repository_not_installed%' OR ${cloud_agent_code_reviews.error_message} LIKE '%app installation required%' THEN 'Action Required' WHEN ${cloud_agent_code_reviews.error_message} IS NULL THEN 'Unknown Error' @@ -101,6 +102,9 @@ END`; const attemptErrorCategoryExpr = sql`CASE WHEN ${cloud_agent_code_review_attempts.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'gitlab_project_access_required', 'byok_invalid_key', 'selected_model_unavailable') THEN 'Action Required' + WHEN ${cloud_agent_code_review_attempts.terminal_reason} = 'workspace_capacity' THEN 'Sandbox Capacity' + WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%sandbox storage full%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%admission rejected%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%storage full%' THEN 'Sandbox Capacity' + WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%connect to the sandbox%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Sandbox connection failed%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%container shut down%' THEN 'Sandbox Connection' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%rate limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Rate limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%429%' THEN 'Rate Limited' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%timeout%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Timeout%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%ETIMEDOUT%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%timed out%' THEN 'Timeout' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%context window%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%token limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%too large%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%maximum context length%' THEN 'Context Window Exceeded' @@ -109,8 +113,6 @@ const attemptErrorCategoryExpr = sql`CASE WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%500%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%502%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%503%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%internal server%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Internal Server%' THEN 'Upstream Server Error' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%ECONNREFUSED%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%ECONNRESET%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%socket hang up%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%network%' THEN 'Network Error' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%parse%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%JSON%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%unexpected token%' THEN 'Parse Error' - WHEN ${cloud_agent_code_review_attempts.terminal_reason} = 'workspace_capacity' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%sandbox storage full%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%admission rejected%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%storage full%' THEN 'Sandbox Capacity' - WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%connect to the sandbox%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Sandbox connection failed%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%container shut down%' THEN 'Sandbox Connection' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%could not be delivered%' THEN 'Delivery Failure' WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%repository_not_installed%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%app installation required%' THEN 'Action Required' WHEN ${cloud_agent_code_review_attempts.error_message} IS NULL THEN 'Unknown Error' diff --git a/services/cloud-agent-next/src/execution/orchestrator.test.ts b/services/cloud-agent-next/src/execution/orchestrator.test.ts index e8679e65a3..11f469db33 100644 --- a/services/cloud-agent-next/src/execution/orchestrator.test.ts +++ b/services/cloud-agent-next/src/execution/orchestrator.test.ts @@ -360,7 +360,7 @@ describe('ExecutionOrchestrator AgentSandbox delivery', () => { await expect(orchestrator.execute(basePlan)).rejects.toBe(finalizingError); }); - it('does not recover the shared sandbox for plain capacity admission rejection', async () => { + it('translates a capacity admission rejection to a retryable sandbox_storage_full failure without recovering the shared sandbox', async () => { const { orchestrator, ensureWrapper, deleteSandbox } = createOrchestrator(); ensureWrapper.mockRejectedValueOnce( new WorkspaceCapacityAdmissionRejectedError({ @@ -371,7 +371,13 @@ describe('ExecutionOrchestrator AgentSandbox delivery', () => { }) ); - await expect(orchestrator.execute(basePlan)).rejects.toThrow('Failed to start wrapper'); + // Must not degrade to a generic WRAPPER_START_FAILED: the subtype is what + // routes the pending flush to the capacity-specific retry backoff. + await expect(orchestrator.execute(basePlan)).rejects.toMatchObject({ + code: 'WORKSPACE_SETUP_FAILED', + retryable: true, + workspaceFailureSubtype: 'sandbox_storage_full', + } satisfies Partial); expect(deleteSandbox).not.toHaveBeenCalled(); }); diff --git a/services/cloud-agent-next/src/execution/orchestrator.ts b/services/cloud-agent-next/src/execution/orchestrator.ts index b616880d9f..578229a71d 100644 --- a/services/cloud-agent-next/src/execution/orchestrator.ts +++ b/services/cloud-agent-next/src/execution/orchestrator.ts @@ -18,6 +18,7 @@ import { ExecutionError } from './errors.js'; import { SessionService } from '../session-service.js'; import { logger } from '../logger.js'; import { WrapperError } from '../kilo/wrapper-client.js'; +import { WorkspaceCapacityAdmissionRejectedError } from '../workspace-errors.js'; import { withDORetry } from '../utils/do-retry.js'; import { withTimeout } from '@kilocode/worker-utils'; import { logSandboxOperationTimeout } from '../sandbox-timeout-logging.js'; @@ -57,6 +58,17 @@ function withWorkspacePreparationTimeout(operation: Promise, step: string) function translateKnownWrapperFailure(error: unknown): Error | undefined { if (error instanceof ExecutionError) return error; + // A full workspace disk surfaces here as a raw WorkspaceCapacityAdmissionRejectedError. + // Translate it into a workspace_setup_failed / sandbox_storage_full ExecutionError + // so it flows through the shared subtype-based classification and gets the + // capacity-specific retry backoff instead of being wrapped as a generic + // wrapperStartFailed (which would lose its identity and the longer budget). + if (error instanceof WorkspaceCapacityAdmissionRejectedError) { + return ExecutionError.workspaceSetupFailed(error.message, error, { + subtype: 'sandbox_storage_full', + retryable: true, + }); + } if (!(error instanceof WrapperError)) return undefined; if (error.code === 'WORKSPACE_SETUP_FAILED') { diff --git a/services/cloud-agent-next/src/session/pending-messages.test.ts b/services/cloud-agent-next/src/session/pending-messages.test.ts index a94a75f5aa..496d024c76 100644 --- a/services/cloud-agent-next/src/session/pending-messages.test.ts +++ b/services/cloud-agent-next/src/session/pending-messages.test.ts @@ -431,6 +431,41 @@ describe('recordPendingFlushFailure', () => { expect(result.nextFlushAttemptAt).toBe(110_000); }); + it('bounds retries when failures alternate between reset-eligible modes', async () => { + const storage = createMemoryStorage(); + let message = makeMessage(); + await storePendingSessionMessage(storage, message); + const now = 100_000; + + // Fresh sandbox-connect failure resets to attempt 1. + let result = await recordPendingFlushFailure(storage, message, 'connect failed', now, { + policy: 'warm-followup', + code: 'SANDBOX_CONNECT_FAILED', + }); + expect(result.attempts).toBe(1); + expect(result.exhausted).toBe(false); + message = result.message; + + // Switching to capacity (both modes reset-eligible) does NOT reset; attempts accumulate. + result = await recordPendingFlushFailure(storage, message, 'sandbox storage full', now, { + policy: 'warm-followup', + code: 'WORKSPACE_SETUP_FAILED', + subtype: 'sandbox_storage_full', + }); + expect(result.attempts).toBe(2); + message = result.message; + + // Switching back to connect keeps accumulating and exceeds the connect budget, + // so the flapping message exhausts instead of resetting forever. + result = await recordPendingFlushFailure(storage, message, 'connect failed', now, { + policy: 'warm-followup', + code: 'SANDBOX_CONNECT_FAILED', + }); + expect(result.attempts).toBe(3); + expect(result.exhausted).toBe(true); + expect(result.nextFlushAttemptAt).toBeUndefined(); + }); + it('retries a sandbox connection failure once after exactly five seconds', async () => { const storage = createMemoryStorage(); let message = makeMessage({ createdAt: 100_000 }); diff --git a/services/cloud-agent-next/src/session/pending-messages.ts b/services/cloud-agent-next/src/session/pending-messages.ts index e905224d73..f43717a2b7 100644 --- a/services/cloud-agent-next/src/session/pending-messages.ts +++ b/services/cloud-agent-next/src/session/pending-messages.ts @@ -490,6 +490,22 @@ export function shouldSkipPendingFlush(message: PendingSessionMessage, now: numb return message.nextFlushAttemptAt !== undefined && message.nextFlushAttemptAt > now; } +/** + * Reset-eligible modes each start a fresh retry budget on entry (sandbox-connect + * has a short reconnect budget; sandbox-capacity has a longer backed-off budget + * for transient disk pressure). Alternating between them must NOT keep resetting, + * so callers only reset when entering one of these from a non-reset-eligible state. + */ +function isResetEligibleFailure( + code: PendingFlushFailureCode | undefined, + subtype: WorkspaceFailureSubtype | undefined +): boolean { + return ( + code === 'SANDBOX_CONNECT_FAILED' || + (code === 'WORKSPACE_SETUP_FAILED' && subtype === 'sandbox_storage_full') + ); +} + export async function recordPendingFlushFailure( storage: SessionQueueStorage, message: PendingSessionMessage, @@ -540,17 +556,15 @@ export async function recordPendingFlushFailure( : options.code === 'WORKSPACE_SETUP_FAILED' ? options.safeFailureMessage : undefined; - // A newly-observed sandbox-capacity failure starts its own retry budget, the - // same way a sandbox-connection failure does, so the full 10s/30s/60s backoff - // applies even if the message already failed under a different code first. - const enteringSandboxCapacityRetry = - flushFailureCode === 'WORKSPACE_SETUP_FAILED' && - failureSubtype === 'sandbox_storage_full' && - message.lastFlushFailureSubtype !== 'sandbox_storage_full'; + // Reset the attempt counter only when a message ENTERS a reset-eligible + // transient mode (sandbox-connect or sandbox-capacity) from a state that is + // not itself reset-eligible, so each fresh sequence gets its full backoff + // budget. When failures alternate between the two reset-eligible modes the + // counter is NOT reset, so attempts accumulate and the message still exhausts + // instead of flapping between modes forever. const attempts = - (flushFailureCode === 'SANDBOX_CONNECT_FAILED' && - message.lastFlushFailureCode !== 'SANDBOX_CONNECT_FAILED') || - enteringSandboxCapacityRetry + isResetEligibleFailure(flushFailureCode, failureSubtype) && + !isResetEligibleFailure(message.lastFlushFailureCode, message.lastFlushFailureSubtype) ? 1 : (message.flushAttempts ?? 0) + 1; const retryDelays = diff --git a/services/cloud-agent-next/src/session/session-message-queue.ts b/services/cloud-agent-next/src/session/session-message-queue.ts index 84df4e94fa..c91daa9a86 100644 --- a/services/cloud-agent-next/src/session/session-message-queue.ts +++ b/services/cloud-agent-next/src/session/session-message-queue.ts @@ -15,7 +15,6 @@ import { logger } from '../logger.js'; import { dispatchedKilocodeModelId } from '../persistence/model-utils.js'; import type { SessionMetadata } from '../persistence/session-metadata.js'; import { isSandboxWorkspaceProbeTimeoutError } from '../sandbox-recovery.js'; -import { WorkspaceCapacityAdmissionRejectedError } from '../workspace-errors.js'; import { WrapperCleanupBlockedError, type WrapperCleanupBlock, @@ -467,26 +466,6 @@ export async function flushNextPendingSessionMessage(params: { remainingCount: totalCount, }; } - if (error instanceof WorkspaceCapacityAdmissionRejectedError) { - // A full workspace disk is transient backpressure, not a review failure. - // Route it through the sandbox_storage_full path so it is classified as a - // capacity failure and retried with a longer backoff (cleanup and other - // reviews finishing free space) instead of dying after one attempt. - const failure = await recordPendingFlushFailure( - params.storage, - message, - error.message, - Date.now(), - { - policy, - code: 'WORKSPACE_SETUP_FAILED', - subtype: 'sandbox_storage_full', - retryable: true, - scheduleTerminalizationRepair: params.scheduleTerminalizationRepair, - } - ); - return toFailureResult(failure, totalCount); - } const code = error instanceof MessageDeliveryRequestValidationError ? error.code diff --git a/services/code-review-infra/src/types.ts b/services/code-review-infra/src/types.ts index e3a1f84cf4..9f46088efe 100644 --- a/services/code-review-infra/src/types.ts +++ b/services/code-review-infra/src/types.ts @@ -111,6 +111,9 @@ export interface CodeReviewStatusResponse { export type CodeReviewStatusResult = CodeReviewStatusResponse | null; +// KEEP IN SYNC with CODE_REVIEW_TERMINAL_REASONS (packages/db/src/schema-types.ts) +// and CloudAgentTerminalReason (packages/worker-utils/src/cloud-agent-next-client.ts). +// A value missing here is silently coerced to undefined by `.catch(undefined)`. const InternalStatusTerminalReasonSchema = z .enum([ 'billing', @@ -127,6 +130,7 @@ const InternalStatusTerminalReasonSchema = z 'timeout', 'upstream_error', 'sandbox_error', + 'workspace_capacity', 'unknown', ]) .nullable() From ca398d575839e916349adf3c5b11c0fb6f6cc6c5 Mon Sep 17 00:00:00 2001 From: St0rmz1 Date: Sat, 25 Jul 2026 19:22:31 -0700 Subject: [PATCH 5/5] fix(code-review): classify orchestrator-path capacity failures as workspace_capacity --- .../code-review-status/[reviewId]/route.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts index cbb68b8108..3759b99504 100644 --- a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts +++ b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts @@ -350,7 +350,14 @@ function normalizePayload(raw: StatusUpdatePayload): { // Infer workspace capacity (transient sandbox disk pressure) so it is // classified distinctly instead of counted as an unknown delivery failure. - if (!terminalReason && isWorkspaceCapacityFailure(failure, raw.errorMessage)) { + // Capacity arrives either with no terminal reason (delivery path) or with a + // generic 'sandbox_error' (orchestrator session-start path), so override that + // one generic value too — otherwise the column stays 'sandbox_error' and only + // the admin router's message match recovers the category. + if ( + (!terminalReason || terminalReason === 'sandbox_error') && + isWorkspaceCapacityFailure(failure, raw.errorMessage) + ) { terminalReason = 'workspace_capacity'; } @@ -382,6 +389,12 @@ function normalizePayload(raw: StatusUpdatePayload): { * `workspace_capacity` terminal reason. Prefers the structured failure subtype * from cloud-agent-next and falls back to the safe error message for older * payloads. + * + * The message fallback is intentionally broad ('admission rejected'); a stricter, + * phrasing-anchored variant lives in + * services/code-review-infra/src/code-review-orchestrator.ts + * (isWorkspaceAdmissionCapacityFailure). Keep the two in mind together if the + * admission-rejection message format ever changes. */ export function isWorkspaceCapacityFailure( failure: CloudAgentSafeFailure | undefined,