feat(code reviewer) better workspace capacity handling

apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts

@@ -368,9 +368,13 @@ function mockCreatedInfraRetryFlow(
 
 // --- Tests ---
 
-import type { POST as POSTType } from './route';
+import type {
+  POST as POSTType,
+  isWorkspaceCapacityFailure as isWorkspaceCapacityFailureType,
+} from './route';
 
 let POST: typeof POSTType;
+let isWorkspaceCapacityFailure: typeof isWorkspaceCapacityFailureType;
 
 beforeEach(async () => {
   jest.clearAllMocks();
@@ -431,7 +435,27 @@ beforeEach(async () => {
   );
   mockDisableCodeReviewForActionRequiredFailure.mockResolvedValue(undefined);
   mockDisableCodeReviewForRepeatedCloneTimeoutsToday.mockResolvedValue(null);
-  ({ POST } = await import('./route'));
+  ({ POST, isWorkspaceCapacityFailure } = await import('./route'));
+});
+
+describe('isWorkspaceCapacityFailure', () => {
+  it('detects a full-disk failure from the structured subtype or the message', () => {
+    expect(
+      isWorkspaceCapacityFailure(undefined, 'Workspace setup failed: sandbox storage full')
+    ).toBe(true);
+    // Orchestrator session-start path reports it inside a "(500)" message.
+    expect(
+      isWorkspaceCapacityFailure(
+        undefined,
+        'initiate failed (500): Workspace admission rejected: 1036 MB available below 2048 MB threshold after cleanup'
+      )
+    ).toBe(true);
+  });
+
+  it('ignores unrelated failures', () => {
+    expect(isWorkspaceCapacityFailure(undefined, 'The message could not be delivered')).toBe(false);
+    expect(isWorkspaceCapacityFailure(undefined, undefined)).toBe(false);
+  });
 });
 
 describe('POST /api/internal/code-review-status/[reviewId]', () => {

apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts

@@ -348,6 +348,19 @@ function normalizePayload(raw: StatusUpdatePayload): {
     terminalReason = 'billing';
   }
 
+  // Infer workspace capacity (transient sandbox disk pressure) so it is
+  // classified distinctly instead of counted as an unknown delivery failure.
+  // Capacity arrives either with no terminal reason (delivery path) or with a
+  // generic 'sandbox_error' (orchestrator session-start path), so override that
+  // one generic value too — otherwise the column stays 'sandbox_error' and only
+  // the admin router's message match recovers the category.
+  if (
+    (!terminalReason || terminalReason === 'sandbox_error') &&
+    isWorkspaceCapacityFailure(failure, raw.errorMessage)
+  ) {
+    terminalReason = 'workspace_capacity';
+  }
+
   if (
     (raw.status === 'failed' || raw.status === 'interrupted') &&
     isModelNotFoundCodeReviewTerminalReason(terminalReason, raw.errorMessage)
@@ -371,6 +384,29 @@ function normalizePayload(raw: StatusUpdatePayload): {
   };
 }
 
+/**
+ * Detects a workspace disk-capacity failure so it is recorded with a distinct
+ * `workspace_capacity` terminal reason. Prefers the structured failure subtype
+ * from cloud-agent-next and falls back to the safe error message for older
+ * payloads.
+ *
+ * The message fallback is intentionally broad ('admission rejected'); a stricter,
+ * phrasing-anchored variant lives in
+ * services/code-review-infra/src/code-review-orchestrator.ts
+ * (isWorkspaceAdmissionCapacityFailure). Keep the two in mind together if the
+ * admission-rejection message format ever changes.
+ */
+export function isWorkspaceCapacityFailure(
+  failure: CloudAgentSafeFailure | undefined,
+  errorMessage: string | undefined
+): boolean {
+  if (failure?.code === 'workspace_setup_failed' && failure.subtype === 'sandbox_storage_full') {
+    return true;
+  }
+  const message = (errorMessage ?? '').toLowerCase();
+  return message.includes('sandbox storage full') || message.includes('admission rejected');
+}
+
 function isBillingCodeReviewTerminalReason(
   terminalReason?: CodeReviewTerminalReason,
   errorMessage?: string | null

apps/web/src/routers/admin-code-reviews-router.test.ts

@@ -341,6 +341,60 @@ describe('adminCodeReviewsRouter', () => {
     expect(exportRows[0]).toHaveProperty('attempt_status');
   });
 
+  it('buckets sandbox capacity and delivery failures instead of Other', async () => {
+    const owner = { type: 'user', id: adminUser.id } satisfies ReviewOwner;
+
+    await db.insert(cloud_agent_code_reviews).values([
+      reviewValues({
+        owner,
+        status: 'failed',
+        createdAt: timestamp(700),
+        terminalReason: 'workspace_capacity',
+        errorMessage: 'Workspace setup failed: sandbox storage full',
+      }),
+      // terminal_reason is authoritative and must win over the generic '%404%'
+      // branch that this admission-rejected message would otherwise match.
+      reviewValues({
+        owner,
+        status: 'failed',
+        createdAt: timestamp(705),
+        terminalReason: 'workspace_capacity',
+        errorMessage: 'Workspace admission rejected: 404 MB available below 2048 MB threshold',
+      }),
+      // Orchestrator session-start path: reports capacity as terminal_reason
+      // 'sandbox_error' with a "(500)" message. The capacity message match must
+      // win over the generic '%500%' Upstream Server Error branch.
+      reviewValues({
+        owner,
+        status: 'failed',
+        createdAt: timestamp(707),
+        terminalReason: 'sandbox_error',
+        errorMessage:
+          'initiate failed (500): Workspace admission rejected: 1036 MB available below 2048 MB threshold after cleanup',
+      }),
+      reviewValues({
+        owner,
+        status: 'failed',
+        createdAt: timestamp(710),
+        errorMessage: 'The message could not be delivered',
+      }),
+    ]);
+
+    const caller = await createCallerForUser(adminUser.id);
+    const errors = await caller.admin.codeReviews.getErrorAnalysis(filterInput());
+
+    expect(errors.categories).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ category: 'Sandbox Capacity', count: 3 }),
+        expect.objectContaining({ category: 'Delivery Failure', count: 1 }),
+      ])
+    );
+    const categoryNames = errors.categories.map(category => category.category);
+    expect(categoryNames).not.toContain('Other');
+    expect(categoryNames).not.toContain('Not Found');
+    expect(categoryNames).not.toContain('Upstream Server Error');
+  });
+
   it('classifies final model-not-found outcomes as cancellations instead of failures', async () => {
     const owner = { type: 'user', id: adminUser.id } satisfies ReviewOwner;
 

apps/web/src/routers/admin-code-reviews-router.ts

@@ -83,6 +83,9 @@ const excludeModelNotFoundAttempt = sql`COALESCE(${cloud_agent_code_review_attem
  */
 const errorCategoryExpr = sql<string>`CASE
   WHEN ${cloud_agent_code_reviews.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'gitlab_project_access_required', 'byok_invalid_key', 'selected_model_unavailable') THEN 'Action Required'
+  WHEN ${cloud_agent_code_reviews.terminal_reason} = 'workspace_capacity' THEN 'Sandbox Capacity'
+  WHEN ${cloud_agent_code_reviews.error_message} LIKE '%sandbox storage full%' OR ${cloud_agent_code_reviews.error_message} LIKE '%admission rejected%' OR ${cloud_agent_code_reviews.error_message} LIKE '%storage full%' THEN 'Sandbox Capacity'
+  WHEN ${cloud_agent_code_reviews.error_message} LIKE '%connect to the sandbox%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Sandbox connection failed%' OR ${cloud_agent_code_reviews.error_message} LIKE '%container shut down%' THEN 'Sandbox Connection'
   WHEN ${cloud_agent_code_reviews.error_message} LIKE '%rate limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Rate limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%429%' THEN 'Rate Limited'
   WHEN ${cloud_agent_code_reviews.error_message} LIKE '%timeout%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Timeout%' OR ${cloud_agent_code_reviews.error_message} LIKE '%ETIMEDOUT%' OR ${cloud_agent_code_reviews.error_message} LIKE '%timed out%' THEN 'Timeout'
   WHEN ${cloud_agent_code_reviews.error_message} LIKE '%context window%' OR ${cloud_agent_code_reviews.error_message} LIKE '%token limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%too large%' OR ${cloud_agent_code_reviews.error_message} LIKE '%maximum context length%' THEN 'Context Window Exceeded'
@@ -91,12 +94,17 @@ const errorCategoryExpr = sql<string>`CASE
   WHEN ${cloud_agent_code_reviews.error_message} LIKE '%500%' OR ${cloud_agent_code_reviews.error_message} LIKE '%502%' OR ${cloud_agent_code_reviews.error_message} LIKE '%503%' OR ${cloud_agent_code_reviews.error_message} LIKE '%internal server%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Internal Server%' THEN 'Upstream Server Error'
   WHEN ${cloud_agent_code_reviews.error_message} LIKE '%ECONNREFUSED%' OR ${cloud_agent_code_reviews.error_message} LIKE '%ECONNRESET%' OR ${cloud_agent_code_reviews.error_message} LIKE '%socket hang up%' OR ${cloud_agent_code_reviews.error_message} LIKE '%network%' THEN 'Network Error'
   WHEN ${cloud_agent_code_reviews.error_message} LIKE '%parse%' OR ${cloud_agent_code_reviews.error_message} LIKE '%JSON%' OR ${cloud_agent_code_reviews.error_message} LIKE '%unexpected token%' THEN 'Parse Error'
+  WHEN ${cloud_agent_code_reviews.error_message} LIKE '%could not be delivered%' THEN 'Delivery Failure'
+  WHEN ${cloud_agent_code_reviews.error_message} LIKE '%repository_not_installed%' OR ${cloud_agent_code_reviews.error_message} LIKE '%app installation required%' THEN 'Action Required'
   WHEN ${cloud_agent_code_reviews.error_message} IS NULL THEN 'Unknown Error'
   ELSE 'Other'
 END`;
 
 const attemptErrorCategoryExpr = sql<string>`CASE
   WHEN ${cloud_agent_code_review_attempts.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'gitlab_project_access_required', 'byok_invalid_key', 'selected_model_unavailable') THEN 'Action Required'
+  WHEN ${cloud_agent_code_review_attempts.terminal_reason} = 'workspace_capacity' THEN 'Sandbox Capacity'
+  WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%sandbox storage full%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%admission rejected%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%storage full%' THEN 'Sandbox Capacity'
+  WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%connect to the sandbox%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Sandbox connection failed%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%container shut down%' THEN 'Sandbox Connection'
   WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%rate limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Rate limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%429%' THEN 'Rate Limited'
   WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%timeout%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Timeout%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%ETIMEDOUT%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%timed out%' THEN 'Timeout'
   WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%context window%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%token limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%too large%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%maximum context length%' THEN 'Context Window Exceeded'
@@ -105,6 +113,8 @@ const attemptErrorCategoryExpr = sql<string>`CASE
   WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%500%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%502%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%503%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%internal server%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Internal Server%' THEN 'Upstream Server Error'
   WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%ECONNREFUSED%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%ECONNRESET%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%socket hang up%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%network%' THEN 'Network Error'
   WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%parse%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%JSON%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%unexpected token%' THEN 'Parse Error'
+  WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%could not be delivered%' THEN 'Delivery Failure'
+  WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%repository_not_installed%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%app installation required%' THEN 'Action Required'
   WHEN ${cloud_agent_code_review_attempts.error_message} IS NULL THEN 'Unknown Error'
   ELSE 'Other'
 END`;

packages/db/src/schema-types.ts

@@ -2000,6 +2000,7 @@ export const CODE_REVIEW_TERMINAL_REASONS = [
   'timeout',
   'upstream_error',
   'sandbox_error',
+  'workspace_capacity',
   'unknown',
 ] as const;
 

packages/worker-utils/src/cloud-agent-next-client.ts

@@ -164,6 +164,7 @@ export type CloudAgentTerminalReason =
   | 'timeout'
   | 'upstream_error'
   | 'sandbox_error'
+  | 'workspace_capacity'
   | 'unknown';
 
 export class CloudAgentNextError extends Error {

services/cloud-agent-next/src/execution/orchestrator.test.ts

@@ -360,7 +360,7 @@ describe('ExecutionOrchestrator AgentSandbox delivery', () => {
     await expect(orchestrator.execute(basePlan)).rejects.toBe(finalizingError);
   });
 
-  it('does not recover the shared sandbox for plain capacity admission rejection', async () => {
+  it('translates a capacity admission rejection to a retryable sandbox_storage_full failure without recovering the shared sandbox', async () => {
     const { orchestrator, ensureWrapper, deleteSandbox } = createOrchestrator();
     ensureWrapper.mockRejectedValueOnce(
       new WorkspaceCapacityAdmissionRejectedError({
@@ -371,7 +371,13 @@ describe('ExecutionOrchestrator AgentSandbox delivery', () => {
       })
     );
 
-    await expect(orchestrator.execute(basePlan)).rejects.toThrow('Failed to start wrapper');
+    // Must not degrade to a generic WRAPPER_START_FAILED: the subtype is what
+    // routes the pending flush to the capacity-specific retry backoff.
+    await expect(orchestrator.execute(basePlan)).rejects.toMatchObject({
+      code: 'WORKSPACE_SETUP_FAILED',
+      retryable: true,
+      workspaceFailureSubtype: 'sandbox_storage_full',
+    } satisfies Partial<ExecutionError>);
     expect(deleteSandbox).not.toHaveBeenCalled();
   });
 

services/cloud-agent-next/src/execution/orchestrator.ts

@@ -18,6 +18,7 @@ import { ExecutionError } from './errors.js';
 import { SessionService } from '../session-service.js';
 import { logger } from '../logger.js';
 import { WrapperError } from '../kilo/wrapper-client.js';
+import { WorkspaceCapacityAdmissionRejectedError } from '../workspace-errors.js';
 import { withDORetry } from '../utils/do-retry.js';
 import { withTimeout } from '@kilocode/worker-utils';
 import { logSandboxOperationTimeout } from '../sandbox-timeout-logging.js';
@@ -57,6 +58,17 @@ function withWorkspacePreparationTimeout<T>(operation: Promise<T>, step: string)
 
 function translateKnownWrapperFailure(error: unknown): Error | undefined {
   if (error instanceof ExecutionError) return error;
+  // A full workspace disk surfaces here as a raw WorkspaceCapacityAdmissionRejectedError.
+  // Translate it into a workspace_setup_failed / sandbox_storage_full ExecutionError
+  // so it flows through the shared subtype-based classification and gets the
+  // capacity-specific retry backoff instead of being wrapped as a generic
+  // wrapperStartFailed (which would lose its identity and the longer budget).
+  if (error instanceof WorkspaceCapacityAdmissionRejectedError) {
+    return ExecutionError.workspaceSetupFailed(error.message, error, {
+      subtype: 'sandbox_storage_full',
+      retryable: true,
+    });
+  }
   if (!(error instanceof WrapperError)) return undefined;
 
   if (error.code === 'WORKSPACE_SETUP_FAILED') {

services/cloud-agent-next/src/session/pending-messages.test.ts

@@ -371,6 +371,101 @@ describe('recordPendingFlushFailure', () => {
     expect(delays).toEqual([2_000, undefined]);
   });
 
+  it('gives a full-disk workspace failure a longer backed-off retry budget', async () => {
+    const storage = createMemoryStorage();
+    let message = makeMessage();
+    await storePendingSessionMessage(storage, message);
+
+    const delays: (number | undefined)[] = [];
+    const now = 100_000;
+
+    // A workspace_setup_failed with the sandbox_storage_full subtype is transient
+    // disk backpressure and should retry several times with growing delays before
+    // exhausting, unlike a plain workspace failure (one warm-followup retry above).
+    for (let i = 0; i < 4; i++) {
+      const result = await recordPendingFlushFailure(
+        storage,
+        message,
+        'sandbox storage full',
+        now,
+        {
+          policy: 'warm-followup',
+          code: 'WORKSPACE_SETUP_FAILED',
+          subtype: 'sandbox_storage_full',
+        }
+      );
+      delays.push(
+        result.nextFlushAttemptAt !== undefined ? result.nextFlushAttemptAt - now : undefined
+      );
+      message = result.message;
+    }
+
+    expect(delays).toEqual([10_000, 30_000, 60_000, undefined]);
+  });
+
+  it('starts the capacity retry budget fresh after a different earlier failure', async () => {
+    const storage = createMemoryStorage();
+    const message = makeMessage({
+      createdAt: 1,
+      flushAttempts: 2,
+      lastFlushFailureCode: 'WRAPPER_START_FAILED',
+    });
+    await storePendingSessionMessage(storage, message);
+
+    const result = await recordPendingFlushFailure(
+      storage,
+      message,
+      'sandbox storage full',
+      100_000,
+      {
+        policy: 'warm-followup',
+        code: 'WORKSPACE_SETUP_FAILED',
+        subtype: 'sandbox_storage_full',
+      }
+    );
+
+    // Reset to attempt 1 so the full 10s/30s/60s budget applies, not truncated
+    // by the two earlier non-capacity attempts.
+    expect(result.attempts).toBe(1);
+    expect(result.exhausted).toBe(false);
+    expect(result.nextFlushAttemptAt).toBe(110_000);
+  });
+
+  it('bounds retries when failures alternate between reset-eligible modes', async () => {
+    const storage = createMemoryStorage();
+    let message = makeMessage();
+    await storePendingSessionMessage(storage, message);
+    const now = 100_000;
+
+    // Fresh sandbox-connect failure resets to attempt 1.
+    let result = await recordPendingFlushFailure(storage, message, 'connect failed', now, {
+      policy: 'warm-followup',
+      code: 'SANDBOX_CONNECT_FAILED',
+    });
+    expect(result.attempts).toBe(1);
+    expect(result.exhausted).toBe(false);
+    message = result.message;
+
+    // Switching to capacity (both modes reset-eligible) does NOT reset; attempts accumulate.
+    result = await recordPendingFlushFailure(storage, message, 'sandbox storage full', now, {
+      policy: 'warm-followup',
+      code: 'WORKSPACE_SETUP_FAILED',
+      subtype: 'sandbox_storage_full',
+    });
+    expect(result.attempts).toBe(2);
+    message = result.message;
+
+    // Switching back to connect keeps accumulating and exceeds the connect budget,
+    // so the flapping message exhausts instead of resetting forever.
+    result = await recordPendingFlushFailure(storage, message, 'connect failed', now, {
+      policy: 'warm-followup',
+      code: 'SANDBOX_CONNECT_FAILED',
+    });
+    expect(result.attempts).toBe(3);
+    expect(result.exhausted).toBe(true);
+    expect(result.nextFlushAttemptAt).toBeUndefined();
+  });
+
   it('retries a sandbox connection failure once after exactly five seconds', async () => {
     const storage = createMemoryStorage();
     let message = makeMessage({ createdAt: 100_000 });