feat(cloud-agent): attribute setup and admission failures by cause

apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.test.ts

@@ -9,15 +9,19 @@ import {
 import { CLOUD_AGENT_FAILURE_REASONS } from '@kilocode/worker-utils/cloud-agent-failure';
 
 describe('getOperationalFailureStats', () => {
-  it('calculates operational failure percentage without counting interruptions', () => {
-    expect(
-      getOperationalFailureStats({
-        completedRuns: 90,
-        failedRuns: 7,
-        setupFailures: 3,
-        interruptedRuns: 25,
-      })
-    ).toEqual({ failureEvents: 10, assessedOutcomes: 100, failureRatePercent: 10 });
+  it('reports run and setup failure rates over their own populations', () => {
+    const stats = getOperationalFailureStats({
+      completedRuns: 90,
+      failedRuns: 7,
+      setupFailures: 3,
+      interruptedRuns: 25,
+      sessionsObserved: 60,
+    });
+
+    expect(stats.runOutcomes).toBe(97);
+    expect(stats.runFailureRatePercent).toBeCloseTo((7 / 97) * 100);
+    expect(stats.sessionsObserved).toBe(60);
+    expect(stats.setupFailureRatePercent).toBeCloseTo((3 / 60) * 100);
   });
 
   it('does not report a percentage when no operational outcomes were assessed', () => {
@@ -27,8 +31,14 @@ describe('getOperationalFailureStats', () => {
         failedRuns: 0,
         setupFailures: 0,
         interruptedRuns: 4,
+        sessionsObserved: 0,
       })
-    ).toEqual({ failureEvents: 0, assessedOutcomes: 0, failureRatePercent: null });
+    ).toEqual({
+      runOutcomes: 0,
+      runFailureRatePercent: null,
+      sessionsObserved: 0,
+      setupFailureRatePercent: null,
+    });
   });
 });
 
@@ -43,6 +53,7 @@ describe('failure responsibility summary', () => {
       failedRuns: 17,
       setupFailures: 3,
       interruptedRuns: 2,
+      sessionsObserved: 20,
       platformFailures: 9,
       userFailures: 6,
       unknownFailures: 5,
@@ -72,6 +83,7 @@ describe('failure responsibility summary', () => {
       failedRuns: 0,
       setupFailures: 0,
       interruptedRuns: 0,
+      sessionsObserved: 0,
       platformFailures: 0,
       userFailures: 0,
       unknownFailures: 0,

apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.ts

@@ -8,6 +8,7 @@ type OperationalHealthSummary = {
   failedRuns: number;
   setupFailures: number;
   interruptedRuns: number;
+  sessionsObserved: number;
 };
 
 type ObservedHealthSummary = OperationalHealthSummary & {
@@ -26,6 +27,9 @@ const FAILURE_REASON_LABELS = {
   setup_command: 'Setup command',
   source_control_authentication: 'Source control authentication',
   source_control_configuration: 'Source control configuration',
+  source_control_clone_timeout: 'Repository clone timed out',
+  source_control_checkout_timeout: 'Repository checkout timed out',
+  source_control_repository_corrupt: 'Repository data is corrupt',
   sandbox_capacity: 'Sandbox capacity',
   sandbox_connectivity: 'Sandbox connectivity',
   runtime_startup: 'Runtime startup',
@@ -44,6 +48,15 @@ const FAILURE_REASON_LABELS = {
   source_control_network: 'Source control network',
   assistant_unknown: 'Unknown assistant failure',
   workspace_unknown: 'Unknown workspace failure',
+  session_import_timeout: 'Session import timed out',
+  session_import_failed: 'Session import failed',
+  setup_command_timeout: 'Setup command timed out',
+  admission_capacity: 'Admission queue full',
+  admission_not_found: 'Session not found at admission',
+  admission_internal: 'Internal admission error',
+  admission_compute_stopping: 'Compute stopping at admission',
+  admission_billing_unavailable: 'Billing unavailable at admission',
+  admission_forbidden: 'Admission forbidden',
   session_coordination: 'Session coordination',
   initial_request_invalid: 'Invalid initial request',
   initial_admission_unknown: 'Unknown initial admission failure',
@@ -87,11 +100,14 @@ export function getObservedHealthStats(summary: ObservedHealthSummary) {
 }
 
 export function getOperationalFailureStats(summary: OperationalHealthSummary) {
-  const failureEvents = summary.failedRuns + summary.setupFailures;
-  const assessedOutcomes = summary.completedRuns + failureEvents;
+  const runOutcomes = summary.failedRuns + summary.completedRuns;
   return {
-    failureEvents,
-    assessedOutcomes,
-    failureRatePercent: assessedOutcomes === 0 ? null : (failureEvents / assessedOutcomes) * 100,
+    runOutcomes,
+    runFailureRatePercent: runOutcomes === 0 ? null : (summary.failedRuns / runOutcomes) * 100,
+    sessionsObserved: summary.sessionsObserved,
+    setupFailureRatePercent:
+      summary.sessionsObserved === 0
+        ? null
+        : (summary.setupFailures / summary.sessionsObserved) * 100,
   };
 }

apps/web/src/routers/admin-cloud-agent-next-router.test.ts

@@ -212,12 +212,13 @@ describe('adminCloudAgentNextRouter', () => {
       completedRuns: 1,
       failedRuns: 2,
       interruptedRuns: 1,
+      sessionsObserved: 3,
       setupFailures: 2,
       platformFailures: 1,
       userFailures: 2,
       unknownFailures: 1,
-      platformFailureRate: 0.5,
-      allFailureRate: 0.8,
+      runFailureRate: 2 / 3,
+      setupFailureRate: 2 / 3,
     });
     expect(health.topErrors).toEqual(
       expect.arrayContaining([
@@ -419,8 +420,8 @@ describe('adminCloudAgentNextRouter', () => {
       startDate: '2035-01-12T00:00:00.000Z',
       endDate: '2035-01-13T00:00:00.000Z',
     });
-    expect(health.summary.platformFailureRate).toBeNull();
-    expect(health.summary.allFailureRate).toBeNull();
+    expect(health.summary.runFailureRate).toBeNull();
+    expect(health.summary.setupFailureRate).toBeNull();
     expect(health.topErrors).toEqual([]);
     expect(health.errorTotals).toEqual({ events: 0, groups: 0 });
   });

apps/web/src/routers/admin-cloud-agent-next-router.ts

@@ -97,6 +97,10 @@ function failureRate(failures: number, completed: number): number | null {
   return denominator === 0 ? null : failures / denominator;
 }
 
+function ratio(part: number, total: number): number | null {
+  return total === 0 ? null : part / total;
+}
+
 export const adminCloudAgentNextRouter = createTRPCRouter({
   getHealthOverview: adminProcedure.input(HealthOverviewFilterSchema).query(async ({ input }) => {
     const sessionStage = sql<string>`COALESCE(${cloud_agent_sessions.failure_stage}, 'unclassified')`;
@@ -115,7 +119,7 @@ export const adminCloudAgentNextRouter = createTRPCRouter({
       input.responsibility === 'all'
         ? undefined
         : sql`${runResponsibility} = ${input.responsibility}`;
-    const [summaryRows, setupRows, runErrorRows] = await Promise.all([
+    const [summaryRows, setupRows, runErrorRows, sessionCountRows] = await Promise.all([
       db
         .select({
           completed: sql<number>`COUNT(*) FILTER (WHERE ${cloud_agent_session_runs.status} = 'completed')`,
@@ -182,18 +186,29 @@ export const adminCloudAgentNextRouter = createTRPCRouter({
           )
         )
         .groupBy(runStage, runCode, runResponsibility, runReason),
+      db
+        .select({ sessionsObserved: sql<number>`COUNT(*)` })
+        .from(cloud_agent_sessions)
+        .where(
+          and(
+            gte(cloud_agent_sessions.created_at, input.startDate),
+            lt(cloud_agent_sessions.created_at, input.endDate),
+            retainedSessionCondition()
+          )
+        ),
     ]);
     const row = summaryRows[0];
     const summary = {
       completedRuns: count(row?.completed),
       failedRuns: count(row?.failed),
       interruptedRuns: count(row?.interrupted),
+      sessionsObserved: count(sessionCountRows[0]?.sessionsObserved),
       setupFailures: 0,
       platformFailures: count(row?.platformFailures),
       userFailures: count(row?.userFailures),
       unknownFailures: count(row?.unknownFailures),
-      platformFailureRate: null as number | null,
-      allFailureRate: null as number | null,
+      runFailureRate: null as number | null,
+      setupFailureRate: null as number | null,
     };
     const setupErrors: HealthError[] = [];
     for (const setupRow of setupRows) {
@@ -246,11 +261,8 @@ export const adminCloudAgentNextRouter = createTRPCRouter({
       groups: errors.length,
       events: errors.reduce((total, error) => total + error.count, 0),
     };
-    summary.platformFailureRate = failureRate(summary.platformFailures, summary.completedRuns);
-    summary.allFailureRate = failureRate(
-      summary.failedRuns + summary.setupFailures,
-      summary.completedRuns
-    );
+    summary.runFailureRate = failureRate(summary.failedRuns, summary.completedRuns);
+    summary.setupFailureRate = ratio(summary.setupFailures, summary.sessionsObserved);
     return { summary, topErrors, errorTotals };
   }),
 

packages/db/src/schema.ts

@@ -6241,6 +6241,9 @@ export type CloudAgentFailureReason =
   | 'setup_command'
   | 'source_control_authentication'
   | 'source_control_configuration'
+  | 'source_control_clone_timeout'
+  | 'source_control_checkout_timeout'
+  | 'source_control_repository_corrupt'
   | 'sandbox_capacity'
   | 'sandbox_connectivity'
   | 'runtime_startup'
@@ -6259,6 +6262,15 @@ export type CloudAgentFailureReason =
   | 'source_control_network'
   | 'assistant_unknown'
   | 'workspace_unknown'
+  | 'session_import_timeout'
+  | 'session_import_failed'
+  | 'setup_command_timeout'
+  | 'admission_capacity'
+  | 'admission_not_found'
+  | 'admission_internal'
+  | 'admission_compute_stopping'
+  | 'admission_billing_unavailable'
+  | 'admission_forbidden'
   | 'session_coordination'
   | 'initial_request_invalid'
   | 'initial_admission_unknown'

packages/worker-utils/src/cloud-agent-failure.test.ts

@@ -74,7 +74,7 @@ describe('classifyCloudAgentFailure', () => {
         code: 'workspace_setup_failed',
         workspaceSubtype: 'git_pack_corrupt',
       })
-    ).toEqual({ responsibility: 'unknown', reason: 'source_control_network' });
+    ).toEqual({ responsibility: 'unknown', reason: 'source_control_repository_corrupt' });
   });
 
   it.each([
@@ -181,6 +181,28 @@ describe('classifyCloudAgentFailure', () => {
     }
   );
 
+  it.each([
+    ['git_clone_timeout', 'unknown', 'source_control_clone_timeout'],
+    ['git_checkout_timeout', 'unknown', 'source_control_checkout_timeout'],
+    ['git_pack_corrupt', 'unknown', 'source_control_repository_corrupt'],
+    ['setup_command_timeout', 'user', 'setup_command_timeout'],
+    ['setup_command_failed', 'user', 'setup_command'],
+    ['kilo_import_timeout', 'unknown', 'session_import_timeout'],
+    ['kilo_import_failed', 'unknown', 'session_import_failed'],
+  ] as const)(
+    'splits collapsed workspace subtype %s into reason %s',
+    (workspaceSubtype, responsibility, reason) => {
+      expect(
+        classifyCloudAgentFailure({
+          source: 'run',
+          stage: 'pre_dispatch',
+          code: 'workspace_setup_failed',
+          workspaceSubtype,
+        })
+      ).toEqual({ responsibility, reason });
+    }
+  );
+
   it('classifies setup failures from structured stage and code only', () => {
     expect(
       classifyCloudAgentFailure({
@@ -197,6 +219,52 @@ describe('classifyCloudAgentFailure', () => {
       })
     ).toEqual({ responsibility: 'platform', reason: 'session_coordination' });
   });
+
+  it('attributes the opaque initial admission rejection from its admission code', () => {
+    expect(
+      classifyCloudAgentFailure({
+        source: 'setup',
+        stage: 'initial_admission',
+        code: 'initial_admission_rejected',
+        admissionCode: 'INTERNAL',
+      })
+    ).toEqual({ responsibility: 'platform', reason: 'admission_internal' });
+    expect(
+      classifyCloudAgentFailure({
+        source: 'setup',
+        stage: 'initial_admission',
+        code: 'initial_admission_rejected',
+        admissionCode: 'SANDBOX_CONNECT_FAILED',
+      })
+    ).toEqual({ responsibility: 'platform', reason: 'sandbox_connectivity' });
+    expect(
+      classifyCloudAgentFailure({
+        source: 'setup',
+        stage: 'initial_admission',
+        code: 'initial_admission_rejected',
+        admissionCode: 'PAYMENT_REQUIRED',
+      })
+    ).toEqual({ responsibility: 'user', reason: 'insufficient_credits' });
+    expect(
+      classifyCloudAgentFailure({
+        source: 'setup',
+        stage: 'initial_admission',
+        code: 'initial_admission_rejected',
+        admissionCode: 'UNKNOWN',
+      })
+    ).toEqual({ responsibility: 'unknown', reason: 'initial_admission_unknown' });
+  });
+
+  it('classifies a full admission queue as platform capacity, not unknown', () => {
+    expect(
+      classifyCloudAgentFailure({
+        source: 'setup',
+        stage: 'initial_admission',
+        code: 'initial_queue_full',
+        admissionCode: 'PENDING_QUEUE_FULL',
+      })
+    ).toEqual({ responsibility: 'platform', reason: 'admission_capacity' });
+  });
 });
 
 describe('CloudAgentSafeFailureSchema', () => {