From 66d2b2b2344d4517615598ebab74374e4f1cfd69 Mon Sep 17 00:00:00 2001 From: Evgeny Shurakov Date: Thu, 10 Sep 2026 17:37:09 +0200 Subject: [PATCH 1/3] feat(cloud-agent): attribute setup and admission failures by cause Cloud Agent session/run telemetry collapsed several distinct failure causes into coarse or unknown buckets, which hid their real cause from the admin health views. - Split previously collapsed workspace reasons into distinct values (clone/checkout timeout, corrupt pack, setup-command timeout, session import timeout/failure) so the classifier keeps the detail. - Preserve the bounded admission failure code across the setup boundary and classify it, so initial_admission_rejected is attributed instead of landing in initial_admission_unknown. Map initial_queue_full to platform/admission_capacity rather than unknown. - Deduplicate the admission failure mapping into one helper shared by the grouped start path and the legacy prepared-admission path. - Separate the health summary into run failure rate (failed runs over run outcomes) and setup failure rate (setup failures over sessions observed) instead of mixing runs and sessions in one rate. No migration: failure_reason/failure_responsibility are unconstrained text columns; only the shared enum values and schema type changed. --- .../health-summary.test.ts | 32 +++++-- .../CloudAgentNextTelemetry/health-summary.ts | 26 ++++- .../admin-cloud-agent-next-router.test.ts | 9 +- .../routers/admin-cloud-agent-next-router.ts | 28 ++++-- packages/db/src/schema.ts | 12 +++ .../src/cloud-agent-failure.test.ts | 70 +++++++++++++- .../worker-utils/src/cloud-agent-failure.ts | 96 ++++++++++++++++++- services/cloud-agent-next/src/router.test.ts | 6 +- .../src/session-prepare.test.ts | 12 ++- .../src/session/admission-failure.ts | 29 ++++++ .../src/session/legacy-prepared-admission.ts | 15 +-- .../src/session/session-registration.ts | 22 +---- .../src/telemetry/report-store.test.ts | 31 +++++- .../src/telemetry/report-store.ts | 9 +- .../src/telemetry/session-reports.ts | 2 + 15 files changed, 328 insertions(+), 71 deletions(-) create mode 100644 services/cloud-agent-next/src/session/admission-failure.ts diff --git a/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.test.ts b/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.test.ts index 13e18b3b8e3..3fc86aab6a9 100644 --- a/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.test.ts +++ b/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.test.ts @@ -9,15 +9,19 @@ import { import { CLOUD_AGENT_FAILURE_REASONS } from '@kilocode/worker-utils/cloud-agent-failure'; describe('getOperationalFailureStats', () => { - it('calculates operational failure percentage without counting interruptions', () => { - expect( - getOperationalFailureStats({ - completedRuns: 90, - failedRuns: 7, - setupFailures: 3, - interruptedRuns: 25, - }) - ).toEqual({ failureEvents: 10, assessedOutcomes: 100, failureRatePercent: 10 }); + it('reports run and setup failure rates over their own populations', () => { + const stats = getOperationalFailureStats({ + completedRuns: 90, + failedRuns: 7, + setupFailures: 3, + interruptedRuns: 25, + sessionsObserved: 60, + }); + + expect(stats.runOutcomes).toBe(97); + expect(stats.runFailureRatePercent).toBeCloseTo((7 / 97) * 100); + expect(stats.sessionsObserved).toBe(60); + expect(stats.setupFailureRatePercent).toBeCloseTo((3 / 60) * 100); }); it('does not report a percentage when no operational outcomes were assessed', () => { @@ -27,8 +31,14 @@ describe('getOperationalFailureStats', () => { failedRuns: 0, setupFailures: 0, interruptedRuns: 4, + sessionsObserved: 0, }) - ).toEqual({ failureEvents: 0, assessedOutcomes: 0, failureRatePercent: null }); + ).toEqual({ + runOutcomes: 0, + runFailureRatePercent: null, + sessionsObserved: 0, + setupFailureRatePercent: null, + }); }); }); @@ -43,6 +53,7 @@ describe('failure responsibility summary', () => { failedRuns: 17, setupFailures: 3, interruptedRuns: 2, + sessionsObserved: 20, platformFailures: 9, userFailures: 6, unknownFailures: 5, @@ -72,6 +83,7 @@ describe('failure responsibility summary', () => { failedRuns: 0, setupFailures: 0, interruptedRuns: 0, + sessionsObserved: 0, platformFailures: 0, userFailures: 0, unknownFailures: 0, diff --git a/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.ts b/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.ts index 98c02c8fa28..14640cf41de 100644 --- a/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.ts +++ b/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.ts @@ -8,6 +8,7 @@ type OperationalHealthSummary = { failedRuns: number; setupFailures: number; interruptedRuns: number; + sessionsObserved: number; }; type ObservedHealthSummary = OperationalHealthSummary & { @@ -26,6 +27,9 @@ const FAILURE_REASON_LABELS = { setup_command: 'Setup command', source_control_authentication: 'Source control authentication', source_control_configuration: 'Source control configuration', + source_control_clone_timeout: 'Repository clone timed out', + source_control_checkout_timeout: 'Repository checkout timed out', + source_control_repository_corrupt: 'Repository data is corrupt', sandbox_capacity: 'Sandbox capacity', sandbox_connectivity: 'Sandbox connectivity', runtime_startup: 'Runtime startup', @@ -44,6 +48,15 @@ const FAILURE_REASON_LABELS = { source_control_network: 'Source control network', assistant_unknown: 'Unknown assistant failure', workspace_unknown: 'Unknown workspace failure', + session_import_timeout: 'Session import timed out', + session_import_failed: 'Session import failed', + setup_command_timeout: 'Setup command timed out', + admission_capacity: 'Admission queue full', + admission_not_found: 'Session not found at admission', + admission_internal: 'Internal admission error', + admission_compute_stopping: 'Compute stopping at admission', + admission_billing_unavailable: 'Billing unavailable at admission', + admission_forbidden: 'Admission forbidden', session_coordination: 'Session coordination', initial_request_invalid: 'Invalid initial request', initial_admission_unknown: 'Unknown initial admission failure', @@ -87,11 +100,14 @@ export function getObservedHealthStats(summary: ObservedHealthSummary) { } export function getOperationalFailureStats(summary: OperationalHealthSummary) { - const failureEvents = summary.failedRuns + summary.setupFailures; - const assessedOutcomes = summary.completedRuns + failureEvents; + const runOutcomes = summary.failedRuns + summary.completedRuns; return { - failureEvents, - assessedOutcomes, - failureRatePercent: assessedOutcomes === 0 ? null : (failureEvents / assessedOutcomes) * 100, + runOutcomes, + runFailureRatePercent: runOutcomes === 0 ? null : (summary.failedRuns / runOutcomes) * 100, + sessionsObserved: summary.sessionsObserved, + setupFailureRatePercent: + summary.sessionsObserved === 0 + ? null + : (summary.setupFailures / summary.sessionsObserved) * 100, }; } diff --git a/apps/web/src/routers/admin-cloud-agent-next-router.test.ts b/apps/web/src/routers/admin-cloud-agent-next-router.test.ts index 4ed7067d51f..3d1e7ce23cb 100644 --- a/apps/web/src/routers/admin-cloud-agent-next-router.test.ts +++ b/apps/web/src/routers/admin-cloud-agent-next-router.test.ts @@ -212,12 +212,13 @@ describe('adminCloudAgentNextRouter', () => { completedRuns: 1, failedRuns: 2, interruptedRuns: 1, + sessionsObserved: 3, setupFailures: 2, platformFailures: 1, userFailures: 2, unknownFailures: 1, - platformFailureRate: 0.5, - allFailureRate: 0.8, + runFailureRate: 2 / 3, + setupFailureRate: 2 / 3, }); expect(health.topErrors).toEqual( expect.arrayContaining([ @@ -419,8 +420,8 @@ describe('adminCloudAgentNextRouter', () => { startDate: '2035-01-12T00:00:00.000Z', endDate: '2035-01-13T00:00:00.000Z', }); - expect(health.summary.platformFailureRate).toBeNull(); - expect(health.summary.allFailureRate).toBeNull(); + expect(health.summary.runFailureRate).toBeNull(); + expect(health.summary.setupFailureRate).toBeNull(); expect(health.topErrors).toEqual([]); expect(health.errorTotals).toEqual({ events: 0, groups: 0 }); }); diff --git a/apps/web/src/routers/admin-cloud-agent-next-router.ts b/apps/web/src/routers/admin-cloud-agent-next-router.ts index cfa5ab17e8b..b3bb9c91af3 100644 --- a/apps/web/src/routers/admin-cloud-agent-next-router.ts +++ b/apps/web/src/routers/admin-cloud-agent-next-router.ts @@ -97,6 +97,10 @@ function failureRate(failures: number, completed: number): number | null { return denominator === 0 ? null : failures / denominator; } +function ratio(part: number, total: number): number | null { + return total === 0 ? null : part / total; +} + export const adminCloudAgentNextRouter = createTRPCRouter({ getHealthOverview: adminProcedure.input(HealthOverviewFilterSchema).query(async ({ input }) => { const sessionStage = sql`COALESCE(${cloud_agent_sessions.failure_stage}, 'unclassified')`; @@ -115,7 +119,7 @@ export const adminCloudAgentNextRouter = createTRPCRouter({ input.responsibility === 'all' ? undefined : sql`${runResponsibility} = ${input.responsibility}`; - const [summaryRows, setupRows, runErrorRows] = await Promise.all([ + const [summaryRows, setupRows, runErrorRows, sessionCountRows] = await Promise.all([ db .select({ completed: sql`COUNT(*) FILTER (WHERE ${cloud_agent_session_runs.status} = 'completed')`, @@ -182,18 +186,29 @@ export const adminCloudAgentNextRouter = createTRPCRouter({ ) ) .groupBy(runStage, runCode, runResponsibility, runReason), + db + .select({ sessionsObserved: sql`COUNT(*)` }) + .from(cloud_agent_sessions) + .where( + and( + gte(cloud_agent_sessions.created_at, input.startDate), + lt(cloud_agent_sessions.created_at, input.endDate), + retainedSessionCondition() + ) + ), ]); const row = summaryRows[0]; const summary = { completedRuns: count(row?.completed), failedRuns: count(row?.failed), interruptedRuns: count(row?.interrupted), + sessionsObserved: count(sessionCountRows[0]?.sessionsObserved), setupFailures: 0, platformFailures: count(row?.platformFailures), userFailures: count(row?.userFailures), unknownFailures: count(row?.unknownFailures), - platformFailureRate: null as number | null, - allFailureRate: null as number | null, + runFailureRate: null as number | null, + setupFailureRate: null as number | null, }; const setupErrors: HealthError[] = []; for (const setupRow of setupRows) { @@ -246,11 +261,8 @@ export const adminCloudAgentNextRouter = createTRPCRouter({ groups: errors.length, events: errors.reduce((total, error) => total + error.count, 0), }; - summary.platformFailureRate = failureRate(summary.platformFailures, summary.completedRuns); - summary.allFailureRate = failureRate( - summary.failedRuns + summary.setupFailures, - summary.completedRuns - ); + summary.runFailureRate = failureRate(summary.failedRuns, summary.completedRuns); + summary.setupFailureRate = ratio(summary.setupFailures, summary.sessionsObserved); return { summary, topErrors, errorTotals }; }), diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 5cd260e932e..4c7000095bb 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -6241,6 +6241,9 @@ export type CloudAgentFailureReason = | 'setup_command' | 'source_control_authentication' | 'source_control_configuration' + | 'source_control_clone_timeout' + | 'source_control_checkout_timeout' + | 'source_control_repository_corrupt' | 'sandbox_capacity' | 'sandbox_connectivity' | 'runtime_startup' @@ -6259,6 +6262,15 @@ export type CloudAgentFailureReason = | 'source_control_network' | 'assistant_unknown' | 'workspace_unknown' + | 'session_import_timeout' + | 'session_import_failed' + | 'setup_command_timeout' + | 'admission_capacity' + | 'admission_not_found' + | 'admission_internal' + | 'admission_compute_stopping' + | 'admission_billing_unavailable' + | 'admission_forbidden' | 'session_coordination' | 'initial_request_invalid' | 'initial_admission_unknown' diff --git a/packages/worker-utils/src/cloud-agent-failure.test.ts b/packages/worker-utils/src/cloud-agent-failure.test.ts index 353e3060dad..414b3ce586e 100644 --- a/packages/worker-utils/src/cloud-agent-failure.test.ts +++ b/packages/worker-utils/src/cloud-agent-failure.test.ts @@ -74,7 +74,7 @@ describe('classifyCloudAgentFailure', () => { code: 'workspace_setup_failed', workspaceSubtype: 'git_pack_corrupt', }) - ).toEqual({ responsibility: 'unknown', reason: 'source_control_network' }); + ).toEqual({ responsibility: 'unknown', reason: 'source_control_repository_corrupt' }); }); it.each([ @@ -181,6 +181,28 @@ describe('classifyCloudAgentFailure', () => { } ); + it.each([ + ['git_clone_timeout', 'unknown', 'source_control_clone_timeout'], + ['git_checkout_timeout', 'unknown', 'source_control_checkout_timeout'], + ['git_pack_corrupt', 'unknown', 'source_control_repository_corrupt'], + ['setup_command_timeout', 'user', 'setup_command_timeout'], + ['setup_command_failed', 'user', 'setup_command'], + ['kilo_import_timeout', 'unknown', 'session_import_timeout'], + ['kilo_import_failed', 'unknown', 'session_import_failed'], + ] as const)( + 'splits collapsed workspace subtype %s into reason %s', + (workspaceSubtype, responsibility, reason) => { + expect( + classifyCloudAgentFailure({ + source: 'run', + stage: 'pre_dispatch', + code: 'workspace_setup_failed', + workspaceSubtype, + }) + ).toEqual({ responsibility, reason }); + } + ); + it('classifies setup failures from structured stage and code only', () => { expect( classifyCloudAgentFailure({ @@ -197,6 +219,52 @@ describe('classifyCloudAgentFailure', () => { }) ).toEqual({ responsibility: 'platform', reason: 'session_coordination' }); }); + + it('attributes the opaque initial admission rejection from its admission code', () => { + expect( + classifyCloudAgentFailure({ + source: 'setup', + stage: 'initial_admission', + code: 'initial_admission_rejected', + admissionCode: 'INTERNAL', + }) + ).toEqual({ responsibility: 'platform', reason: 'admission_internal' }); + expect( + classifyCloudAgentFailure({ + source: 'setup', + stage: 'initial_admission', + code: 'initial_admission_rejected', + admissionCode: 'SANDBOX_CONNECT_FAILED', + }) + ).toEqual({ responsibility: 'platform', reason: 'sandbox_connectivity' }); + expect( + classifyCloudAgentFailure({ + source: 'setup', + stage: 'initial_admission', + code: 'initial_admission_rejected', + admissionCode: 'PAYMENT_REQUIRED', + }) + ).toEqual({ responsibility: 'user', reason: 'insufficient_credits' }); + expect( + classifyCloudAgentFailure({ + source: 'setup', + stage: 'initial_admission', + code: 'initial_admission_rejected', + admissionCode: 'UNKNOWN', + }) + ).toEqual({ responsibility: 'unknown', reason: 'initial_admission_unknown' }); + }); + + it('classifies a full admission queue as platform capacity, not unknown', () => { + expect( + classifyCloudAgentFailure({ + source: 'setup', + stage: 'initial_admission', + code: 'initial_queue_full', + admissionCode: 'PENDING_QUEUE_FULL', + }) + ).toEqual({ responsibility: 'platform', reason: 'admission_capacity' }); + }); }); describe('CloudAgentSafeFailureSchema', () => { diff --git a/packages/worker-utils/src/cloud-agent-failure.ts b/packages/worker-utils/src/cloud-agent-failure.ts index 4faec148083..de2e2b2b8cb 100644 --- a/packages/worker-utils/src/cloud-agent-failure.ts +++ b/packages/worker-utils/src/cloud-agent-failure.ts @@ -57,6 +57,39 @@ export const WORKSPACE_FAILURE_SUBTYPES = [ export const WorkspaceFailureSubtypeSchema = z.enum(WORKSPACE_FAILURE_SUBTYPES); export type WorkspaceFailureSubtype = z.infer; +/** + * Bounded admission-result codes carried across the setup failure boundary so + * the classifier can attribute an `initial_admission` rejection. `UNKNOWN` is a + * runtime sentinel for a code this version does not recognise. + */ +export const CLOUD_AGENT_ADMISSION_FAILURE_CODES = [ + 'NOT_FOUND', + 'BAD_REQUEST', + 'INTERNAL', + 'PAYMENT_REQUIRED', + 'COMPUTE_STOPPING', + 'BILLING_UNAVAILABLE', + 'PENDING_QUEUE_FULL', + 'FORBIDDEN', + 'MODEL_VALIDATION_UNAVAILABLE', + 'SANDBOX_CONNECT_FAILED', + 'WORKSPACE_SETUP_FAILED', + 'KILO_SERVER_FAILED', + 'WRAPPER_START_FAILED', + 'WRAPPER_FINALIZING', + 'UNKNOWN', +] as const; +export const CloudAgentAdmissionFailureCodeSchema = z.enum(CLOUD_AGENT_ADMISSION_FAILURE_CODES); +export type CloudAgentAdmissionFailureCode = z.infer; + +const KNOWN_ADMISSION_FAILURE_CODES = new Set(CLOUD_AGENT_ADMISSION_FAILURE_CODES); + +export function toAdmissionFailureCode(code: string | undefined): CloudAgentAdmissionFailureCode { + return code !== undefined && KNOWN_ADMISSION_FAILURE_CODES.has(code) + ? (code as CloudAgentAdmissionFailureCode) + : 'UNKNOWN'; +} + export const CLOUD_AGENT_FAILURE_RESPONSIBILITIES = ['platform', 'user', 'unknown'] as const; export const CloudAgentFailureResponsibilitySchema = z.enum(CLOUD_AGENT_FAILURE_RESPONSIBILITIES); export type CloudAgentFailureResponsibility = z.infer; @@ -69,6 +102,9 @@ export const CLOUD_AGENT_FAILURE_REASONS = [ 'setup_command', 'source_control_authentication', 'source_control_configuration', + 'source_control_clone_timeout', + 'source_control_checkout_timeout', + 'source_control_repository_corrupt', 'sandbox_capacity', 'sandbox_connectivity', 'runtime_startup', @@ -87,6 +123,15 @@ export const CLOUD_AGENT_FAILURE_REASONS = [ 'source_control_network', 'assistant_unknown', 'workspace_unknown', + 'session_import_timeout', + 'session_import_failed', + 'setup_command_timeout', + 'admission_capacity', + 'admission_not_found', + 'admission_internal', + 'admission_compute_stopping', + 'admission_billing_unavailable', + 'admission_forbidden', 'session_coordination', 'initial_request_invalid', 'initial_admission_unknown', @@ -143,6 +188,7 @@ type SetupFailureFacts = { | 'initial_queue_full' | 'invalid_initial_intent' | 'do_rpc_outcome_unknown'; + admissionCode?: CloudAgentAdmissionFailureCode; }; function classified( @@ -162,6 +208,7 @@ function classifyWorkspaceFailure( case 'git_branch_missing': return classified('user', 'source_control_configuration'); case 'setup_command_timeout': + return classified('user', 'setup_command_timeout'); case 'setup_command_failed': return classified('user', 'setup_command'); case 'sandbox_storage_full': @@ -169,18 +216,60 @@ function classifyWorkspaceFailure( case 'git_rate_limited': return classified('platform', 'rate_limited'); case 'git_clone_timeout': + return classified('unknown', 'source_control_clone_timeout'); case 'git_checkout_timeout': + return classified('unknown', 'source_control_checkout_timeout'); case 'git_network_failed': - case 'git_pack_corrupt': return classified('unknown', 'source_control_network'); + case 'git_pack_corrupt': + return classified('unknown', 'source_control_repository_corrupt'); case 'kilo_import_timeout': + return classified('unknown', 'session_import_timeout'); case 'kilo_import_failed': + return classified('unknown', 'session_import_failed'); case 'workspace_setup_unknown': case undefined: return classified('unknown', 'workspace_unknown'); } } +function classifyAdmissionFailure( + admissionCode: CloudAgentAdmissionFailureCode | undefined +): CloudAgentFailureClassification { + switch (admissionCode) { + case 'NOT_FOUND': + return classified('platform', 'admission_not_found'); + case 'INTERNAL': + return classified('platform', 'admission_internal'); + case 'PAYMENT_REQUIRED': + return classified('user', 'insufficient_credits'); + case 'COMPUTE_STOPPING': + return classified('platform', 'admission_compute_stopping'); + case 'BILLING_UNAVAILABLE': + return classified('platform', 'admission_billing_unavailable'); + case 'PENDING_QUEUE_FULL': + return classified('platform', 'admission_capacity'); + case 'FORBIDDEN': + return classified('user', 'admission_forbidden'); + case 'MODEL_VALIDATION_UNAVAILABLE': + return classified('platform', 'managed_model_configuration'); + case 'SANDBOX_CONNECT_FAILED': + return classified('platform', 'sandbox_connectivity'); + case 'WORKSPACE_SETUP_FAILED': + return classified('unknown', 'workspace_unknown'); + case 'KILO_SERVER_FAILED': + case 'WRAPPER_START_FAILED': + return classified('platform', 'runtime_startup'); + case 'WRAPPER_FINALIZING': + return classified('platform', 'session_coordination'); + case 'BAD_REQUEST': + return classified('user', 'initial_request_invalid'); + case 'UNKNOWN': + case undefined: + return classified('unknown', 'initial_admission_unknown'); + } +} + function classifyAssistantFailure(input: RunFailureFacts): CloudAgentFailureClassification { if (input.code === 'payment_required' || input.assistantReason === 'insufficient_credits') { return classified('user', 'insufficient_credits'); @@ -238,8 +327,11 @@ export function classifyCloudAgentFailure( if (input.code === 'invalid_initial_intent') { return classified('user', 'initial_request_invalid'); } + if (input.code === 'initial_queue_full') { + return classified('platform', 'admission_capacity'); + } if (input.stage === 'initial_admission') { - return classified('unknown', 'initial_admission_unknown'); + return classifyAdmissionFailure(input.admissionCode); } return classified('unknown', 'unclassified'); } diff --git a/services/cloud-agent-next/src/router.test.ts b/services/cloud-agent-next/src/router.test.ts index 624012bd6b2..3d44a4ab192 100644 --- a/services/cloud-agent-next/src/router.test.ts +++ b/services/cloud-agent-next/src/router.test.ts @@ -2785,7 +2785,11 @@ describe('legacy V2 execution response compatibility', () => { expect(recordCloudAgentSessionFailure).toHaveBeenCalledWith( expect.objectContaining({ - failure: { stage: 'initial_admission', code: 'initial_queue_full' }, + failure: { + stage: 'initial_admission', + code: 'initial_queue_full', + admissionCode: 'PENDING_QUEUE_FULL', + }, }) ); }); diff --git a/services/cloud-agent-next/src/session-prepare.test.ts b/services/cloud-agent-next/src/session-prepare.test.ts index 0e3e1fa5d62..3ab9c579a98 100644 --- a/services/cloud-agent-next/src/session-prepare.test.ts +++ b/services/cloud-agent-next/src/session-prepare.test.ts @@ -1858,7 +1858,11 @@ describe('start endpoint', () => { expect(doStub.admitSubmittedMessage).not.toHaveBeenCalled(); expect(recordSessionFailureMock).toHaveBeenCalledWith( expect.objectContaining({ - failure: { stage: 'initial_admission', code: 'initial_admission_rejected' }, + failure: { + stage: 'initial_admission', + code: 'initial_admission_rejected', + admissionCode: 'INTERNAL', + }, }), expect.any(Object) ); @@ -1884,7 +1888,11 @@ describe('start endpoint', () => { expect(recordSessionFailureMock).toHaveBeenCalledWith( expect.objectContaining({ - failure: { stage: 'initial_admission', code: 'initial_admission_rejected' }, + failure: { + stage: 'initial_admission', + code: 'initial_admission_rejected', + admissionCode: 'INTERNAL', + }, }), expect.any(Object) ); diff --git a/services/cloud-agent-next/src/session/admission-failure.ts b/services/cloud-agent-next/src/session/admission-failure.ts new file mode 100644 index 00000000000..81f80540a1d --- /dev/null +++ b/services/cloud-agent-next/src/session/admission-failure.ts @@ -0,0 +1,29 @@ +import { + toAdmissionFailureCode, + type CloudAgentAdmissionFailureCode, +} from '@kilocode/worker-utils/cloud-agent-failure'; +import type { SessionMessageAdmissionResult } from '../execution/types.js'; + +export type InitialAdmissionFailure = { + stage: 'initial_admission'; + code: 'initial_admission_rejected' | 'initial_queue_full' | 'invalid_initial_intent'; + admissionCode: CloudAgentAdmissionFailureCode; +}; + +/** + * Maps a durable admission result to the bounded session-establishment failure + * shape and preserves the underlying admission code so the telemetry classifier + * can attribute the rejection instead of lumping it into an opaque catch-all. + */ +export function initialAdmissionFailure( + result: Extract +): InitialAdmissionFailure { + const admissionCode = toAdmissionFailureCode(result.code); + if (result.code === 'PENDING_QUEUE_FULL') { + return { stage: 'initial_admission', code: 'initial_queue_full', admissionCode }; + } + if (result.code === 'BAD_REQUEST') { + return { stage: 'initial_admission', code: 'invalid_initial_intent', admissionCode }; + } + return { stage: 'initial_admission', code: 'initial_admission_rejected', admissionCode }; +} diff --git a/services/cloud-agent-next/src/session/legacy-prepared-admission.ts b/services/cloud-agent-next/src/session/legacy-prepared-admission.ts index 3d6f07b150a..cb6202bd345 100644 --- a/services/cloud-agent-next/src/session/legacy-prepared-admission.ts +++ b/services/cloud-agent-next/src/session/legacy-prepared-admission.ts @@ -11,6 +11,7 @@ import type { SessionId } from '../types/ids.js'; import { withDORetry } from '../utils/do-retry.js'; import { resolveSessionStub } from '../sandbox-session/session-stub.js'; import { projectAdmissionToPublicAck, throwAdmissionError } from './queue-message.js'; +import { initialAdmissionFailure } from './admission-failure.js'; export type LegacyPreparedInitialAdmissionInput = { cloudAgentSessionId: string; @@ -39,18 +40,6 @@ export async function replayLegacyPreparedInitialMessageIfAlreadyAdmitted( return projectAdmissionToPublicAck(sessionId, result); } -function legacyInitialAdmissionFailure( - result: Extract -) { - if (result.code === 'PENDING_QUEUE_FULL') { - return { stage: 'initial_admission', code: 'initial_queue_full' } as const; - } - if (result.code === 'BAD_REQUEST') { - return { stage: 'initial_admission', code: 'invalid_initial_intent' } as const; - } - return { stage: 'initial_admission', code: 'initial_admission_rejected' } as const; -} - async function recordSetupFailure(record: () => Promise): Promise { try { await record(); @@ -93,7 +82,7 @@ export async function admitLegacyPreparedInitialMessage( recordCloudAgentSessionFailure( { cloudAgentSessionId: input.cloudAgentSessionId, - failure: legacyInitialAdmissionFailure(result), + failure: initialAdmissionFailure(result), }, ctx.env ) diff --git a/services/cloud-agent-next/src/session/session-registration.ts b/services/cloud-agent-next/src/session/session-registration.ts index d17a252b026..62f61579b76 100644 --- a/services/cloud-agent-next/src/session/session-registration.ts +++ b/services/cloud-agent-next/src/session/session-registration.ts @@ -62,6 +62,7 @@ import { resolveSharedSandboxAssignment } from '../shared-sandbox-route.js'; import { generateKiloSessionId } from '../utils/kilo-session-id.js'; import { sha256Hex } from '../utils/sha256.js'; import { createMessageId } from './message-id.js'; +import { initialAdmissionFailure } from './admission-failure.js'; import type { MessageResultRPCResponse } from './message-result.js'; import type { AcceptedExecutionTurn, @@ -162,15 +163,6 @@ export function executionTurnSubmissionFromAcceptedTurn( }; } -type SessionEstablishmentFailure = - | { stage: 'sandbox_identity'; code: 'sandbox_id_derivation_failed' } - | { stage: 'registration'; code: 'do_registration_rejected' } - | { - stage: 'initial_admission'; - code: 'initial_admission_rejected' | 'initial_queue_full' | 'invalid_initial_intent'; - } - | { stage: 'transport'; code: 'do_rpc_outcome_unknown' }; - type NewSessionAllocation = SessionRegistrationResult & { reportingCreatedAt?: string; credentialContainment: CredentialContainment; @@ -405,18 +397,6 @@ function sessionCreateSettledOutboxEvent(params: { }; } -function initialAdmissionFailure( - result: Extract -): Extract { - if (result.code === 'PENDING_QUEUE_FULL') { - return { stage: 'initial_admission', code: 'initial_queue_full' }; - } - if (result.code === 'BAD_REQUEST') { - return { stage: 'initial_admission', code: 'invalid_initial_intent' }; - } - return { stage: 'initial_admission', code: 'initial_admission_rejected' }; -} - async function recordPostSetupFailure(record: () => Promise): Promise { try { await record(); diff --git a/services/cloud-agent-next/src/telemetry/report-store.test.ts b/services/cloud-agent-next/src/telemetry/report-store.test.ts index c1d4d997a15..bd907bb9223 100644 --- a/services/cloud-agent-next/src/telemetry/report-store.test.ts +++ b/services/cloud-agent-next/src/telemetry/report-store.test.ts @@ -236,7 +236,11 @@ describe('cloud agent reporting store', () => { const result = await store.recordSessionFailure({ cloudAgentSessionId, occurredAt, - failure: { stage: 'initial_admission', code: 'initial_queue_full' }, + failure: { + stage: 'initial_admission', + code: 'initial_queue_full', + admissionCode: 'PENDING_QUEUE_FULL', + }, diagnostic: { errorMessageRedacted: 'Initial queue is full', errorExpiresAt: '2026-06-01T12:00:00.000Z', @@ -247,12 +251,33 @@ describe('cloud agent reporting store', () => { failure_at: occurredAt, failure_stage: 'initial_admission', failure_code: 'initial_queue_full', - failure_responsibility: 'unknown', - failure_reason: 'initial_admission_unknown', + failure_responsibility: 'platform', + failure_reason: 'admission_capacity', error_message_redacted: 'Initial queue is full', }); }); + it('attributes an admission rejection from the preserved admission code', async () => { + const fake = makeDb([], [[{ cloudAgentSessionId }]]); + const store = createCloudAgentReportStore(fake.db as never); + const result = await store.recordSessionFailure({ + cloudAgentSessionId, + occurredAt, + failure: { + stage: 'initial_admission', + code: 'initial_admission_rejected', + admissionCode: 'INTERNAL', + }, + }); + expect(result).toEqual({ applied: true }); + expect(fake.updates.find(call => call.table === cloud_agent_sessions)?.values).toMatchObject({ + failure_stage: 'initial_admission', + failure_code: 'initial_admission_rejected', + failure_responsibility: 'platform', + failure_reason: 'admission_internal', + }); + }); + it('preserves the first typed pre-run failure fact on later reports', async () => { const fake = makeDb([], [[]]); const store = createCloudAgentReportStore(fake.db as never); diff --git a/services/cloud-agent-next/src/telemetry/report-store.ts b/services/cloud-agent-next/src/telemetry/report-store.ts index ea6d415b8bf..6e297ec6480 100644 --- a/services/cloud-agent-next/src/telemetry/report-store.ts +++ b/services/cloud-agent-next/src/telemetry/report-store.ts @@ -6,7 +6,10 @@ import type { CloudAgentRunStateReport, } from '@kilocode/worker-utils/cloud-agent-queue-report'; import { cloud_agent_session_runs, cloud_agent_sessions } from '@kilocode/db/schema'; -import { classifyCloudAgentFailure } from '@kilocode/worker-utils/cloud-agent-failure'; +import { + classifyCloudAgentFailure, + CloudAgentAdmissionFailureCodeSchema, +} from '@kilocode/worker-utils/cloud-agent-failure'; import { SESSION_ID_RE } from '../shared/protocol.js'; export const CLOUD_AGENT_REPORT_RETENTION_DAYS = 90; @@ -29,6 +32,7 @@ const sessionFailureSchema = z.discriminatedUnion('stage', [ z.object({ stage: z.literal('initial_admission'), code: z.enum(['initial_admission_rejected', 'initial_queue_full', 'invalid_initial_intent']), + admissionCode: CloudAgentAdmissionFailureCodeSchema, }), z.object({ stage: z.literal('transport'), code: z.literal('do_rpc_outcome_unknown') }), ]); @@ -404,6 +408,9 @@ export function createCloudAgentReportStore(db: WorkerDb) { source: 'setup', stage: input.failure.stage, code: input.failure.code, + ...(input.failure.stage === 'initial_admission' + ? { admissionCode: input.failure.admissionCode } + : {}), }); let applied = false; const result = await withReportingSessionLock(db, input.cloudAgentSessionId, async tx => { diff --git a/services/cloud-agent-next/src/telemetry/session-reports.ts b/services/cloud-agent-next/src/telemetry/session-reports.ts index 519f32dc393..ed4f87eddf8 100644 --- a/services/cloud-agent-next/src/telemetry/session-reports.ts +++ b/services/cloud-agent-next/src/telemetry/session-reports.ts @@ -2,6 +2,7 @@ import type { Env } from '../types.js'; import type { SessionMetadata } from '../persistence/session-metadata.js'; import { getPgDb } from '../db/pg.js'; import { CLOUD_AGENT_REPORT_RETENTION_DAYS, createCloudAgentReportStore } from './report-store.js'; +import type { CloudAgentAdmissionFailureCode } from '@kilocode/worker-utils/cloud-agent-failure'; export type CloudAgentSessionFailure = | { stage: 'sandbox_identity'; code: 'sandbox_id_derivation_failed' } @@ -9,6 +10,7 @@ export type CloudAgentSessionFailure = | { stage: 'initial_admission'; code: 'initial_admission_rejected' | 'initial_queue_full' | 'invalid_initial_intent'; + admissionCode: CloudAgentAdmissionFailureCode; } | { stage: 'transport'; code: 'do_rpc_outcome_unknown' }; From 69349ac6f6d3b41d0a02febfc982023ed817eb49 Mon Sep 17 00:00:00 2001 From: Evgeny Shurakov Date: Thu, 10 Sep 2026 21:39:45 +0200 Subject: [PATCH 2/3] fix(cloud-agent): make setup failure rate a true proportion The rate divided sessions created in the window by setup failures counted by failure_at, so a session created just before the window that failed inside it was in the numerator but not the denominator, letting the rate exceed 100%. Count sessions exposed to setup in the window (created in it, or failed in it) so failures stay a subset. --- .../admin-cloud-agent-next-router.test.ts | 4 ++-- .../routers/admin-cloud-agent-next-router.ts | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/web/src/routers/admin-cloud-agent-next-router.test.ts b/apps/web/src/routers/admin-cloud-agent-next-router.test.ts index 3d1e7ce23cb..ad631ce2402 100644 --- a/apps/web/src/routers/admin-cloud-agent-next-router.test.ts +++ b/apps/web/src/routers/admin-cloud-agent-next-router.test.ts @@ -212,13 +212,13 @@ describe('adminCloudAgentNextRouter', () => { completedRuns: 1, failedRuns: 2, interruptedRuns: 1, - sessionsObserved: 3, + sessionsObserved: 4, setupFailures: 2, platformFailures: 1, userFailures: 2, unknownFailures: 1, runFailureRate: 2 / 3, - setupFailureRate: 2 / 3, + setupFailureRate: 2 / 4, }); expect(health.topErrors).toEqual( expect.arrayContaining([ diff --git a/apps/web/src/routers/admin-cloud-agent-next-router.ts b/apps/web/src/routers/admin-cloud-agent-next-router.ts index b3bb9c91af3..57a82a72d5e 100644 --- a/apps/web/src/routers/admin-cloud-agent-next-router.ts +++ b/apps/web/src/routers/admin-cloud-agent-next-router.ts @@ -1,7 +1,7 @@ import { adminProcedure, createTRPCRouter } from '@/lib/trpc/init'; import { db } from '@/lib/drizzle'; import { cloud_agent_session_runs, cloud_agent_sessions } from '@kilocode/db/schema'; -import { and, desc, eq, gte, isNotNull, lt, sql, type SQL } from 'drizzle-orm'; +import { and, desc, eq, gte, isNotNull, lt, or, sql, type SQL } from 'drizzle-orm'; import * as z from 'zod'; import { CloudAgentFailureReasonSchema, @@ -191,8 +191,19 @@ export const adminCloudAgentNextRouter = createTRPCRouter({ .from(cloud_agent_sessions) .where( and( - gte(cloud_agent_sessions.created_at, input.startDate), - lt(cloud_agent_sessions.created_at, input.endDate), + // Setup rate denominator: sessions exposed to setup in the window — + // created in it, or whose session-level setup failure occurred in it. + // The failure set (failure_at) is always a subset of this population. + or( + and( + gte(cloud_agent_sessions.created_at, input.startDate), + lt(cloud_agent_sessions.created_at, input.endDate) + ), + and( + gte(cloud_agent_sessions.failure_at, input.startDate), + lt(cloud_agent_sessions.failure_at, input.endDate) + ) + ), retainedSessionCondition() ) ), From cefe34bdf1019a860efc31162a0c9fc919af901e Mon Sep 17 00:00:00 2001 From: Evgeny Shurakov Date: Thu, 10 Sep 2026 21:39:50 +0200 Subject: [PATCH 3/3] feat(cloud-agent): classify assistant limits, interrupts, and wrapper failures Split the remaining undifferentiated failure buckets by cause and ownership so the admin health view attributes them correctly: - assistant limits: assistant_invalid_request/context_limit/output_limit by provider ownership (byok user, managed platform, unknown unknown), assistant_content_filter user, assistant_structured_output platform - source-control clone/checkout/network/corrupt and session-import failures are platform infrastructure - user_interrupt user; container_shutdown and system_interrupt platform - wrapper_disconnected, wrapper_startup, wrapper_crash, assistant_no_reply - provider_ownership_unknown when an ownership-dependent assistant failure has no known provider ownership Adds the matching code-review terminal reasons, labels, and tests. The old generic reason values stay as deprecated union entries so historical rows keep a label during retention. --- .../health-summary.test.ts | 14 +- .../CloudAgentNextTelemetry/health-summary.ts | 13 ++ .../terminal-reason-from-failure.test.ts | 10 +- .../terminal-reason-from-failure.ts | 8 +- .../src/routers/admin-code-reviews-router.ts | 4 + packages/db/src/schema-types.ts | 4 + packages/db/src/schema.ts | 25 ++- .../src/cloud-agent-failure.test.ts | 142 +++++++++++++----- .../worker-utils/src/cloud-agent-failure.ts | 98 +++++++++--- .../src/telemetry/queue-reports.test.ts | 75 ++++----- 10 files changed, 279 insertions(+), 114 deletions(-) diff --git a/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.test.ts b/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.test.ts index 3fc86aab6a9..5837e504598 100644 --- a/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.test.ts +++ b/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.test.ts @@ -107,7 +107,19 @@ describe('failure responsibility summary', () => { ['content_filter', 'Content filter'], ['structured_output', 'Invalid structured output'], ['invalid_request', 'Model request rejected'], - ['request_timeout', 'Request timed out'], + ['assistant_invalid_request', 'Assistant invalid request'], + ['assistant_context_limit', 'Assistant context limit'], + ['assistant_output_limit', 'Assistant output limit'], + ['assistant_content_filter', 'Assistant content filter'], + ['assistant_structured_output', 'Assistant structured output'], + ['provider_ownership_unknown', 'Unknown provider ownership'], + ['user_interrupt', 'User interrupt'], + ['container_shutdown', 'Container shutdown'], + ['system_interrupt', 'System interrupt'], + ['wrapper_disconnected', 'Wrapper disconnected'], + ['wrapper_startup', 'Wrapper startup failure'], + ['wrapper_crash', 'Wrapper crash after activity'], + ['assistant_no_reply', 'Assistant no reply'], ] as const)('labels %s as %s', (reason, label) => { expect(failureReasonLabel(reason)).toBe(label); }); diff --git a/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.ts b/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.ts index 14640cf41de..a8d0e53d5a7 100644 --- a/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.ts +++ b/apps/web/src/app/admin/components/CloudAgentNextTelemetry/health-summary.ts @@ -40,12 +40,25 @@ const FAILURE_REASON_LABELS = { managed_model_configuration: 'Managed model configuration', provider_unavailable: 'Provider unavailable', request_timeout: 'Request timed out', + assistant_invalid_request: 'Assistant invalid request', + assistant_context_limit: 'Assistant context limit', + assistant_output_limit: 'Assistant output limit', + assistant_content_filter: 'Assistant content filter', + assistant_structured_output: 'Assistant structured output', + provider_ownership_unknown: 'Unknown provider ownership', invalid_request: 'Model request rejected', context_limit: 'Context limit', output_limit: 'Output limit', content_filter: 'Content filter', structured_output: 'Invalid structured output', source_control_network: 'Source control network', + wrapper_disconnected: 'Wrapper disconnected', + wrapper_startup: 'Wrapper startup failure', + wrapper_crash: 'Wrapper crash after activity', + assistant_no_reply: 'Assistant no reply', + user_interrupt: 'User interrupt', + container_shutdown: 'Container shutdown', + system_interrupt: 'System interrupt', assistant_unknown: 'Unknown assistant failure', workspace_unknown: 'Unknown workspace failure', session_import_timeout: 'Session import timed out', diff --git a/apps/web/src/lib/code-reviews/terminal-reason-from-failure.test.ts b/apps/web/src/lib/code-reviews/terminal-reason-from-failure.test.ts index 316463f264b..74046734b85 100644 --- a/apps/web/src/lib/code-reviews/terminal-reason-from-failure.test.ts +++ b/apps/web/src/lib/code-reviews/terminal-reason-from-failure.test.ts @@ -83,13 +83,13 @@ describe('terminalReasonFromCloudAgentFailure', () => { ); it.each([ - ['context_limit', 'assistant_failed'], - ['output_limit', 'assistant_failed'], - ['content_filter', 'assistant_failed'], - ['structured_output', 'assistant_failed'], + ['context_limit', 'assistant_context_limit'], + ['output_limit', 'assistant_output_limit'], + ['content_filter', 'assistant_content_filter'], + ['structured_output', 'assistant_structured_output'], ['timeout', 'assistant_timeout'], ['invalid_request', 'assistant_invalid_request'], - ] as const)('keeps %s mapped to the existing %s category', (assistantReason, expected) => { + ] as const)('maps assistant reason %s to %s', (assistantReason, expected) => { expect( terminalReasonFromCloudAgentFailure({ code: 'assistant_error', diff --git a/apps/web/src/lib/code-reviews/terminal-reason-from-failure.ts b/apps/web/src/lib/code-reviews/terminal-reason-from-failure.ts index 3b08ebd1396..5628b392495 100644 --- a/apps/web/src/lib/code-reviews/terminal-reason-from-failure.ts +++ b/apps/web/src/lib/code-reviews/terminal-reason-from-failure.ts @@ -89,10 +89,10 @@ const ASSISTANT_REASON_REASONS = { timeout: 'assistant_timeout', provider_authentication: 'assistant_unauthorized', invalid_request: 'assistant_invalid_request', - context_limit: 'assistant_failed', - output_limit: 'assistant_failed', - content_filter: 'assistant_failed', - structured_output: 'assistant_failed', + context_limit: 'assistant_context_limit', + output_limit: 'assistant_output_limit', + content_filter: 'assistant_content_filter', + structured_output: 'assistant_structured_output', insufficient_credits: 'billing', model_unavailable: 'model_not_found', unknown: 'assistant_failed', diff --git a/apps/web/src/routers/admin-code-reviews-router.ts b/apps/web/src/routers/admin-code-reviews-router.ts index 91abbdb9646..fd9371d7810 100644 --- a/apps/web/src/routers/admin-code-reviews-router.ts +++ b/apps/web/src/routers/admin-code-reviews-router.ts @@ -111,6 +111,10 @@ const TERMINAL_REASON_LABELS: Record = { assistant_timeout: 'Assistant Timeout', assistant_unauthorized: 'Assistant Unauthorized', assistant_invalid_request: 'Assistant Invalid Request', + assistant_context_limit: 'Assistant Context Limit', + assistant_output_limit: 'Assistant Output Limit', + assistant_content_filter: 'Assistant Content Filter', + assistant_structured_output: 'Assistant Structured Output', assistant_failed: 'Assistant Error', assistant_no_reply: 'Assistant No Reply', wrapper_failed: 'Agent Wrapper Failure', diff --git a/packages/db/src/schema-types.ts b/packages/db/src/schema-types.ts index c2cfbd50cd2..728a8783cdf 100644 --- a/packages/db/src/schema-types.ts +++ b/packages/db/src/schema-types.ts @@ -2368,6 +2368,10 @@ export const CODE_REVIEW_TERMINAL_REASONS = [ 'assistant_timeout', 'assistant_unauthorized', 'assistant_invalid_request', + 'assistant_context_limit', + 'assistant_output_limit', + 'assistant_content_filter', + 'assistant_structured_output', 'assistant_no_reply', 'wrapper_failed', 'runtime_startup_failed', diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 4c7000095bb..0e8f44c2ef4 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -6254,13 +6254,21 @@ export type CloudAgentFailureReason = | 'managed_model_configuration' | 'provider_unavailable' | 'request_timeout' - | 'invalid_request' - | 'context_limit' - | 'output_limit' - | 'content_filter' - | 'structured_output' + | 'assistant_invalid_request' + | 'assistant_context_limit' + | 'assistant_output_limit' + | 'assistant_content_filter' + | 'assistant_structured_output' + | 'provider_ownership_unknown' | 'source_control_network' | 'assistant_unknown' + | 'wrapper_disconnected' + | 'wrapper_startup' + | 'wrapper_crash' + | 'assistant_no_reply' + | 'user_interrupt' + | 'container_shutdown' + | 'system_interrupt' | 'workspace_unknown' | 'session_import_timeout' | 'session_import_failed' @@ -6274,6 +6282,13 @@ export type CloudAgentFailureReason = | 'session_coordination' | 'initial_request_invalid' | 'initial_admission_unknown' + // Deprecated producer values retained so historical text rows still resolve to + // a label. Keep in sync with @kilocode/worker-utils/cloud-agent-failure. + | 'invalid_request' + | 'context_limit' + | 'output_limit' + | 'content_filter' + | 'structured_output' | 'unclassified'; export const cloud_agent_sessions = pgTable( diff --git a/packages/worker-utils/src/cloud-agent-failure.test.ts b/packages/worker-utils/src/cloud-agent-failure.test.ts index 414b3ce586e..8bfc6da084e 100644 --- a/packages/worker-utils/src/cloud-agent-failure.test.ts +++ b/packages/worker-utils/src/cloud-agent-failure.test.ts @@ -49,7 +49,7 @@ describe('classifyCloudAgentFailure', () => { }); }); - it('preserves ambiguous assistant and source-control failures as unknown', () => { + it('preserves ambiguous assistant failures as unknown', () => { expect( classifyCloudAgentFailure({ source: 'run', @@ -59,6 +59,9 @@ describe('classifyCloudAgentFailure', () => { providerOwnership: 'unknown', }) ).toEqual({ responsibility: 'unknown', reason: 'assistant_unknown' }); + }); + + it('attributes source-control infrastructure failures to the platform', () => { expect( classifyCloudAgentFailure({ source: 'run', @@ -66,7 +69,7 @@ describe('classifyCloudAgentFailure', () => { code: 'workspace_setup_failed', workspaceSubtype: 'git_network_failed', }) - ).toEqual({ responsibility: 'unknown', reason: 'source_control_network' }); + ).toEqual({ responsibility: 'platform', reason: 'source_control_network' }); expect( classifyCloudAgentFailure({ source: 'run', @@ -74,18 +77,18 @@ describe('classifyCloudAgentFailure', () => { code: 'workspace_setup_failed', workspaceSubtype: 'git_pack_corrupt', }) - ).toEqual({ responsibility: 'unknown', reason: 'source_control_repository_corrupt' }); + ).toEqual({ responsibility: 'platform', reason: 'source_control_repository_corrupt' }); }); it.each([ ['provider_authentication', 'byok', 'user', 'provider_authentication'], ['provider_authentication', 'managed', 'platform', 'managed_provider_authentication'], - ['provider_authentication', 'unknown', 'unknown', 'provider_authentication'], - ['provider_authentication', undefined, 'unknown', 'provider_authentication'], + ['provider_authentication', 'unknown', 'unknown', 'provider_ownership_unknown'], + ['provider_authentication', undefined, 'unknown', 'provider_ownership_unknown'], ['provider_unavailable', 'byok', 'unknown', 'provider_unavailable'], ['provider_unavailable', 'managed', 'platform', 'managed_provider_unavailable'], - ['provider_unavailable', 'unknown', 'unknown', 'provider_unavailable'], - ['provider_unavailable', undefined, 'unknown', 'provider_unavailable'], + ['provider_unavailable', 'unknown', 'unknown', 'provider_ownership_unknown'], + ['provider_unavailable', undefined, 'unknown', 'provider_ownership_unknown'], ] as const)( 'classifies %s with %s ownership without losing the known cause', (assistantReason, providerOwnership, responsibility, reason) => { @@ -102,41 +105,75 @@ describe('classifyCloudAgentFailure', () => { ); it.each([ - ['managed', 'platform'], - ['byok', 'unknown'], - ['unknown', 'unknown'], - [undefined, 'unknown'], - ] as const)('retains request_timeout with %s ownership', (providerOwnership, responsibility) => { - const failure = classifyCloudAgentFailure({ - source: 'run', - stage: 'agent_activity', - code: 'assistant_error', - assistantReason: 'timeout', - providerOwnership, - }); - - expect(failure).toEqual({ responsibility, reason: 'request_timeout' }); - expect(CloudAgentFailureReasonSchema.parse(failure.reason)).toBe('request_timeout'); - }); - - it.each([ - 'context_limit', - 'output_limit', - 'content_filter', - 'structured_output', - 'invalid_request', - ] as const)('retains %s without inferring responsibility from ownership', assistantReason => { - for (const providerOwnership of [...CLOUD_AGENT_PROVIDER_OWNERSHIPS, undefined]) { + ['managed', 'platform', 'request_timeout'], + ['byok', 'unknown', 'request_timeout'], + ['unknown', 'unknown', 'provider_ownership_unknown'], + [undefined, 'unknown', 'provider_ownership_unknown'], + ] as const)( + 'retains request_timeout with %s ownership', + (providerOwnership, responsibility, reason) => { const failure = classifyCloudAgentFailure({ source: 'run', stage: 'agent_activity', code: 'assistant_error', - assistantReason, + assistantReason: 'timeout', providerOwnership, }); - expect(failure).toEqual({ responsibility: 'unknown', reason: assistantReason }); - expect(CloudAgentFailureReasonSchema.parse(failure.reason)).toBe(assistantReason); + expect(failure).toEqual({ responsibility, reason }); + expect(CloudAgentFailureReasonSchema.parse(failure.reason)).toBe(reason); + } + ); + + it.each([ + ['invalid_request', 'assistant_invalid_request'], + ['context_limit', 'assistant_context_limit'], + ['output_limit', 'assistant_output_limit'], + ] as const)( + 'attributes %s by provider ownership without collapsing the cause', + (assistantReason, expectedReason) => { + const cases = [ + ['byok', 'user'], + ['managed', 'platform'], + ['unknown', 'unknown'], + [undefined, 'unknown'], + ] as const; + + for (const [providerOwnership, responsibility] of cases) { + const failure = classifyCloudAgentFailure({ + source: 'run', + stage: 'agent_activity', + code: 'assistant_error', + assistantReason, + providerOwnership, + }); + + expect(failure).toEqual({ responsibility, reason: expectedReason }); + expect(CloudAgentFailureReasonSchema.parse(failure.reason)).toBe(expectedReason); + } + } + ); + + it('attributes content filter to the user and structured output to the platform', () => { + for (const providerOwnership of [...CLOUD_AGENT_PROVIDER_OWNERSHIPS, undefined]) { + expect( + classifyCloudAgentFailure({ + source: 'run', + stage: 'agent_activity', + code: 'assistant_error', + assistantReason: 'content_filter', + providerOwnership, + }) + ).toEqual({ responsibility: 'user', reason: 'assistant_content_filter' }); + expect( + classifyCloudAgentFailure({ + source: 'run', + stage: 'agent_activity', + code: 'assistant_error', + assistantReason: 'structured_output', + providerOwnership, + }) + ).toEqual({ responsibility: 'platform', reason: 'assistant_structured_output' }); } }); @@ -182,13 +219,13 @@ describe('classifyCloudAgentFailure', () => { ); it.each([ - ['git_clone_timeout', 'unknown', 'source_control_clone_timeout'], - ['git_checkout_timeout', 'unknown', 'source_control_checkout_timeout'], - ['git_pack_corrupt', 'unknown', 'source_control_repository_corrupt'], + ['git_clone_timeout', 'platform', 'source_control_clone_timeout'], + ['git_checkout_timeout', 'platform', 'source_control_checkout_timeout'], + ['git_pack_corrupt', 'platform', 'source_control_repository_corrupt'], ['setup_command_timeout', 'user', 'setup_command_timeout'], ['setup_command_failed', 'user', 'setup_command'], - ['kilo_import_timeout', 'unknown', 'session_import_timeout'], - ['kilo_import_failed', 'unknown', 'session_import_failed'], + ['kilo_import_timeout', 'platform', 'session_import_timeout'], + ['kilo_import_failed', 'platform', 'session_import_failed'], ] as const)( 'splits collapsed workspace subtype %s into reason %s', (workspaceSubtype, responsibility, reason) => { @@ -203,6 +240,31 @@ describe('classifyCloudAgentFailure', () => { } ); + it.each([ + ['wrapper_disconnected', 'wrapper_disconnected'], + ['wrapper_no_output', 'wrapper_liveness'], + ['wrapper_ping_timeout', 'wrapper_liveness'], + ['wrapper_error_before_activity', 'wrapper_startup'], + ['wrapper_error_after_activity', 'wrapper_crash'], + ['missing_assistant_reply', 'assistant_no_reply'], + ] as const)('splits wrapper code %s into platform reason %s', (code, reason) => { + expect(classifyCloudAgentFailure({ source: 'run', stage: 'agent_activity', code })).toEqual({ + responsibility: 'platform', + reason, + }); + }); + + it.each([ + ['user_interrupt', 'user', 'user_interrupt'], + ['container_shutdown', 'platform', 'container_shutdown'], + ['system_interrupt', 'platform', 'system_interrupt'], + ] as const)('attributes interruption code %s to %s/%s', (code, responsibility, reason) => { + expect(classifyCloudAgentFailure({ source: 'run', stage: 'interruption', code })).toEqual({ + responsibility, + reason, + }); + }); + it('classifies setup failures from structured stage and code only', () => { expect( classifyCloudAgentFailure({ diff --git a/packages/worker-utils/src/cloud-agent-failure.ts b/packages/worker-utils/src/cloud-agent-failure.ts index de2e2b2b8cb..784797b7013 100644 --- a/packages/worker-utils/src/cloud-agent-failure.ts +++ b/packages/worker-utils/src/cloud-agent-failure.ts @@ -115,13 +115,21 @@ export const CLOUD_AGENT_FAILURE_REASONS = [ 'managed_model_configuration', 'provider_unavailable', 'request_timeout', - 'invalid_request', - 'context_limit', - 'output_limit', - 'content_filter', - 'structured_output', + 'assistant_invalid_request', + 'assistant_context_limit', + 'assistant_output_limit', + 'assistant_content_filter', + 'assistant_structured_output', + 'provider_ownership_unknown', 'source_control_network', 'assistant_unknown', + 'wrapper_disconnected', + 'wrapper_startup', + 'wrapper_crash', + 'assistant_no_reply', + 'user_interrupt', + 'container_shutdown', + 'system_interrupt', 'workspace_unknown', 'session_import_timeout', 'session_import_failed', @@ -135,6 +143,14 @@ export const CLOUD_AGENT_FAILURE_REASONS = [ 'session_coordination', 'initial_request_invalid', 'initial_admission_unknown', + // Deprecated producer values: the classifier now emits the `assistant_*` + // reasons above. Retained so historical rows in the unconstrained text column + // still resolve to a label instead of rendering blank for the retention window. + 'invalid_request', + 'context_limit', + 'output_limit', + 'content_filter', + 'structured_output', 'unclassified', ] as const; export const CloudAgentFailureReasonSchema = z.enum(CLOUD_AGENT_FAILURE_REASONS); @@ -216,17 +232,17 @@ function classifyWorkspaceFailure( case 'git_rate_limited': return classified('platform', 'rate_limited'); case 'git_clone_timeout': - return classified('unknown', 'source_control_clone_timeout'); + return classified('platform', 'source_control_clone_timeout'); case 'git_checkout_timeout': - return classified('unknown', 'source_control_checkout_timeout'); + return classified('platform', 'source_control_checkout_timeout'); case 'git_network_failed': - return classified('unknown', 'source_control_network'); + return classified('platform', 'source_control_network'); case 'git_pack_corrupt': - return classified('unknown', 'source_control_repository_corrupt'); + return classified('platform', 'source_control_repository_corrupt'); case 'kilo_import_timeout': - return classified('unknown', 'session_import_timeout'); + return classified('platform', 'session_import_timeout'); case 'kilo_import_failed': - return classified('unknown', 'session_import_failed'); + return classified('platform', 'session_import_failed'); case 'workspace_setup_unknown': case undefined: return classified('unknown', 'workspace_unknown'); @@ -270,6 +286,17 @@ function classifyAdmissionFailure( } } +/** + * Assistant limits whose responsibility depends on whose credential the model + * ran under: the user's own key makes the request shape/quota their problem, + * our managed key makes it ours, and an unknown owner stays unknown. + */ +const OWNERSHIP_DEPENDENT_ASSISTANT_LIMITS = { + invalid_request: 'assistant_invalid_request', + context_limit: 'assistant_context_limit', + output_limit: 'assistant_output_limit', +} as const; + function classifyAssistantFailure(input: RunFailureFacts): CloudAgentFailureClassification { if (input.code === 'payment_required' || input.assistantReason === 'insufficient_credits') { return classified('user', 'insufficient_credits'); @@ -287,27 +314,42 @@ function classifyAssistantFailure(input: RunFailureFacts): CloudAgentFailureClas if (input.providerOwnership === 'managed') { return classified('platform', 'managed_provider_authentication'); } - return classified('unknown', 'provider_authentication'); + return classified('unknown', 'provider_ownership_unknown'); } if (input.assistantReason === 'timeout') { - return classified( - input.providerOwnership === 'managed' ? 'platform' : 'unknown', - 'request_timeout' - ); + if (input.providerOwnership === 'managed') { + return classified('platform', 'request_timeout'); + } + return input.providerOwnership === 'byok' + ? classified('unknown', 'request_timeout') + : classified('unknown', 'provider_ownership_unknown'); } if (input.assistantReason === 'provider_unavailable') { - return input.providerOwnership === 'managed' - ? classified('platform', 'managed_provider_unavailable') - : classified('unknown', 'provider_unavailable'); + if (input.providerOwnership === 'managed') { + return classified('platform', 'managed_provider_unavailable'); + } + return input.providerOwnership === 'byok' + ? classified('unknown', 'provider_unavailable') + : classified('unknown', 'provider_ownership_unknown'); } if ( input.assistantReason === 'invalid_request' || input.assistantReason === 'context_limit' || - input.assistantReason === 'output_limit' || - input.assistantReason === 'content_filter' || - input.assistantReason === 'structured_output' + input.assistantReason === 'output_limit' ) { - return classified('unknown', input.assistantReason); + const responsibility = + input.providerOwnership === 'byok' + ? 'user' + : input.providerOwnership === 'managed' + ? 'platform' + : 'unknown'; + return classified(responsibility, OWNERSHIP_DEPENDENT_ASSISTANT_LIMITS[input.assistantReason]); + } + if (input.assistantReason === 'content_filter') { + return classified('user', 'assistant_content_filter'); + } + if (input.assistantReason === 'structured_output') { + return classified('platform', 'assistant_structured_output'); } return classified('unknown', 'assistant_unknown'); } @@ -349,12 +391,16 @@ export function classifyCloudAgentFailure( case 'delivery_failure_unknown': return classified('platform', 'delivery'); case 'wrapper_disconnected': + return classified('platform', 'wrapper_disconnected'); case 'wrapper_no_output': case 'wrapper_ping_timeout': + return classified('platform', 'wrapper_liveness'); case 'wrapper_error_before_activity': + return classified('platform', 'wrapper_startup'); case 'wrapper_error_after_activity': + return classified('platform', 'wrapper_crash'); case 'missing_assistant_reply': - return classified('platform', 'wrapper_liveness'); + return classified('platform', 'assistant_no_reply'); case 'assistant_error': case 'payment_required': case 'model_missing': @@ -362,9 +408,11 @@ export function classifyCloudAgentFailure( case 'unclassified': return classified('unknown', 'unclassified'); case 'user_interrupt': + return classified('user', 'user_interrupt'); case 'container_shutdown': + return classified('platform', 'container_shutdown'); case 'system_interrupt': - return classified('unknown', 'unclassified'); + return classified('platform', 'system_interrupt'); } } diff --git a/services/cloud-agent-next/src/telemetry/queue-reports.test.ts b/services/cloud-agent-next/src/telemetry/queue-reports.test.ts index 66cff650009..cb635f76b03 100644 --- a/services/cloud-agent-next/src/telemetry/queue-reports.test.ts +++ b/services/cloud-agent-next/src/telemetry/queue-reports.test.ts @@ -70,7 +70,7 @@ describe('Cloud Agent report emitter', () => { failureStage: 'agent_activity', failureCode: 'wrapper_error_after_activity', failureResponsibility: 'platform', - failureReason: 'wrapper_liveness', + failureReason: 'wrapper_crash', diagnostic: { errorMessageRedacted: 'Wrapper failed after agent activity', errorExpiresAt: new Date(5 + 30 * 24 * 60 * 60 * 1000).toISOString(), @@ -165,38 +165,45 @@ describe('Cloud Agent report emitter', () => { }); it.each([ - ['wrapper_ping_timeout', 'Wrapper health check timed out'], - ['wrapper_no_output', 'Wrapper made no execution progress during the watchdog window'], - ['wrapper_disconnected', 'Wrapper disconnected before completion'], - ] as const)('reports %s before and after activity', async (failureCode, expectedDiagnostic) => { - for (const failureStage of ['post_dispatch_no_activity', 'agent_activity'] as const) { - const reports: CloudAgentQueueReport[] = []; - await emitRunStateReport({ - queue: { send: async report => void reports.push(report) }, - cloudAgentSessionId: 'agent_report', - state: { - ...state, + ['wrapper_ping_timeout', 'Wrapper health check timed out', 'wrapper_liveness'], + [ + 'wrapper_no_output', + 'Wrapper made no execution progress during the watchdog window', + 'wrapper_liveness', + ], + ['wrapper_disconnected', 'Wrapper disconnected before completion', 'wrapper_disconnected'], + ] as const)( + 'reports %s before and after activity', + async (failureCode, expectedDiagnostic, expectedFailureReason) => { + for (const failureStage of ['post_dispatch_no_activity', 'agent_activity'] as const) { + const reports: CloudAgentQueueReport[] = []; + await emitRunStateReport({ + queue: { send: async report => void reports.push(report) }, + cloudAgentSessionId: 'agent_report', + state: { + ...state, + failureStage, + failureCode, + agentActivityObservedAt: failureStage === 'agent_activity' ? 4 : undefined, + }, + }); + + expect(reports).toHaveLength(1); + expect(reports[0].run).toMatchObject({ failureStage, failureCode, - agentActivityObservedAt: failureStage === 'agent_activity' ? 4 : undefined, - }, - }); - - expect(reports).toHaveLength(1); - expect(reports[0].run).toMatchObject({ - failureStage, - failureCode, - failureResponsibility: 'platform', - failureReason: 'wrapper_liveness', - diagnostic: { errorMessageRedacted: expectedDiagnostic }, - }); - if (failureStage === 'agent_activity') { - expect(reports[0].run.agentActivityObservedAt).toBe(new Date(4).toISOString()); - } else { - expect(reports[0].run).not.toHaveProperty('agentActivityObservedAt'); + failureResponsibility: 'platform', + failureReason: expectedFailureReason, + diagnostic: { errorMessageRedacted: expectedDiagnostic }, + }); + if (failureStage === 'agent_activity') { + expect(reports[0].run.agentActivityObservedAt).toBe(new Date(4).toISOString()); + } else { + expect(reports[0].run).not.toHaveProperty('agentActivityObservedAt'); + } } } - }); + ); it.each([ [ @@ -213,18 +220,18 @@ describe('Cloud Agent report emitter', () => { ], ['provider_unavailable', 'Assistant service is unavailable', 'managed_provider_unavailable'], ['timeout', 'Assistant request timed out', 'request_timeout'], - ['invalid_request', 'Assistant request was invalid', 'invalid_request'], - ['context_limit', 'The model context limit was exceeded', 'context_limit'], - ['output_limit', 'The model output limit was reached', 'output_limit'], + ['invalid_request', 'Assistant request was invalid', 'assistant_invalid_request'], + ['context_limit', 'The model context limit was exceeded', 'assistant_context_limit'], + ['output_limit', 'The model output limit was reached', 'assistant_output_limit'], [ 'content_filter', 'The model provider blocked the response under its content policy', - 'content_filter', + 'assistant_content_filter', ], [ 'structured_output', 'The model response did not match the required format', - 'structured_output', + 'assistant_structured_output', ], ['unknown', 'Assistant request failed', 'assistant_unknown'], [undefined, 'Assistant request failed', 'assistant_unknown'],