diff --git a/apps/web/src/app/admin/api/session-traces/hooks.ts b/apps/web/src/app/admin/api/session-traces/hooks.ts index 1cd840b5eb..e72c7044fb 100644 --- a/apps/web/src/app/admin/api/session-traces/hooks.ts +++ b/apps/web/src/app/admin/api/session-traces/hooks.ts @@ -11,6 +11,16 @@ export function useAdminSessionTrace(sessionId: string | null, enabled = true) { }); } +export function useAdminSessionContainerTelemetry(sessionId: string | null, enabled = true) { + const trpc = useTRPC(); + return useQuery({ + ...trpc.admin.sessionTraces.getContainerMetrics.queryOptions({ session_id: sessionId ?? '' }), + enabled: enabled && !!sessionId, + staleTime: 5 * 60 * 1000, + refetchOnWindowFocus: false, + }); +} + export function useAdminSessionMessages(sessionId: string | null, enabled = true) { const trpc = useTRPC(); return useQuery({ diff --git a/apps/web/src/app/admin/components/SessionContainerTelemetry.test.ts b/apps/web/src/app/admin/components/SessionContainerTelemetry.test.ts new file mode 100644 index 0000000000..0be44573a7 --- /dev/null +++ b/apps/web/src/app/admin/components/SessionContainerTelemetry.test.ts @@ -0,0 +1,104 @@ +import React from 'react'; +import { renderToStaticMarkup } from 'react-dom/server'; +import { describe, expect, it, jest } from '@jest/globals'; + +import { + containerMetricSeriesLabel, + SessionContainerTelemetryContent, + type SessionContainerInfo, +} from './SessionContainerTelemetry'; + +const info: SessionContainerInfo = { + cloudAgentSessionId: 'agent_1', + sandboxId: 'ses-1', + scope: 'isolated', + windowStartAt: '2026-07-31T08:43:00.000Z', + windowEndAt: '2026-07-31T08:51:00.000Z', + runs: [], + intervals: [ + { + id: 'interval-1', + service: 'cloud-agent-next-sandbox-small-containment', + sandboxId: 'ses-1', + cloudflareInstanceId: 'durable-object-id', + containerClass: 'SandboxSmallContainment', + startedAt: '2026-07-31T08:43:00.000Z', + lastSeenAt: '2026-07-31T08:51:00.000Z', + stoppedAt: '2026-07-31T08:51:00.000Z', + status: 'closed', + closeReason: 'exit', + exitCode: 0, + sku: { id: 'cloud-agent-small-2026-07', name: 'Cloud Agent Small', description: null }, + capacity: { vcpu: 2, memoryBytes: 6 * 1024 ** 3, diskBytes: 10_000_000_000 }, + capacitySource: 'recorded', + }, + ], +}; + +describe('SessionContainerTelemetry', () => { + it('renders workload summaries with normalized CPU utilization', () => { + const metricsQuery = { + isLoading: false, + isError: false, + data: { + available: true as const, + partial: false, + issues: [], + rows: [ + { + windowKey: 'interval-1', + timestamp: '2026-07-31T08:49:00.000Z', + applicationId: 'app-1', + instanceId: 'durable-object-id', + placementId: 'placement-1', + location: 'ord02', + region: 'WNAM', + avg: { + cpuUtilization: 0.72, + memory: 4 * 1024 ** 3, + rxBandwidthBps: 10, + txBandwidthBps: 20, + containerUptime: 300, + }, + max: { memory: 5 * 1024 ** 3, diskUsage: 100, diskUsagePercentage: 1 }, + quantiles: { cpuUtilizationP95: 0.9, memoryP95: 4.8 * 1024 ** 3 }, + sum: { cpuTimeSec: 30, rxBytes: 100, txBytes: 200 }, + }, + ], + }, + }; + + const consoleWarn = jest.spyOn(console, 'warn').mockImplementation(() => undefined); + const html = renderToStaticMarkup( + React.createElement(SessionContainerTelemetryContent, { info, metricsQuery }) + ); + consoleWarn.mockRestore(); + + expect(html).toContain('Peak memory'); + expect(html).toContain('5.0 GiB / 6.0 GiB'); + expect(html).toContain('90.0%'); + expect(containerMetricSeriesLabel('interval-1', 'placement-1')).toBe( + 'placement-1 · interval-1' + ); + expect(html).toContain('
{ + const metricsQuery = { + isLoading: false, + isError: false, + data: { available: false as const, reason: 'no_container_intervals' as const }, + }; + const html = renderToStaticMarkup( + React.createElement(SessionContainerTelemetryContent, { + info: { ...info, sandboxId: null, scope: 'unknown' }, + metricsQuery, + }) + ); + + expect(html).toContain('Unknown container scope'); + expect(html).not.toContain('This container is shared'); + }); +}); diff --git a/apps/web/src/app/admin/components/SessionContainerTelemetry.tsx b/apps/web/src/app/admin/components/SessionContainerTelemetry.tsx new file mode 100644 index 0000000000..498d98bb25 --- /dev/null +++ b/apps/web/src/app/admin/components/SessionContainerTelemetry.tsx @@ -0,0 +1,436 @@ +'use client'; + +import React from 'react'; +import type { inferRouterOutputs } from '@trpc/server'; +import { + CartesianGrid, + Legend, + Line, + LineChart, + ReferenceLine, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from 'recharts'; + +import { Alert, AlertDescription } from '@/components/ui/alert'; +import { Badge } from '@/components/ui/badge'; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'; +import { Skeleton } from '@/components/ui/skeleton'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import type { RootRouter } from '@/routers/root-router'; + +type RouterOutputs = inferRouterOutputs; +type SessionContainerTelemetryResult = + RouterOutputs['admin']['sessionTraces']['getContainerMetrics']; +export type SessionContainerInfo = NonNullable; +type SessionContainerMetrics = SessionContainerTelemetryResult['metrics']; +type MetricsQueryState = { + isLoading: boolean; + isError: boolean; + error?: { message: string } | null; + data?: SessionContainerMetrics; +}; + +function formatBytes(value: number | null): string { + if (value === null || !Number.isFinite(value)) return '-'; + const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + let amount = value; + let unit = 0; + while (amount >= 1024 && unit < units.length - 1) { + amount /= 1024; + unit += 1; + } + return `${amount >= 10 || unit === 0 ? amount.toFixed(0) : amount.toFixed(1)} ${units[unit]}`; +} + +function formatPercent(value: number | null): string { + return value === null || !Number.isFinite(value) ? '-' : `${(value * 100).toFixed(1)}%`; +} + +function formatTime(value: string): string { + return new Date(value).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }); +} + +export function containerMetricSeriesLabel(windowKey: string, placementId: string): string { + const placement = placementId.length > 12 ? `${placementId.slice(0, 8)}...` : placementId; + const window = windowKey.length > 12 ? `${windowKey.slice(0, 8)}...` : windowKey; + return `${placement} · ${window}`; +} + +export function SessionContainerTelemetryContent({ + info, + metricsQuery, +}: { + info: SessionContainerInfo; + metricsQuery: MetricsQueryState; +}) { + const metrics = metricsQuery.data; + const rows = metrics?.available ? metrics.rows : []; + const latestInterval = info.intervals.at(-1); + const knownMemoryCapacities = info.intervals.flatMap(interval => + interval.capacity ? [interval.capacity.memoryBytes] : [] + ); + const memoryCapacities = new Set(knownMemoryCapacities); + // A global limit is only valid when every plotted interval has the same known capacity. + const memoryCapacity = + knownMemoryCapacities.length === info.intervals.length && memoryCapacities.size === 1 + ? knownMemoryCapacities[0] + : null; + const seriesColors = [ + 'var(--chart-1)', + 'var(--chart-2)', + 'var(--chart-3)', + 'var(--chart-4)', + 'var(--chart-5)', + ]; + const placementsByWindow = new Map>(); + for (const row of rows) { + const placements = placementsByWindow.get(row.windowKey) ?? new Set(); + placements.add(row.placementId); + placementsByWindow.set(row.windowKey, placements); + } + let nextSeriesIndex = 0; + const series = [...placementsByWindow].flatMap(([windowKey, placements]) => + [...placements].map(placementId => { + const seriesIndex = nextSeriesIndex; + nextSeriesIndex += 1; + return { + windowKey, + placementId, + label: containerMetricSeriesLabel(windowKey, placementId), + color: seriesColors[seriesIndex % seriesColors.length], + memoryMaxKey: `memoryMax${seriesIndex}`, + memoryP95Key: `memoryP95${seriesIndex}`, + cpuAverageKey: `cpuAverage${seriesIndex}`, + cpuP95Key: `cpuP95${seriesIndex}`, + }; + }) + ); + const seriesByWindow = new Map>(); + for (const item of series) { + const placements = seriesByWindow.get(item.windowKey) ?? new Map(); + placements.set(item.placementId, item); + seriesByWindow.set(item.windowKey, placements); + } + const chartPoints = new Map>(); + for (const row of rows) { + const point = chartPoints.get(row.timestamp) ?? { + timestamp: row.timestamp, + }; + const item = seriesByWindow.get(row.windowKey)?.get(row.placementId); + if (item) { + point[item.memoryMaxKey] = row.max.memory === null ? null : row.max.memory / 1024 ** 3; + point[item.memoryP95Key] = + row.quantiles.memoryP95 === null ? null : row.quantiles.memoryP95 / 1024 ** 3; + point[item.cpuAverageKey] = + row.avg.cpuUtilization === null ? null : row.avg.cpuUtilization * 100; + point[item.cpuP95Key] = + row.quantiles.cpuUtilizationP95 === null ? null : row.quantiles.cpuUtilizationP95 * 100; + } + chartPoints.set(row.timestamp, point); + } + const chartData = [...chartPoints.values()].sort( + (left, right) => Date.parse(String(left.timestamp)) - Date.parse(String(right.timestamp)) + ); + const peakMemory = rows.reduce( + (peak, row) => (row.max.memory === null ? peak : Math.max(peak ?? 0, row.max.memory)), + null + ); + const peakCpuP95 = rows.reduce( + (peak, row) => + row.quantiles.cpuUtilizationP95 === null + ? peak + : Math.max(peak ?? 0, row.quantiles.cpuUtilizationP95), + null + ); + const placements = new Set(rows.map(row => row.placementId)); + + return ( + + +
+
+ Container Metrics + + Cloudflare workload samples for the container intervals attributed to this session + +
+ + {info.scope === 'isolated' + ? 'Isolated container' + : info.scope === 'shared' + ? 'Shared container' + : 'Unknown container scope'} + +
+ + + {info.scope === 'shared' && ( + + + This container is shared. Workload samples may include activity from other sessions. + + + )} + + {metricsQuery.isLoading ? ( +
+
+ {Array.from({ length: 4 }, (_, index) => ( + + ))} +
+ +
+ ) : metricsQuery.isError ? ( + + + {metricsQuery.error?.message ?? 'Container metrics failed to load.'} + + + ) : !metrics?.available ? ( +

+ {metrics?.reason === 'no_provider_identity' + ? 'No Cloudflare instance identity was recorded for these container intervals.' + : metrics?.reason === 'no_overlapping_intervals' + ? 'No container intervals overlap the recorded session activity window.' + : metrics?.reason === 'ambiguous_application' + ? 'Cloudflare returned more than one container application for this instance ID.' + : metrics?.reason === 'no_container_intervals' + ? 'No metered container intervals were recorded for this session.' + : 'Container metrics are not available for this session.'} +

+ ) : rows.length === 0 ? ( +

+ No Cloudflare workload samples are available for these container intervals. +

+ ) : ( + <> + {metrics.partial && ( + + + Cloudflare returned partial metrics. {metrics.issues.join(' ')} + + + )} + +
+
+

Peak memory

+

+ {formatBytes(peakMemory)} + {memoryCapacity ? ` / ${formatBytes(memoryCapacity)}` : ''} +

+
+
+

Peak CPU P95

+

{formatPercent(peakCpuP95)}

+
+
+

Placements

+

{placements.size}

+
+
+

Container intervals

+

{info.intervals.length}

+
+
+ +
+
+

Memory

+
+ + + + formatTime(String(value))} + minTickGap={24} + /> + `${Number(value).toFixed(1)} GiB`} + width={62} + /> + [ + `${Number(value).toFixed(2)} GiB`, + String(name), + ]} + labelFormatter={label => new Date(String(label)).toLocaleString()} + /> + + {memoryCapacity && ( + + )} + {series.flatMap(item => [ + , + , + ])} + + +
+
+ +
+

CPU Utilization

+
+ + + + formatTime(String(value))} + minTickGap={24} + /> + `${Number(value).toFixed(0)}%`} + width={48} + /> + [`${Number(value).toFixed(1)}%`, String(name)]} + labelFormatter={label => new Date(String(label)).toLocaleString()} + /> + + {series.flatMap(item => [ + , + , + ])} + + +
+
+
+ +
+ Metric samples ({rows.length}) +
+ + + + + Time + Memory max + Memory P95 + CPU avg + CPU P95 + Disk + RX / TX + Placement + + + + {rows.map((row, index) => ( + + + {new Date(row.timestamp).toLocaleString()} + + + {formatBytes(row.max.memory)} + + + {formatBytes(row.quantiles.memoryP95)} + + + {formatPercent(row.avg.cpuUtilization)} + + + {formatPercent(row.quantiles.cpuUtilizationP95)} + + + {formatBytes(row.max.diskUsage)} + + + {formatBytes(row.sum.rxBytes)} / {formatBytes(row.sum.txBytes)} + + {row.placementId} + + ))} + +
+ Cloudflare container workload metric samples +
+
+
+ + )} + + {latestInterval?.capacitySource === 'configured' && ( +

+ Capacity is inferred from the recorded container service for this historical interval. +

+ )} + + + ); +} diff --git a/apps/web/src/app/admin/components/SessionTraceViewer.tsx b/apps/web/src/app/admin/components/SessionTraceViewer.tsx index 436fce3ee1..f60a2cfff5 100644 --- a/apps/web/src/app/admin/components/SessionTraceViewer.tsx +++ b/apps/web/src/app/admin/components/SessionTraceViewer.tsx @@ -16,14 +16,26 @@ import { MessageErrorBoundary as V2MessageErrorBoundary } from '@/components/clo import { isNewSession } from '@/lib/cloud-agent/session-type'; import { useAdminSessionTrace, + useAdminSessionContainerTelemetry, useAdminSessionMessages, useAdminApiConversationHistory, useAdminResolveCloudAgentSession, } from '@/app/admin/api/session-traces/hooks'; -import { Search, User, Calendar, Globe, GitBranch, Loader2, Download } from 'lucide-react'; +import { + Search, + User, + Calendar, + Globe, + GitBranch, + Loader2, + Download, + Server, + Tag, +} from 'lucide-react'; import type { CloudMessage, Message } from '@/components/cloud-agent/types'; import type { StoredMessage } from '@/components/cloud-agent-next/types'; import { useAdminPermissions } from '@/app/admin/useAdminPermissions'; +import { SessionContainerTelemetryContent } from './SessionContainerTelemetry'; const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; const SES_PREFIX = 'ses_'; @@ -120,6 +132,16 @@ export function SessionTraceViewer() { }, [sessionIdFromUrl]); const sessionQuery = useAdminSessionTrace(searchedSessionId, canViewSessions); + const cloudAgentSessionIdentity = sessionQuery.data + ? (sessionQuery.data.cloud_agent_session_id ?? + ('cloud_agent_session_scope_id' in sessionQuery.data + ? sessionQuery.data.cloud_agent_session_scope_id + : null)) + : null; + const containerTelemetryQuery = useAdminSessionContainerTelemetry( + searchedSessionId, + canViewSessions && !!cloudAgentSessionIdentity + ); const messagesQuery = useAdminSessionMessages(searchedSessionId, canViewSessions); const apiHistoryQuery = useAdminApiConversationHistory(searchedSessionId, canViewSessions); @@ -214,6 +236,7 @@ export function SessionTraceViewer() { return null; }, [v2Messages]); + const latestContainerInterval = containerTelemetryQuery.data?.info?.intervals.at(-1); const breadcrumbs = ( @@ -380,10 +403,54 @@ export function SessionTraceViewer() { )} + {latestContainerInterval?.cloudflareInstanceId && ( +
+ + Container: + + {latestContainerInterval.cloudflareInstanceId} + +
+ )} + {latestContainerInterval && ( +
+ + SKU: + + {latestContainerInterval.sku.name}{' '} + + ({latestContainerInterval.sku.id}) + + +
+ )} )} + {containerTelemetryQuery.data?.info && + containerTelemetryQuery.data.info.intervals.length > 0 && ( + + )} + + {cloudAgentSessionIdentity && containerTelemetryQuery.isError && ( + + + {containerTelemetryQuery.error?.message || 'Container telemetry failed to load.'} + + + )} + {sessionQuery.data && ( diff --git a/apps/web/src/lib/cloudflare/container-capacity.ts b/apps/web/src/lib/cloudflare/container-capacity.ts new file mode 100644 index 0000000000..3e5027ee57 --- /dev/null +++ b/apps/web/src/lib/cloudflare/container-capacity.ts @@ -0,0 +1,53 @@ +const MEBIBYTE_BYTES = 1024 ** 2; +const MEGABYTE_BYTES = 1_000_000; + +// Production Cloud Agent values mirror the top-level services/cloud-agent-next/wrangler.jsonc +// entries and SANDBOX_CAPACITIES in services/cloud-agent-next/src/container-usage-context.ts. +// Development intentionally uses different named instance types and does not query Analytics. +// Keep container-capacity-parity.test.ts aligned when adding a container class. + +export type ContainerCapacity = { + vcpu: number; + memoryBytes: number; + diskBytes: number; +}; + +export function containerCapacityForService(service: string): ContainerCapacity | null { + switch (service) { + case 'gastown': + case 'cloud-agent-next-sandbox': + case 'cloud-agent-next-sandbox-containment': + return { + vcpu: 4, + memoryBytes: 12_288 * MEBIBYTE_BYTES, + diskBytes: 20_000 * MEGABYTE_BYTES, + }; + case 'cloud-agent-next-sandbox-small': + case 'cloud-agent-next-sandbox-dind': + case 'cloud-agent-next-sandbox-small-containment': + return { vcpu: 2, memoryBytes: 6_144 * MEBIBYTE_BYTES, diskBytes: 10_000 * MEGABYTE_BYTES }; + case 'cloud-agent-next-sandbox-code-review': + case 'cloud-agent-next-sandbox-code-review-containment': + return { vcpu: 1, memoryBytes: 4_096 * MEBIBYTE_BYTES, diskBytes: 8_000 * MEGABYTE_BYTES }; + default: + return null; + } +} + +export function sharedContainerCapacity(services: Set): ContainerCapacity | null { + let shared: ContainerCapacity | null = null; + for (const service of services) { + const capacity = containerCapacityForService(service); + if (!capacity) return null; + if ( + shared && + (shared.vcpu !== capacity.vcpu || + shared.memoryBytes !== capacity.memoryBytes || + shared.diskBytes !== capacity.diskBytes) + ) { + return null; + } + shared = capacity; + } + return shared; +} diff --git a/apps/web/src/lib/cloudflare/container-metrics-analytics.test.ts b/apps/web/src/lib/cloudflare/container-metrics-analytics.test.ts new file mode 100644 index 0000000000..3f212c28d5 --- /dev/null +++ b/apps/web/src/lib/cloudflare/container-metrics-analytics.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it, jest } from '@jest/globals'; + +jest.mock('@/lib/config.server', () => ({ + CLOUDFLARE_ACCOUNT_ID: '', + CLOUDFLARE_ANALYTICS_API_TOKEN: '', +})); + +import { queryContainerMetricsAnalytics } from './container-metrics-analytics'; + +const REQUIRED_FIELDS = [ + 'dimensions_datetimeMinute', + 'dimensions_applicationId', + 'dimensions_instanceId', + 'dimensions_placementId', + 'dimensions_location', + 'dimensions_region', + 'avg_cpuUtilization', + 'avg_memory', + 'avg_rxBandwidthBps', + 'avg_txBandwidthBps', + 'avg_containerUptime', + 'max_memory', + 'max_diskUsage', + 'max_diskUsagePercentage', + 'quantiles_cpuUtilizationP95', + 'quantiles_memoryP95', + 'sum_cpuTimeSec', + 'sum_rxBytes', + 'sum_txBytes', +]; + +const input = { + windows: [ + { + key: 'interval-1', + instanceId: 'durable-object-id', + start: '2026-07-31T08:43:00.000Z', + end: '2026-07-31T08:51:00.000Z', + }, + ], +}; + +function response(body: unknown) { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); +} + +function settingsBody(availableFields = REQUIRED_FIELDS) { + return { + data: { + viewer: { + accounts: [ + { + settings: { + containersMetricsAdaptiveGroups: { + enabled: true, + availableFields, + maxPageSize: 1_000, + maxNumberOfFields: 30, + notOlderThan: 2_678_400, + maxDuration: 86_400, + }, + }, + }, + ], + }, + }, + errors: null, + }; +} + +describe('queryContainerMetricsAnalytics', () => { + it('queries workload metrics by Durable Object ID and normalizes minute samples', async () => { + const requests: Array<{ query: string; variables: Record }> = []; + const fetch = jest.fn(async (_url: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as { + query: string; + variables: Record; + }; + requests.push(body); + if (requests.length === 1) return response(settingsBody()); + return response({ + data: { + viewer: { + accounts: [ + { + m0: [ + { + dimensions: { + datetimeMinute: '2026-07-31T08:49:00.000Z', + applicationId: 'application-id', + instanceId: 'durable-object-id', + placementId: 'placement-id', + location: 'ord02', + region: 'WNAM', + }, + avg: { + cpuUtilization: 0.72, + memory: 4_000, + rxBandwidthBps: 10, + txBandwidthBps: 20, + containerUptime: 300, + }, + max: { memory: 5_000, diskUsage: 6_000, diskUsagePercentage: 12 }, + quantiles: { cpuUtilizationP95: 0.9, memoryP95: 4_800 }, + sum: { cpuTimeSec: 30, rxBytes: 100, txBytes: 200 }, + }, + ], + }, + ], + }, + }, + errors: null, + }); + }) as typeof globalThis.fetch; + + const result = await queryContainerMetricsAnalytics(input, { + fetch, + accountId: 'account-id', + apiToken: 'analytics-token', + now: () => new Date('2026-08-01T00:00:00.000Z'), + }); + + expect(requests[1]?.query).toContain('containersMetricsAdaptiveGroups'); + expect(requests[1]?.query).toContain('orderBy: [datetimeMinute_ASC]'); + expect(requests[1]?.variables).toEqual({ + accountTag: 'account-id', + datetimeStart0: input.windows[0]?.start, + datetimeEnd0: input.windows[0]?.end, + instanceIds0: ['durable-object-id'], + }); + expect(result).toEqual({ + partial: false, + issues: [], + rows: [ + expect.objectContaining({ + windowKey: 'interval-1', + timestamp: '2026-07-31T08:49:00.000Z', + instanceId: 'durable-object-id', + placementId: 'placement-id', + max: expect.objectContaining({ memory: 5_000 }), + }), + ], + }); + expect(JSON.stringify(result)).not.toContain('analytics-token'); + }); + + it('reports single-window unscoped GraphQL errors as partial results', async () => { + let requestCount = 0; + const fetch = jest.fn(async () => { + requestCount += 1; + if (requestCount === 1) return response(settingsBody()); + return response({ + data: { viewer: { accounts: [{ m0: [] }] } }, + errors: [{ message: 'Account data is incomplete', path: ['viewer', 'accounts'] }], + }); + }) as typeof globalThis.fetch; + + await expect( + queryContainerMetricsAnalytics(input, { + fetch, + accountId: 'account-id', + apiToken: 'analytics-token', + now: () => new Date('2026-08-01T00:00:00.000Z'), + }) + ).resolves.toEqual({ + rows: [], + partial: true, + issues: ['Account data is incomplete'], + }); + }); + + it('fails before the workload query when required fields are unavailable', async () => { + const fetch = jest.fn(async () => + response(settingsBody(REQUIRED_FIELDS.slice(1))) + ) as typeof globalThis.fetch; + + await expect( + queryContainerMetricsAnalytics(input, { + fetch, + accountId: 'account-id', + apiToken: 'analytics-token', + }) + ).rejects.toMatchObject({ code: 'fields_unavailable' }); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it('rejects malformed provider timestamps as an invalid response shape', async () => { + let requestCount = 0; + const fetch = jest.fn(async () => { + requestCount += 1; + if (requestCount === 1) return response(settingsBody()); + return response({ + data: { + viewer: { + accounts: [ + { + m0: [ + { + dimensions: { + datetimeMinute: 'not-a-datetime', + applicationId: 'application-id', + instanceId: 'durable-object-id', + placementId: 'placement-id', + location: 'ord02', + region: 'WNAM', + }, + avg: { + cpuUtilization: 0.72, + memory: 4_000, + rxBandwidthBps: 10, + txBandwidthBps: 20, + containerUptime: 300, + }, + max: { memory: 5_000, diskUsage: 6_000, diskUsagePercentage: 12 }, + quantiles: { cpuUtilizationP95: 0.9, memoryP95: 4_800 }, + sum: { cpuTimeSec: 30, rxBytes: 100, txBytes: 200 }, + }, + ], + }, + ], + }, + }, + errors: null, + }); + }) as typeof globalThis.fetch; + + await expect( + queryContainerMetricsAnalytics(input, { + fetch, + accountId: 'account-id', + apiToken: 'analytics-token', + now: () => new Date('2026-08-01T00:00:00.000Z'), + }) + ).rejects.toMatchObject({ code: 'invalid_response_shape' }); + }); +}); diff --git a/apps/web/src/lib/cloudflare/container-metrics-analytics.ts b/apps/web/src/lib/cloudflare/container-metrics-analytics.ts new file mode 100644 index 0000000000..e2042662fd --- /dev/null +++ b/apps/web/src/lib/cloudflare/container-metrics-analytics.ts @@ -0,0 +1,514 @@ +import 'server-only'; + +import * as z from 'zod'; + +import { CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_ANALYTICS_API_TOKEN } from '@/lib/config.server'; + +const GRAPHQL_URL = 'https://api.cloudflare.com/client/v4/graphql'; +const DATASET = 'containersMetricsAdaptiveGroups' as const; +const DEFAULT_TIMEOUT_MS = 10_000; +const MAX_WINDOWS = 20; +const MAX_WINDOWS_PER_REQUEST = 10; +const MAX_RESPONSE_BYTES = 2 * 1024 * 1024; + +export type ContainerMetricsWindow = { + key: string; + instanceId: string; + start: string; + end: string; +}; + +export type ContainerMetricsRow = { + windowKey: string; + timestamp: string; + applicationId: string; + instanceId: string; + placementId: string; + location: string | null; + region: string | null; + avg: { + cpuUtilization: number | null; + memory: number | null; + rxBandwidthBps: number | null; + txBandwidthBps: number | null; + containerUptime: number | null; + }; + max: { + memory: number | null; + diskUsage: number | null; + diskUsagePercentage: number | null; + }; + quantiles: { + cpuUtilizationP95: number | null; + memoryP95: number | null; + }; + sum: { + cpuTimeSec: number | null; + rxBytes: number | null; + txBytes: number | null; + }; +}; + +export type ContainerMetricsResult = { + rows: ContainerMetricsRow[]; + partial: boolean; + issues: string[]; +}; + +export type ContainerMetricsAnalyticsOptions = { + fetch?: typeof fetch; + accountId?: string; + apiToken?: string; + timeoutMs?: number; + now?: () => Date; +}; + +export class ContainerMetricsAnalyticsError extends Error { + readonly code: string; + + constructor(code: string, message: string) { + super(message); + this.name = 'ContainerMetricsAnalyticsError'; + this.code = code; + } +} + +const DatasetSettingsSchema = z.object({ + enabled: z.boolean(), + availableFields: z.array(z.string()), + maxPageSize: z.number().int().positive(), + maxNumberOfFields: z.number().int().positive(), + notOlderThan: z.number().int().nonnegative(), + maxDuration: z.number().int().positive(), +}); + +const GraphqlErrorsSchema = z + .array( + z + .object({ + message: z.string().optional(), + path: z.array(z.union([z.string(), z.number()])).optional(), + }) + .passthrough() + ) + .nullish(); + +const SettingsResponseSchema = z.object({ + data: z + .object({ + viewer: z + .object({ + accounts: z + .array( + z.object({ + settings: z + .object({ containersMetricsAdaptiveGroups: DatasetSettingsSchema.optional() }) + .optional(), + }) + ) + .optional(), + }) + .optional(), + }) + .nullish(), + errors: GraphqlErrorsSchema, +}); + +const NullableNumber = z.number().nullable(); +const MetricsGroupSchema = z.object({ + dimensions: z.object({ + datetimeMinute: z.string().datetime(), + applicationId: z.string(), + instanceId: z.string(), + placementId: z.string(), + location: z.string().nullable(), + region: z.string().nullable(), + }), + avg: z.object({ + cpuUtilization: NullableNumber, + memory: NullableNumber, + rxBandwidthBps: NullableNumber, + txBandwidthBps: NullableNumber, + containerUptime: NullableNumber, + }), + max: z.object({ + memory: NullableNumber, + diskUsage: NullableNumber, + diskUsagePercentage: NullableNumber, + }), + quantiles: z.object({ + cpuUtilizationP95: NullableNumber, + memoryP95: NullableNumber, + }), + sum: z.object({ + cpuTimeSec: NullableNumber, + rxBytes: NullableNumber, + txBytes: NullableNumber, + }), +}); + +const MetricsResponseSchema = z.object({ + data: z + .object({ + viewer: z + .object({ accounts: z.array(z.record(z.string(), z.array(MetricsGroupSchema).nullable())) }) + .optional(), + }) + .nullish(), + errors: GraphqlErrorsSchema, +}); + +const SETTINGS_QUERY = ` +query ContainerMetricsSettings($accountTag: String!) { + viewer { + accounts(filter: { accountTag: $accountTag }) { + settings { + containersMetricsAdaptiveGroups { + enabled + availableFields + maxPageSize + maxNumberOfFields + notOlderThan + maxDuration + } + } + } + } +}`; + +const REQUIRED_FIELDS = [ + 'dimensions_datetimeMinute', + 'dimensions_applicationId', + 'dimensions_instanceId', + 'dimensions_placementId', + 'dimensions_location', + 'dimensions_region', + 'avg_cpuUtilization', + 'avg_memory', + 'avg_rxBandwidthBps', + 'avg_txBandwidthBps', + 'avg_containerUptime', + 'max_memory', + 'max_diskUsage', + 'max_diskUsagePercentage', + 'quantiles_cpuUtilizationP95', + 'quantiles_memoryP95', + 'sum_cpuTimeSec', + 'sum_rxBytes', + 'sum_txBytes', +]; + +type QueryPlan = ContainerMetricsWindow & { part: number }; + +function metricsQuery(limit: number, plans: QueryPlan[]): string { + const variables = plans + .map( + (_plan, index) => + `$datetimeStart${index}: Time!\n $datetimeEnd${index}: Time!\n $instanceIds${index}: [String!]` + ) + .join('\n '); + const fields = plans + .map( + (_plan, index) => ` + m${index}: containersMetricsAdaptiveGroups( + limit: ${limit} + filter: { + datetime_geq: $datetimeStart${index} + datetime_lt: $datetimeEnd${index} + instanceId_in: $instanceIds${index} + } + orderBy: [datetimeMinute_ASC] + ) { + dimensions { datetimeMinute applicationId instanceId placementId location region } + avg { cpuUtilization memory rxBandwidthBps txBandwidthBps containerUptime } + max { memory diskUsage diskUsagePercentage } + quantiles { cpuUtilizationP95 memoryP95 } + sum { cpuTimeSec rxBytes txBytes } + }` + ) + .join('\n'); + return ` +query ContainerMetrics( + $accountTag: String! + ${variables} +) { + viewer { + accounts(filter: { accountTag: $accountTag }) { + ${fields} + } + } +}`; +} + +function parseTime(value: string, field: string): number { + const parsed = Date.parse(value); + if (!Number.isFinite(parsed)) { + throw new ContainerMetricsAnalyticsError('invalid_input', `${field} must be an ISO datetime.`); + } + return parsed; +} + +function errorMessages(errors: z.infer): string[] { + return (errors ?? []).map(error => error.message ?? 'Unknown GraphQL error'); +} + +async function postGraphql(args: { + fetchImpl: typeof fetch; + token: string; + timeoutMs: number; + query: string; + variables: Record; +}): Promise { + let response: Response; + try { + response = await args.fetchImpl(GRAPHQL_URL, { + method: 'POST', + headers: { authorization: `Bearer ${args.token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ query: args.query, variables: args.variables }), + signal: AbortSignal.timeout(args.timeoutMs), + }); + } catch (error) { + const timedOut = + error instanceof Error && (error.name === 'AbortError' || error.name === 'TimeoutError'); + throw new ContainerMetricsAnalyticsError( + timedOut ? 'timeout' : 'network_error', + timedOut + ? `Cloudflare Analytics request timed out after ${args.timeoutMs}ms.` + : 'Cloudflare Analytics request failed before receiving a response.' + ); + } + const text = await response.text(); + if (new TextEncoder().encode(text).byteLength > MAX_RESPONSE_BYTES) { + throw new ContainerMetricsAnalyticsError( + 'response_too_large', + 'Cloudflare Analytics response exceeded the safety limit.' + ); + } + if (!response.ok) { + throw new ContainerMetricsAnalyticsError( + 'http_error', + `Cloudflare Analytics returned HTTP ${response.status}. Verify the token scope and retry.` + ); + } + try { + return JSON.parse(text) as unknown; + } catch { + throw new ContainerMetricsAnalyticsError( + 'invalid_json', + 'Cloudflare Analytics returned a non-JSON response.' + ); + } +} + +function splitWindow( + window: ContainerMetricsWindow, + startMs: number, + endMs: number, + maxDurationSeconds: number +): QueryPlan[] { + const plans: QueryPlan[] = []; + const durationMs = maxDurationSeconds * 1_000; + let part = 0; + for (let cursor = startMs; cursor < endMs; cursor += durationMs) { + plans.push({ + ...window, + start: new Date(cursor).toISOString(), + end: new Date(Math.min(cursor + durationMs, endMs)).toISOString(), + part, + }); + part += 1; + } + return plans; +} + +export async function queryContainerMetricsAnalytics( + input: { windows: ContainerMetricsWindow[] }, + options: ContainerMetricsAnalyticsOptions = {} +): Promise { + const accountId = options.accountId ?? CLOUDFLARE_ACCOUNT_ID; + const token = options.apiToken ?? CLOUDFLARE_ANALYTICS_API_TOKEN; + if (!accountId || !token) { + throw new ContainerMetricsAnalyticsError( + 'missing_config', + 'Cloudflare container metrics are not configured for the web app.' + ); + } + if (input.windows.length > MAX_WINDOWS) { + throw new ContainerMetricsAnalyticsError( + 'request_limit_exceeded', + `At most ${MAX_WINDOWS} container metric windows may be queried at once.` + ); + } + const keys = new Set(); + for (const window of input.windows) { + if (!window.key || !window.instanceId || keys.has(window.key)) { + throw new ContainerMetricsAnalyticsError( + 'invalid_input', + 'Container metric windows require unique keys and instance IDs.' + ); + } + keys.add(window.key); + if ( + parseTime(window.end, `end for ${window.key}`) <= + parseTime(window.start, `start for ${window.key}`) + ) { + throw new ContainerMetricsAnalyticsError( + 'invalid_input', + `end must be after start for ${window.key}.` + ); + } + } + + const fetchImpl = options.fetch ?? fetch; + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const settingsBody = await postGraphql({ + fetchImpl, + token, + timeoutMs, + query: SETTINGS_QUERY, + variables: { accountTag: accountId }, + }); + const settingsResponse = SettingsResponseSchema.safeParse(settingsBody); + if (!settingsResponse.success) { + throw new ContainerMetricsAnalyticsError( + 'invalid_response_shape', + 'Cloudflare Analytics settings response had an unexpected shape.' + ); + } + const settingsErrors = errorMessages(settingsResponse.data.errors); + const settings = + settingsResponse.data.data?.viewer?.accounts?.[0]?.settings?.containersMetricsAdaptiveGroups; + if (!settings || !settings.enabled) { + throw new ContainerMetricsAnalyticsError( + 'dataset_unavailable', + settingsErrors[0] ?? `Cloudflare Analytics dataset ${DATASET} is unavailable.` + ); + } + const missingFields = REQUIRED_FIELDS.filter(field => !settings.availableFields.includes(field)); + if (missingFields.length > 0 || settings.maxNumberOfFields < REQUIRED_FIELDS.length) { + throw new ContainerMetricsAnalyticsError( + 'fields_unavailable', + `Cloudflare Analytics dataset ${DATASET} is missing required workload fields.` + ); + } + + const issues = [...settingsErrors]; + let partial = settingsErrors.length > 0; + const oldestAllowed = + (options.now ?? (() => new Date()))().getTime() - settings.notOlderThan * 1_000; + const plans = input.windows.flatMap(window => { + const start = parseTime(window.start, `start for ${window.key}`); + const end = parseTime(window.end, `end for ${window.key}`); + if (end <= oldestAllowed) { + partial = true; + issues.push(`Container metrics for ${window.key} are outside Cloudflare retention.`); + return []; + } + const retainedStart = Math.max(start, oldestAllowed); + if (retainedStart !== start) { + partial = true; + issues.push( + `Container metrics for ${window.key} are partially outside Cloudflare retention.` + ); + } + return splitWindow(window, retainedStart, end, settings.maxDuration); + }); + if (plans.length > MAX_WINDOWS) { + throw new ContainerMetricsAnalyticsError( + 'request_limit_exceeded', + `The requested time range requires more than ${MAX_WINDOWS} Cloudflare metric windows.` + ); + } + + const rows: ContainerMetricsRow[] = []; + for (let batchStart = 0; batchStart < plans.length; batchStart += MAX_WINDOWS_PER_REQUEST) { + const batch = plans.slice(batchStart, batchStart + MAX_WINDOWS_PER_REQUEST); + const variables: Record = { accountTag: accountId }; + for (const [index, plan] of batch.entries()) { + variables[`datetimeStart${index}`] = plan.start; + variables[`datetimeEnd${index}`] = plan.end; + variables[`instanceIds${index}`] = [plan.instanceId]; + } + const body = await postGraphql({ + fetchImpl, + token, + timeoutMs, + query: metricsQuery(settings.maxPageSize, batch), + variables, + }); + const response = MetricsResponseSchema.safeParse(body); + if (!response.success) { + throw new ContainerMetricsAnalyticsError( + 'invalid_response_shape', + 'Cloudflare Analytics metrics response had an unexpected shape.' + ); + } + const account = response.data.data?.viewer?.accounts?.[0]; + const errors = response.data.errors ?? []; + const unscopedErrors = errors.filter( + error => !error.path?.some(part => /^m\d+$/.test(String(part))) + ); + if (unscopedErrors.length > 0 && batch.length > 1) { + throw new ContainerMetricsAnalyticsError( + 'graphql_error', + `Cloudflare Analytics returned an unscoped batch error: ${errorMessages(unscopedErrors).join('; ')}` + ); + } + if (!account) { + throw new ContainerMetricsAnalyticsError( + 'graphql_error', + errorMessages(errors)[0] ?? 'Cloudflare Analytics metrics query returned no account data.' + ); + } + for (const [index, plan] of batch.entries()) { + const alias = `m${index}`; + const aliasErrors = errors.filter( + error => + error.path?.includes(alias) || + error.path === undefined || + (batch.length === 1 && unscopedErrors.includes(error)) + ); + const groups = account[alias]; + if (!groups) { + if (aliasErrors.length === 0) { + throw new ContainerMetricsAnalyticsError( + 'invalid_response_shape', + `Cloudflare Analytics metrics response omitted ${alias}.` + ); + } + partial = true; + issues.push(...errorMessages(aliasErrors)); + continue; + } + if (aliasErrors.length > 0 || groups.length >= settings.maxPageSize) { + partial = true; + issues.push( + ...(aliasErrors.length > 0 + ? errorMessages(aliasErrors) + : [`Container metric window ${plan.key} reached the Cloudflare page limit.`]) + ); + } + rows.push( + ...groups.map(group => ({ + windowKey: plan.key, + timestamp: new Date(group.dimensions.datetimeMinute).toISOString(), + applicationId: group.dimensions.applicationId, + instanceId: group.dimensions.instanceId, + placementId: group.dimensions.placementId, + location: group.dimensions.location, + region: group.dimensions.region, + avg: group.avg, + max: group.max, + quantiles: group.quantiles, + sum: group.sum, + })) + ); + } + } + + rows.sort( + (left, right) => + left.timestamp.localeCompare(right.timestamp) || left.windowKey.localeCompare(right.windowKey) + ); + return { rows, partial, issues }; +} diff --git a/apps/web/src/routers/admin-router.ts b/apps/web/src/routers/admin-router.ts index 842b4ed69e..c259dc8cc1 100644 --- a/apps/web/src/routers/admin-router.ts +++ b/apps/web/src/routers/admin-router.ts @@ -41,6 +41,11 @@ import { adminKiloclawProvidersRouter } from '@/routers/admin-kiloclaw-providers import { adminFeatureInterestRouter } from '@/routers/admin-feature-interest-router'; import { adminCodeReviewsRouter } from '@/routers/admin-code-reviews-router'; import { adminCloudAgentNextRouter } from '@/routers/admin-cloud-agent-next-router'; +import { + getSessionContainerInfo, + getSessionContainerMetricsForInfo, +} from '@/routers/admin/session-container-telemetry'; +import { ContainerMetricsAnalyticsError } from '@/lib/cloudflare/container-metrics-analytics'; import { adminAIAttributionRouter } from '@/routers/admin-ai-attribution-router'; import { ossSponsorshipRouter } from '@/routers/admin/oss-sponsorship-router'; import { contributorChampionsRouter } from '@/routers/admin/contributor-champions-router'; @@ -2205,6 +2210,36 @@ export const adminRouter = createTRPCRouter({ }; }), + getContainerInfo: sessionViewerProcedure + .input(z.object({ session_id: sessionIdSchema })) + .query(({ input }) => getSessionContainerInfo(input.session_id)), + + getContainerMetrics: sessionViewerProcedure + .input(z.object({ session_id: sessionIdSchema })) + .query(async ({ input }) => { + try { + const info = await getSessionContainerInfo(input.session_id); + const metrics = info + ? await getSessionContainerMetricsForInfo(info) + : { available: false as const, reason: 'not_cloud_agent_session' as const }; + return { info, metrics }; + } catch (error) { + if (error instanceof ContainerMetricsAnalyticsError) { + const preconditionCodes = new Set([ + 'missing_config', + 'dataset_unavailable', + 'fields_unavailable', + 'request_limit_exceeded', + ]); + throw new TRPCError({ + code: preconditionCodes.has(error.code) ? 'PRECONDITION_FAILED' : 'BAD_GATEWAY', + message: error.message, + }); + } + throw error; + } + }), + getMessages: sessionViewerProcedure .input(z.object({ session_id: sessionIdSchema })) .query(async ({ input }) => { diff --git a/apps/web/src/routers/admin-session-traces.test.ts b/apps/web/src/routers/admin-session-traces.test.ts index 6041e20e91..71f974a0fc 100644 --- a/apps/web/src/routers/admin-session-traces.test.ts +++ b/apps/web/src/routers/admin-session-traces.test.ts @@ -11,8 +11,18 @@ jest.mock('@/lib/session-ingest-client', () => ({ import { db } from '@/lib/drizzle'; import { createCallerForUser } from '@/routers/test-utils'; +import { + getSessionContainerMetrics, + getSessionContainerMetricsForInfo, +} from '@/routers/admin/session-container-telemetry'; import { insertTestUser } from '@/tests/helpers/user.helper'; -import { cliSessions, cli_sessions_v2 } from '@kilocode/db/schema'; +import { + cliSessions, + cli_sessions_v2, + cloud_agent_sessions, + cloud_billing_sku, + container_usage_interval, +} from '@kilocode/db/schema'; async function insertAdmin(overrides: Parameters[0] = {}) { return insertTestUser({ is_admin: true, ...overrides }); @@ -42,6 +52,16 @@ describe('admin.sessionTraces authorization', () => { (caller: Awaited>) => caller.admin.sessionTraces.getMessages({ session_id: crypto.randomUUID() }), ], + [ + 'getContainerInfo', + (caller: Awaited>) => + caller.admin.sessionTraces.getContainerInfo({ session_id: crypto.randomUUID() }), + ], + [ + 'getContainerMetrics', + (caller: Awaited>) => + caller.admin.sessionTraces.getContainerMetrics({ session_id: crypto.randomUUID() }), + ], [ 'getApiConversationHistory', (caller: Awaited>) => @@ -171,4 +191,111 @@ describe('admin.sessionTraces authorization', () => { }); expect(mockFetchSessionSnapshot).toHaveBeenCalledWith(sessionId, owner.id); }); + + test('a session viewer can read Cloud Agent container identity, SKU, and recorded capacity', async () => { + const owner = await insertTestUser(); + const viewer = await insertAdmin({ can_view_sessions: true }); + const sessionId = `ses_${crypto.randomUUID()}`; + const cloudAgentSessionId = `agent_${crypto.randomUUID()}`; + const sandboxId = `ses-${'a'.repeat(48)}`; + await db.insert(cli_sessions_v2).values({ + session_id: sessionId, + kilo_user_id: owner.id, + cloud_agent_session_id: cloudAgentSessionId, + created_at: '2026-07-31T08:43:36.040Z', + updated_at: '2026-07-31T08:55:07.000Z', + }); + await db.insert(cloud_agent_sessions).values({ + cloud_agent_session_id: cloudAgentSessionId, + kilo_session_id: sessionId, + initial_message_id: `msg_${crypto.randomUUID()}`, + sandbox_id: sandboxId, + created_at: '2026-07-31T08:43:36.040Z', + }); + await db.insert(cloud_billing_sku).values({ + id: 'cloud-agent-small-test', + name: 'Cloud Agent Small', + unit: 'second', + rate_cents_per_unit: '0.001', + }); + await db.insert(container_usage_interval).values({ + id: `cloud-agent-next-sandbox-small-containment:${sandboxId}:1`, + service: 'cloud-agent-next-sandbox-small-containment', + instance_id: sandboxId, + start_epoch_ms: 1, + cloud_billing_sku_id: 'cloud-agent-small-test', + context_fingerprint: 'b'.repeat(64), + subject_type: 'user', + subject_id: owner.id, + actor_type: 'user', + actor_id: owner.id, + session_id: cloudAgentSessionId, + started_at: '2026-07-31T08:20:00.000Z', + last_seen_at: '2026-07-31T09:30:00.000Z', + metadata: { + durable_object_id: 'durable-object-id', + container_class: 'SandboxSmallContainment', + vcpu: '2', + memory_mib: '6144', + disk_mb: '10000', + }, + }); + + const caller = await createCallerForUser(viewer.id); + await expect( + caller.admin.sessionTraces.getContainerInfo({ session_id: sessionId }) + ).resolves.toMatchObject({ + cloudAgentSessionId, + sandboxId, + scope: 'isolated', + intervals: [ + { + cloudflareInstanceId: 'durable-object-id', + containerClass: 'SandboxSmallContainment', + sku: { id: 'cloud-agent-small-test', name: 'Cloud Agent Small' }, + capacity: { vcpu: 2, memoryBytes: 6_442_450_944, diskBytes: 10_000_000_000 }, + capacitySource: 'recorded', + }, + ], + }); + + let providerWindows: unknown; + await expect( + getSessionContainerMetrics(sessionId, async input => { + providerWindows = input.windows; + return { rows: [], partial: false, issues: [] }; + }) + ).resolves.toMatchObject({ available: true }); + expect(providerWindows).toEqual([ + { + key: `cloud-agent-next-sandbox-small-containment:${sandboxId}:1`, + instanceId: 'durable-object-id', + start: '2026-07-31T08:33:36.040Z', + end: '2026-07-31T09:05:07.000Z', + }, + ]); + + const info = await caller.admin.sessionTraces.getContainerInfo({ session_id: sessionId }); + expect(info).not.toBeNull(); + if (!info) throw new Error('Expected container info'); + await expect( + getSessionContainerMetricsForInfo({ + ...info, + windowStartAt: '2026-07-31T10:00:00.000Z', + windowEndAt: '2026-07-31T10:10:00.000Z', + }) + ).resolves.toEqual({ available: false, reason: 'no_overlapping_intervals' }); + + const childSessionId = `ses_${crypto.randomUUID()}`; + await db.insert(cli_sessions_v2).values({ + session_id: childSessionId, + kilo_user_id: owner.id, + cloud_agent_session_scope_id: cloudAgentSessionId, + created_at: '2026-07-31T08:44:00.000Z', + updated_at: '2026-07-31T08:50:00.000Z', + }); + await expect( + caller.admin.sessionTraces.getContainerInfo({ session_id: childSessionId }) + ).resolves.toMatchObject({ cloudAgentSessionId, sandboxId }); + }); }); diff --git a/apps/web/src/routers/admin/cloud-billing-skus-router.ts b/apps/web/src/routers/admin/cloud-billing-skus-router.ts index 69dda541f7..b86fcb212b 100644 --- a/apps/web/src/routers/admin/cloud-billing-skus-router.ts +++ b/apps/web/src/routers/admin/cloud-billing-skus-router.ts @@ -10,6 +10,7 @@ import { queryContainerUsageAnalytics, type ContainerUsageAnalyticsResult, } from '@/lib/cloudflare/container-usage-analytics'; +import { sharedContainerCapacity } from '@/lib/cloudflare/container-capacity'; import { cloud_billing_sku, container_usage_interval, @@ -109,8 +110,6 @@ const usageReconciliationSchema = z const BILLING_HEALTH_WINDOW_MS = 24 * 60 * 60 * 1_000; const STALE_OPEN_INTERVAL_MS = 15 * 60 * 1_000; const PROVIDER_BOUNDARY_PADDING_MS = 5_000; -const MEBIBYTE_BYTES = 1024 ** 2; -const MEGABYTE_BYTES = 1_000_000; const CAPACITY_CROSS_CHECK_RELATIVE_TOLERANCE = 1e-6; const COMPARISON_METHOD = 'Cloudflare memory and disk byte-seconds are queried with five seconds of boundary tolerance, normalized by configured instance capacity, and compared with accepted meter seconds for each exact meter run. CPU time is included as a secondary usage diagnostic.'; @@ -269,44 +268,6 @@ function providerIdentityForMeterRow(row: { }; } -// Cloud Agent values mirror services/cloud-agent-next/wrangler.jsonc. Gastown -// uses Cloudflare standard-4, documented as 12 GiB of provisioned memory. -type ProvisionedCapacity = { memoryBytes: number; diskBytes: number }; - -function provisionedCapacityForService(service: string): ProvisionedCapacity | null { - switch (service) { - case 'gastown': - case 'cloud-agent-next-sandbox': - case 'cloud-agent-next-sandbox-containment': - return { memoryBytes: 12_288 * MEBIBYTE_BYTES, diskBytes: 20_000 * MEGABYTE_BYTES }; - case 'cloud-agent-next-sandbox-small': - case 'cloud-agent-next-sandbox-dind': - case 'cloud-agent-next-sandbox-small-containment': - return { memoryBytes: 6_144 * MEBIBYTE_BYTES, diskBytes: 10_000 * MEGABYTE_BYTES }; - case 'cloud-agent-next-sandbox-code-review': - case 'cloud-agent-next-sandbox-code-review-containment': - return { memoryBytes: 4_096 * MEBIBYTE_BYTES, diskBytes: 8_000 * MEGABYTE_BYTES }; - default: - return null; - } -} - -function sharedProvisionedCapacity(services: Set): ProvisionedCapacity | null { - let shared: ProvisionedCapacity | null = null; - for (const service of services) { - const capacity = provisionedCapacityForService(service); - if (!capacity) return null; - if ( - shared && - (shared.memoryBytes !== capacity.memoryBytes || shared.diskBytes !== capacity.diskBytes) - ) { - return null; - } - shared = capacity; - } - return shared; -} - function addMeterReconciliationRow( rows: Map, meter: { @@ -412,7 +373,7 @@ function normalizedReconciliationRows( statusDetail = 'Cloudflare returned only part of the required provider data.'; } else { matchedProvider = candidates[0] ?? null; - const provisionedCapacity = sharedProvisionedCapacity(meter.services); + const provisionedCapacity = sharedContainerCapacity(meter.services); if (!provisionedCapacity) { status = 'comparison_unavailable'; statusDetail = 'The recorded service has no single verified provisioned-capacity mapping.'; diff --git a/apps/web/src/routers/admin/session-container-telemetry.ts b/apps/web/src/routers/admin/session-container-telemetry.ts new file mode 100644 index 0000000000..59ca8f64fb --- /dev/null +++ b/apps/web/src/routers/admin/session-container-telemetry.ts @@ -0,0 +1,345 @@ +import 'server-only'; + +import { db } from '@/lib/drizzle'; +import { isNewSession } from '@/lib/cloud-agent/session-type'; +import { + containerCapacityForService, + type ContainerCapacity, +} from '@/lib/cloudflare/container-capacity'; +import { + queryContainerMetricsAnalytics, + type ContainerMetricsResult, +} from '@/lib/cloudflare/container-metrics-analytics'; +import { + cliSessions, + cli_sessions_v2, + cloud_agent_session_runs, + cloud_agent_sessions, + cloud_billing_sku, + container_usage_interval, +} from '@kilocode/db/schema'; +import { and, asc, eq, gt, like, lt, or } from 'drizzle-orm'; +import * as z from 'zod'; + +const SESSION_METRICS_PADDING_MS = 10 * 60 * 1_000; + +const capacityMetadataSchema = z + .object({ + durable_object_id: z.string().min(1), + container_class: z.string().optional(), + vcpu: z.string().optional(), + memory_mib: z.string().optional(), + disk_mb: z.string().optional(), + }) + .passthrough(); + +type SessionReference = { + cloudAgentSessionId: string; + subjectType: 'user' | 'org'; + subjectId: string; + createdAt: string; + updatedAt: string; +}; + +export type SessionContainerInterval = { + id: string; + service: string; + sandboxId: string; + cloudflareInstanceId: string | null; + containerClass: string | null; + startedAt: string; + lastSeenAt: string; + stoppedAt: string | null; + status: 'open' | 'closed'; + closeReason: string | null; + exitCode: number | null; + sku: { + id: string; + name: string; + description: string | null; + }; + capacity: ContainerCapacity | null; + capacitySource: 'recorded' | 'configured' | null; +}; + +export type SessionContainerInfo = { + cloudAgentSessionId: string; + sandboxId: string | null; + scope: 'isolated' | 'shared' | 'unknown'; + windowStartAt: string; + windowEndAt: string; + intervals: SessionContainerInterval[]; + runs: Array<{ + messageId: string; + status: string; + queuedAt: string | null; + dispatchAcceptedAt: string | null; + agentActivityObservedAt: string | null; + terminalAt: string | null; + }>; +}; + +function iso(value: string | null): string | null { + return value ? new Date(value).toISOString() : null; +} + +async function resolveSessionReference(sessionId: string): Promise { + if (isNewSession(sessionId)) { + const [session] = await db + .select({ + cloudAgentSessionId: cli_sessions_v2.cloud_agent_session_id, + cloudAgentSessionScopeId: cli_sessions_v2.cloud_agent_session_scope_id, + kiloUserId: cli_sessions_v2.kilo_user_id, + organizationId: cli_sessions_v2.organization_id, + createdAt: cli_sessions_v2.created_at, + updatedAt: cli_sessions_v2.updated_at, + }) + .from(cli_sessions_v2) + .where(eq(cli_sessions_v2.session_id, sessionId)) + .limit(1); + const cloudAgentSessionId = session?.cloudAgentSessionId ?? session?.cloudAgentSessionScopeId; + if (!session || !cloudAgentSessionId) return null; + return { + cloudAgentSessionId, + subjectType: session.organizationId ? 'org' : 'user', + subjectId: session.organizationId ?? session.kiloUserId, + createdAt: new Date(session.createdAt).toISOString(), + updatedAt: new Date(session.updatedAt).toISOString(), + }; + } + + const [session] = await db + .select({ + cloudAgentSessionId: cliSessions.cloud_agent_session_id, + kiloUserId: cliSessions.kilo_user_id, + organizationId: cliSessions.organization_id, + createdAt: cliSessions.created_at, + updatedAt: cliSessions.updated_at, + }) + .from(cliSessions) + .where(eq(cliSessions.session_id, sessionId)) + .limit(1); + if (!session?.cloudAgentSessionId) return null; + return { + cloudAgentSessionId: session.cloudAgentSessionId, + subjectType: session.organizationId ? 'org' : 'user', + subjectId: session.organizationId ?? session.kiloUserId, + createdAt: new Date(session.createdAt).toISOString(), + updatedAt: new Date(session.updatedAt).toISOString(), + }; +} + +function recordedCapacity( + metadata: z.infer +): ContainerCapacity | null { + const vcpu = Number(metadata.vcpu); + const memoryMiB = Number(metadata.memory_mib); + const diskMB = Number(metadata.disk_mb); + if ( + !Number.isInteger(vcpu) || + vcpu <= 0 || + !Number.isInteger(memoryMiB) || + memoryMiB <= 0 || + !Number.isInteger(diskMB) || + diskMB <= 0 + ) { + return null; + } + return { + vcpu, + memoryBytes: memoryMiB * 1024 ** 2, + diskBytes: diskMB * 1_000_000, + }; +} + +export async function getSessionContainerInfo( + sessionId: string +): Promise { + const reference = await resolveSessionReference(sessionId); + if (!reference) return null; + + const [report] = await db + .select({ sandboxId: cloud_agent_sessions.sandbox_id }) + .from(cloud_agent_sessions) + .where(eq(cloud_agent_sessions.cloud_agent_session_id, reference.cloudAgentSessionId)) + .limit(1); + const sandboxId = report?.sandboxId ?? null; + const runRows = await db + .select({ + messageId: cloud_agent_session_runs.message_id, + status: cloud_agent_session_runs.status, + queuedAt: cloud_agent_session_runs.queued_at, + dispatchAcceptedAt: cloud_agent_session_runs.dispatch_accepted_at, + agentActivityObservedAt: cloud_agent_session_runs.agent_activity_observed_at, + terminalAt: cloud_agent_session_runs.terminal_at, + }) + .from(cloud_agent_session_runs) + .where(eq(cloud_agent_session_runs.cloud_agent_session_id, reference.cloudAgentSessionId)) + .orderBy(asc(cloud_agent_session_runs.queued_at), asc(cloud_agent_session_runs.message_id)); + const observedTimes = runRows.flatMap(run => + [run.queuedAt, run.dispatchAcceptedAt, run.agentActivityObservedAt, run.terminalAt].flatMap( + value => (value ? [Date.parse(value)] : []) + ) + ); + const windowStartMs = + Math.min(Date.parse(reference.createdAt), ...observedTimes) - SESSION_METRICS_PADDING_MS; + const windowEndMs = + Math.max(Date.parse(reference.updatedAt), ...observedTimes) + SESSION_METRICS_PADDING_MS; + const windowStartAt = new Date(windowStartMs).toISOString(); + const windowEndAt = new Date(windowEndMs).toISOString(); + const overlapsSessionWindow = and( + lt(container_usage_interval.started_at, windowEndAt), + gt(container_usage_interval.last_seen_at, windowStartAt) + ); + const intervalIdentityCondition = sandboxId + ? or( + eq(container_usage_interval.session_id, reference.cloudAgentSessionId), + eq(container_usage_interval.instance_id, sandboxId) + ) + : eq(container_usage_interval.session_id, reference.cloudAgentSessionId); + + const intervalRows = await db + .select({ + id: container_usage_interval.id, + service: container_usage_interval.service, + sandboxId: container_usage_interval.instance_id, + metadata: container_usage_interval.metadata, + startedAt: container_usage_interval.started_at, + lastSeenAt: container_usage_interval.last_seen_at, + stoppedAt: container_usage_interval.stopped_at, + status: container_usage_interval.status, + closeReason: container_usage_interval.close_reason, + exitCode: container_usage_interval.exit_code, + skuId: cloud_billing_sku.id, + skuName: cloud_billing_sku.name, + skuDescription: cloud_billing_sku.description, + }) + .from(container_usage_interval) + .innerJoin( + cloud_billing_sku, + eq(container_usage_interval.cloud_billing_sku_id, cloud_billing_sku.id) + ) + .where( + and( + eq(container_usage_interval.subject_type, reference.subjectType), + eq(container_usage_interval.subject_id, reference.subjectId), + like(container_usage_interval.service, 'cloud-agent-next%'), + overlapsSessionWindow, + intervalIdentityCondition + ) + ) + .orderBy(asc(container_usage_interval.started_at), asc(container_usage_interval.id)); + + const intervals = intervalRows.map(row => { + const parsedMetadata = capacityMetadataSchema.safeParse(row.metadata); + const metadata = parsedMetadata.success ? parsedMetadata.data : null; + const recorded = metadata ? recordedCapacity(metadata) : null; + const configured = containerCapacityForService(row.service); + return { + id: row.id, + service: row.service, + sandboxId: row.sandboxId, + cloudflareInstanceId: metadata?.durable_object_id ?? null, + containerClass: metadata?.container_class ?? null, + startedAt: new Date(row.startedAt).toISOString(), + lastSeenAt: new Date(row.lastSeenAt).toISOString(), + stoppedAt: iso(row.stoppedAt), + status: row.status, + closeReason: row.closeReason, + exitCode: row.exitCode, + sku: { id: row.skuId, name: row.skuName, description: row.skuDescription }, + capacity: recorded ?? configured, + capacitySource: recorded + ? ('recorded' as const) + : configured + ? ('configured' as const) + : null, + } satisfies SessionContainerInterval; + }); + + return { + cloudAgentSessionId: reference.cloudAgentSessionId, + sandboxId, + scope: !sandboxId + ? 'unknown' + : sandboxId.startsWith('ses-') || + sandboxId.startsWith('crv-') || + sandboxId.startsWith('dind-') + ? 'isolated' + : 'shared', + windowStartAt, + windowEndAt, + intervals, + runs: runRows.map(run => ({ + messageId: run.messageId, + status: run.status, + queuedAt: iso(run.queuedAt), + dispatchAcceptedAt: iso(run.dispatchAcceptedAt), + agentActivityObservedAt: iso(run.agentActivityObservedAt), + terminalAt: iso(run.terminalAt), + })), + }; +} + +export type SessionContainerMetrics = + | { + available: false; + reason: + | 'not_cloud_agent_session' + | 'no_container_intervals' + | 'no_provider_identity' + | 'no_overlapping_intervals' + | 'ambiguous_application'; + } + | ({ available: true } & ContainerMetricsResult); + +export async function getSessionContainerMetricsForInfo( + info: SessionContainerInfo, + queryMetrics: typeof queryContainerMetricsAnalytics = queryContainerMetricsAnalytics +): Promise { + if (info.intervals.length === 0) return { available: false, reason: 'no_container_intervals' }; + const intervalsWithProviderIdentity = info.intervals.filter( + ( + interval + ): interval is SessionContainerInterval & { + cloudflareInstanceId: string; + } => interval.cloudflareInstanceId !== null + ); + if (intervalsWithProviderIdentity.length === 0) { + return { available: false, reason: 'no_provider_identity' }; + } + const windows = intervalsWithProviderIdentity.flatMap(interval => { + const start = new Date( + Math.max(Date.parse(interval.startedAt), Date.parse(info.windowStartAt)) + ).toISOString(); + const end = new Date( + Math.min(Date.parse(interval.stoppedAt ?? interval.lastSeenAt), Date.parse(info.windowEndAt)) + ).toISOString(); + if (Date.parse(end) <= Date.parse(start)) return []; + return [ + { + key: interval.id, + instanceId: interval.cloudflareInstanceId, + start, + end, + }, + ]; + }); + if (windows.length === 0) return { available: false, reason: 'no_overlapping_intervals' }; + const metrics = await queryMetrics({ windows }); + const applicationIds = new Set(metrics.rows.map(row => row.applicationId)); + if (applicationIds.size > 1) { + return { available: false, reason: 'ambiguous_application' }; + } + return { available: true, ...metrics }; +} + +export async function getSessionContainerMetrics( + sessionId: string, + queryMetrics: typeof queryContainerMetricsAnalytics = queryContainerMetricsAnalytics +): Promise { + const info = await getSessionContainerInfo(sessionId); + return info + ? getSessionContainerMetricsForInfo(info, queryMetrics) + : { available: false, reason: 'not_cloud_agent_session' }; +} diff --git a/services/cloud-agent-next/src/container-capacity-parity.test.ts b/services/cloud-agent-next/src/container-capacity-parity.test.ts new file mode 100644 index 0000000000..4f7de40df8 --- /dev/null +++ b/services/cloud-agent-next/src/container-capacity-parity.test.ts @@ -0,0 +1,54 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { parse } from 'jsonc-parser'; +import { describe, expect, it } from 'vitest'; + +import { containerCapacityForService } from '../../../apps/web/src/lib/cloudflare/container-capacity.js'; +import { SANDBOX_CAPACITIES, type SandboxClassName } from './container-usage-context.js'; + +type WranglerContainer = { + class_name: SandboxClassName; + instance_type: { + vcpu: number; + memory_mib: number; + disk_mb: number; + }; +}; + +type WranglerConfig = { + containers: WranglerContainer[]; +}; + +const SERVICE_BY_CLASS: Record = { + Sandbox: 'cloud-agent-next-sandbox', + SandboxContainment: 'cloud-agent-next-sandbox-containment', + SandboxSmall: 'cloud-agent-next-sandbox-small', + SandboxSmallContainment: 'cloud-agent-next-sandbox-small-containment', + SandboxDIND: 'cloud-agent-next-sandbox-dind', + SandboxCodeReview: 'cloud-agent-next-sandbox-code-review', + SandboxCodeReviewContainment: 'cloud-agent-next-sandbox-code-review-containment', +}; + +describe('production container capacity parity', () => { + it('keeps Wrangler, usage metadata, and web reconciliation capacities aligned', () => { + const config = parse( + fs.readFileSync(path.join(process.cwd(), 'wrangler.jsonc'), 'utf8') + ) as WranglerConfig; + + expect(config.containers).toHaveLength(Object.keys(SANDBOX_CAPACITIES).length); + for (const container of config.containers) { + const expected = { + vcpu: container.instance_type.vcpu, + memoryMiB: container.instance_type.memory_mib, + diskMB: container.instance_type.disk_mb, + }; + expect(SANDBOX_CAPACITIES[container.class_name]).toEqual(expected); + expect(containerCapacityForService(SERVICE_BY_CLASS[container.class_name])).toEqual({ + vcpu: expected.vcpu, + memoryBytes: expected.memoryMiB * 1024 ** 2, + diskBytes: expected.diskMB * 1_000_000, + }); + } + }); +}); diff --git a/services/cloud-agent-next/src/container-usage-context.test.ts b/services/cloud-agent-next/src/container-usage-context.test.ts index bd9a0deb95..1d20d70601 100644 --- a/services/cloud-agent-next/src/container-usage-context.test.ts +++ b/services/cloud-agent-next/src/container-usage-context.test.ts @@ -5,6 +5,7 @@ import { assertSandboxBillingAllocation, buildSandboxBillingInput, configureSandboxBillingInput, + SANDBOX_CAPACITIES, SANDBOX_USAGE_SKUS, } from './container-usage-context.js'; @@ -31,6 +32,18 @@ describe('container usage context', () => { }); }); + it('snapshots the configured capacity for every sandbox class', () => { + expect(SANDBOX_CAPACITIES).toEqual({ + Sandbox: { vcpu: 4, memoryMiB: 12_288, diskMB: 20_000 }, + SandboxContainment: { vcpu: 4, memoryMiB: 12_288, diskMB: 20_000 }, + SandboxSmall: { vcpu: 2, memoryMiB: 6_144, diskMB: 10_000 }, + SandboxSmallContainment: { vcpu: 2, memoryMiB: 6_144, diskMB: 10_000 }, + SandboxDIND: { vcpu: 2, memoryMiB: 6_144, diskMB: 10_000 }, + SandboxCodeReview: { vcpu: 1, memoryMiB: 4_096, diskMB: 8_000 }, + SandboxCodeReviewContainment: { vcpu: 1, memoryMiB: 4_096, diskMB: 8_000 }, + }); + }); + it.each([ { name: 'personal human', diff --git a/services/cloud-agent-next/src/container-usage-context.ts b/services/cloud-agent-next/src/container-usage-context.ts index 1b48f9218f..72410211a7 100644 --- a/services/cloud-agent-next/src/container-usage-context.ts +++ b/services/cloud-agent-next/src/container-usage-context.ts @@ -20,6 +20,22 @@ export const SANDBOX_USAGE_SKUS = { } as const; export type SandboxClassName = keyof typeof SANDBOX_USAGE_SKUS; + +// Production values mirror this service's top-level wrangler.jsonc entries and +// apps/web/src/lib/cloudflare/container-capacity.ts. The parity test reads all three sources. +// Development intentionally uses different named instance types and does not query Analytics. +export const SANDBOX_CAPACITIES: Record< + SandboxClassName, + { vcpu: number; memoryMiB: number; diskMB: number } +> = { + Sandbox: { vcpu: 4, memoryMiB: 12_288, diskMB: 20_000 }, + SandboxContainment: { vcpu: 4, memoryMiB: 12_288, diskMB: 20_000 }, + SandboxSmall: { vcpu: 2, memoryMiB: 6_144, diskMB: 10_000 }, + SandboxSmallContainment: { vcpu: 2, memoryMiB: 6_144, diskMB: 10_000 }, + SandboxDIND: { vcpu: 2, memoryMiB: 6_144, diskMB: 10_000 }, + SandboxCodeReview: { vcpu: 1, memoryMiB: 4_096, diskMB: 8_000 }, + SandboxCodeReviewContainment: { vcpu: 1, memoryMiB: 4_096, diskMB: 8_000 }, +}; export type SandboxBillingInput = Omit & { sandboxId: SandboxId; }; diff --git a/services/cloud-agent-next/src/container-usage.test.ts b/services/cloud-agent-next/src/container-usage.test.ts index 3e105f11d9..3f28a36098 100644 --- a/services/cloud-agent-next/src/container-usage.test.ts +++ b/services/cloud-agent-next/src/container-usage.test.ts @@ -196,6 +196,9 @@ describe('MeteredSandbox', () => { origin: 'cloud-agent', container_class: 'SandboxSmallContainment', durable_object_id: 'do-id', + vcpu: '2', + memory_mib: '6144', + disk_mb: '10000', }, }) ); @@ -241,6 +244,9 @@ describe('MeteredSandbox', () => { metadata: { container_class: 'SandboxDIND', durable_object_id: 'do-id', + vcpu: '2', + memory_mib: '6144', + disk_mb: '10000', origin: 'cloud-agent', }, }) diff --git a/services/cloud-agent-next/src/container-usage.ts b/services/cloud-agent-next/src/container-usage.ts index 998b827359..c54fd561d8 100644 --- a/services/cloud-agent-next/src/container-usage.ts +++ b/services/cloud-agent-next/src/container-usage.ts @@ -17,6 +17,7 @@ import type { Env } from './types.js'; import { assertSandboxBillingAllocation, parseSandboxBillingInput, + SANDBOX_CAPACITIES, SANDBOX_USAGE_SKUS, type SandboxBillingInput, type SandboxClassName, @@ -376,6 +377,7 @@ export abstract class MeteredSandbox extends StockSandbox { input: SandboxBillingInput, trigger: ContainerStartTrigger ): Promise { + const capacity = SANDBOX_CAPACITIES[this.sandboxClassName]; const previousStartEpochMs = (await this.ctx.storage.get(LAST_START_EPOCH_STORAGE_KEY)) ?? -1; const startEpochMs = Math.max(Date.now(), previousStartEpochMs + 1); @@ -391,6 +393,9 @@ export abstract class MeteredSandbox extends StockSandbox { metadata: { container_class: this.sandboxClassName, durable_object_id: this.ctx.id.toString(), + vcpu: String(capacity.vcpu), + memory_mib: String(capacity.memoryMiB), + disk_mb: String(capacity.diskMB), ...(input.metadata?.origin ? { origin: input.metadata.origin } : {}), }, startEpochMs,