From 9902eb9b992cd92f1f695e728d66efbc324f4168 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:51:30 -0700 Subject: [PATCH 1/2] test: require bounded anonymous dev diagnostic ingestion --- web/tests/dev-backend-diagnostics.test.ts | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 web/tests/dev-backend-diagnostics.test.ts diff --git a/web/tests/dev-backend-diagnostics.test.ts b/web/tests/dev-backend-diagnostics.test.ts new file mode 100644 index 000000000000..95ea08dc353a --- /dev/null +++ b/web/tests/dev-backend-diagnostics.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from 'bun:test'; +import { makeDevBackendDiagnosticsHandler } from '../services/observability/devBackendDiagnostics'; +const now = 1_800_000_000_000; +const event = { eventId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', tag: 'pr-123-check', revision: 'a'.repeat(40), startedAtMs: now, durationMs: 25, attempt: 0, outcome: 'unreachable', errorNumber: -1004 }; +const request = (value: unknown) => new Request('https://cmux.test/api/observability/dev-backend', {method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify(value)}); +describe('dev app diagnostics', () => { + test('accepts a signed-out fixed-schema event only after delivery', async () => { + let delivered = false; + const handler = makeDevBackendDiagnosticsHandler({allowed:async()=>true,deliver:async events=>{expect(events[0]).toEqual(event);delivered=true;},now:()=>now}); + const response = await handler(request({version:1,events:[event]})); + expect(response.status).toBe(202);expect(delivered).toBe(true); + expect(await response.json()).toEqual({eventIds:[event.eventId]}); + }); + test('does not acknowledge an unavailable collector', async () => { + const handler = makeDevBackendDiagnosticsHandler({allowed:async()=>true,deliver:async()=>{throw Error('unavailable');},now:()=>now}); + expect((await handler(request({version:1,events:[event]}))).status).toBe(503); + }); + test('rejects arbitrary data, malformed tags and expired events before delivery', async () => { + let delivered = false; + const handler = makeDevBackendDiagnosticsHandler({allowed:async()=>true,deliver:async()=>{delivered=true;},now:()=>now}); + for (const invalid of [{...event,message:'private'}, {...event,tag:'../private'}, {...event,startedAtMs:0}, {...event,outcome:'anything'}]) { + expect((await handler(request({version:1,events:[invalid]}))).status).toBe(400); + } + expect(delivered).toBe(false); + }); + test('rate limits before parsing or delivery', async () => { + const handler = makeDevBackendDiagnosticsHandler({allowed:async()=>false,deliver:async()=>{throw Error('must not deliver');},now:()=>now}); + expect((await handler(request({version:1,events:[event]}))).status).toBe(429); + }); +}); From d9703b86e8d1b92dc6f2bfdb7b50f1ea4d7767b3 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:51:41 -0700 Subject: [PATCH 2/2] feat: collect app-owned development backend outcomes --- .../api/observability/dev-backend/route.ts | 18 ++++++ web/services/observability/DEV-BACKEND.md | 29 +++++++++ .../observability/devBackendDiagnostics.ts | 64 +++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 web/app/api/observability/dev-backend/route.ts create mode 100644 web/services/observability/DEV-BACKEND.md create mode 100644 web/services/observability/devBackendDiagnostics.ts diff --git a/web/app/api/observability/dev-backend/route.ts b/web/app/api/observability/dev-backend/route.ts new file mode 100644 index 000000000000..7249463d777e --- /dev/null +++ b/web/app/api/observability/dev-backend/route.ts @@ -0,0 +1,18 @@ +import { checkRateLimit } from '@vercel/firewall'; +import { deliverDevBackendDiagnostics, makeDevBackendDiagnosticsHandler } from '../../../../services/observability/devBackendDiagnostics'; + +// Dev-only payloads may arrive before sign-in, including when the tag's own +// backend is unreachable. Only this server holds the ingestion credential. +export const POST = makeDevBackendDiagnosticsHandler({ + allowed: async request => { + if (process.env.VERCEL !== '1') return true; + const rule = process.env.CMUX_CLOUD_DIAGNOSTICS_RATE_LIMIT_ID?.trim(); + if (!rule) throw new Error('dev_diagnostics_rate_limit_unconfigured'); + const result = await checkRateLimit(rule, {request}); + if (result.rateLimited || result.error === 'blocked') return false; + if (result.error) throw new Error('dev_diagnostics_rate_limit_unavailable'); + return true; + }, + deliver: events => deliverDevBackendDiagnostics(events), + now: Date.now, +}); diff --git a/web/services/observability/DEV-BACKEND.md b/web/services/observability/DEV-BACKEND.md new file mode 100644 index 000000000000..69364c355348 --- /dev/null +++ b/web/services/observability/DEV-BACKEND.md @@ -0,0 +1,29 @@ +# Development backend diagnostics + +Tagged DEBUG apps report real connection outcomes from `DevBackendStartup`. +`DevBackendDiagnostics` owns a private outbox of at most 100 records and drops +records older than 24 hours. Failed uploads retry every 60 seconds while the +app is alive. The next connection attempt resumes retained records after a +restart. Release builds and test hosts do not enable this sender; the normal +telemetry consent policy applies. + +`https://cmux.com/api/observability/dev-backend` is independent of GCP and does +not require sign-in. The public route accepts only a bounded versioned schema +and applies `CMUX_CLOUD_DIAGNOSTICS_RATE_LIMIT_ID` before parsing. Its fixed +Axiom destination is `cmux-dev-otel-traces`, authorized by the server-only +`CMUX_DEV_BACKEND_DIAGNOSTICS_TOKEN`. Neither token nor account information is +included in the app or event. Submitted tag/revision fields are untrusted +operational observations, not authenticated identities. + +The route returns an event-ID receipt only after Axiom acknowledges the full +batch. Ambiguous delivery may repeat an event; deduplicate `event_id` in +queries. `record_type == "dev_backend_app_outcome"` distinguishes these app +reports from legacy external monitor records. + +For a tagged isolated app, `debug.dev_backend.check` runs the same connection +path as the Cloud panel against that app's configured backend. It accepts no +URL argument and is denied by the default remote-relay policy. Inspect the +real connection result, the app's outbox and Axiom to verify delivery. + +No cron job, LaunchAgent, process scanner or independent host service is +installed by this implementation. diff --git a/web/services/observability/devBackendDiagnostics.ts b/web/services/observability/devBackendDiagnostics.ts new file mode 100644 index 000000000000..a4e7f9088a01 --- /dev/null +++ b/web/services/observability/devBackendDiagnostics.ts @@ -0,0 +1,64 @@ +import { z } from 'zod'; +import { readBoundedJsonObject } from '../apns/routePolicy'; + +const eventSchema = z.strictObject({ + eventId: z.uuid(), tag: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/), + revision: z.string().regex(/^(?:[0-9a-f]{7,64}|unknown)$/), + startedAtMs: z.number().int().nonnegative(), durationMs: z.number().int().min(0).max(3_600_000), + attempt: z.number().int().min(0).max(10_000), + outcome: z.enum(['ready', 'unreachable', 'timeout', 'startup_failed', 'invalid_response']), + errorNumber: z.number().int().min(-65_535).max(65_535).optional(), + httpStatus: z.number().int().min(100).max(599).optional(), +}); +const batchSchema = z.strictObject({version:z.literal(1),events:z.array(eventSchema).min(1).max(20)}); +export type DevBackendEvent = z.infer; + +export function makeDevBackendDiagnosticsHandler(dependencies: { + allowed: (request: Request) => Promise; + deliver: (events: DevBackendEvent[]) => Promise; + now: () => number; +}) { + return async (request: Request): Promise => { + try { + if (!await dependencies.allowed(request)) return response(429, {error:'rate_limited'}); + if (request.headers.get('content-type')?.split(';')[0].trim() !== 'application/json') return response(415, {error:'unsupported_content_type'}); + const encoding = request.headers.get('content-encoding'); + if (encoding && encoding !== 'identity') return response(415, {error:'unsupported_encoding'}); + const body = await readBoundedJsonObject(request, 16 * 1024); + if (!body.ok) return response(body.error === 'request_too_large' ? 413 : 400, {error:'invalid_diagnostics'}); + const batch = batchSchema.safeParse(body.value); + if (!batch.success) return response(400, {error:'invalid_diagnostics'}); + const now = dependencies.now(); + if (batch.data.events.some(event => event.startedAtMs < now - 86_400_000 || event.startedAtMs > now + 300_000)) return response(400, {error:'invalid_timestamp'}); + if (new Set(batch.data.events.map(event => event.eventId)).size !== batch.data.events.length) return response(400, {error:'duplicate_event'}); + await dependencies.deliver(batch.data.events); + return response(202, {eventIds:batch.data.events.map(event => event.eventId)}); + } catch { + return response(503, {error:'diagnostics_unavailable'}); + } + }; +} + +/** Anonymous operational observations: no account identity, free text or client-selected destination. */ +export async function deliverDevBackendDiagnostics(events: DevBackendEvent[], env = process.env, doFetch: typeof fetch = fetch) { + const token = env.CMUX_DEV_BACKEND_DIAGNOSTICS_TOKEN?.trim(); + if (!token) throw new Error('dev_diagnostics_unconfigured'); + const rows = events.map(event => ({ + _time: new Date(event.startedAtMs).toISOString(), event_id:event.eventId, + record_type:'dev_backend_app_outcome', source:'cmux-mac-dev', + client_tag:event.tag, client_revision:event.revision, + outcome:event.outcome, duration_ms:event.durationMs, attempt:event.attempt, + error_number:event.errorNumber, http_status:event.httpStatus, + })); + const result = await doFetch('https://api.axiom.co/v1/datasets/cmux-dev-otel-traces/ingest', { + method:'POST',redirect:'error',signal:AbortSignal.timeout(10_000), + headers:{authorization:`Bearer ${token}`,'content-type':'application/json'},body:JSON.stringify(rows), + }); + if (!result.ok) throw new Error('dev_diagnostics_delivery_failed'); + const receipt = await result.json() as {ingested?:number;failed?:number}; + if (receipt.ingested !== rows.length || Number(receipt.failed ?? 0) !== 0) throw new Error('dev_diagnostics_partial_delivery'); +} + +function response(status: number, body: unknown): Response { + return Response.json(body, {status,headers:{'cache-control':'no-store', ...(status === 429 || status === 503 ? {'retry-after':'60'} : {})}}); +}