Add bounded collector for dev app backend diagnostics

web/app/api/observability/dev-backend/route.ts

@@ -0,0 +1,18 @@
+import { checkRateLimit } from '@vercel/firewall';
+import { deliverDevBackendDiagnostics, makeDevBackendDiagnosticsHandler } from '../../../../services/observability/devBackendDiagnostics';
+
+// Dev-only payloads may arrive before sign-in, including when the tag's own
+// backend is unreachable. Only this server holds the ingestion credential.
+export const POST = makeDevBackendDiagnosticsHandler({
+  allowed: async request => {
+    if (process.env.VERCEL !== '1') return true;
+    const rule = process.env.CMUX_CLOUD_DIAGNOSTICS_RATE_LIMIT_ID?.trim();
+    if (!rule) throw new Error('dev_diagnostics_rate_limit_unconfigured');
+    const result = await checkRateLimit(rule, {request});
+    if (result.rateLimited || result.error === 'blocked') return false;
+    if (result.error) throw new Error('dev_diagnostics_rate_limit_unavailable');
+    return true;
+  },
+  deliver: events => deliverDevBackendDiagnostics(events),
+  now: Date.now,
+});

web/services/observability/DEV-BACKEND.md

@@ -0,0 +1,29 @@
+# Development backend diagnostics
+
+Tagged DEBUG apps report real connection outcomes from `DevBackendStartup`.
+`DevBackendDiagnostics` owns a private outbox of at most 100 records and drops
+records older than 24 hours. Failed uploads retry every 60 seconds while the
+app is alive. The next connection attempt resumes retained records after a
+restart. Release builds and test hosts do not enable this sender; the normal
+telemetry consent policy applies.
+
+`https://cmux.com/api/observability/dev-backend` is independent of GCP and does
+not require sign-in. The public route accepts only a bounded versioned schema
+and applies `CMUX_CLOUD_DIAGNOSTICS_RATE_LIMIT_ID` before parsing. Its fixed
+Axiom destination is `cmux-dev-otel-traces`, authorized by the server-only
+`CMUX_DEV_BACKEND_DIAGNOSTICS_TOKEN`. Neither token nor account information is
+included in the app or event. Submitted tag/revision fields are untrusted
+operational observations, not authenticated identities.
+
+The route returns an event-ID receipt only after Axiom acknowledges the full
+batch. Ambiguous delivery may repeat an event; deduplicate `event_id` in
+queries. `record_type == "dev_backend_app_outcome"` distinguishes these app
+reports from legacy external monitor records.
+
+For a tagged isolated app, `debug.dev_backend.check` runs the same connection
+path as the Cloud panel against that app's configured backend. It accepts no
+URL argument and is denied by the default remote-relay policy. Inspect the
+real connection result, the app's outbox and Axiom to verify delivery.
+
+No cron job, LaunchAgent, process scanner or independent host service is
+installed by this implementation.

web/services/observability/devBackendDiagnostics.ts

@@ -0,0 +1,64 @@
+import { z } from 'zod';
+import { readBoundedJsonObject } from '../apns/routePolicy';
+
+const eventSchema = z.strictObject({
+  eventId: z.uuid(), tag: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/),
+  revision: z.string().regex(/^(?:[0-9a-f]{7,64}|unknown)$/),
+  startedAtMs: z.number().int().nonnegative(), durationMs: z.number().int().min(0).max(3_600_000),
+  attempt: z.number().int().min(0).max(10_000),
+  outcome: z.enum(['ready', 'unreachable', 'timeout', 'startup_failed', 'invalid_response']),
+  errorNumber: z.number().int().min(-65_535).max(65_535).optional(),
+  httpStatus: z.number().int().min(100).max(599).optional(),
+});
+const batchSchema = z.strictObject({version:z.literal(1),events:z.array(eventSchema).min(1).max(20)});
+export type DevBackendEvent = z.infer<typeof eventSchema>;
+
+export function makeDevBackendDiagnosticsHandler(dependencies: {
+  allowed: (request: Request) => Promise<boolean>;
+  deliver: (events: DevBackendEvent[]) => Promise<void>;
+  now: () => number;
+}) {
+  return async (request: Request): Promise<Response> => {
+    try {
+      if (!await dependencies.allowed(request)) return response(429, {error:'rate_limited'});
+      if (request.headers.get('content-type')?.split(';')[0].trim() !== 'application/json') return response(415, {error:'unsupported_content_type'});
+      const encoding = request.headers.get('content-encoding');
+      if (encoding && encoding !== 'identity') return response(415, {error:'unsupported_encoding'});
+      const body = await readBoundedJsonObject(request, 16 * 1024);
+      if (!body.ok) return response(body.error === 'request_too_large' ? 413 : 400, {error:'invalid_diagnostics'});
+      const batch = batchSchema.safeParse(body.value);
+      if (!batch.success) return response(400, {error:'invalid_diagnostics'});
+      const now = dependencies.now();
+      if (batch.data.events.some(event => event.startedAtMs < now - 86_400_000 || event.startedAtMs > now + 300_000)) return response(400, {error:'invalid_timestamp'});
+      if (new Set(batch.data.events.map(event => event.eventId)).size !== batch.data.events.length) return response(400, {error:'duplicate_event'});
+      await dependencies.deliver(batch.data.events);
+      return response(202, {eventIds:batch.data.events.map(event => event.eventId)});
+    } catch {
+      return response(503, {error:'diagnostics_unavailable'});
+    }
+  };
+}
+
+/** Anonymous operational observations: no account identity, free text or client-selected destination. */
+export async function deliverDevBackendDiagnostics(events: DevBackendEvent[], env = process.env, doFetch: typeof fetch = fetch) {
+  const token = env.CMUX_DEV_BACKEND_DIAGNOSTICS_TOKEN?.trim();
+  if (!token) throw new Error('dev_diagnostics_unconfigured');
+  const rows = events.map(event => ({
+    _time: new Date(event.startedAtMs).toISOString(), event_id:event.eventId,
+    record_type:'dev_backend_app_outcome', source:'cmux-mac-dev',
+    client_tag:event.tag, client_revision:event.revision,
+    outcome:event.outcome, duration_ms:event.durationMs, attempt:event.attempt,
+    error_number:event.errorNumber, http_status:event.httpStatus,
+  }));
+  const result = await doFetch('https://api.axiom.co/v1/datasets/cmux-dev-otel-traces/ingest', {
+    method:'POST',redirect:'error',signal:AbortSignal.timeout(10_000),
+    headers:{authorization:`Bearer ${token}`,'content-type':'application/json'},body:JSON.stringify(rows),
+  });
+  if (!result.ok) throw new Error('dev_diagnostics_delivery_failed');
+  const receipt = await result.json() as {ingested?:number;failed?:number};
+  if (receipt.ingested !== rows.length || Number(receipt.failed ?? 0) !== 0) throw new Error('dev_diagnostics_partial_delivery');
+}
+
+function response(status: number, body: unknown): Response {
+  return Response.json(body, {status,headers:{'cache-control':'no-store', ...(status === 429 || status === 503 ? {'retry-after':'60'} : {})}});
+}

web/tests/dev-backend-diagnostics.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, test } from 'bun:test';
+import { makeDevBackendDiagnosticsHandler } from '../services/observability/devBackendDiagnostics';
+const now = 1_800_000_000_000;
+const event = { eventId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', tag: 'pr-123-check', revision: 'a'.repeat(40), startedAtMs: now, durationMs: 25, attempt: 0, outcome: 'unreachable', errorNumber: -1004 };
+const request = (value: unknown) => new Request('https://cmux.test/api/observability/dev-backend', {method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify(value)});
+describe('dev app diagnostics', () => {
+  test('accepts a signed-out fixed-schema event only after delivery', async () => {
+    let delivered = false;
+    const handler = makeDevBackendDiagnosticsHandler({allowed:async()=>true,deliver:async events=>{expect(events[0]).toEqual(event);delivered=true;},now:()=>now});
+    const response = await handler(request({version:1,events:[event]}));
+    expect(response.status).toBe(202);expect(delivered).toBe(true);
+    expect(await response.json()).toEqual({eventIds:[event.eventId]});
+  });
+  test('does not acknowledge an unavailable collector', async () => {
+    const handler = makeDevBackendDiagnosticsHandler({allowed:async()=>true,deliver:async()=>{throw Error('unavailable');},now:()=>now});
+    expect((await handler(request({version:1,events:[event]}))).status).toBe(503);
+  });
+  test('rejects arbitrary data, malformed tags and expired events before delivery', async () => {
+    let delivered = false;
+    const handler = makeDevBackendDiagnosticsHandler({allowed:async()=>true,deliver:async()=>{delivered=true;},now:()=>now});
+    for (const invalid of [{...event,message:'private'}, {...event,tag:'../private'}, {...event,startedAtMs:0}, {...event,outcome:'anything'}]) {
+      expect((await handler(request({version:1,events:[invalid]}))).status).toBe(400);
+    }
+    expect(delivered).toBe(false);
+  });
+  test('rate limits before parsing or delivery', async () => {
+    const handler = makeDevBackendDiagnosticsHandler({allowed:async()=>false,deliver:async()=>{throw Error('must not deliver');},now:()=>now});
+    expect((await handler(request({version:1,events:[event]}))).status).toBe(429);
+  });
+});