From 38db74c457a6d9e8723a26676d050bdd67f6dbb2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 4 Aug 2026 23:24:59 +0000 Subject: [PATCH 1/6] Add component-level /health/components endpoint to the main worker --- .../health-components-handler.node.test.ts | 131 +++++++++++++++++ .../src/app/handlers/health-components.ts | 138 ++++++++++++++++++ packages/worker/src/app/router.ts | 2 + packages/worker/src/app/routes.ts | 1 + 4 files changed, 272 insertions(+) create mode 100644 packages/worker/src/app/handlers/health-components-handler.node.test.ts create mode 100644 packages/worker/src/app/handlers/health-components.ts diff --git a/packages/worker/src/app/handlers/health-components-handler.node.test.ts b/packages/worker/src/app/handlers/health-components-handler.node.test.ts new file mode 100644 index 0000000000..8cb1672253 --- /dev/null +++ b/packages/worker/src/app/handlers/health-components-handler.node.test.ts @@ -0,0 +1,131 @@ +import { expect, test, vi } from 'vitest' +import { RequestContext } from 'remix/router' +import { + collectHealthComponents, + createHealthComponentsHandler, + healthComponentIds, + type HealthComponentsReport, +} from '#app/handlers/health-components.ts' + +function createHealthyBindings() { + return { + APP_COMMIT_SHA: 'abc123', + APP_DB: { + prepare: () => ({ first: async () => ({ 1: 1 }) }), + } as unknown as D1Database, + AUDIT_DB: { + prepare: () => ({ first: async () => ({ 1: 1 }) }), + } as unknown as D1Database, + OAUTH_KV: { get: async () => null } as unknown as KVNamespace, + COMMUNITY_ASSETS: { head: async () => null } as unknown as R2Bucket, + } +} + +function createRequestContext() { + return new RequestContext( + new Request('https://example.com/health/components'), + ) +} + +test('collectHealthComponents reports every component healthy with latency', async () => { + const report = await collectHealthComponents(createHealthyBindings()) + + expect(report.ok).toBe(true) + expect(report.commitSha).toBe('abc123') + expect(report.components.map((component) => component.id)).toEqual([ + ...healthComponentIds, + ]) + for (const component of report.components) { + expect(component.ok).toBe(true) + expect(component.latencyMs).toBeGreaterThanOrEqual(0) + expect(component.error).toBeUndefined() + } +}) + +test('collectHealthComponents marks a throwing binding as failed without failing the rest', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + const bindings = createHealthyBindings() + bindings.APP_DB = { + prepare: () => ({ + first: async () => { + throw new Error('database is unavailable') + }, + }), + } as unknown as D1Database + const report = await collectHealthComponents(bindings) + + expect(report.ok).toBe(false) + const appDb = report.components.find( + (component) => component.id === 'app_db', + ) + expect(appDb).toMatchObject({ ok: false, error: 'error' }) + const others = report.components.filter( + (component) => component.id !== 'app_db', + ) + for (const component of others) { + expect(component.ok).toBe(true) + } + } finally { + warnSpy.mockRestore() + } +}) + +test('collectHealthComponents marks missing bindings as unavailable', async () => { + const report = await collectHealthComponents({ + APP_COMMIT_SHA: undefined, + }) + + expect(report.ok).toBe(false) + expect(report.commitSha).toBeNull() + for (const component of report.components) { + expect(component).toMatchObject({ ok: false, error: 'unavailable' }) + } +}) + +test('handler returns 200 when healthy and memoizes results within the cache window', async () => { + let prepareCalls = 0 + const bindings = createHealthyBindings() + bindings.APP_DB = { + prepare: () => { + prepareCalls += 1 + return { first: async () => ({ 1: 1 }) } + }, + } as unknown as D1Database + const handler = createHealthComponentsHandler(bindings) + + const first = await handler.handler(createRequestContext()) + const second = await handler.handler(createRequestContext()) + + expect(first.status).toBe(200) + expect(first.headers.get('Cache-Control')).toBe('no-store') + expect(second.status).toBe(200) + expect(prepareCalls).toBe(1) + + const body = (await first.json()) as HealthComponentsReport + expect(body.ok).toBe(true) + expect(body.checkedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/) +}) + +test('handler returns 503 when any component fails', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + const bindings = createHealthyBindings() + bindings.OAUTH_KV = { + get: async () => { + throw new Error('kv is down') + }, + } as unknown as KVNamespace + const handler = createHealthComponentsHandler(bindings) + + const response = await handler.handler(createRequestContext()) + + expect(response.status).toBe(503) + const body = (await response.json()) as HealthComponentsReport + expect(body.ok).toBe(false) + const kv = body.components.find((component) => component.id === 'kv') + expect(kv).toMatchObject({ ok: false, error: 'error' }) + } finally { + warnSpy.mockRestore() + } +}) diff --git a/packages/worker/src/app/handlers/health-components.ts b/packages/worker/src/app/handlers/health-components.ts new file mode 100644 index 0000000000..ea5e0f7df6 --- /dev/null +++ b/packages/worker/src/app/handlers/health-components.ts @@ -0,0 +1,138 @@ +import { type Action } from 'remix/router' +import { type routes } from '#app/routes.ts' +import { type AppEnv } from '#worker/env-schema.ts' + +/** + * Component-level health used by the public status page prober + * (`packages/status`). Each check is a cheap read against one storage binding, + * so a failing component points at that subsystem rather than the worker as a + * whole. Results are memoized briefly so public traffic cannot amplify load on + * the underlying bindings. + */ + +const componentCheckTimeoutMs = 3_000 +const resultCacheTtlMs = 10_000 + +export const healthComponentIds = [ + 'app_db', + 'audit_db', + 'kv', + 'assets', +] as const + +export type HealthComponentId = (typeof healthComponentIds)[number] + +export type HealthComponentResult = { + id: HealthComponentId + ok: boolean + latencyMs: number + error?: 'timeout' | 'unavailable' | 'error' +} + +export type HealthComponentsReport = { + ok: boolean + commitSha: string | null + checkedAt: string + components: Array +} + +type HealthComponentsEnv = { + APP_COMMIT_SHA: AppEnv['APP_COMMIT_SHA'] + APP_DB?: D1Database + AUDIT_DB?: D1Database + OAUTH_KV?: KVNamespace + COMMUNITY_ASSETS?: R2Bucket +} + +async function runComponentCheck( + id: HealthComponentId, + check: (() => Promise) | null, +): Promise { + const startedAt = Date.now() + if (!check) { + return { id, ok: false, latencyMs: 0, error: 'unavailable' } + } + let timeoutHandle: ReturnType | undefined + const timeout = new Promise<'timeout'>((resolve) => { + timeoutHandle = setTimeout( + () => resolve('timeout'), + componentCheckTimeoutMs, + ) + }) + try { + const outcome = await Promise.race([ + check().then(() => 'ok' as const), + timeout, + ]) + const latencyMs = Date.now() - startedAt + if (outcome === 'timeout') { + console.warn('health-component-timeout', JSON.stringify({ id })) + return { id, ok: false, latencyMs, error: 'timeout' } + } + return { id, ok: true, latencyMs } + } catch (error) { + const latencyMs = Date.now() - startedAt + console.warn( + 'health-component-failed', + JSON.stringify({ + id, + message: error instanceof Error ? error.message : String(error), + }), + ) + return { id, ok: false, latencyMs, error: 'error' } + } finally { + clearTimeout(timeoutHandle) + } +} + +export async function collectHealthComponents( + env: HealthComponentsEnv, +): Promise { + const appDb = env.APP_DB + const auditDb = env.AUDIT_DB + const oauthKv = env.OAUTH_KV + const assets = env.COMMUNITY_ASSETS + const components = await Promise.all([ + runComponentCheck( + 'app_db', + appDb ? () => appDb.prepare('SELECT 1').first() : null, + ), + runComponentCheck( + 'audit_db', + auditDb ? () => auditDb.prepare('SELECT 1').first() : null, + ), + runComponentCheck( + 'kv', + oauthKv ? () => oauthKv.get('health-component-probe') : null, + ), + runComponentCheck( + 'assets', + assets ? () => assets.head('health-component-probe') : null, + ), + ]) + return { + ok: components.every((component) => component.ok), + commitSha: env.APP_COMMIT_SHA ?? null, + checkedAt: new Date().toISOString(), + components, + } +} + +export function createHealthComponentsHandler(appEnv: HealthComponentsEnv) { + let cached: { report: HealthComponentsReport; expiresAt: number } | null = + null + return { + middleware: [], + async handler() { + const now = Date.now() + if (!cached || cached.expiresAt <= now) { + const report = await collectHealthComponents(appEnv) + cached = { report, expiresAt: now + resultCacheTtlMs } + } + return Response.json(cached.report, { + status: cached.report.ok ? 200 : 503, + headers: { 'Cache-Control': 'no-store' }, + }) + }, + } satisfies Action +} diff --git a/packages/worker/src/app/router.ts b/packages/worker/src/app/router.ts index 64167444fd..0b4450a839 100644 --- a/packages/worker/src/app/router.ts +++ b/packages/worker/src/app/router.ts @@ -162,6 +162,7 @@ import { createBlogRssHandler, } from '#app/handlers/blog.tsx' import { createHealthHandler } from '#app/handlers/health.ts' +import { createHealthComponentsHandler } from '#app/handlers/health-components.ts' import { createSentryTunnelHandler } from '#app/handlers/sentry-tunnel.ts' import { createHomeHandler } from '#app/handlers/home.ts' import { createLoginHandler } from '#app/handlers/login.ts' @@ -213,6 +214,7 @@ export function createAppRouter(env: Env) { actions: { home: createHomeHandler(env), health: createHealthHandler(env), + healthComponents: createHealthComponentsHandler(env), sentryTunnel: createSentryTunnelHandler(env), login: createLoginHandler(env), ogPageImage: createOgPageImageHandler(env), diff --git a/packages/worker/src/app/routes.ts b/packages/worker/src/app/routes.ts index d6df46c433..2674a24f53 100644 --- a/packages/worker/src/app/routes.ts +++ b/packages/worker/src/app/routes.ts @@ -137,6 +137,7 @@ export const routes = route({ timeline: '/timeline', timelineApi: '/timeline.json', health: '/health', + healthComponents: '/health/components', sentryTunnel: post('/sentry-tunnel'), login: '/login', ogPageImage: '/og/:page.png', From c5dd9e4d777523d44f3d69aaec9296f981b29332 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 4 Aug 2026 23:25:03 +0000 Subject: [PATCH 2/6] Add packages/status: public status page worker with prober, incidents, and alert email policy --- packages/status/alert-email.ts | 80 ++++ packages/status/email-policy.node.test.ts | 127 ++++++ packages/status/email-policy.ts | 129 ++++++ packages/status/incidents.node.test.ts | 71 ++++ packages/status/incidents.ts | 71 ++++ packages/status/package.json | 15 + packages/status/probes.node.test.ts | 134 ++++++ packages/status/probes.ts | 210 ++++++++++ packages/status/readme.md | 35 ++ packages/status/status-page.node.test.ts | 108 +++++ packages/status/status-page.ts | 199 +++++++++ packages/status/status-store.ts | 475 ++++++++++++++++++++++ packages/status/status-types.ts | 67 +++ packages/status/tsconfig.json | 14 + packages/status/vitest.config.ts | 8 + packages/status/worker.ts | 42 ++ packages/status/wrangler.jsonc | 50 +++ 17 files changed, 1835 insertions(+) create mode 100644 packages/status/alert-email.ts create mode 100644 packages/status/email-policy.node.test.ts create mode 100644 packages/status/email-policy.ts create mode 100644 packages/status/incidents.node.test.ts create mode 100644 packages/status/incidents.ts create mode 100644 packages/status/package.json create mode 100644 packages/status/probes.node.test.ts create mode 100644 packages/status/probes.ts create mode 100644 packages/status/readme.md create mode 100644 packages/status/status-page.node.test.ts create mode 100644 packages/status/status-page.ts create mode 100644 packages/status/status-store.ts create mode 100644 packages/status/status-types.ts create mode 100644 packages/status/tsconfig.json create mode 100644 packages/status/vitest.config.ts create mode 100644 packages/status/worker.ts create mode 100644 packages/status/wrangler.jsonc diff --git a/packages/status/alert-email.ts b/packages/status/alert-email.ts new file mode 100644 index 0000000000..b912aa7c93 --- /dev/null +++ b/packages/status/alert-email.ts @@ -0,0 +1,80 @@ +/** + * Sends operator alert email through the Cloudflare Email REST API — the same + * mechanism the main worker uses for ops alerts. When credentials are absent + * (local dev), the send is skipped but reported so the caller can keep its + * notification state machine consistent. + */ + +const defaultApiBaseUrl = 'https://api.cloudflare.com' + +export type AlertEmailConfig = { + accountId?: string + apiToken?: string + apiBaseUrl?: string + fetcher?: typeof fetch +} + +export type AlertEmailMessage = { + from: string + to: string + subject: string + text: string + html: string +} + +export type AlertEmailResult = { + delivered: boolean + skipped?: boolean + error?: string +} + +export async function sendAlertEmail( + config: AlertEmailConfig, + message: AlertEmailMessage, +): Promise { + const accountId = config.accountId?.trim() + const apiToken = config.apiToken?.trim() + if (!accountId || !apiToken) { + console.info( + 'status-alert-email-unconfigured', + JSON.stringify({ subject: message.subject }), + ) + return { delivered: false, skipped: true } + } + const apiBaseUrl = config.apiBaseUrl?.trim() || defaultApiBaseUrl + const endpoint = new URL( + `client/v4/accounts/${accountId}/email/sending/send`, + apiBaseUrl.endsWith('/') ? apiBaseUrl : `${apiBaseUrl}/`, + ) + const fetcher = config.fetcher ?? fetch + let response: Response + try { + response = await fetcher(endpoint.toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(message), + }) + } catch (error) { + const detail = error instanceof Error ? error.message : String(error) + console.warn('status-alert-email-request-failed', detail) + return { delivered: false, error: detail } + } + const payload = (await response.json().catch(() => null)) as { + success?: boolean + errors?: Array<{ message?: string }> + } | null + if (!response.ok || payload?.success !== true) { + const detail = + payload?.errors?.[0]?.message ?? + `Cloudflare Email API returned HTTP ${response.status}.` + console.warn( + 'status-alert-email-failed', + JSON.stringify({ status: response.status, detail }), + ) + return { delivered: false, error: detail } + } + return { delivered: true } +} diff --git a/packages/status/email-policy.node.test.ts b/packages/status/email-policy.node.test.ts new file mode 100644 index 0000000000..673d205079 --- /dev/null +++ b/packages/status/email-policy.node.test.ts @@ -0,0 +1,127 @@ +import { expect, test } from 'vitest' +import { + composeStatusEmail, + decideStatusEmail, + defaultDailyEmailLimit, + reminderIntervalMs, + type EmailPolicyInput, + type OpenIncidentSummary, +} from './email-policy.ts' + +const baseNow = Date.UTC(2026, 7, 4, 12, 0, 0) + +function openIncident( + overrides: Partial = {}, +): OpenIncidentSummary { + return { + component: 'app', + startedAt: baseNow - 5 * 60 * 1000, + detail: 'HTTP 500', + ...overrides, + } +} + +function input(overrides: Partial = {}): EmailPolicyInput { + return { + now: baseNow, + openIncidents: [], + lastNotifiedState: 'ok', + lastEmailSentAt: null, + emailsSentToday: 0, + dailyLimit: defaultDailyEmailLimit, + ...overrides, + } +} + +test('first incident of an episode sends the incident email', () => { + const decision = decideStatusEmail(input({ openIncidents: [openIncident()] })) + expect(decision).toEqual({ kind: 'incident_opened' }) +}) + +test('further incidents while one is already announced stay paused', () => { + const decision = decideStatusEmail( + input({ + openIncidents: [openIncident(), openIncident({ component: 'kv' })], + lastNotifiedState: 'incident', + lastEmailSentAt: baseNow - 60 * 60 * 1000, + emailsSentToday: 1, + }), + ) + expect(decision).toBeNull() +}) + +test('a reminder goes out once 24 hours pass without an email', () => { + const decision = decideStatusEmail( + input({ + openIncidents: [openIncident()], + lastNotifiedState: 'incident', + lastEmailSentAt: baseNow - reminderIntervalMs, + }), + ) + expect(decision).toEqual({ kind: 'daily_reminder' }) +}) + +test('all clear sends once every incident is resolved', () => { + const decision = decideStatusEmail( + input({ + lastNotifiedState: 'incident', + lastEmailSentAt: baseNow - 30 * 60 * 1000, + emailsSentToday: 1, + }), + ) + expect(decision).toEqual({ kind: 'all_clear' }) +}) + +test('no email when everything is healthy and already announced', () => { + expect(decideStatusEmail(input())).toBeNull() +}) + +test('the daily cap suppresses every kind of email', () => { + const capped = { emailsSentToday: defaultDailyEmailLimit } + expect( + decideStatusEmail(input({ ...capped, openIncidents: [openIncident()] })), + ).toBeNull() + expect( + decideStatusEmail(input({ ...capped, lastNotifiedState: 'incident' })), + ).toBeNull() +}) + +test('a capped all-clear is sent on the first uncapped tick', () => { + // While capped, lastNotifiedState stays 'incident'. Next day the counter + // resets and the pending all-clear goes out. + const decision = decideStatusEmail( + input({ + lastNotifiedState: 'incident', + lastEmailSentAt: baseNow - 6 * 60 * 60 * 1000, + emailsSentToday: 0, + }), + ) + expect(decision).toEqual({ kind: 'all_clear' }) +}) + +test('composed emails carry component names, status page link, and escape html', () => { + const incident = openIncident({ detail: '' }) + for (const kind of [ + 'incident_opened', + 'daily_reminder', + 'all_clear', + ] as const) { + const content = composeStatusEmail({ + kind, + openIncidents: [incident], + statusPageUrl: 'https://status.heykody.dev', + now: baseNow, + }) + expect(content.subject).toContain('[kody status]') + expect(content.text).toContain('https://status.heykody.dev') + expect(content.html).toContain('https://status.heykody.dev') + expect(content.html).not.toContain('