diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 30a11c98dd..ccd784b251 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -34,6 +34,7 @@ jobs: should_deploy: ${{ steps.sha_guard.outputs.should_deploy }} deploy_backup_control_plane: ${{ steps.sha_guard.outputs.deploy_backup_control_plane }} + deploy_status_worker: ${{ steps.sha_guard.outputs.deploy_status_worker }} steps: - name: πŸ“¦ Checkout uses: actions/checkout@v6.0.2 @@ -68,6 +69,7 @@ jobs: if [ "$HEAD_SHA" != "$DEPLOY_SHA" ]; then echo "should_deploy=false" >> "$GITHUB_OUTPUT" echo "deploy_backup_control_plane=false" >> "$GITHUB_OUTPUT" + echo "deploy_status_worker=false" >> "$GITHUB_OUTPUT" echo "Skipping deploy: target SHA is not current main HEAD." echo "target=$DEPLOY_SHA" echo "head=$HEAD_SHA" @@ -75,12 +77,14 @@ jobs: fi echo "should_deploy=true" >> "$GITHUB_OUTPUT" - # Always redeploy the DR control plane on manual workflow_dispatch. - # On push-driven deploys, only when control-plane (or shared backup - # contracts) changed in the commit being deployed. + # Always redeploy the DR control plane and status worker on manual + # workflow_dispatch. On push-driven deploys, only when their files + # changed in the commit being deployed. if [ "$EVENT_NAME" = "workflow_dispatch" ]; then echo "deploy_backup_control_plane=true" >> "$GITHUB_OUTPUT" + echo "deploy_status_worker=true" >> "$GITHUB_OUTPUT" echo "Backup control plane deploy: forced (workflow_dispatch)." + echo "Status worker deploy: forced (workflow_dispatch)." exit 0 fi @@ -93,6 +97,15 @@ jobs: echo "Backup control plane deploy: skipped (no relevant path changes)." fi + if git diff --name-only "${DEPLOY_SHA}^" "${DEPLOY_SHA}" | grep -E \ + '^(\.github/workflows/deploy\.yml|packages/status/)'; then + echo "deploy_status_worker=true" >> "$GITHUB_OUTPUT" + echo "Status worker deploy: path filter matched." + else + echo "deploy_status_worker=false" >> "$GITHUB_OUTPUT" + echo "Status worker deploy: skipped (no relevant path changes)." + fi + deploy: runs-on: ubuntu-latest name: πŸš€ Deploy to production @@ -364,6 +377,107 @@ jobs: exit 1 fi + deploy-status-worker: + runs-on: ubuntu-latest + name: πŸ“Ÿ Deploy status worker + timeout-minutes: 8 + # Deploys after the main worker so the status prober never probes a + # production worker that predates the endpoints it expects (for example + # /health/components), which would open spurious incidents. + needs: + - sha-guard + - deploy + if: >- + needs.sha-guard.outputs.should_deploy == 'true' && + needs.sha-guard.outputs.deploy_status_worker == 'true' + environment: + name: production + url: https://status.heykody.dev + steps: + - name: πŸ“¦ Checkout + uses: actions/checkout@v6.0.2 + with: + ref: ${{ needs.sha-guard.outputs.deploy_sha }} + fetch-depth: 0 + + - name: 🟒 Setup Node + uses: actions/setup-node@v6 + with: + node-version: 26 + cache: npm + + - name: πŸ“₯ Install Dependencies + run: npm ci + + - name: ☁️ Deploy status worker + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + DEPLOY_COMMIT_SHA: ${{ needs.sha-guard.outputs.deploy_sha }} + run: | + set -euo pipefail + + deploy_attempts=3 + deploy_delay_seconds=10 + for attempt in $(seq 1 "$deploy_attempts"); do + echo "Status worker deploy attempt $attempt/$deploy_attempts" + set +e + npm run status:deploy -- \ + --var "BUILD_COMMIT:${DEPLOY_COMMIT_SHA}" 2>&1 | tee status-deploy.log + deploy_status="${PIPESTATUS[0]}" + set -e + if [ "$deploy_status" -eq 0 ]; then + break + fi + if [ "$attempt" -eq "$deploy_attempts" ]; then + echo "Status worker deploy failed after $deploy_attempts attempts." >&2 + exit "$deploy_status" + fi + echo "Retrying in ${deploy_delay_seconds}s." + sleep "$deploy_delay_seconds" + deploy_delay_seconds=$((deploy_delay_seconds * 2)) + done + + - name: πŸ” Sync status worker alert email secret + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + run: | + set -euo pipefail + # The status worker sends alert email through the Cloudflare Email + # REST API using the same account token the deploy uses. + printf '%s' "$CLOUDFLARE_API_TOKEN" | npx wrangler secret put \ + CLOUDFLARE_API_TOKEN --config packages/status/wrangler.jsonc + + - name: 🩺 Healthcheck (status worker) + shell: bash + env: + EXPECTED_COMMIT_SHA: ${{ needs.sha-guard.outputs.deploy_sha }} + run: | + set -euo pipefail + HEALTHCHECK_URL="https://status.heykody.dev/health" + echo "Healthcheck URL: $HEALTHCHECK_URL" + + attempts=20 + delay_seconds=3 + for i in $(seq 1 "$attempts"); do + echo "Attempt $i/$attempts" + if curl --fail --silent --show-error --location --max-time 10 \ + --header "Accept: application/json" \ + "$HEALTHCHECK_URL" > status-health.json && \ + node -e "const fs = require('node:fs'); const json = JSON.parse(fs.readFileSync('status-health.json','utf8')); const expected = process.env.EXPECTED_COMMIT_SHA; if (json?.ok !== true) { console.error('status-healthcheck-unexpected-response', json); process.exit(1); } if (json?.commit !== expected) { console.error('status-healthcheck-unexpected-commit', { expected, actual: json?.commit }); process.exit(1); } console.log('status-healthcheck-ok', json);"; then + exit 0 + fi + sleep "$delay_seconds" + done + + echo "Status worker healthcheck failed after ${attempts} attempts." >&2 + if [ -f status-health.json ]; then + echo "Last response body:" >&2 + cat status-health.json >&2 + fi + exit 1 + deploy-backup-control-plane: runs-on: ubuntu-latest name: πŸ›Ÿ Deploy DR backup control plane diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index d2ae9f5365..577ca4d89f 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -65,6 +65,10 @@ jobs: if: ${{ !cancelled() }} run: npm run backup:build + - name: πŸ“Ÿ Status build + if: ${{ !cancelled() }} + run: npm run status:build + - name: πŸ—ΊοΈ Primitives if: ${{ !cancelled() }} run: npm run primitives:check diff --git a/docs/contributing/architecture/primitives.yaml b/docs/contributing/architecture/primitives.yaml index dbfdc2bde7..44d1a7bb4b 100644 --- a/docs/contributing/architecture/primitives.yaml +++ b/docs/contributing/architecture/primitives.yaml @@ -63,6 +63,19 @@ primitives: docs: - docs/contributing/architecture/remote-connectors.md + - id: status-page + group: surfaces + name: Public status page + summary: + Independently deployed status worker (status.heykody.dev) probing public + endpoints every minute, storing history in its own Durable Object, and + emailing operator alerts under a capped policy. + code: + - packages/status/ + docs: + - docs/contributing/setup-manifest.md + - docs/contributing/decisions/0004-status-page-separate-worker.md + - id: scheduled-cron group: surfaces name: Scheduled handler diff --git a/docs/contributing/decisions/0004-status-page-separate-worker.md b/docs/contributing/decisions/0004-status-page-separate-worker.md new file mode 100644 index 0000000000..8f1351ebf3 --- /dev/null +++ b/docs/contributing/decisions/0004-status-page-separate-worker.md @@ -0,0 +1,41 @@ +# 0004: Status page as a separate worker with its own storage + +- **Status:** accepted +- **Date:** 2026-08-04 + +## Context + +The platform needed a public status page. A status page's core job is to stay up +while the product is down, so serving it from the main `kody` worker would +defeat the point: a bad deploy, a worker-level exception, or an `APP_DB` outage +would take the status page down with the product. Third-party hosted status +services were considered and rejected as a new external vendor for a small, +self-contained need; the repo already had a second-worker precedent in +`packages/backup-control-plane/`. + +## Decision + +The status page is an independently deployed worker (`packages/status/`, +`status.heykody.dev`) that observes the product strictly from the outside via +public endpoints, and stores probe history, incidents, and notification state in +its own Durable Object β€” never in `APP_DB`. The main worker exposes +`GET /health/components` (cheap per-binding checks) so the prober can report +storage subsystems individually. Operator alert email goes through the +Cloudflare Email REST API under a strict policy: one email per outage episode, +one reminder per day while unresolved, one all-clear on recovery, all under a +daily cap. + +Status data is global operational telemetry, not user data, so the per-user +isolation invariant is not implicated; the status worker holds no user state and +no `APP_DB` access. + +## Consequences + +The status page survives main-worker deploy failures, code regressions, and D1 +outages, but shares Cloudflare as a platform β€” a full Cloudflare outage takes +both down. If that residual risk ever matters, add an external ping service on +top; do not move the status page into the main worker. The status worker +duplicates a small amount of email-sending code rather than importing from +`packages/worker` (import boundaries keep it dependency-free). Incident records +are probe-derived only; manually posted incident narratives are a possible later +addition, not built now. diff --git a/docs/contributing/decisions/index.md b/docs/contributing/decisions/index.md index 194c3dffa3..a1a0316281 100644 --- a/docs/contributing/decisions/index.md +++ b/docs/contributing/decisions/index.md @@ -22,3 +22,4 @@ behavior (see [documentation principles](../documentation.md)). - [0001 β€” No user-facing package versioning or import pins](./0001-no-package-versioning.md) - [0002 β€” Data placement: D1, per-user Durable Objects, Analytics Engine](./0002-data-placement.md) - [0003 β€” Repos are the base primitive; packages are an explicit extension](./0003-repos-as-base-primitive.md) +- [0004 β€” Status page as a separate worker with its own storage](./0004-status-page-separate-worker.md) diff --git a/docs/contributing/setup-manifest.md b/docs/contributing/setup-manifest.md index b8e6639b43..810c0876b9 100644 --- a/docs/contributing/setup-manifest.md +++ b/docs/contributing/setup-manifest.md @@ -235,6 +235,25 @@ verifying public key, trusted restore baseline id/digest, Access Offline CLI restore still trusts only the checked-in manifest public-key, production-identity, and restore-baseline registries. +### Status page worker + +The public status page (`packages/status/`, served at `status.heykody.dev` via a +wrangler custom domain on the production zone) is an independently deployed +Worker with a cron trigger and one `StatusStore` Durable Object (SQLite). It +probes public endpoints on the main worker and `kodyapps.dev` every minute and +never touches `APP_DB` (see decision record +[0004](./decisions/0004-status-page-separate-worker.md)). + +Code deploys are automated by the production deploy workflow +(`.github/workflows/deploy.yml` job `deploy-status-worker`) when a `main` push +changes `packages/status/`, and on every manual `workflow_dispatch` of that +workflow. The job deploys with the production-account `CLOUDFLARE_API_TOKEN`, +sets `BUILD_COMMIT` to the deploy SHA, and syncs the same token as the Worker +secret `CLOUDFLARE_API_TOKEN` so the status worker can send operator alert email +through the Cloudflare Email REST API (from `ALERT_EMAIL_FROM` to +`ALERT_EMAIL_TO`, both non-secret vars in `packages/status/wrangler.jsonc`). +Without that secret, alert sends are skipped and logged. + ## Optional Cloudflare offerings The default footprint stays intentionally small. If you want to add additional diff --git a/package-lock.json b/package-lock.json index e312fd703c..e2d139a8ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "workspaces": [ "packages/backup-control-plane", "packages/shared", + "packages/status", "packages/worker", "packages/mock-servers/*" ], @@ -1858,6 +1859,10 @@ "resolved": "packages/mock-servers/cloudflare", "link": true }, + "node_modules/@kody/status": { + "resolved": "packages/status", + "link": true + }, "node_modules/@kody/worker": { "resolved": "packages/worker", "link": true @@ -11418,6 +11423,15 @@ } } }, + "packages/status": { + "name": "@kody/status", + "devDependencies": { + "@cloudflare/workers-types": "^5.20260801.1", + "@types/node": "^26.1.2", + "typescript": "^6.0.3", + "vitest": "^4.1.10" + } + }, "packages/worker": { "name": "@kody/worker", "dependencies": { diff --git a/package.json b/package.json index c7db0cefe3..cf33b8f9f4 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,8 @@ "build": "nx run worker:build", "backup:build": "wrangler deploy --dry-run --config packages/backup-control-plane/wrangler.jsonc", "backup:deploy": "wrangler deploy --config packages/backup-control-plane/wrangler.jsonc", + "status:build": "wrangler deploy --dry-run --config packages/status/wrangler.jsonc", + "status:deploy": "wrangler deploy --config packages/status/wrangler.jsonc", "backup:readiness": "node tools/disaster-recovery/canonical-readiness-cli.ts", "backup:resources": "node tools/ci/backup-resources-cli.ts", "backup:restore-drill": "node tools/disaster-recovery/d1-restore-drill-cli.ts", @@ -52,13 +54,13 @@ "preview": "nx run worker:build-client && npm run migrate:local && node --env-file=packages/worker/.env ./wrangler-env.ts dev --local", "preview:e2e": "node --env-file=packages/worker/.env tools/prepare-e2e-env.ts && nx run worker:build-client && npm run migrate:e2e && node --env-file=packages/worker/.env ./wrangler-env.ts dev --local --persist-to .wrangler/state/e2e", "generate-types": "node --env-file=packages/worker/.env ./wrangler-env.ts types ./packages/worker/worker-configuration.d.ts", - "typecheck": "nx run worker:typecheck && tsc --noEmit -p packages/backup-control-plane/tsconfig.json", + "typecheck": "nx run worker:typecheck && tsc --noEmit -p packages/backup-control-plane/tsconfig.json && tsc --noEmit -p packages/status/tsconfig.json", "test": "nx run worker:test", "test:node": "nx run worker:test-node", "test:workers": "nx run worker:test-workers", "test:push": "npm run test && npm run test:e2e:run", "inspect": "npx -y @mcpjam/inspector inspector", - "validate": "concurrently -n format,lint,typecheck,test,e2e,mcp,backup-build,primitives,migrations,deploy-guardrails,docs -c green,yellow,magenta,blue,cyan,red,blueBright,white,gray,greenBright,redBright \"npm run format:check\" \"npm run lint\" \"npm run typecheck\" \"npm run test\" \"npm run test:e2e:run\" \"npm run test:mcp\" \"npm run backup:build\" \"npm run primitives:check\" \"npm run migrations:check\" \"npm run deploy-guardrails:check\" \"npm run docs:check-temporal\"", + "validate": "concurrently -n format,lint,typecheck,test,e2e,mcp,backup-build,status-build,primitives,migrations,deploy-guardrails,docs -c green,yellow,magenta,blue,cyan,red,blueBright,yellowBright,white,gray,redBright,greenBright \"npm run format:check\" \"npm run lint\" \"npm run typecheck\" \"npm run test\" \"npm run test:e2e:run\" \"npm run test:mcp\" \"npm run backup:build\" \"npm run status:build\" \"npm run primitives:check\" \"npm run migrations:check\" \"npm run deploy-guardrails:check\" \"npm run docs:check-temporal\"", "validate:fix": "npm run format && npm run lint:fix", "test:e2e:ensure": "node tools/ensure-playwright-browser.ts", "test:e2e:a11y": "playwright test e2e/a11y.spec.ts", @@ -109,6 +111,7 @@ "workspaces": [ "packages/backup-control-plane", "packages/shared", + "packages/status", "packages/worker", "packages/mock-servers/*" ], diff --git a/packages/status/alert-email.ts b/packages/status/alert-email.ts new file mode 100644 index 0000000000..b912aa7c93 --- /dev/null +++ b/packages/status/alert-email.ts @@ -0,0 +1,80 @@ +/** + * Sends operator alert email through the Cloudflare Email REST API β€” the same + * mechanism the main worker uses for ops alerts. When credentials are absent + * (local dev), the send is skipped but reported so the caller can keep its + * notification state machine consistent. + */ + +const defaultApiBaseUrl = 'https://api.cloudflare.com' + +export type AlertEmailConfig = { + accountId?: string + apiToken?: string + apiBaseUrl?: string + fetcher?: typeof fetch +} + +export type AlertEmailMessage = { + from: string + to: string + subject: string + text: string + html: string +} + +export type AlertEmailResult = { + delivered: boolean + skipped?: boolean + error?: string +} + +export async function sendAlertEmail( + config: AlertEmailConfig, + message: AlertEmailMessage, +): Promise { + const accountId = config.accountId?.trim() + const apiToken = config.apiToken?.trim() + if (!accountId || !apiToken) { + console.info( + 'status-alert-email-unconfigured', + JSON.stringify({ subject: message.subject }), + ) + return { delivered: false, skipped: true } + } + const apiBaseUrl = config.apiBaseUrl?.trim() || defaultApiBaseUrl + const endpoint = new URL( + `client/v4/accounts/${accountId}/email/sending/send`, + apiBaseUrl.endsWith('/') ? apiBaseUrl : `${apiBaseUrl}/`, + ) + const fetcher = config.fetcher ?? fetch + let response: Response + try { + response = await fetcher(endpoint.toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(message), + }) + } catch (error) { + const detail = error instanceof Error ? error.message : String(error) + console.warn('status-alert-email-request-failed', detail) + return { delivered: false, error: detail } + } + const payload = (await response.json().catch(() => null)) as { + success?: boolean + errors?: Array<{ message?: string }> + } | null + if (!response.ok || payload?.success !== true) { + const detail = + payload?.errors?.[0]?.message ?? + `Cloudflare Email API returned HTTP ${response.status}.` + console.warn( + 'status-alert-email-failed', + JSON.stringify({ status: response.status, detail }), + ) + return { delivered: false, error: detail } + } + return { delivered: true } +} diff --git a/packages/status/email-policy.node.test.ts b/packages/status/email-policy.node.test.ts new file mode 100644 index 0000000000..6062d6bc0e --- /dev/null +++ b/packages/status/email-policy.node.test.ts @@ -0,0 +1,127 @@ +import { expect, test } from 'vitest' +import { + composeStatusEmail, + decideStatusEmail, + defaultDailyEmailLimit, + reminderIntervalMs, + type EmailPolicyInput, + type OpenIncidentSummary, +} from './email-policy.ts' + +const baseNow = Date.UTC(2026, 7, 4, 12, 0, 0) + +function openIncident( + overrides: Partial = {}, +): OpenIncidentSummary { + return { + component: 'app', + startedAt: baseNow - 5 * 60 * 1000, + detail: 'HTTP 500', + ...overrides, + } +} + +function input(overrides: Partial = {}): EmailPolicyInput { + return { + now: baseNow, + openIncidents: [], + lastNotifiedState: 'ok', + lastEmailSentAt: null, + emailsSentToday: 0, + dailyLimit: defaultDailyEmailLimit, + ...overrides, + } +} + +test('first incident of an episode sends the incident email', () => { + const decision = decideStatusEmail(input({ openIncidents: [openIncident()] })) + expect(decision).toEqual({ kind: 'incident_opened' }) +}) + +test('further incidents while one is already announced stay paused', () => { + const decision = decideStatusEmail( + input({ + openIncidents: [openIncident(), openIncident({ component: 'kv' })], + lastNotifiedState: 'incident', + lastEmailSentAt: baseNow - 60 * 60 * 1000, + emailsSentToday: 1, + }), + ) + expect(decision).toBeNull() +}) + +test('a reminder goes out once 24 hours pass without an email', () => { + const decision = decideStatusEmail( + input({ + openIncidents: [openIncident()], + lastNotifiedState: 'incident', + lastEmailSentAt: baseNow - reminderIntervalMs, + }), + ) + expect(decision).toEqual({ kind: 'daily_reminder' }) +}) + +test('all clear sends once every incident is resolved', () => { + const decision = decideStatusEmail( + input({ + lastNotifiedState: 'incident', + lastEmailSentAt: baseNow - 30 * 60 * 1000, + emailsSentToday: 1, + }), + ) + expect(decision).toEqual({ kind: 'all_clear' }) +}) + +test('no email when everything is healthy and no incident was announced', () => { + expect(decideStatusEmail(input())).toBeNull() +}) + +test('the daily cap suppresses every kind of email', () => { + const capped = { emailsSentToday: defaultDailyEmailLimit } + expect( + decideStatusEmail(input({ ...capped, openIncidents: [openIncident()] })), + ).toBeNull() + expect( + decideStatusEmail(input({ ...capped, lastNotifiedState: 'incident' })), + ).toBeNull() +}) + +test('a capped all-clear is sent on the first uncapped tick', () => { + // While capped, lastNotifiedState stays 'incident'. Next day the counter + // resets and the pending all-clear goes out. + const decision = decideStatusEmail( + input({ + lastNotifiedState: 'incident', + lastEmailSentAt: baseNow - 6 * 60 * 60 * 1000, + emailsSentToday: 0, + }), + ) + expect(decision).toEqual({ kind: 'all_clear' }) +}) + +test('composed emails carry component names, status page link, and escape html', () => { + const incident = openIncident({ detail: '' }) + for (const kind of [ + 'incident_opened', + 'daily_reminder', + 'all_clear', + ] as const) { + const content = composeStatusEmail({ + kind, + openIncidents: [incident], + statusPageUrl: 'https://status.heykody.dev', + now: baseNow, + }) + expect(content.subject).toContain('[kody status]') + expect(content.text).toContain('https://status.heykody.dev') + expect(content.html).toContain('https://status.heykody.dev') + expect(content.html).not.toContain('