Add a public status page: status.heykody.dev worker + /health/components + operator email alerts

.github/workflows/deploy.yml

@@ -34,6 +34,7 @@ jobs:
       should_deploy: ${{ steps.sha_guard.outputs.should_deploy }}
       deploy_backup_control_plane:
         ${{ steps.sha_guard.outputs.deploy_backup_control_plane }}
+      deploy_status_worker: ${{ steps.sha_guard.outputs.deploy_status_worker }}
     steps:
       - name: 📦 Checkout
         uses: actions/checkout@v6.0.2
@@ -68,19 +69,22 @@ jobs:
           if [ "$HEAD_SHA" != "$DEPLOY_SHA" ]; then
             echo "should_deploy=false" >> "$GITHUB_OUTPUT"
             echo "deploy_backup_control_plane=false" >> "$GITHUB_OUTPUT"
+            echo "deploy_status_worker=false" >> "$GITHUB_OUTPUT"
             echo "Skipping deploy: target SHA is not current main HEAD."
             echo "target=$DEPLOY_SHA"
             echo "head=$HEAD_SHA"
             exit 0
           fi
           echo "should_deploy=true" >> "$GITHUB_OUTPUT"
 
-          # Always redeploy the DR control plane on manual workflow_dispatch.
-          # On push-driven deploys, only when control-plane (or shared backup
-          # contracts) changed in the commit being deployed.
+          # Always redeploy the DR control plane and status worker on manual
+          # workflow_dispatch. On push-driven deploys, only when their files
+          # changed in the commit being deployed.
           if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
             echo "deploy_backup_control_plane=true" >> "$GITHUB_OUTPUT"
+            echo "deploy_status_worker=true" >> "$GITHUB_OUTPUT"
             echo "Backup control plane deploy: forced (workflow_dispatch)."
+            echo "Status worker deploy: forced (workflow_dispatch)."
             exit 0
           fi
 
@@ -93,6 +97,15 @@ jobs:
             echo "Backup control plane deploy: skipped (no relevant path changes)."
           fi
 
+          if git diff --name-only "${DEPLOY_SHA}^" "${DEPLOY_SHA}" | grep -E \
+            '^(\.github/workflows/deploy\.yml|packages/status/)'; then
+            echo "deploy_status_worker=true" >> "$GITHUB_OUTPUT"
+            echo "Status worker deploy: path filter matched."
+          else
+            echo "deploy_status_worker=false" >> "$GITHUB_OUTPUT"
+            echo "Status worker deploy: skipped (no relevant path changes)."
+          fi
+
   deploy:
     runs-on: ubuntu-latest
     name: 🚀 Deploy to production
@@ -364,6 +377,107 @@ jobs:
             exit 1
           fi
 
+  deploy-status-worker:
+    runs-on: ubuntu-latest
+    name: 📟 Deploy status worker
+    timeout-minutes: 8
+    # Deploys after the main worker so the status prober never probes a
+    # production worker that predates the endpoints it expects (for example
+    # /health/components), which would open spurious incidents.
+    needs:
+      - sha-guard
+      - deploy
+    if: >-
+      needs.sha-guard.outputs.should_deploy == 'true' &&
+      needs.sha-guard.outputs.deploy_status_worker == 'true'
+    environment:
+      name: production
+      url: https://status.heykody.dev
+    steps:
+      - name: 📦 Checkout
+        uses: actions/checkout@v6.0.2
+        with:
+          ref: ${{ needs.sha-guard.outputs.deploy_sha }}
+          fetch-depth: 0
+
+      - name: 🟢 Setup Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: 26
+          cache: npm
+
+      - name: 📥 Install Dependencies
+        run: npm ci
+
+      - name: ☁️ Deploy status worker
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}
+          DEPLOY_COMMIT_SHA: ${{ needs.sha-guard.outputs.deploy_sha }}
+        run: |
+          set -euo pipefail
+
+          deploy_attempts=3
+          deploy_delay_seconds=10
+          for attempt in $(seq 1 "$deploy_attempts"); do
+            echo "Status worker deploy attempt $attempt/$deploy_attempts"
+            set +e
+            npm run status:deploy -- \
+              --var "BUILD_COMMIT:${DEPLOY_COMMIT_SHA}" 2>&1 | tee status-deploy.log
+            deploy_status="${PIPESTATUS[0]}"
+            set -e
+            if [ "$deploy_status" -eq 0 ]; then
+              break
+            fi
+            if [ "$attempt" -eq "$deploy_attempts" ]; then
+              echo "Status worker deploy failed after $deploy_attempts attempts." >&2
+              exit "$deploy_status"
+            fi
+            echo "Retrying in ${deploy_delay_seconds}s."
+            sleep "$deploy_delay_seconds"
+            deploy_delay_seconds=$((deploy_delay_seconds * 2))
+          done
+
+      - name: 🔐 Sync status worker alert email secret
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}
+        run: |
+          set -euo pipefail
+          # The status worker sends alert email through the Cloudflare Email
+          # REST API using the same account token the deploy uses.
+          printf '%s' "$CLOUDFLARE_API_TOKEN" | npx wrangler secret put \
+            CLOUDFLARE_API_TOKEN --config packages/status/wrangler.jsonc
+
+      - name: 🩺 Healthcheck (status worker)
+        shell: bash
+        env:
+          EXPECTED_COMMIT_SHA: ${{ needs.sha-guard.outputs.deploy_sha }}
+        run: |
+          set -euo pipefail
+          HEALTHCHECK_URL="https://status.heykody.dev/health"
+          echo "Healthcheck URL: $HEALTHCHECK_URL"
+
+          attempts=20
+          delay_seconds=3
+          for i in $(seq 1 "$attempts"); do
+            echo "Attempt $i/$attempts"
+            if curl --fail --silent --show-error --location --max-time 10 \
+              --header "Accept: application/json" \
+              "$HEALTHCHECK_URL" > status-health.json && \
+              node -e "const fs = require('node:fs'); const json = JSON.parse(fs.readFileSync('status-health.json','utf8')); const expected = process.env.EXPECTED_COMMIT_SHA; if (json?.ok !== true) { console.error('status-healthcheck-unexpected-response', json); process.exit(1); } if (json?.commit !== expected) { console.error('status-healthcheck-unexpected-commit', { expected, actual: json?.commit }); process.exit(1); } console.log('status-healthcheck-ok', json);"; then
+              exit 0
+            fi
+            sleep "$delay_seconds"
+          done
+
+          echo "Status worker healthcheck failed after ${attempts} attempts." >&2
+          if [ -f status-health.json ]; then
+            echo "Last response body:" >&2
+            cat status-health.json >&2
+          fi
+          exit 1
+
   deploy-backup-control-plane:
     runs-on: ubuntu-latest
     name: 🛟 Deploy DR backup control plane

.github/workflows/validate.yml

@@ -65,6 +65,10 @@ jobs:
         if: ${{ !cancelled() }}
         run: npm run backup:build
 
+      - name: 📟 Status build
+        if: ${{ !cancelled() }}
+        run: npm run status:build
+
       - name: 🗺️ Primitives
         if: ${{ !cancelled() }}
         run: npm run primitives:check

docs/contributing/architecture/primitives.yaml

@@ -63,6 +63,19 @@ primitives:
     docs:
       - docs/contributing/architecture/remote-connectors.md
 
+  - id: status-page
+    group: surfaces
+    name: Public status page
+    summary:
+      Independently deployed status worker (status.heykody.dev) probing public
+      endpoints every minute, storing history in its own Durable Object, and
+      emailing operator alerts under a capped policy.
+    code:
+      - packages/status/
+    docs:
+      - docs/contributing/setup-manifest.md
+      - docs/contributing/decisions/0004-status-page-separate-worker.md
+
   - id: scheduled-cron
     group: surfaces
     name: Scheduled handler

docs/contributing/decisions/0004-status-page-separate-worker.md

@@ -0,0 +1,41 @@
+# 0004: Status page as a separate worker with its own storage
+
+- **Status:** accepted
+- **Date:** 2026-08-04
+
+## Context
+
+The platform needed a public status page. A status page's core job is to stay up
+while the product is down, so serving it from the main `kody` worker would
+defeat the point: a bad deploy, a worker-level exception, or an `APP_DB` outage
+would take the status page down with the product. Third-party hosted status
+services were considered and rejected as a new external vendor for a small,
+self-contained need; the repo already had a second-worker precedent in
+`packages/backup-control-plane/`.
+
+## Decision
+
+The status page is an independently deployed worker (`packages/status/`,
+`status.heykody.dev`) that observes the product strictly from the outside via
+public endpoints, and stores probe history, incidents, and notification state in
+its own Durable Object — never in `APP_DB`. The main worker exposes
+`GET /health/components` (cheap per-binding checks) so the prober can report
+storage subsystems individually. Operator alert email goes through the
+Cloudflare Email REST API under a strict policy: one email per outage episode,
+one reminder per day while unresolved, one all-clear on recovery, all under a
+daily cap.
+
+Status data is global operational telemetry, not user data, so the per-user
+isolation invariant is not implicated; the status worker holds no user state and
+no `APP_DB` access.
+
+## Consequences
+
+The status page survives main-worker deploy failures, code regressions, and D1
+outages, but shares Cloudflare as a platform — a full Cloudflare outage takes
+both down. If that residual risk ever matters, add an external ping service on
+top; do not move the status page into the main worker. The status worker
+duplicates a small amount of email-sending code rather than importing from
+`packages/worker` (import boundaries keep it dependency-free). Incident records
+are probe-derived only; manually posted incident narratives are a possible later
+addition, not built now.

docs/contributing/decisions/index.md

@@ -22,3 +22,4 @@ behavior (see [documentation principles](../documentation.md)).
 - [0001 — No user-facing package versioning or import pins](./0001-no-package-versioning.md)
 - [0002 — Data placement: D1, per-user Durable Objects, Analytics Engine](./0002-data-placement.md)
 - [0003 — Repos are the base primitive; packages are an explicit extension](./0003-repos-as-base-primitive.md)
+- [0004 — Status page as a separate worker with its own storage](./0004-status-page-separate-worker.md)

docs/contributing/setup-manifest.md

@@ -235,6 +235,25 @@ verifying public key, trusted restore baseline id/digest, Access
 Offline CLI restore still trusts only the checked-in manifest public-key,
 production-identity, and restore-baseline registries.
 
+### Status page worker
+
+The public status page (`packages/status/`, served at `status.heykody.dev` via a
+wrangler custom domain on the production zone) is an independently deployed
+Worker with a cron trigger and one `StatusStore` Durable Object (SQLite). It
+probes public endpoints on the main worker and `kodyapps.dev` every minute and
+never touches `APP_DB` (see decision record
+[0004](./decisions/0004-status-page-separate-worker.md)).
+
+Code deploys are automated by the production deploy workflow
+(`.github/workflows/deploy.yml` job `deploy-status-worker`) when a `main` push
+changes `packages/status/`, and on every manual `workflow_dispatch` of that
+workflow. The job deploys with the production-account `CLOUDFLARE_API_TOKEN`,
+sets `BUILD_COMMIT` to the deploy SHA, and syncs the same token as the Worker
+secret `CLOUDFLARE_API_TOKEN` so the status worker can send operator alert email
+through the Cloudflare Email REST API (from `ALERT_EMAIL_FROM` to
+`ALERT_EMAIL_TO`, both non-secret vars in `packages/status/wrangler.jsonc`).
+Without that secret, alert sends are skipped and logged.
+
 ## Optional Cloudflare offerings
 
 The default footprint stays intentionally small. If you want to add additional

package.json

@@ -35,6 +35,8 @@
     "build": "nx run worker:build",
     "backup:build": "wrangler deploy --dry-run --config packages/backup-control-plane/wrangler.jsonc",
     "backup:deploy": "wrangler deploy --config packages/backup-control-plane/wrangler.jsonc",
+    "status:build": "wrangler deploy --dry-run --config packages/status/wrangler.jsonc",
+    "status:deploy": "wrangler deploy --config packages/status/wrangler.jsonc",
     "backup:readiness": "node tools/disaster-recovery/canonical-readiness-cli.ts",
     "backup:resources": "node tools/ci/backup-resources-cli.ts",
     "backup:restore-drill": "node tools/disaster-recovery/d1-restore-drill-cli.ts",
@@ -52,13 +54,13 @@
     "preview": "nx run worker:build-client && npm run migrate:local && node --env-file=packages/worker/.env ./wrangler-env.ts dev --local",
     "preview:e2e": "node --env-file=packages/worker/.env tools/prepare-e2e-env.ts && nx run worker:build-client && npm run migrate:e2e && node --env-file=packages/worker/.env ./wrangler-env.ts dev --local --persist-to .wrangler/state/e2e",
     "generate-types": "node --env-file=packages/worker/.env ./wrangler-env.ts types ./packages/worker/worker-configuration.d.ts",
-    "typecheck": "nx run worker:typecheck && tsc --noEmit -p packages/backup-control-plane/tsconfig.json",
+    "typecheck": "nx run worker:typecheck && tsc --noEmit -p packages/backup-control-plane/tsconfig.json && tsc --noEmit -p packages/status/tsconfig.json",
     "test": "nx run worker:test",
     "test:node": "nx run worker:test-node",
     "test:workers": "nx run worker:test-workers",
     "test:push": "npm run test && npm run test:e2e:run",
     "inspect": "npx -y @mcpjam/inspector inspector",
-    "validate": "concurrently -n format,lint,typecheck,test,e2e,mcp,backup-build,primitives,migrations,deploy-guardrails,docs -c green,yellow,magenta,blue,cyan,red,blueBright,white,gray,greenBright,redBright \"npm run format:check\" \"npm run lint\" \"npm run typecheck\" \"npm run test\" \"npm run test:e2e:run\" \"npm run test:mcp\" \"npm run backup:build\" \"npm run primitives:check\" \"npm run migrations:check\" \"npm run deploy-guardrails:check\" \"npm run docs:check-temporal\"",
+    "validate": "concurrently -n format,lint,typecheck,test,e2e,mcp,backup-build,status-build,primitives,migrations,deploy-guardrails,docs -c green,yellow,magenta,blue,cyan,red,blueBright,yellowBright,white,gray,redBright,greenBright \"npm run format:check\" \"npm run lint\" \"npm run typecheck\" \"npm run test\" \"npm run test:e2e:run\" \"npm run test:mcp\" \"npm run backup:build\" \"npm run status:build\" \"npm run primitives:check\" \"npm run migrations:check\" \"npm run deploy-guardrails:check\" \"npm run docs:check-temporal\"",
     "validate:fix": "npm run format && npm run lint:fix",
     "test:e2e:ensure": "node tools/ensure-playwright-browser.ts",
     "test:e2e:a11y": "playwright test e2e/a11y.spec.ts",
@@ -109,6 +111,7 @@
   "workspaces": [
     "packages/backup-control-plane",
     "packages/shared",
+    "packages/status",
     "packages/worker",
     "packages/mock-servers/*"
   ],