diff --git a/.github/workflows/cloud-vm-migrate.yml b/.github/workflows/cloud-vm-migrate.yml index d32fa3cd0ee7..560583293302 100644 --- a/.github/workflows/cloud-vm-migrate.yml +++ b/.github/workflows/cloud-vm-migrate.yml @@ -8,9 +8,7 @@ on: required: true default: staging type: choice - options: - - staging - - production + options: [staging, production] cleanup_iroh_challenges: description: Prune expired, consumed, and duplicate Iroh registration challenges after migration required: false @@ -19,7 +17,6 @@ on: permissions: contents: read - id-token: write concurrency: group: cloud-vm-migrate-${{ inputs.target }} @@ -35,24 +32,15 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - # Staging must rehearse the dispatched branch before it can merge. - # Production remains pinned to reviewed main for both preflight and - # the prerequisite staging migration. ref: ${{ inputs.target == 'production' && 'refs/heads/main' || github.sha }} - - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - - name: Install dependencies run: bun install --frozen-lockfile - - name: Cloud VM migration preflight run: bun run cloud-vm:preflight -- --schema-only . - - name: Rehearse Iroh cleanup on isolated Postgres if: ${{ inputs.cleanup_iroh_challenges }} - # db:test discovers every CMUX_DB_TEST suite, including - # iroh-challenge-cleanup-db.test.ts (audit, apply, and rerun). run: bun run cloud-vm:preflight -- . migrate-staging: @@ -64,54 +52,31 @@ jobs: run: working-directory: web env: - AWS_REGION: ${{ vars.AWS_REGION || 'us-west-2' }} - AWS_ROLE_ARN: ${{ secrets.AWS_MIGRATION_ROLE_ARN }} CMUX_CLOUD_VM_ENV_SOURCE: process - CMUX_DB_DRIVER: aws-rds-iam - CMUX_DB_SSL_REJECT_UNAUTHORIZED: ${{ vars.CMUX_DB_SSL_REJECT_UNAUTHORIZED || 'true' }} - PGDATABASE: ${{ vars.PGDATABASE }} - PGHOST: ${{ vars.PGHOST }} - PGPORT: ${{ vars.PGPORT || '5432' }} - PGUSER: ${{ vars.PGUSER }} + CMUX_DB_DRIVER: url + DATABASE_URL: ${{ secrets.DATABASE_URL }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.target == 'production' && 'refs/heads/main' || github.sha }} - - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - - name: Install dependencies run: bun install --frozen-lockfile - - - name: Prepare AWS web identity credentials + - name: Validate PlanetScale credentials run: | set -euo pipefail - if [ -z "${AWS_ROLE_ARN:-}" ]; then - echo "::error::Missing cloud-vm-staging secret AWS_MIGRATION_ROLE_ARN" + if [ -z "${DATABASE_URL:-}" ]; then + echo "::error::Missing cloud-vm-staging secret DATABASE_URL" exit 1 fi - for key in PGHOST PGPORT PGUSER PGDATABASE AWS_REGION; do - if [ -z "${!key:-}" ]; then - echo "::error::Missing cloud-vm-staging variable $key" - exit 1 - fi - done - token_file="$RUNNER_TEMP/aws-web-identity-token" - curl -fsSL \ - -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ - "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sts.amazonaws.com" \ - | jq -e -r '.value' > "$token_file" - echo "AWS_WEB_IDENTITY_TOKEN_FILE=$token_file" >> "$GITHUB_ENV" - echo "AWS_ROLE_ARN=$AWS_ROLE_ARN" >> "$GITHUB_ENV" - echo "AWS_REGION=$AWS_REGION" >> "$GITHUB_ENV" - export AWS_WEB_IDENTITY_TOKEN_FILE="$token_file" - aws sts get-caller-identity --no-cli-pager >/dev/null - + case "$DATABASE_URL" in + postgres://*|postgresql://*) ;; + *) echo "::error::DATABASE_URL must use the PostgreSQL URL scheme"; exit 1 ;; + esac - name: Apply staging migration run: bun run cloud-vm:migrate -- staging - - name: Clean up staging Iroh challenges if: ${{ inputs.cleanup_iroh_challenges }} run: bun run cloud-vm:cleanup-iroh -- staging --apply @@ -125,54 +90,31 @@ jobs: run: working-directory: web env: - AWS_REGION: ${{ vars.AWS_REGION || 'us-west-2' }} - AWS_ROLE_ARN: ${{ secrets.AWS_MIGRATION_ROLE_ARN }} CMUX_CLOUD_VM_ENV_SOURCE: process - CMUX_DB_DRIVER: aws-rds-iam - CMUX_DB_SSL_REJECT_UNAUTHORIZED: ${{ vars.CMUX_DB_SSL_REJECT_UNAUTHORIZED || 'true' }} - PGDATABASE: ${{ vars.PGDATABASE }} - PGHOST: ${{ vars.PGHOST }} - PGPORT: ${{ vars.PGPORT || '5432' }} - PGUSER: ${{ vars.PGUSER }} + CMUX_DB_DRIVER: url + DATABASE_URL: ${{ secrets.DATABASE_URL }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: refs/heads/main - - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - - name: Install dependencies run: bun install --frozen-lockfile - - - name: Prepare AWS web identity credentials + - name: Validate PlanetScale credentials run: | set -euo pipefail - if [ -z "${AWS_ROLE_ARN:-}" ]; then - echo "::error::Missing cloud-vm-production secret AWS_MIGRATION_ROLE_ARN" + if [ -z "${DATABASE_URL:-}" ]; then + echo "::error::Missing cloud-vm-production secret DATABASE_URL" exit 1 fi - for key in PGHOST PGPORT PGUSER PGDATABASE AWS_REGION; do - if [ -z "${!key:-}" ]; then - echo "::error::Missing cloud-vm-production variable $key" - exit 1 - fi - done - token_file="$RUNNER_TEMP/aws-web-identity-token" - curl -fsSL \ - -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ - "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sts.amazonaws.com" \ - | jq -e -r '.value' > "$token_file" - echo "AWS_WEB_IDENTITY_TOKEN_FILE=$token_file" >> "$GITHUB_ENV" - echo "AWS_ROLE_ARN=$AWS_ROLE_ARN" >> "$GITHUB_ENV" - echo "AWS_REGION=$AWS_REGION" >> "$GITHUB_ENV" - export AWS_WEB_IDENTITY_TOKEN_FILE="$token_file" - aws sts get-caller-identity --no-cli-pager >/dev/null - + case "$DATABASE_URL" in + postgres://*|postgresql://*) ;; + *) echo "::error::DATABASE_URL must use the PostgreSQL URL scheme"; exit 1 ;; + esac - name: Apply production migration run: bun run cloud-vm:migrate -- production - - name: Clean up production Iroh challenges if: ${{ inputs.cleanup_iroh_challenges }} run: bun run cloud-vm:cleanup-iroh -- production --apply diff --git a/CLAUDE.md b/CLAUDE.md index 189f29899752..81552933fd06 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,9 @@ # cmux agent notes +## Database provider + +cmux Cloud uses PlanetScale PostgreSQL, organization `cmux`, database `cmux-prod`. Branches are `main` (production), `staging`, and `development`. Vercel uses a PlanetScale `DATABASE_URL`; migration jobs use `DATABASE_URL` and `bun run cloud-vm:migrate -- `. Aurora/RDS IAM and AWS migration-role instructions are retired. AWS KMS access for coderouter encryption is separate from database access. For PlanetScale CLI work, run `pscale auth check --format json` and pass `--org cmux` plus the confirmed branch. + ## Setup `./scripts/setup.sh` initializes submodules, builds GhosttyKit, and installs the pbxproj normalization pre-commit hook. diff --git a/docs/cloud-vm-backend-rollout-todo.md b/docs/cloud-vm-backend-rollout-todo.md index 77272b5140dd..186aad4b4494 100644 --- a/docs/cloud-vm-backend-rollout-todo.md +++ b/docs/cloud-vm-backend-rollout-todo.md @@ -1,3 +1,5 @@ +> **Provider update (2026-09-16):** This rollout checklist is historical. cmux Cloud now uses PlanetScale PostgreSQL (`cmux` / `cmux-prod`, `main` production, `staging` staging, `development` development). Do not execute the old AWS Aurora/RDS steps below. Use `skills/cmux-backend/references/cloud-vm-control-plane.md` and the current `cloud-vm-migrate.yml` workflow. + # Cloud VM Backend Rollout Todo This is the scoped todo list for making the Cloud VM backend production-ready with application logic running in the existing Vercel `manaflow/cmux` project. diff --git a/docs/iroh-v2/ACCEPTANCE.md b/docs/iroh-v2/ACCEPTANCE.md index 365bc7819cd0..4f8c284e1ce4 100644 --- a/docs/iroh-v2/ACCEPTANCE.md +++ b/docs/iroh-v2/ACCEPTANCE.md @@ -76,7 +76,7 @@ Capture continuous video in bounded segments, dense frame splits around launches | A13 | Every HTTP status and socket error has explicit client behavior. | Table-driven behavior tests cover 101, successful 2xx/204 as applicable, redirects/unexpected content, malformed response, 400 validation, 401 expiry/auth, 403 permission, 404 missing, 409 identity/revision conflict, 413 size, 429 with retry delay, storage full/quota, unsupported schema, 5xx, timeout, cancellation and socket-close codes. Retry only retryable work and reuse mutation request IDs. Never sign out on a transient server error. | | A14 | Generous user-based limits and finite buffers protect resources without charging terminal packets or internal calls. | Separate operation buckets; registration 300/hour burst 50; two phones/five Macs fit initial enrollment. 16 KiB inbound frame, 64 KiB snapshot chunks; 1,024 messages/2 MiB per connection, 4,096/8 MiB per user, including transport-buffered bytes. Slow subscriber safely resyncs without losing revocation. No delivered-message or local IROH-attempt quota. | | A15 | Drizzle schema activation gates traffic; immutable migrations; DB usage constraints reject atomically. | Real Miniflare/workerd persistence, populated upgrade, repeated activation, partial failure rollback, unsupported-schema refusal, compatible code rollback, storage guard and concurrent quota tests. No successful migration marker on failure. Offline devices do not expire from inactivity. | -| A16 | Shared PostgreSQL owns global EndpointID reservation and genuinely shared records; team SQLite owns team-local state. PlanetScale is the recommended target after the Aurora migration. | Unique constraint and recovery tests, one authoritative home per record, no periodic global lookup on relay renewal, plus a migration receipt and live uniqueness check before cutover. Per-user product storage allowance and team physical bounds are explicit and tested. | +| A16 | Shared PostgreSQL owns global EndpointID reservation and genuinely shared records; team SQLite owns team-local state. PlanetScale is the current shared PostgreSQL provider. | Unique constraint and recovery tests, one authoritative home per record, no periodic global lookup on relay renewal, plus a migration receipt and live uniqueness check before cutover. Per-user product storage allowance and team physical bounds are explicit and tested. | | A17 | Dashboard uses verified team directory/settings/revocations and filtered ordered changes. | Two-team/two-user browser tests; permission-filtered list and revisions, changed metadata, revocation, cursor-gap snapshot and team-switch cache isolation. Same operations and limit policy as native clients. | | A18 | Complete bounded observability works on deployed backends and relays. | Query success/denial/rate-limit/error completion events for HTTP and sockets plus enrollment, renewal, DB, revision, socket and deployment events. Confirm Cloudflare metrics, Axiom ingestion and Sentry test exception. Sink outage never blocks response, memory/time are bounded and loss is visible. Durable authority audit survives export failure. No credentials/bodies/terminal bytes/SQL parameters in records. | | A19 | Relays validate 30-minute credentials locally using public verification keys and correct audience. | Real relay rejects expired/forged/wrong-audience credentials; key rotation supports still-valid issuances; no backend call per handshake. Logs cover result and aggregate bytes. Load test per-user relay guard separately from exact cross-team backend counters. | diff --git a/docs/iroh-v2/design/IROH-DECISIONS.md b/docs/iroh-v2/design/IROH-DECISIONS.md index 3b7bad3c4ec1..f51a73a20cc8 100644 --- a/docs/iroh-v2/design/IROH-DECISIONS.md +++ b/docs/iroh-v2/design/IROH-DECISIONS.md @@ -1,5 +1,7 @@ # IROH v2 decisions +> Provider update (2026-09-16): cmux Cloud now uses PlanetScale Postgres. Aurora references below describe the historical design and do not authorize AWS database operations. Use `skills/cmux-backend/references/cloud-vm-control-plane.md` for the current database workflow. + Updated 15 September 2026, revision 23. Accepted directions are recorded here; unresolved implementation details are listed at the end. Revision 23 clarifies that new Macs must support older iOS apps; new iOS apps only need to support new Macs. ## Backend and scope diff --git a/docs/presence-service.md b/docs/presence-service.md index b246b03e8ba6..48449111aac1 100644 --- a/docs/presence-service.md +++ b/docs/presence-service.md @@ -39,7 +39,7 @@ GET /v1/presence/subscribe -> forward w/ verified team ------> WS (hibernation) - **State machine** (`src/core.ts`): pure and synchronous. A team's presence is a map of app instances keyed by `(deviceId, tag)`, the same identity as - the Aurora registry (`devices.device_uuid` + `device_app_instances.tag`). + the PlanetScale Postgres registry (`devices.device_uuid` + `device_app_instances.tag`). Online is set by a heartbeat; offline is an explicit event, produced either by a goodbye heartbeat (`stopping: true`, clean shutdown) or by the DO alarm when heartbeats stop. @@ -79,8 +79,8 @@ GET /v1/presence/subscribe -> forward w/ verified team ------> WS (hibernation) ## Migrations and durability Presence is deliberately ephemeral. The durable source of device identity is -the Aurora `devices` / `device_app_instances` registry -(https://github.com/manaflow-ai/cmux/pull/5626); this service adds no Aurora +the PlanetScale Postgres `devices` / `device_app_instances` registry +(https://github.com/manaflow-ai/cmux/pull/5626); this service adds no Postgres columns and therefore ships no Drizzle migration. DO storage keeps the live instance map plus a 24h offline tail for "last seen", pruned by the same alarm, and the durable per-device owner pins. Losing the service's storage @@ -91,7 +91,7 @@ The service's own schema story is the `[[migrations]]` block in `wrangler.toml`: Durable Object class migrations are applied by `wrangler deploy` in the deploy-on-push workflow, atomically with the code, so storage classes can never lag the deployed code the way the prod Aurora -migrations once lagged the web deploy. If presence ever does need an Aurora +migrations once lagged the web deploy. If presence ever does need a Postgres column, the Drizzle migration must land in `web/db/migrations` and is applied by the `web-db-migrations` CI job and the cloud-vm migrate workflow (`.github/workflows/cloud-vm-migrate.yml`), per the cloud-vm-ops runbook. diff --git a/plans/feat-do-device-list/DESIGN.md b/plans/feat-do-device-list/DESIGN.md index a71b40864be5..ca5666454938 100644 --- a/plans/feat-do-device-list/DESIGN.md +++ b/plans/feat-do-device-list/DESIGN.md @@ -1,5 +1,7 @@ # Local-first sync for cmux (and the iOS device list as its first consumer) +> Provider update (2026-09-16): cmux Cloud now uses PlanetScale Postgres. Aurora references below describe the historical design and do not authorize AWS database operations. Use `skills/cmux-backend/references/cloud-vm-control-plane.md` for the current database workflow. + Status: proposed. Phase 1 ships the generic sync substrate plus the device-list consumer behind a flag, with the Aurora registry kept intact as a fallback. diff --git a/skills/cmux-backend/SKILL.md b/skills/cmux-backend/SKILL.md index 70e6f6a0fd89..8377ea51d351 100644 --- a/skills/cmux-backend/SKILL.md +++ b/skills/cmux-backend/SKILL.md @@ -12,8 +12,8 @@ description: "Backend TypeScript and Cloud VM development rules for cmux. Use wh - Plain TypeScript is for trivial data shapes, constants, config files, frontend React, and small glue where Effect would add ceremony without improving failure handling. - Cloud VM backend logic stays in Vercel route handlers and Effect services backed by Postgres. Do not reintroduce Rivet or a raw actor protocol unless a later architecture doc explicitly changes the control plane. - Postgres is the source of truth for VM lifecycle, active VM limits, idempotency, and usage events. -- Production and staging Cloud VM Postgres use the Vercel Marketplace AWS Aurora PostgreSQL OIDC/RDS IAM path, with runtime env `CMUX_DB_DRIVER=aws-rds-iam`, `AWS_ROLE_ARN`, `AWS_REGION`, `PGHOST`, `PGPORT`, `PGUSER`, `PGDATABASE`. -- Run production/staging migrations with `bun db:migrate:aws-rds-iam`; never from Vercel build or route startup. Local dev keeps the `CMUX_PORT`-derived Docker Postgres path from `bun dev`. +- Production and staging Cloud VM Postgres use PlanetScale PostgreSQL database `cmux-prod` in organization `cmux`. The runtime reads `DATABASE_URL` with `CMUX_DB_DRIVER=url`; migration jobs use the protected `DATABASE_URL` secret. AWS credentials are not database credentials. +- Run production/staging migrations with `bun run cloud-vm:migrate -- staging` followed by `-- production`; never from Vercel build or route startup. Local dev keeps the `CMUX_PORT`-derived Docker Postgres path from `bun dev`. - Cloud VM create pricing gates use Stack Auth team payment items when enabled. ## Secrets diff --git a/skills/cmux-backend/references/cloud-vm-control-plane.md b/skills/cmux-backend/references/cloud-vm-control-plane.md index 4be2fc9463b0..ed3c1ec46ef8 100644 --- a/skills/cmux-backend/references/cloud-vm-control-plane.md +++ b/skills/cmux-backend/references/cloud-vm-control-plane.md @@ -10,11 +10,11 @@ Cloud VM backend logic lives in Vercel route handlers and Effect services. Reque ## Migrations -Production and staging: `bun db:migrate:aws-rds-iam`. Never run Drizzle migrations from Vercel build or route startup; that makes deploy behavior non-deterministic and couples app availability to schema mutation. Local development keeps the `CMUX_PORT`-derived Docker Postgres path from `bun dev`. +Production and staging: `bun run cloud-vm:migrate -- staging` followed by `-- production`. Never run Drizzle migrations from Vercel build or route startup; that makes deploy behavior non-deterministic and couples app availability to schema mutation. Local development keeps the `CMUX_PORT`-derived Docker Postgres path from `bun dev`. -## AWS RDS IAM runtime +## PlanetScale PostgreSQL runtime -Production and staging use the Vercel Marketplace AWS Aurora PostgreSQL OIDC/RDS IAM path with `CMUX_DB_DRIVER=aws-rds-iam`, `AWS_ROLE_ARN`, `AWS_REGION`, `PGHOST`, `PGPORT`, `PGUSER`, `PGDATABASE`. Do not invent parallel env names for the same settings; each new name is another migration and deploy surface. +Production and staging use PlanetScale Postgres database `cmux-prod` in organization `cmux`. Production is branch `main`; staging is `staging`; development is `development`. The application reads `DATABASE_URL` and uses `CMUX_DB_DRIVER=url`. Migration jobs use the protected `DATABASE_URL` secret. AWS credentials are not database credentials. ## Pricing and active limits diff --git a/web/package.json b/web/package.json index 82e02761e929..b31177839b93 100644 --- a/web/package.json +++ b/web/package.json @@ -17,7 +17,7 @@ "stripe:backfill-subscriptions": "bun scripts/stripe/backfill-subscriptions-from-stripe.ts", "billing:backfill-founders-lockout": "bun scripts/backfill-founders-lockout.ts", "cloud-vm:env:audit": "bun scripts/cloud-vm/audit-vercel-env.mjs", - "cloud-vm:migrate": "bun scripts/cloud-vm/migrate-vercel-aurora-iam.mjs", + "cloud-vm:migrate": "bun scripts/cloud-vm/migrate-planetscale.mjs", "cloud-vm:cleanup-iroh": "bun scripts/cloud-vm/cleanup-iroh-challenges.ts", "cloud-vm:preflight": "bash scripts/cloud-vm/verify-migration-preflight.sh", "cloud-vm:smoke": "bun scripts/cloud-vm/smoke-vm-api.mjs", @@ -33,7 +33,7 @@ "db:down": "bash scripts/db-local.sh down", "db:generate": "bunx drizzle-kit generate --config drizzle.config.ts", "db:migrate": "bash scripts/db-local.sh migrate", - "db:migrate:aws-rds-iam": "bun scripts/migrate-aws-rds-iam.ts", + "db:migrate:planetscale": "bun scripts/cloud-vm/migrate-planetscale.mjs", "db:ready": "bash scripts/db-local.sh ready", "db:reset": "bash scripts/db-local.sh reset", "db:status": "bash scripts/db-local.sh status", diff --git a/web/scripts/cloud-vm/aurora-operator.mjs b/web/scripts/cloud-vm/aurora-operator.mjs deleted file mode 100644 index fef899659661..000000000000 --- a/web/scripts/cloud-vm/aurora-operator.mjs +++ /dev/null @@ -1,46 +0,0 @@ -import { execFileSync } from "node:child_process"; -import { createRequire } from "node:module"; -import path from "node:path"; -import { parseBoolean, requireEnvKeys } from "./projects.mjs"; - -// Uses the operator AWS credential chain, never a pulled Vercel OIDC token. -export function createAuroraOperatorPool(webDir, env, label) { - const { Pool } = createRequire(path.join(webDir, "package.json"))("pg"); - requireEnvKeys(env, ["AWS_REGION", "PGHOST", "PGPORT", "PGUSER", "PGDATABASE"], `${label} maintenance`); - if ((env.CMUX_DB_SSL_CA_PEM || env.CMUX_DB_SSL_CA_PEM_BASE64) && process.env.CMUX_ALLOW_DB_CA_OVERRIDE !== "1") { - throw new Error( - "CMUX_DB_SSL_CA_PEM(_BASE64) is set. Current Vercel Aurora RDS certs chain to Amazon Root CA 1, so Node's default trust store should be used. Remove the override, redeploy, then retry. Set CMUX_ALLOW_DB_CA_OVERRIDE=1 only for a verified private CA.", - ); - } - const pgPort = Number(env.PGPORT); - if (!Number.isInteger(pgPort) || pgPort <= 0 || pgPort > 65535) { - throw new Error(`invalid PGPORT for ${label} maintenance: ${env.PGPORT}`); - } - if (!parseBoolean(env.CMUX_DB_SSL_REJECT_UNAUTHORIZED, true)) { - throw new Error("Operator database connections require certificate validation"); - } - - const authToken = execFileSync(process.env.AWS_CLI ?? "aws", [ - "rds", - "generate-db-auth-token", - "--hostname", - env.PGHOST, - "--port", - String(pgPort), - "--region", - env.AWS_REGION, - "--username", - env.PGUSER, - ], { encoding: "utf8" }).trim(); - - return new Pool({ - host: env.PGHOST, - port: pgPort, - user: env.PGUSER, - database: env.PGDATABASE, - password: authToken, - ssl: { rejectUnauthorized: true }, - max: 1, - connectionTimeoutMillis: 15_000, - }); -} diff --git a/web/scripts/cloud-vm/cleanup-iroh-challenges.ts b/web/scripts/cloud-vm/cleanup-iroh-challenges.ts index d12b50df62a0..dbdc04859a36 100644 --- a/web/scripts/cloud-vm/cleanup-iroh-challenges.ts +++ b/web/scripts/cloud-vm/cleanup-iroh-challenges.ts @@ -1,7 +1,7 @@ import * as Effect from "effect/Effect"; import * as Cause from "effect/Cause"; import * as Option from "effect/Option"; -import { createAuroraOperatorPool } from "./aurora-operator.mjs"; +import { createPlanetScaleOperatorPool } from "./planetscale-operator.mjs"; import { cleanupIrohChallenges, IrohChallengeCleanupError } from "./iroh-challenge-cleanup"; import { loadTargetEnv, parseWebDirAndTarget } from "./projects.mjs"; @@ -10,7 +10,7 @@ const { webDir, project, rest } = parseWebDirAndTarget(process.argv.slice(2), us if (rest.some((arg: string) => arg !== "--apply")) throw new Error(usage); const program = Effect.acquireUseRelease( - Effect.try(() => createAuroraOperatorPool(webDir, loadTargetEnv(project), project.label)), + Effect.try(() => createPlanetScaleOperatorPool(webDir, loadTargetEnv(project), project.label)), (pool) => cleanupIrohChallenges(pool, { apply: rest.includes("--apply") }).pipe( Effect.tap((result) => Effect.sync(() => { console.log(JSON.stringify({ target: project.label, ...result })); @@ -24,6 +24,6 @@ if (result._tag === "Failure") { const error = Cause.failureOption(result.cause); console.error(Option.isSome(error) && error.value instanceof IrohChallengeCleanupError ? error.value.message - : "Iroh challenge cleanup did not complete. Check operator AWS authentication and database access; committed batches are safe to rerun."); + : "Iroh challenge cleanup did not complete. Check PlanetScale authentication and database access; committed batches are safe to rerun."); process.exitCode = 1; } diff --git a/web/scripts/cloud-vm/iroh-challenge-cleanup.md b/web/scripts/cloud-vm/iroh-challenge-cleanup.md index 8f4a2a5aa75f..98e87a70d678 100644 --- a/web/scripts/cloud-vm/iroh-challenge-cleanup.md +++ b/web/scripts/cloud-vm/iroh-challenge-cleanup.md @@ -5,7 +5,7 @@ legacy and IRX clients use this broker. No client update or schema change is required. The slot key is `(user_id, client_namespace, device_uuid, tag)`; `app_instance_id` is replaced when a build restarts. -Run from `web/` with the same operator AWS access as `cloud-vm:migrate`: +Run from `web/` with the same PlanetScale connection URL as `cloud-vm:migrate`: ```sh bun run cloud-vm:cleanup-iroh -- staging @@ -34,7 +34,7 @@ Ongoing bounded storage depends on keeping the replacement issuer deployed. The protected `Cloud VM DB migration` workflow also accepts `cleanup_iroh_challenges=true`. It tests the data migration on isolated Postgres, then runs staging before production using the existing environment -protections and AWS identity. Production source remains pinned to `main`. +protections and PlanetScale credentials. Production source remains pinned to `main`. Deletion makes space reusable after PostgreSQL vacuuming; allocated bytes need not fall immediately. This command does not perform a blocking diff --git a/web/scripts/cloud-vm/migrate-planetscale.mjs b/web/scripts/cloud-vm/migrate-planetscale.mjs new file mode 100644 index 000000000000..972acfb214e7 --- /dev/null +++ b/web/scripts/cloud-vm/migrate-planetscale.mjs @@ -0,0 +1,39 @@ +#!/usr/bin/env node +import { createRequire } from "node:module"; +import path from "node:path"; +import { loadTargetEnv, parseWebDirAndTarget } from "./projects.mjs"; +import { createPlanetScaleOperatorPool, PlanetScaleOperatorConfigError } from "./planetscale-operator.mjs"; + +const usage = "Usage: migrate-planetscale.mjs [web-dir] [--check]"; +const { webDir, project, rest } = parseWebDirAndTarget(process.argv.slice(2), usage); +if (rest.some((arg) => arg !== "--check")) { + console.error(usage); + process.exit(2); +} +const checkOnly = rest.includes("--check"); +try { + const pool = createPlanetScaleOperatorPool(webDir, loadTargetEnv(project), project.projectName); + try { + if (checkOnly) { + await pool.query("begin read only"); + try { + await pool.query("select 1"); + } finally { + await pool.query("rollback"); + } + } else { + const requireFromWeb = createRequire(path.join(webDir, "package.json")); + const { drizzle } = requireFromWeb("drizzle-orm/node-postgres"); + const { migrate } = requireFromWeb("drizzle-orm/node-postgres/migrator"); + await migrate(drizzle({ client: pool }), { migrationsFolder: path.join(webDir, "db/migrations") }); + } + } finally { + await pool.end(); + } + console.log(`${project.label} PlanetScale ${checkOnly ? "connection verified (read only)" : "migration applied"}`); +} catch (error) { + // Driver errors may contain credentials, SQL, or customer rows. + console.error(error instanceof PlanetScaleOperatorConfigError ? error.message : + `${project.label} PlanetScale ${checkOnly ? "connection check" : "migration"} failed; check target credentials, access, and migration state.`); + process.exitCode = 1; +} diff --git a/web/scripts/cloud-vm/migrate-vercel-aurora-iam.mjs b/web/scripts/cloud-vm/migrate-vercel-aurora-iam.mjs deleted file mode 100755 index 073e6e2ec49d..000000000000 --- a/web/scripts/cloud-vm/migrate-vercel-aurora-iam.mjs +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env node -import { createRequire } from "node:module"; -import path from "node:path"; -import { - loadTargetEnv, - parseWebDirAndTarget, -} from "./projects.mjs"; - -import { createAuroraOperatorPool } from "./aurora-operator.mjs"; - -const usage = "Usage: migrate-vercel-aurora-iam.mjs [web-dir] "; -const { webDir, project } = parseWebDirAndTarget(process.argv.slice(2), usage); -const pkgPath = path.join(webDir, "package.json"); -const migrationsFolder = path.join(webDir, "db/migrations"); -const requireFromWeb = createRequire(pkgPath); -const { drizzle } = requireFromWeb("drizzle-orm/node-postgres"); -const { migrate } = requireFromWeb("drizzle-orm/node-postgres/migrator"); - -try { - const env = loadTargetEnv(project); - const pool = createAuroraOperatorPool(webDir, env, project.projectName); - - try { - const db = drizzle({ client: pool }); - await migrate(db, { migrationsFolder }); - } finally { - await pool.end(); - } - - console.log(`${project.label} migration applied`); -} catch (error) { - console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); -} diff --git a/web/scripts/cloud-vm/planetscale-operator.mjs b/web/scripts/cloud-vm/planetscale-operator.mjs new file mode 100644 index 000000000000..1b40c7a2d3e9 --- /dev/null +++ b/web/scripts/cloud-vm/planetscale-operator.mjs @@ -0,0 +1,43 @@ +import { createRequire } from "node:module"; +import path from "node:path"; + +export class PlanetScaleOperatorConfigError extends Error {} + +/** Operator jobs use a direct, certificate-verified PlanetScale connection. */ +export function planetScaleOperatorOptions(env, label) { + const raw = env.DIRECT_DATABASE_URL?.trim() || env.DATABASE_URL?.trim(); + const fail = (message) => { throw new PlanetScaleOperatorConfigError(`${label}: ${message}`); }; + if (!raw) fail("DATABASE_URL or DIRECT_DATABASE_URL is required for PlanetScale maintenance"); + let url; + try { url = new URL(raw); } catch { fail("invalid database URL"); } + if (!["postgres:", "postgresql:"].includes(url.protocol)) fail("expected a PostgreSQL URL"); + if (!url.hostname.endsWith(".pg.psdb.cloud")) fail("expected a PlanetScale Postgres endpoint"); + if (!url.username || !url.password) fail("database URL must contain operator credentials"); + if (url.port && !["5432", "6432"].includes(url.port)) fail("unsupported PlanetScale port"); + const sslmode = url.searchParams.get("sslmode"); + if (sslmode && !["require", "verify-ca", "verify-full"].includes(sslmode)) { + fail("operator connections require certificate verification"); + } + if (env.CMUX_DB_SSL_REJECT_UNAUTHORIZED && + !["true", "1", "yes", "on"].includes(env.CMUX_DB_SSL_REJECT_UNAUTHORIZED.trim().toLowerCase())) { + fail("operator connections require certificate verification"); + } + // The deployed URL uses PgBouncer (6432). Migrations and cleanup jobs need + // the same branch's direct port for session settings and advisory locks. + url.port = "5432"; + // pg parses query parameters after Pool options. Operator connections own + // their TLS and endpoint policy; URL parameters must not override either. + url.search = ""; + return { + connectionString: url.href, + ssl: { rejectUnauthorized: true }, + max: 1, + connectionTimeoutMillis: 15_000, + }; +} + +export function createPlanetScaleOperatorPool(webDir, env, label) { + const options = planetScaleOperatorOptions(env, label); + const { Pool } = createRequire(path.join(webDir, "package.json"))("pg"); + return new Pool(options); +} diff --git a/web/scripts/cloud-vm/projects.mjs b/web/scripts/cloud-vm/projects.mjs index f70e6999f349..fe5817b79601 100755 --- a/web/scripts/cloud-vm/projects.mjs +++ b/web/scripts/cloud-vm/projects.mjs @@ -23,8 +23,8 @@ export const projects = { }; export const requiredRuntimeEnvKeys = [ + // AWS_REGION is used by KMS and other AWS SDK clients, never for the database. "AWS_REGION", - "AWS_ROLE_ARN", // Without the Slack sink every triggered VM alert drops silently while the // alert cron keeps returning 200, so an unset webhook is an observability // outage, not a tuning choice. The only waiver is a recorded operator @@ -54,10 +54,7 @@ export const requiredRuntimeEnvKeys = [ "FREESTYLE_API_KEY", "NEXT_PUBLIC_STACK_PROJECT_ID", "NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY", - "PGDATABASE", - "PGHOST", - "PGPORT", - "PGUSER", + "DATABASE_URL", "STACK_SECRET_SERVER_KEY", ]; @@ -66,6 +63,7 @@ export const requiredRuntimeEnvKeys = [ // stack-token form that the runtime client accepts without making operators // store two credentials. export const requiredRuntimeEnvAlternativeGroups = [ + { requiredKeys: ["DATABASE_URL"], alternatives: [["DIRECT_DATABASE_URL"]] }, { requiredKeys: ["FREESTYLE_API_KEY"], alternatives: [["FREESTYLE_STACK_ACCESS_TOKEN", "FREESTYLE_TEAM_ID"]], @@ -106,7 +104,6 @@ export const recommendedRuntimeEnvKeys = [ // VALUES are audited by freeProvisioningAudit.mjs (a permissive value fails). // CMUX_ALERTS_SINK_UNCONFIGURED_ACK is absent for the same reason; its VALUE // is audited by alertSinkAudit.mjs. - "CMUX_DB_SSL_REJECT_UNAUTHORIZED", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_HEADERS", "OTEL_SERVICE_NAME", diff --git a/web/scripts/cloud-vm/verify-migration-preflight.sh b/web/scripts/cloud-vm/verify-migration-preflight.sh index 9b5a95caf19f..44a8e8bba705 100755 --- a/web/scripts/cloud-vm/verify-migration-preflight.sh +++ b/web/scripts/cloud-vm/verify-migration-preflight.sh @@ -22,7 +22,7 @@ cd "$web_dir" required_scripts=( db:check db:migrate - db:migrate:aws-rds-iam + db:migrate:planetscale db:test ) diff --git a/web/scripts/migrate-aws-rds-iam.ts b/web/scripts/migrate-aws-rds-iam.ts index 0007d4c6c391..cdcfb2f727c6 100644 --- a/web/scripts/migrate-aws-rds-iam.ts +++ b/web/scripts/migrate-aws-rds-iam.ts @@ -1,34 +1,3 @@ -import { migrate } from "drizzle-orm/node-postgres/migrator"; -import { drizzle } from "drizzle-orm/node-postgres"; -import { cloudDbConfig } from "../db/config"; -import { createAwsRdsIamPool } from "../db/client"; -import * as schema from "../db/schema"; - -async function main() { - const config = cloudDbConfig(); - if (config.driver !== "aws-rds-iam") { - throw new Error("CMUX_DB_DRIVER=aws-rds-iam is required for this migration command"); - } - - const pool = createAwsRdsIamPool(config); - try { - const db = drizzle({ client: pool, schema }); - await migrate(db, { migrationsFolder: "db/migrations" }); - } finally { - await pool.end(); - } -} - -main().catch((error) => { - const messages: string[] = []; - let current: unknown = error; - for (let depth = 0; depth < 4 && current; depth++) { - messages.push(current instanceof Error ? current.message : String(current)); - current = typeof current === "object" && current !== null && "cause" in current - ? current.cause - : undefined; - } - const message = messages.join(": "); - console.error(`aws-rds-iam migration failed: ${message}`); - process.exit(1); -}); +// Retained as a fail-closed entry point for stale operator instructions. +console.error("Aurora/RDS IAM migrations are retired. Use bun run cloud-vm:migrate -- for PlanetScale."); +process.exitCode = 1; diff --git a/web/services/coderouter/README.md b/web/services/coderouter/README.md index 9ae6b76644f0..88451a828d13 100644 --- a/web/services/coderouter/README.md +++ b/web/services/coderouter/README.md @@ -42,7 +42,7 @@ PostHog events go to the main cmux project (`POSTHOG_PROJECT_KEY` / `POSTHOG_HOS Route outcomes, failures, tokens, models, providers, latency, and Cloud VM attribution are stored in ClickHouse `route_events` and `usage_events`. This avoids a second usage ledger in PostHog and keeps billing and product reporting on one authoritative dataset. -Fault classification (`classifyCoderouterFault`) decides who is paged. `operator` (RDS, KMS, config, an unhandled throw): `$exception` at `error` level. `upstream` (provider 5xx/429 that survived failover, transport timeouts) and `tenant` (no usable account): `warning`. `caller` (bad token, 4xx): trace only, no exception. Fingerprints are `coderouter:::` for route outcomes and `coderouter.:` for reported failures, so one condition is one PostHog issue. +Fault classification (`classifyCoderouterFault`) decides who is paged. `operator` (PlanetScale, KMS, config, an unhandled throw): `$exception` at `error` level. `upstream` (provider 5xx/429 that survived failover, transport timeouts) and `tenant` (no usable account): `warning`. `caller` (bad token, 4xx): trace only, no exception. Fingerprints are `coderouter:::` for route outcomes and `coderouter.:` for reported failures, so one condition is one PostHog issue. Unhandled throws in a route are no longer swallowed as a bare 503: the wrapper reports `route_crash` with the real stack (PostHog `$exception`, Sentry), then answers with the surface's own 503 shape. @@ -61,7 +61,7 @@ Investigating one failure: take the `x-coderouter-request-id`, query ClickHouse | key | condition | severity | env | | --- | --- | --- | --- | | `coderouter-health` | health is `degraded` or `down` | warning / critical | | -| `coderouter-operator-failures` | `provider_unavailable` from our side (RDS/KMS/config), ≥ 1 | critical | `CMUX_CODEROUTER_ALERT_OPERATOR_FAILURES_5M` | +| `coderouter-operator-failures` | `provider_unavailable` from our side (PlanetScale/KMS/config), ≥ 1 | critical | `CMUX_CODEROUTER_ALERT_OPERATOR_FAILURES_5M` | | `coderouter-upstream-failures` | provider 5xx/transport after failover, ≥ 5 | warning | `CMUX_CODEROUTER_ALERT_UPSTREAM_FAILURES_5M` | | `coderouter-no-usable-account` | tenants with no healthy account, ≥ 10 (names the teams) | warning | `CMUX_CODEROUTER_ALERT_NO_ACCOUNT_5M` | | `coderouter-auth-rejected` | unauthorized requests ≥ 25 | warning | `CMUX_CODEROUTER_ALERT_AUTH_REJECTED_5M` | diff --git a/web/services/vms/README.md b/web/services/vms/README.md index 1b8be69b3cf0..6af1225460dd 100644 --- a/web/services/vms/README.md +++ b/web/services/vms/README.md @@ -248,19 +248,14 @@ Provider SDKs remain Promise-based adapters under `drivers/`, but all route-visi Vercel runs the Next.js application and all VM REST routes. Postgres is the persistent control plane. There is no Rivet deployment for this feature. -Production and staging use Vercel Marketplace AWS Aurora PostgreSQL with OIDC federation and RDS IAM auth. The runtime does not need a long-lived database password. +Production and staging use PlanetScale PostgreSQL. The Vercel runtime and explicit migration jobs use the PlanetScale connection URL. Set these Vercel environment variables per production/staging environment: -- `CMUX_DB_DRIVER=aws-rds-iam`. -- `AWS_ROLE_ARN`, IAM role Vercel assumes. -- `AWS_REGION`, Aurora region. -- `PGHOST`, Aurora cluster endpoint. -- `PGPORT`, usually `5432`. -- `PGUSER`, IAM-enabled Postgres role. -- `PGDATABASE`, app database name. +- `CMUX_DB_DRIVER=url`. +- `DATABASE_URL`, a PlanetScale PostgreSQL connection URL. Keep it in the Vercel project secret store. - `CMUX_DB_POOL_MAX`, small pool size for Vercel Functions. Start with `5`. -- `CMUX_DB_SSL_REJECT_UNAUTHORIZED`, optional. Leave unset for the current Vercel Marketplace Aurora databases so Node uses its default trust store. +- Preserve `sslmode=verify-full` on the PlanetScale URL. - `CMUX_VM_CREATE_ENABLED`, global create kill switch. Set `0` to block new paid creates while keeping list, attach, and delete available. - `CMUX_VM_ALLOW_FREE_PROVISIONING`, explicit opt-out of the paid-plan Cloud VM gate. Leave unset @@ -301,7 +296,9 @@ Set these Vercel environment variables per production/staging environment: Local development keeps using Docker Postgres through `DATABASE_URL`, derived from `CMUX_PORT`. -Run production/staging migrations explicitly, never during Vercel build or route startup. The local operator path pulls deployed Vercel env. The GitHub Actions path uses the minimal DB metadata copied into protected GitHub environments, generates an RDS IAM auth token, and applies Drizzle migrations: +Use `bun run cloud-vm:migrate -- staging --check` to verify access without changing schema. Operator jobs use the branch's direct port (5432) and verify its TLS certificate. `DIRECT_DATABASE_URL` takes precedence when set. With process-provided credentials, set `CMUX_CLOUD_VM_ENV_SOURCE=process`; otherwise the command pulls the selected Vercel project. + +Run production/staging migrations explicitly, never during Vercel build or route startup. The local operator path pulls the selected Vercel project `DATABASE_URL`. The GitHub Actions path reads the protected `DATABASE_URL` secret and applies Drizzle migrations: ```bash bun run cloud-vm:migrate -- staging @@ -392,12 +389,9 @@ They use these GitHub Environments: Each environment needs: -- variable `AWS_REGION`, usually `us-west-2` -- variables `PGHOST`, `PGPORT`, `PGUSER`, and `PGDATABASE` -- variable `CMUX_DB_SSL_REJECT_UNAUTHORIZED`, usually `true` +- secret `DATABASE_URL` for the target branch - variables `NEXT_PUBLIC_STACK_PROJECT_ID` and `NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY` - secret `STACK_SECRET_SERVER_KEY` for smoke workflows -- secret `AWS_MIGRATION_ROLE_ARN` for migration workflows Production migration runs staging migration first on the same commit, then waits on the protected production environment approval. diff --git a/web/tests/aurora-operator.test.ts b/web/tests/aurora-operator.test.ts deleted file mode 100644 index efacf93c851a..000000000000 --- a/web/tests/aurora-operator.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { expect, test } from "bun:test"; -import { createAuroraOperatorPool } from "../scripts/cloud-vm/aurora-operator.mjs"; - -test("operator rejects disabled certificate validation before creating an AWS token", () => { - expect(() => createAuroraOperatorPool(process.cwd(), { - AWS_REGION: "us-west-2", - PGHOST: "database.invalid", - PGPORT: "5432", - PGUSER: "test", - PGDATABASE: "test", - CMUX_DB_SSL_REJECT_UNAUTHORIZED: "false", - }, "test")).toThrow("Operator database connections require certificate validation"); -}); diff --git a/web/tests/cloud-vm-env-audit.test.ts b/web/tests/cloud-vm-env-audit.test.ts index 0ba79b77a53a..59a3939e6511 100644 --- a/web/tests/cloud-vm-env-audit.test.ts +++ b/web/tests/cloud-vm-env-audit.test.ts @@ -463,3 +463,12 @@ describe("free-provisioning override audit", () => { } }); }); + + +describe("PlanetScale database env audit", () => { + test("accepts deployed and direct URLs without demanding AWS database metadata", () => { + expect(requiredRuntimeEnvKeySatisfied("DATABASE_URL", new Set(["DATABASE_URL"]))).toBe(true); + expect(requiredRuntimeEnvKeySatisfied("DATABASE_URL", new Set(["DIRECT_DATABASE_URL"]))).toBe(true); + expect(requiredRuntimeEnvKeySatisfied("DATABASE_URL", new Set(["AWS_REGION", "PGHOST"]))).toBe(false); + }); +}); diff --git a/web/tests/planetscale-operator.test.ts b/web/tests/planetscale-operator.test.ts new file mode 100644 index 000000000000..771e3a7ac85d --- /dev/null +++ b/web/tests/planetscale-operator.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, test } from "bun:test"; +import { createPlanetScaleOperatorPool, planetScaleOperatorOptions } from "../scripts/cloud-vm/planetscale-operator.mjs"; + +const pooled = "postgres://operator:secret@staging.pg.psdb.cloud:6432/postgres?sslmode=verify-full"; + +describe("PlanetScale operator connection", () => { + test("uses the deployed URL without AWS credentials and selects the direct port", async () => { + const pool = createPlanetScaleOperatorPool(process.cwd(), { DATABASE_URL: pooled }, "staging"); + try { + const url = new URL(pool.options.connectionString!); + expect(url.hostname).toBe("staging.pg.psdb.cloud"); + expect(url.port).toBe("5432"); + expect(url.username).toBe("operator"); + expect(pool.options.ssl).toEqual({ rejectUnauthorized: true }); + expect(pool.options.max).toBe(1); + } finally { await pool.end(); } + }); + + test("prefers the explicitly configured direct URL", () => { + const direct = pooled.replace("staging.pg", "direct.pg").replace(":6432", ":5432"); + expect(planetScaleOperatorOptions({ DATABASE_URL: pooled, DIRECT_DATABASE_URL: direct }, "staging") + .connectionString).toContain("direct.pg.psdb.cloud:5432"); + }); + + test("URL parameters cannot override certificate verification or the target", () => { + const options = planetScaleOperatorOptions({ DATABASE_URL: `${pooled}&ssl=true&sslrootcert=other.pem&host=other.invalid&port=1234` }, "staging"); + const url = new URL(options.connectionString); + expect([...url.searchParams.keys()]).toEqual([]); + expect(url.hostname).toBe("staging.pg.psdb.cloud"); + expect(url.port).toBe("5432"); + expect(options.ssl).toEqual({ rejectUnauthorized: true }); + }); + + test.each(["disable", "allow", "prefer", "no-verify"])("refuses insecure SSL mode %s", (mode) => { + expect(() => planetScaleOperatorOptions({ DATABASE_URL: pooled.replace("verify-full", mode) }, "staging")) + .toThrow("require certificate verification"); + }); + + test("refuses the obsolete TLS bypass", () => { + expect(() => planetScaleOperatorOptions({ DATABASE_URL: pooled, CMUX_DB_SSL_REJECT_UNAUTHORIZED: "false" }, "staging")) + .toThrow("require certificate verification"); + }); + + test.each([ + "postgres://operator:secret@example.com/postgres", + "postgres://operator:secret@staging.pg.psdb.cloud.evil.test/postgres", + "mysql://operator:secret@staging.pg.psdb.cloud/postgres", + "postgres://operator@staging.pg.psdb.cloud/postgres", + "postgres://operator:secret@staging.pg.psdb.cloud:1234/postgres", + "not-a-url-with-secret", + ])("refuses invalid operator target without exposing its value", (url) => { + let message = ""; + try { planetScaleOperatorOptions({ DATABASE_URL: url }, "staging"); } + catch (error) { message = (error as Error).message; } + expect(message).not.toBe(""); + expect(message).not.toContain("secret"); + expect(message).not.toContain(url); + }); + + test("missing URLs never fall back to the old AWS environment", () => { + expect(() => planetScaleOperatorOptions({ PGHOST: "database.invalid", AWS_REGION: "us-west-2" }, "staging")) + .toThrow("DATABASE_URL or DIRECT_DATABASE_URL is required"); + }); +});