Address PeerDB CDC review comments

deploy/README.md

@@ -118,7 +118,7 @@ CI (main) -> build dofek + dofek-ml (same tag)
               -> bootstrap stack if <stack>_db is missing
               -> wait for postgres writable
               -> migrate (one-shot container on <stack>_default)
-              -> docker stack deploy dofek
+              -> docker stack deploy <stack>
 ```
 
 1. **Build**: GitHub Actions builds the `server` and `ml` images and pushes them to GHCR with the same tag.
@@ -128,6 +128,7 @@ CI (main) -> build dofek + dofek-ml (same tag)
       The template escapes embedded newlines only when `secret.IsMultilineEncodingEnabled` is true.
       - Must include `CREDENTIAL_ENCRYPTION_KEY_BASE64` (base64-encoded 32-byte key).
       - Must include `CLICKHOUSE_PASSWORD` for the ClickHouse service. The deploy workflow URL-encodes it into `CLICKHOUSE_PASSWORD_ENCODED` for app `CLICKHOUSE_URL` interpolation.
+      - Must include `POSTGRES_PASSWORD`; PeerDB's catalog database and internal MinIO stage use this existing secret.
       - Optional: `CREDENTIAL_ENCRYPTION_KEY_NAMESPACE` (default `dofek`) and `CREDENTIAL_ENCRYPTION_KEY_NAME` (default `provider-credentials`).
    2. Point Docker CLI at the remote daemon with `DOCKER_HOST=ssh://root@<host>`.
    3. Login to GHCR on the CI runner.
@@ -140,7 +141,7 @@ CI (main) -> build dofek + dofek-ml (same tag)
       When `CLICKHOUSE_URL` is present, this also runs tracked ClickHouse
       analytics migrations before the stack update.
    8. Validate required host bind-mount directories before deploying the stack. This must fail before `docker stack deploy` if paths such as `/mnt/dofek-data/redis` are missing, because Swarm rejects tasks with missing bind sources.
-   9. `docker stack deploy -c deploy/stack.yml --with-registry-auth --prune --detach=false dofek` — swarm performs a single stack-wide update, including `training-export-worker`, and CI waits for the rollout to converge before continuing. The deploy workflow bounds this wait at 20 minutes so a wedged Swarm rollback fails CI instead of running indefinitely.
+   9. `docker stack deploy -c deploy/stack.yml --with-registry-auth --prune --detach=false <stack>` — swarm performs a single stack-wide update, including `training-export-worker`, and CI waits for the rollout to converge before continuing. The deploy workflow bounds this wait at 20 minutes so a wedged Swarm rollback fails CI instead of running indefinitely.
       The workflow parses the Infisical dotenv file inside a child process for stack interpolation. Do not append the full dotenv file to `GITHUB_ENV`; GitHub Actions prints step environments and can expose Infisical-only secrets that GitHub does not automatically mask.
    10. Wait for PeerDB and run the one-shot ClickHouse CDC setup command. The command loads `src/db/peerdb/metric-stream-cdc.sql`, substitutes deployment connection values, and creates the Postgres peer, ClickHouse peer, and `dofek_metric_stream_cdc` mirror if they do not already exist.
    11. Run the materialized-view sync planner. It triggers the refresh webhook only when one of these is true:

deploy/stack.yml

@@ -221,16 +221,23 @@ services:
   peerdb-minio:
     image: minio/minio:latest@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
     environment:
-      MINIO_ROOT_USER: _peerdb_minioadmin
-      MINIO_ROOT_PASSWORD: _peerdb_minioadmin
+      MINIO_ROOT_USER: peerdb
+      MINIO_ROOT_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
       PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdbbucket
     volumes:
       - /mnt/dofek-data/peerdb-minio:/data
     entrypoint: >
       /bin/sh -euc "
       minio server /data --console-address=:36987 &
-      sleep 2;
-      mc alias set peerdb-minio http://peerdb-minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD;
+      attempts=0;
+      until mc alias set peerdb-minio http://peerdb-minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD; do
+        attempts=$$((attempts + 1));
+        if [ $$attempts -ge 30 ]; then
+          echo peerdb-minio did not become ready in time >&2;
+          exit 1;
+        fi;
+        sleep 1;
+      done;
       mc mb --ignore-existing peerdb-minio/$$PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME;
       wait
       "
@@ -248,8 +255,8 @@ services:
       TEMPORAL_CLIENT_KEY: ""
       PEERDB_TEMPORAL_NAMESPACE: default
       PEERDB_ALLOWED_TARGETS: ""
-      PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: _peerdb_minioadmin
-      PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: _peerdb_minioadmin
+      PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: peerdb
+      PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
       PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1
       PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://peerdb-minio:9000
       PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdbbucket

docs/production-incident-baseline.md

@@ -57,7 +57,7 @@ Two separate root causes blocked CI:
   made the dashboard assertion verify the rendered section rather than a brittle
   canvas selector.
 
-### Remaining Risk
+### Remaining Risk (native backfill and CDC transition)
 
 Any future date-sensitive E2E seeds that use UTC string slicing can still drift
  around local-midnight boundaries, and any new cached server query path added to
@@ -2116,3 +2116,48 @@ analytics can switch to `peerdb.metric_stream`. The next operational step is to
 compare row counts and recent-row freshness between Postgres,
 `postgres_fitness.metric_stream`, and `peerdb.metric_stream`, then cut
 `analytics.deduped_sensor` over in a separate migration.
+
+## 2026-05-01: Production Deploy Migration Timed Out During Metric Stream Backfill
+
+### Symptoms
+
+Production deploy run `25237109231` failed in job `74005587158` during the
+`Run migrations` step. The migration container exited with code 1 before
+`docker stack deploy` ran, so the new web stack image was not released.
+
+### Evidence
+
+The first fatal job line was `Migration failed (exit code 1).` The last
+migration log line was:
+
+```text
+error: [migrate] Error: Timeout error.
+```
+
+Immediately before the error, Postgres migrations had applied 0 new files and
+materialized view sync had completed with `synced=0 skipped=7 refreshed=0`.
+The next code path is `runClickHouseMigrations()`.
+
+### Root Cause
+
+ClickHouse migration `0006_backfill_native_metric_stream` copied each full
+Timescale chunk with one `INSERT INTO ... SELECT FROM postgresql(...)` command.
+Production chunks were large enough for a ClickHouse HTTP command to exceed the
+client's default 30 second request timeout. Because the migration also dropped
+`postgres_fitness` and the backfill progress table on every retry, a retry
+would restart the native backfill instead of continuing from completed ranges.
+
+### Fix or Mitigation
+
+The migration now splits Timescale chunk ranges into one-hour ClickHouse
+backfill windows and records those bounded windows in
+`analytics.metric_stream_backfill_chunks`. It only drops `postgres_fitness` and
+the progress table when `postgres_fitness` is still backed by a non-native
+database engine; retries against an already-native database preserve completed
+backfill windows and continue from the first missing window.
+
+### Remaining Risk
+
+Very dense one-hour windows can still take longer than expected, but future
+failures will now identify the exact window being copied and retries will not
+discard completed native backfill progress.

docs/superpowers/plans/2026-05-01-peerdb-clickhouse-cdc.md

@@ -10,7 +10,7 @@
 
 ---
 
-### Task 1: Add PeerDB CDC Setup Logic
+## Task 1: Add PeerDB CDC Setup Logic
 
 **Files:**
 - Create: `src/db/clickhouse-cdc.ts`
@@ -31,7 +31,7 @@
 
 - [ ] Add a direct-run entrypoint in `src/db/setup-clickhouse-cdc.ts`.
 
-### Task 2: Add PeerDB Services To Swarm
+## Task 2: Add PeerDB Services To Swarm
 
 **Files:**
 - Modify: `deploy/stack.yml`
@@ -59,7 +59,7 @@
   - Wait for PeerDB SQL on `peerdb:9900`.
   - Run `src/db/setup-clickhouse-cdc.ts` in a one-shot app container on the swarm network.
 
-### Task 3: Verify And Document
+## Task 3: Verify And Document
 
 **Files:**
 - Modify: `docs/clickhouse-metric-stream.md`

src/db/clickhouse-cdc.test.ts

@@ -110,7 +110,7 @@ describe("PeerDB ClickHouse CDC setup", () => {
     });
 
     expect(clickHouseCommands).toEqual(["CREATE DATABASE IF NOT EXISTS peerdb"]);
-    expect(peerDbQueries).toEqual(["host = 'db', password = 'pa\\'ss\\\\word'"]);
+    expect(peerDbQueries).toEqual(["host = 'db', password = 'pa''ss\\word'"]);
   });
 
   it("fails when the PeerDB SQL template references an unknown placeholder", async () => {
@@ -208,10 +208,10 @@ describe("PeerDB ClickHouse CDC setup", () => {
     });
     const peerDbQuery = String(peerDbClientMocks.query.mock.calls[0]?.[0]);
     expect(peerDbQuery).toContain("host = 'postgres.example'");
-    expect(peerDbQuery).toContain("port = '6543'");
-    expect(peerDbQuery).toContain("password = 'pg\\'credential'");
+    expect(peerDbQuery).toContain("port = 6543");
+    expect(peerDbQuery).toContain("password = 'pg''credential'");
     expect(peerDbQuery).toContain("host = 'clickhouse'");
-    expect(peerDbQuery).toContain("password = 'click\\\\credential'");
+    expect(peerDbQuery).toContain("password = 'click\\credential'");
     expect(peerDbQuery).not.toContain("{{");
     expect(peerDbClientMocks.end).toHaveBeenCalledTimes(1);
     expect(clickHouseClientMocks.close).toHaveBeenCalledTimes(1);
@@ -256,4 +256,37 @@ describe("PeerDB ClickHouse CDC setup", () => {
     );
     expect(peerDbClientMocks.Client).not.toHaveBeenCalled();
   });
+
+  it("requires database URL credentials and database name for PeerDB setup", async () => {
+    process.env.CLICKHOUSE_URL = credentialedUrl(
+      "http",
+      "analytics",
+      "fixture",
+      "clickhouse.example:8123",
+      "",
+    );
+    process.env.POSTGRES_PASSWORD = "peerdb fixture";
+
+    process.env.DATABASE_URL = "postgres://postgres.example:6543/fitness";
+    await expect(setupClickHouseCdcFromEnv()).rejects.toThrow(
+      "DATABASE_URL must include username for PeerDB setup",
+    );
+
+    process.env.DATABASE_URL = "postgres://health@postgres.example:6543/fitness";
+    await expect(setupClickHouseCdcFromEnv()).rejects.toThrow(
+      "DATABASE_URL must include password for PeerDB setup",
+    );
+
+    process.env.DATABASE_URL = credentialedUrl(
+      "postgres",
+      "health",
+      "fixture",
+      "postgres.example:6543",
+      "",
+    );
+    await expect(setupClickHouseCdcFromEnv()).rejects.toThrow(
+      "DATABASE_URL must include database name for PeerDB setup",
+    );
+    expect(peerDbClientMocks.Client).not.toHaveBeenCalled();
+  });
 });

src/db/clickhouse-cdc.ts

@@ -33,7 +33,7 @@ interface RuntimeConfig {
 }
 
 function peerDbStringLiteral(value: string): string {
-  return `'${value.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`;
+  return `'${value.replaceAll("'", "''")}'`;
 }
 
 function requireEnv(name: string): string {
@@ -62,10 +62,28 @@ function parseClickHouseUrl(urlString: string): URL {
   return url;
 }
 
+function requireUrlComponent(value: string, component: string, envName: string): string {
+  if (!value) {
+    throw new Error(`${envName} must include ${component} for PeerDB setup`);
+  }
+  return value;
+}
+
 function buildRuntimeConfig(): RuntimeConfig {
   const databaseUrl = new URL(requireEnv("DATABASE_URL"));
   const clickHouseUrl = parseClickHouseUrl(requireEnv("CLICKHOUSE_URL"));
   const postgresCredential = requireEnv("POSTGRES_PASSWORD");
+  const postgresDatabase = requireUrlComponent(
+    databaseUrl.pathname.replace(/^\//, ""),
+    "database name",
+    "DATABASE_URL",
+  );
+  const postgresUser = decodeURIComponent(
+    requireUrlComponent(databaseUrl.username, "username", "DATABASE_URL"),
+  );
+  const postgresPassword = decodeURIComponent(
+    requireUrlComponent(databaseUrl.password, "password", "DATABASE_URL"),
+  );
 
   return {
     peerDbUrl: buildDefaultPeerDbUrl(postgresCredential),
@@ -76,11 +94,11 @@ function buildRuntimeConfig(): RuntimeConfig {
       clickHouseHost: "clickhouse",
       clickHousePort: 9000,
       clickHouseUser: decodeURIComponent(clickHouseUrl.username),
-      postgresCredential: decodeURIComponent(databaseUrl.password),
-      postgresDatabase: databaseUrl.pathname.replace(/^\//, ""),
+      postgresCredential: postgresPassword,
+      postgresDatabase,
       postgresHost: databaseUrl.hostname,
       postgresPort: Number(databaseUrl.port || 5432),
-      postgresUser: decodeURIComponent(databaseUrl.username),
+      postgresUser,
     },
   };
 }
@@ -95,7 +113,7 @@ function buildTemplateReplacements(values: PeerDbSqlTemplateValues): Record<stri
     POSTGRES_CREDENTIAL: peerDbStringLiteral(values.postgresCredential),
     POSTGRES_DATABASE: peerDbStringLiteral(values.postgresDatabase),
     POSTGRES_HOST: peerDbStringLiteral(values.postgresHost),
-    POSTGRES_PORT: peerDbStringLiteral(String(values.postgresPort)),
+    POSTGRES_PORT: String(values.postgresPort),
     POSTGRES_USER: peerDbStringLiteral(values.postgresUser),
   };
 }