Fix deploy CDC recovery and ClickHouse staleness

.github/workflows/deploy-web-stack.yml

@@ -392,95 +392,43 @@ jobs:
         env:
           IMAGE_TAG: ${{ steps.resolve.outputs.tag }}
         run: |
-          migration_output_file="$RUNNER_TEMP/migrate.log"
           migration_container="${STACK_NAME}_migrate_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}"
-          migration_log_pid=""
-          stop_migration_log_stream() {
-            if [ -n "$migration_log_pid" ] && kill -0 "$migration_log_pid" >/dev/null 2>&1; then
-              kill "$migration_log_pid" >/dev/null 2>&1 || true
-              wait "$migration_log_pid" >/dev/null 2>&1 || true
-            fi
-            migration_log_pid=""
-          }
           cleanup_migration_container() {
-            stop_migration_log_stream
             timeout 60s docker rm -f "$migration_container" >/dev/null 2>&1 || true
           }
           trap cleanup_migration_container EXIT
 
           echo "Starting migration container ${migration_container}..."
           cleanup_migration_container
-          docker run --detach --name "$migration_container" --network "${STACK_NAME}_default" \
-            --env-file "$INFISICAL_ENV_FILE" \
-            --env "CLICKHOUSE_PASSWORD_ENCODED=$CLICKHOUSE_PASSWORD_ENCODED" \
-            --entrypoint sh \
-            "ghcr.io/asherlc/dofek:${IMAGE_TAG}" \
-            -euc 'export DATABASE_URL="postgres://health:${POSTGRES_PASSWORD}@db:5432/health"; export CLICKHOUSE_URL="http://default:${CLICKHOUSE_PASSWORD_ENCODED}@clickhouse:8123"; exec node --experimental-transform-types --enable-source-maps --disable-warning=ExperimentalWarning src/db/run-migrate.ts'
-
-          echo "Streaming migration container logs..."
-          docker logs --follow "$migration_container" &
-          migration_log_pid=$!
-
           # Heavy data migrations (e.g. hypertable backfills moving body
           # measurements into fitness.metric_stream) can legitimately run for
           # hours on prod. Sized to fit the slowest known migration with
           # headroom; if a future migration would exceed this, restructure it
-          # rather than bumping again. For SQL-level visibility into what a
-          # slow migration is doing, query pg_stat_statements (already in
-          # shared_preload_libraries) or pg_stat_progress_create_index from
-          # the running db service — don't rebuild that here.
+          # rather than bumping again. Keep this as the foreground docker run:
+          # polling docker inspect over SSH can false-fail when ClickHouse
+          # background refreshes saturate the single-node host.
           migration_timeout_seconds=14400
-          migration_start_seconds=$SECONDS
-          while true; do
-            inspect_error_file="$RUNNER_TEMP/migrate-inspect.log"
-            set +e
-            container_state="$(timeout 30s docker inspect --format '{{.State.Running}} {{.State.ExitCode}}' "$migration_container" 2>"$inspect_error_file")"
-            inspect_status=$?
-            set -e
-            if [ "$inspect_status" -eq 124 ]; then
-              echo "::error::Timed out inspecting migration container ${migration_container}"
-              cat "$inspect_error_file" || true
-              exit 1
-            fi
-            if [ -z "$container_state" ]; then
-              echo "::error::Migration container ${migration_container} disappeared before completion"
-              cat "$inspect_error_file" || true
-              exit 1
-            fi
-
-            migration_running="$(printf '%s\n' "$container_state" | awk '{print $1}')"
-            migration_exit_code="$(printf '%s\n' "$container_state" | awk '{print $2}')"
-            if [ "$migration_running" = "false" ]; then
-              break
-            fi
-
-            elapsed_seconds=$((SECONDS - migration_start_seconds))
-            if [ "$elapsed_seconds" -ge "$migration_timeout_seconds" ]; then
-              echo "::error::Migration exceeded ${migration_timeout_seconds}s"
-              echo "Migration output before timeout (last 120 lines):"
-              timeout 60s docker logs --tail 120 "$migration_container" || true
-              cleanup_migration_container
-              exit 1
-            fi
-
-            echo "Migration still running after ${elapsed_seconds}s..."
-            sleep 15
-          done
-
-          stop_migration_log_stream
-          timeout 60s docker logs "$migration_container" >"$migration_output_file" 2>&1 || true
-          if [ "$migration_exit_code" -eq 0 ]; then
-            echo "Migration succeeded."
-            trap - EXIT
+          set +e
+          timeout "${migration_timeout_seconds}s" docker run --rm --name "$migration_container" --network "${STACK_NAME}_default" \
+            --env-file "$INFISICAL_ENV_FILE" \
+            --env "CLICKHOUSE_PASSWORD_ENCODED=$CLICKHOUSE_PASSWORD_ENCODED" \
+            --entrypoint sh \
+            "ghcr.io/asherlc/dofek:${IMAGE_TAG}" \
+            -euc 'export DATABASE_URL="postgres://health:${POSTGRES_PASSWORD}@db:5432/health"; export CLICKHOUSE_URL="http://default:${CLICKHOUSE_PASSWORD_ENCODED}@clickhouse:8123"; exec node --experimental-transform-types --enable-source-maps --disable-warning=ExperimentalWarning src/db/run-migrate.ts'
+          migration_status=$?
+          set -e
+          trap - EXIT
+          if [ "$migration_status" -eq 124 ]; then
+            echo "::error::Migration exceeded ${migration_timeout_seconds}s"
             cleanup_migration_container
-          else
-            echo "Migration failed (exit code ${migration_exit_code})."
-            echo "Migration output (last 60 lines):"
-            tail -n 60 "$migration_output_file" || true
+            exit 1
+          fi
+          if [ "$migration_status" -ne 0 ]; then
             cleanup_migration_container
             echo "::error::Migration failed"
-            exit 1
+            exit "$migration_status"
           fi
+          echo "Migration succeeded."
 
       - name: Deploy stack
         id: deploy_stack
@@ -526,6 +474,54 @@ jobs:
         if: ${{ steps.deploy_stack.outcome == 'success' }}
         run: docker system df
 
+      - name: Wait for Postgres writable after stack deploy
+        run: |
+          readiness_attempts=36
+          command_timeout_seconds=15
+          retry_sleep_seconds=5
+          readiness_timeout_seconds=$((readiness_attempts * command_timeout_seconds + (readiness_attempts - 1) * retry_sleep_seconds))
+          for attempt in $(seq 1 "$readiness_attempts"); do
+            recovery_output=$(timeout "${command_timeout_seconds}s" docker run --rm --network "${STACK_NAME}_default" \
+              --env-file "$INFISICAL_ENV_FILE" \
+              postgres:18-alpine sh -euc \
+              "PGPASSWORD=\"\$POSTGRES_PASSWORD\" \
+              psql --host=db --username=health --dbname=health --no-align --tuples-only --quiet \
+              --command \"SELECT CASE WHEN pg_is_in_recovery() THEN 'recovery' ELSE 'writable' END;\"" 2>&1 || true)
+            recovery_state="$(printf '%s\n' "$recovery_output" | tail -n 1 | tr -d '[:space:]')"
+            if [ "$recovery_state" = "writable" ]; then
+              echo "Postgres is writable after stack deploy."
+              break
+            fi
+            if [ "$attempt" -eq "$readiness_attempts" ]; then
+              echo "::error::Postgres did not become writable within ${readiness_timeout_seconds}s after stack deploy"
+              exit 1
+            fi
+            echo "Postgres post-stack readiness attempt ${attempt}/${readiness_attempts}: retrying in ${retry_sleep_seconds}s..."
+            sleep "$retry_sleep_seconds"
+          done
+
+      - name: Wait for ClickHouse after stack deploy
+        run: |
+          readiness_attempts=36
+          command_timeout_seconds=15
+          retry_sleep_seconds=5
+          readiness_timeout_seconds=$((readiness_attempts * command_timeout_seconds + (readiness_attempts - 1) * retry_sleep_seconds))
+          for attempt in $(seq 1 "$readiness_attempts"); do
+            clickhouse_output=$(timeout "${command_timeout_seconds}s" docker run --rm --network "${STACK_NAME}_default" \
+              postgres:18-alpine sh -euc \
+              "wget -qO- http://clickhouse:8123/ping" 2>&1 || true)
+            if [ "$clickhouse_output" = "Ok." ]; then
+              echo "ClickHouse is reachable after stack deploy."
+              break
+            fi
+            if [ "$attempt" -eq "$readiness_attempts" ]; then
+              echo "::error::ClickHouse did not become reachable within ${readiness_timeout_seconds}s after stack deploy"
+              exit 1
+            fi
+            echo "ClickHouse post-stack readiness attempt ${attempt}/${readiness_attempts}: retrying in ${retry_sleep_seconds}s..."
+            sleep "$retry_sleep_seconds"
+          done
+
       - name: Wait for PeerDB
         run: |
           for attempt in $(seq 1 36); do

deploy/README.md

@@ -273,6 +273,13 @@ If management subdomains return `404 page not found`, use:
 - `docs/traefik-subdomain-404-runbook.md`
 - `docs/review-apps.md` for PR-specific shared-front-door routes
 
+### Deployment Runbook: Stale ClickHouse Body Measurements
+
+If yesterday's or today's body weight exists in Postgres but is missing from
+ClickHouse-backed body measurement reads, use:
+
+- `docs/clickhouse-body-measurement-staleness-runbook.md`
+
 ## Management UIs
 - **Portainer**: `https://portainer.dofek.asherlc.com` (Protected by Authentik)
 - **Netdata**: `https://netdata.dofek.asherlc.com` (Protected by Authentik)

deploy/stack.yml

@@ -529,7 +529,7 @@ services:
     deploy:
       resources:
         limits:
-          memory: 400M
+          memory: 768M
       labels:
         - traefik.enable=true
         - traefik.http.routers.netdata.rule=${NETDATA_HOST_RULE:-Host(`netdata.dofek.asherlc.com`)}