[server] Fix ClickHouse provider deletion acknowledgements

analytics/profiles.yml

@@ -17,6 +17,7 @@ dofek:
       user: "{{ env_var('DBT_CLICKHOUSE_USER', 'default') }}"
       password: "{{ env_var('DBT_CLICKHOUSE_PASSWORD', env_var('CLICKHOUSE_PASSWORD')) }}"
       secure: "{{ env_var('DBT_CLICKHOUSE_SECURE', 'false') | lower == 'true' }}"
+      send_receive_timeout: 300
     lint:
       type: clickhouse
       schema: "{{ env_var('DBT_CLICKHOUSE_SCHEMA', 'analytics') }}"

deploy/README.md

@@ -53,6 +53,7 @@ Dofek production is deployed as a **single-node Docker Swarm** stack on Oracle C
 - The `db` service has a 2 GiB container memory limit to prevent one PostgreSQL workload from exhausting the single-node host. If it hits that limit, treat it as a query/workload incident rather than increasing the cap by default.
 - PostgreSQL runs `timescale/timescaledb-ha:pg18.3-ts2.26.4-all` so TimescaleDB and PostGIS are both available. It is configured with `max_connections=40`, `work_mem=4MB`, `maintenance_work_mem=64MB`, `max_locks_per_transaction=4096` for large Timescale chunk scans, and logical replication settings needed by ClickHouse change-data capture. Production keeps six logical slots/senders and caps each slot at 64 GiB of retained WAL so PeerDB has recovery headroom without allowing an inactive slot to retain unbounded WAL.
 - ClickHouse has a 13 GiB container memory limit, a 1 CPU Swarm limit, and a checked-in `clickhouse_memory_limits_13g` profile with a 13 GiB `max_server_memory_usage` cap so large analytics queries fail or throttle inside ClickHouse instead of triggering host-level OOM kills or CPU-starving SSH/Docker on the single-node host. It also loads checked-in server profile settings from `deploy/clickhouse/users.d/` and server settings from `deploy/clickhouse/config.d/`; the production stack mounts these as Docker Swarm configs so app-managed geospatial `Nullable(Point)` columns, bounded memory, and seven-day system-log TTL retention work without manual server changes. Docker Swarm config contents are immutable, so changing a checked-in config file must also rotate the config key in `deploy/stack.yml` (for example `clickhouse_memory_limits_13g`) instead of reusing the same config name with new contents.
+- The default ClickHouse profile cancels read-only HTTP queries when their client disconnects and applies a four-minute elapsed-time ceiling to every query. This keeps a timed-out web/dbt client from leaving server work alive across later retry cycles; ClickHouse documents both [`cancel_http_readonly_queries_on_client_close`](https://clickhouse.com/docs/operations/settings/settings#cancel_http_readonly_queries_on_client_close) and the elapsed-time behavior controlled by [`max_execution_time`](https://clickhouse.com/docs/operations/settings/settings#max_execution_time) with `timeout_before_checking_execution_speed=0`.
 - All production entrypoint modes that run dbt use `--threads 1 --select $DBT_SAFE_MODELS` to avoid concurrent or unsafe ClickHouse model builds on the single-node host. `analytics-worker` also has a 0.5 CPU Swarm limit and currently runs the dbt-native microbatched `sensor_scalar_sample` and `deduped_sensor` models plus the dirty-keyed dashboard and cycling serving models every 15 minutes. `analytics.activity_summary` is served from the incremental `analytics.activity_summary_rows` table through a thin view; `analytics.cycling_activity` and `analytics.daily_cycling` serve the cycling page; and `analytics.daily_recovery`, `analytics.daily_strain`, `analytics.daily_sleep`, and `analytics.weekly_healthspan` serve dashboard recovery, strain, sleep, and healthspan reads. After both dbt groups succeed, the analytics worker sequentially refreshes every live app query cache key registered in Redis, preserving the previous cached value unless recomputation succeeds. Failed model builds or cache refreshes enter the bounded retry path instead of being reported as a successful analytics cycle. dbt documents `build` as running selected models and their tests, and Redis documents TTL-based key expiration in its official references: <https://docs.getdbt.com/reference/commands/build> and <https://redis.io/docs/latest/commands/expire/>. The `cdc-health` service runs `scripts/check-clickhouse-cdc.ts` every five minutes so PeerDB slot loss and stale mirrors are continuously reported instead of being discovered only during dashboard debugging.
 - Netdata has a 768 MiB container memory limit and a checked-in `deploy/netdata/netdata.conf` that bounds dbengine retention to two tiers: one day of per-second data capped at 96 MiB and seven days of per-minute data capped at 128 MiB. The stack mounts this file as a Docker Swarm config, so changing it must also rotate the config key in `deploy/stack.yml` (for example `netdata_db_limits_v2`).
 - PeerDB uses an internal catalog Postgres service, Temporal, worker services, and a private MinIO staging bucket. Its persistent catalog and staging data live under `/mnt/dofek-data/peerdb-catalog` and `/mnt/dofek-data/peerdb-minio`. The catalog uses the PostgreSQL 18 image layout: mount the host directory at `/var/lib/postgresql`, not `/var/lib/postgresql/data`, so the image can manage its versioned data directory. Production mirrors use 100,000-row CDC batches and single-worker 100,000-row initial snapshot partitions so PeerDB can stay inside its fixed memory limits at the cost of slower catch-up.

deploy/clickhouse/users.d/default-query-guardrails.xml

@@ -0,0 +1,10 @@
+<clickhouse>
+  <profiles>
+    <default>
+      <allow_experimental_nullable_tuple_type>1</allow_experimental_nullable_tuple_type>
+      <cancel_http_readonly_queries_on_client_close>1</cancel_http_readonly_queries_on_client_close>
+      <max_execution_time>240</max_execution_time>
+      <timeout_before_checking_execution_speed>0</timeout_before_checking_execution_speed>
+    </default>
+  </profiles>
+</clickhouse>

deploy/stack.yml

@@ -253,8 +253,8 @@ services:
     volumes:
       - /mnt/dofek-data/clickhouse:/var/lib/clickhouse
     configs:
-      - source: clickhouse_nullable_tuple_settings
-        target: /etc/clickhouse-server/users.d/allow-experimental-nullable-tuple-type.xml
+      - source: clickhouse_default_query_guardrails_v2
+        target: /etc/clickhouse-server/users.d/default-query-guardrails.xml
         mode: 0444
       - source: clickhouse_system_log_ttl
         target: /etc/clickhouse-server/config.d/system-log-ttl.xml
@@ -701,8 +701,8 @@ networks:
     attachable: true
 
 configs:
-  clickhouse_nullable_tuple_settings:
-    file: ./clickhouse/users.d/allow-experimental-nullable-tuple-type.xml
+  clickhouse_default_query_guardrails_v2:
+    file: ./clickhouse/users.d/default-query-guardrails.xml
   clickhouse_system_log_ttl:
     file: ./clickhouse/config.d/system-log-ttl.xml
   clickhouse_memory_limits_13g:

docs/clickhouse-metric-stream.md

@@ -3,7 +3,7 @@
 `metric_stream` samples publish to Redpanda first. Redpanda Connect archives the
 topic to Cloudflare R2 for long-term replay, and
 `metric-stream-clickhouse-sink` writes the analytics copy into
-`postgres_fitness.metric_stream`. Postgres is no longer the normal forward
+`ingest.metric_stream`. Postgres is no longer the normal forward
 ingestion path for metric-stream samples, and PeerDB does not mirror
 `fitness.metric_stream`.
 
@@ -13,7 +13,7 @@ Redpanda metric-stream-v1
         | R2 archive              | ClickHouse sink
         v                         v
 Cloudflare R2 replay archive
-ClickHouse postgres_fitness.metric_stream
+ClickHouse ingest.metric_stream
         |
         | dbt microbatch projection by recorded_at
         v
@@ -124,14 +124,18 @@ ClickHouse `Nullable(Point)` columns require
 `allow_experimental_nullable_tuple_type=1`. The app ClickHouse client sends
 that setting with its requests, and Docker deployments also load the checked-in
 server profile at
-`deploy/clickhouse/users.d/allow-experimental-nullable-tuple-type.xml`.
+`deploy/clickhouse/users.d/default-query-guardrails.xml`.
 
 ClickHouse migrations create and update the databases and read models:
 
-- `postgres_fitness.metric_stream`: a ClickHouse-native `MergeTree` copy of the
+- `ingest.metric_stream`: a ClickHouse-native `ReplacingMergeTree` copy of the
   raw metric stream populated by `metric-stream-clickhouse-sink`. Historical
   rows may still have been backfilled from Postgres, but new forward rows come
-  from Redpanda events.
+  from Redpanda events. Its `version` and `is_deleted` columns encode replacement
+  order and logical deletion so current-state queries can select the latest live
+  row ([ReplacingMergeTree](https://clickhouse.com/docs/en/guides/replacing-merge-tree)).
+- `ingest.metric_stream_delete_acknowledgement`: one receipt per version 2
+  deletion event, written only after the sink's tombstone insert completes.
 - `postgres_fitness`: app-managed native ClickHouse raw mirrors with PeerDB CDC
   metadata columns for lower-volume Postgres-backed raw tables, including
   activity, sleep, daily metrics, provider inventory, and sensor priority
@@ -148,29 +152,34 @@ ClickHouse migrations create and update the databases and read models:
 - `analytics.sensor_scalar_sample`: a narrow dbt `microbatch` incremental
   `ReplacingMergeTree` projection of activity sensor scalar channels. It uses
   `recorded_at` as its dbt event time, writes one current row per raw
-  `metric_stream.id`, and collapses row versions with `_peerdb_version`
-  inside the bounded batch query.
+  `metric_stream.id`, and maps the raw stream's `version` into its
+  `_peerdb_version` projection column inside the bounded batch query.
 - `analytics.deduped_sensor`: an activity-agnostic dbt `microbatch`
   incremental `ReplacingMergeTree` table containing the best live scalar sample
   per `(user_id, channel, recorded_at)` according to mirrored sensor
   provider/device priority tables. It uses `recorded_at` as its dbt event time
   and has no `activity_id`; activity reads join samples to activities by time
   window.
 - `analytics.deduped_location`: a normal view over
-  `postgres_fitness.metric_stream` location rows. The Redpanda ClickHouse sink
+  `ingest.metric_stream` location rows. The Redpanda ClickHouse sink
   converts EWKT point payloads into ClickHouse point-compatible values.
 - `analytics.activity_summary`: a normal view over `analytics.deduped_sensor`,
   `analytics.deduped_location`, and `analytics.v_activity`.
 - `analytics.activity_trend_daily`: a normal view with one activity-linked
   sensor trend row per user and UTC day. It is derived from
   `analytics.deduped_sensor`.
 
-Because `postgres_fitness.metric_stream` is an existing app-managed ClickHouse
-table and several read models expect CDC-compatible metadata, it keeps the
-metadata columns even though PeerDB no longer feeds it:
-`_peerdb_synced_at`, `_peerdb_is_deleted`, and `_peerdb_version`. The deploy CDC
-setup command repairs these columns idempotently with `ALTER TABLE ... ADD
-COLUMN IF NOT EXISTS` before PeerDB validates the mirror.
+### Deletion protocol
+
+New metric-stream deletions are version 2 Redpanda events with a unique event
+ID. The ClickHouse sink first filters `ingest.metric_stream` to the deletion
+scope, selects the latest version of each matching ID, inserts a newer
+`is_deleted = 1` version, and then writes the event ID to
+`ingest.metric_stream_delete_acknowledgement`. The provider deletion worker
+waits for that acknowledgement before rebuilding dbt models and invalidating
+the user's analytics cache; it does not scan the full raw table to infer that
+the event was applied. Archived version 1 deletion events remain replayable,
+but do not have acknowledgement IDs.
 
 The native-table backfill is resumable within a successful migration attempt,
 but migration `0006_backfill_native_metric_stream` intentionally drops the

docs/production-incident-baseline.md

@@ -13713,3 +13713,60 @@ Drizzle schema and runtime Zod schemas. Findings and remediations:
   and verify the named activity plus its cache after deployment. Separately,
   `analytics.deduped_location` still references the retired
   `postgres_fitness.metric_stream` table and should be removed or migrated.
+
+## 2026-07-17 — Provider Deletion Analytics Refresh Timed Out
+
+- **Symptoms:** After deleting every Garmin Dump provider record, Postgres was
+  empty and the PeerDB-backed ClickHouse mirrors had no live rows, but provider
+  ClickHouse reads did not complete and the provider deletion analytics job
+  remained at `Waiting for provider deletes to reach analytics...`.
+- **User impact:** Garmin Dump records were deleted from the source and raw
+  mirrors, but derived ClickHouse analytics and their cached API responses were
+  not rebuilt or invalidated.
+- **Evidence:** The exact failing operation was the provider deletion worker's
+  `SELECT count() FROM ingest.metric_stream FINAL WHERE user_id = ... AND
+  provider_id = 'garmin-dump' AND is_deleted = 0`. The first fatal worker log
+  line was `[worker] Job failed: Timeout error.` The BullMQ job for user
+  `f923fed7-d934-4cd9-8cb9-8e83020d0e69` remained active with progress at 20%.
+  ClickHouse query logs showed abandoned scans reading 86,789,817 rows and
+  later failing with code 210 (`I/O error: Broken pipe`) after their clients had
+  disconnected. `system.processes` contained 15 concurrent queries using 9.18
+  GiB, including six abandoned dbt power-curve builds running for 3.6 to 17.9
+  hours and repeated provider deletion scans. In contrast, the provider-scoped
+  PeerDB mirror check completed in about 0.5 seconds, Postgres contained zero
+  Garmin Dump rows, all three replication slots were active with `wal_status =
+  'reserved'`, the CDC health service reported healthy, and the Redpanda
+  ClickHouse sink consumer had zero lag after consuming the deletion event.
+  ClickHouse documents that `FINAL` applies merge-time deduplication during
+  query execution
+  ([ReplacingMergeTree query-time deduplication](https://clickhouse.com/docs/en/guides/replacing-merge-tree#querying-replacingmergetree)).
+  The deletion event itself was emitted and consumed at `2026-07-17 23:39:44`,
+  but its `INSERT INTO ingest.metric_stream SELECT ... FROM
+  ingest.metric_stream FINAL` finished in 11 ms after reading and writing zero
+  rows. A provider-filtered latest-version query subsequently found 3,130,320
+  live Garmin Dump IDs and zero tombstoned IDs. A real ClickHouse integration
+  test reproduced the same successful-command/no-tombstone behavior.
+- **Root cause:** The sink's self-referential `INSERT ... SELECT ... FINAL`
+  command did not produce tombstones, yet Kafka consumption completed. The
+  downstream worker then attempted to detect completion with repeated unbounded
+  `FINAL` scans over the 86.8-million-row table. Timed-out HTTP and dbt clients
+  did not cancel their server queries, so retries accumulated and saturated
+  ClickHouse before the worker could rebuild read models or invalidate caches.
+- **Fix / mitigation:** Canceled eight identified abandoned dbt queries,
+  reducing ClickHouse from 15 queries and 9.18 GiB to short-lived queries using
+  about 17 MiB. Replaced the no-op tombstone SQL with a provider-filtered
+  latest-version aggregation, added versioned deletion event IDs and a small
+  ClickHouse acknowledgement table, and changed the worker to poll that receipt
+  instead of rescanning `metric_stream`. The default ClickHouse profile now
+  cancels disconnected read-only HTTP queries and enforces a four-minute elapsed
+  execution ceiling below dbt's five-minute receive timeout.
+- **Validation:** The executable ClickHouse regression failed with
+  `is_deleted = 0` before the SQL change and passes with `is_deleted = 1` after
+  it. Three ClickHouse integration tests, 219 focused unit tests, the complete
+  changed-file test suite, all-package TypeScript checks, lint, analytics SQL
+  lint, analytics policy checks, XML validation, and diff validation pass.
+- **Remaining risk / follow-up:** Deploy the migration, sink, worker, and rotated
+  ClickHouse profile together. Re-emit the Garmin Dump provider deletion after
+  deployment, verify its event ID appears in
+  `ingest.metric_stream_delete_acknowledgement`, confirm zero live logical IDs,
+  and verify the rebuilt analytics plus user cache.

packages/server/src/routers/provider-detail.test.ts

@@ -1563,7 +1563,10 @@ describe("providerDetailRouter", () => {
           await fn({ execute: txExecute });
         });
       const mockExecute = vi.fn().mockResolvedValueOnce([{ id: "strava" }]);
-      mockReplaceMetricStreamBatch.mockResolvedValueOnce(0);
+      mockReplaceMetricStreamBatch.mockResolvedValueOnce({
+        deletedEventId: "30000000-0000-4000-8000-000000000001",
+        rowCount: 0,
+      });
       mockEnqueueProviderDeleteAnalyticsRefresh.mockResolvedValueOnce(undefined);
       const caller = createCaller({
         db: { execute: mockExecute, transaction: mockTransaction },
@@ -1582,7 +1585,11 @@ describe("providerDetailRouter", () => {
         "provider-data-delete",
       );
       expect(mockTransaction).toHaveBeenCalledTimes(1);
-      expect(mockEnqueueProviderDeleteAnalyticsRefresh).toHaveBeenCalledWith("user-1", "strava");
+      expect(mockEnqueueProviderDeleteAnalyticsRefresh).toHaveBeenCalledWith(
+        "user-1",
+        "strava",
+        "30000000-0000-4000-8000-000000000001",
+      );
       expect(mockProviderDataDeletesInc).toHaveBeenCalledWith({ provider_id: "strava" });
       const deleteSql = txExecute.mock.calls.map((call) => extractSqlText(call[0])).join("\n");
       expect(deleteSql).not.toContain("fitness.oauth_token");

packages/server/src/routers/provider-detail.ts

@@ -254,14 +254,18 @@ export const providerDetailRouter = router({
         });
       }
 
-      await replaceMetricStreamBatch(
+      const metricStreamReplacement = await replaceMetricStreamBatch(
         ctx.db,
         { userId: ctx.userId, providerId: input.providerId },
         [],
         "provider-data-delete",
       );
       await repo.deleteAllProviderRecords(input.providerId);
-      await enqueueProviderDeleteAnalyticsRefresh(ctx.userId, input.providerId);
+      await enqueueProviderDeleteAnalyticsRefresh(
+        ctx.userId,
+        input.providerId,
+        metricStreamReplacement.deletedEventId,
+      );
       providerDataDeletesTotal.inc({ provider_id: input.providerId });
       return { success: true };
     }),

scripts/backfill-ride-with-gps-track-points.ts

@@ -242,7 +242,13 @@ export async function applyRideWithGpsActivityBackfillPlan(
     `);
   }
 
-  return replaceMetricStreamBatch(db, { activityId: plan.id }, plan.metricRows, SOURCE_TYPE_API);
+  const result = await replaceMetricStreamBatch(
+    db,
+    { activityId: plan.id },
+    plan.metricRows,
+    SOURCE_TYPE_API,
+  );
+  return result.rowCount;
 }
 
 async function runBackfill(options: BackfillOptions): Promise<void> {