diff --git a/charts/insight/templates/clickhouse-migrate-job.yaml b/charts/insight/templates/clickhouse-migrate-job.yaml new file mode 100644 index 000000000..981da7d15 --- /dev/null +++ b/charts/insight/templates/clickhouse-migrate-job.yaml @@ -0,0 +1,94 @@ +{{/* +============================================================================== + ClickHouse gold-view migrations +============================================================================== + +Helm Hook Job that applies the gold-view migrations +(`src/ingestion/scripts/migrations/*.sql`) against the external ClickHouse. + +Why a Hook Job: the migrations used to be applied by `scripts/init.sh`, +which `kubectl exec`s into a bundled CH StatefulSet. #1428 dropped the +bundled L2 subcharts (CH is now always external), so that path can no +longer reach ClickHouse and nothing else applied the migrations — the +`clickhouse-init-svcdbs` Hook only runs `CREATE DATABASE`. This Job closes +that gap: it runs the toolbox image (which bundles the migrations, +`create-bronze-placeholders.sh`, and `lib/ch-exec.sh`) and dials +`clickhouse.host` directly over the HTTP interface (CLICKHOUSE_URL selects +the HTTP backend in lib/ch-exec.sh). + +Hook timing: `post-install,post-upgrade`. The post-* phase runs after the +main release (so the `clickhouse-init-svcdbs` pre-install Hook has created +the app database and the app pods have rolled). post-* over pre-* is safe +because gold-view consumers (analytics-api) resolve VIEW source tables +lazily at query time, so views materialising shortly after pod startup is +fine. (hook-weight "5" is inert today — it only orders hooks within the +same phase, and this is currently the only post-* hook; kept as a +placeholder for ordering future post-* hooks.) + +Failure behavior: a failed migration is NOT silently tolerated. Helm blocks +on the hook Job, so a non-zero exit fails `helm upgrade` (and, under the +gitops `--rollback-on-failure` default, rolls the release back). That is +intentional — a broken migration must surface loudly. `backoffLimit` is low +because migration SQL errors are deterministic, not transient: retrying a +bad statement just delays the inevitable failure. + +Idempotent: migrations are re-run on every upgrade and are CREATE OR +REPLACE / IF NOT EXISTS; placeholders are guarded by `ch_table_exists`. +No migration ledger — same contract as the legacy init.sh path. + +Gated on `clickhouse.runMigrations` (default true) AND the availability of +`ingestion.toolboxImage` (the image that ships the migration SQL). +*/}} +{{- if and .Values.clickhouse.runMigrations .Values.ingestion.toolboxImage }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "insight.fullname" . }}-clickhouse-migrate + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "5" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + {{- include "insight.labels" . | nindent 4 }} + app.kubernetes.io/component: clickhouse-migrate +spec: + # Low on purpose: migration SQL errors are deterministic (see header). + # A couple of attempts still absorbs a transient CH blip mid-run. + backoffLimit: 2 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "insight.fullname" . }}-clickhouse-migrate + app.kubernetes.io/component: clickhouse-migrate + spec: + # Suppress the legacy `_SERVICE_HOST/PORT/…` Docker-link env-vars + # kubelet auto-injects for every Service in the namespace at pod-start. + enableServiceLinks: false + restartPolicy: OnFailure + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: migrate + image: {{ .Values.ingestion.toolboxImage | quote }} + imagePullPolicy: IfNotPresent + command: [bash] + args: + - "-c" + - "bash /ingestion/scripts/apply-ch-migrations.sh" + env: + # CLICKHOUSE_URL selects the HTTP backend in lib/ch-exec.sh. + - name: CLICKHOUSE_URL + value: {{ include "insight.clickhouse.url" . | quote }} + - name: CLICKHOUSE_DATABASE + value: {{ include "insight.clickhouse.database" . | quote }} + - name: CLICKHOUSE_USER + value: {{ required "clickhouse.username is required" .Values.clickhouse.username | quote }} + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ required "clickhouse.passwordSecret.name is required" .Values.clickhouse.passwordSecret.name | quote }} + key: {{ required "clickhouse.passwordSecret.key is required" .Values.clickhouse.passwordSecret.key | quote }} +{{- end }} diff --git a/charts/insight/values.yaml b/charts/insight/values.yaml index ad45e0140..86e8a6df9 100644 --- a/charts/insight/values.yaml +++ b/charts/insight/values.yaml @@ -270,6 +270,13 @@ clickhouse: # at startup live here. initDatabases: - insight + # When true (default), a post-install/post-upgrade Hook Job + # (clickhouse-migrate-job) applies the gold-view migrations + # (src/ingestion/scripts/migrations/*.sql) + ADR-0007 placeholders against + # the external ClickHouse, using ingestion.toolboxImage. Set false to own + # CH migrations out-of-band (e.g. a managed-warehouse change process). + # The Job also no-ops if an overlay explicitly clears ingestion.toolboxImage. + runMigrations: true # ─── MariaDB ───────────────────────────────────────────────────────────── mariadb: host: "" # MUST be set (external L2 MariaDB host) diff --git a/docs/domain/ingestion/specs/DESIGN.md b/docs/domain/ingestion/specs/DESIGN.md index bc3a84d0a..30f7f6870 100644 --- a/docs/domain/ingestion/specs/DESIGN.md +++ b/docs/domain/ingestion/specs/DESIGN.md @@ -670,15 +670,15 @@ Key deployment decisions: - Airbyte ships its own bundled PostgreSQL for connector metadata (managed by the `airbyte/airbyte` chart, not by the umbrella). - Helm charts: Airbyte via `airbyte/airbyte`, Argo via `argo/argo-workflows`, the Insight platform via `charts/insight` in this repo. On Cyberfabric-operated and local clusters all three are installed by the gitops Makefile (`cd deploy/gitops && make deploy ENV=` — Airbyte/Argo as L2 system releases, the umbrella as the L3 app release); external consumers install them with their own tooling. - ClickHouse, MariaDB, Redis and Redpanda are deployed as separate L2 releases in `insight-infra` (gitops `make system-*`); the umbrella no longer bundles them as subcharts (removed in chart `0.2.0`). The umbrella consumes them through the unified `.host / .port / .database / .username / .passwordSecret` shape — see `charts/insight/values.yaml`. -- ClickHouse is a StatefulSet (`insight-clickhouse`) deployed as a separate L2 release (gitops `make system-clickhouse`); access is via Service `insight-clickhouse:8123` inside the cluster. Health probes use HTTP GET `/ping` (not `clickhouse-client` exec — avoids CLI flag parsing issues with auto-generated passwords). -- MariaDB per-service databases are provisioned via the bundled bitnami `mariadb.initdbScriptsConfigMap` (see `charts/insight/templates/mariadb-initdb-scripts.yaml`) — bitnami runs every script in that ConfigMap on the FIRST MariaDB pod boot, mounted at `/docker-entrypoint-initdb.d`. The data lives in the PVC after that, so restarts and helm upgrades are no-ops. Each owning service then runs its own SeaORM migrations at startup — see §4.4.2 and [ADR-0006](ADR/0006-service-owned-migrations.md). +- ClickHouse runs as a separate L2 release in `insight-infra` (gitops `make system-clickhouse`); the umbrella addresses it through `clickhouse.host`/`.port` (default `clickhouse.insight-infra.svc.cluster.local:8123`) over the HTTP interface. Health probes use HTTP GET `/ping` (not `clickhouse-client` exec — avoids CLI flag parsing issues with auto-generated passwords). +- MariaDB per-service databases are provisioned by a Helm Hook Job (`charts/insight/templates/mariadb-init-svcdbs-job.yaml`, `pre-install,pre-upgrade`) that dials the external `mariadb.host` and runs `CREATE DATABASE` + `GRANT` for each owned database — idempotent (`IF NOT EXISTS`) across helm upgrades. Each owning service then runs its own SeaORM migrations at startup — see §4.4.2 and [ADR-0006](ADR/0006-service-owned-migrations.md). - CDK connector build script (`airbyte-toolkit/build-connector.sh`) uses `CLUSTER_NAME` env var (default `insight`) for Kind image loading — not hardcoded. - Airbyte port-forward uses `nohup ... & disown` to avoid blocking the terminal. - Argo `dbt-run` WorkflowTemplate uses locally-built `insight-toolbox:local` image (with `imagePullPolicy: IfNotPresent`) — not `ghcr.io/constructorfabric/insight-toolbox:latest`. Local builds via `tools/toolbox/build.sh` pick up dbt model changes without requiring a registry push. Template also accepts `full_refresh` parameter (pass `--full-refresh` to recreate tables from scratch). - CoreDNS is patched to use public DNS upstream (`8.8.8.8`, `8.8.4.4`) — WSL's `/etc/resolv.conf` points to an internal WSL nameserver that cannot reliably resolve external domains (e.g. `login.microsoftonline.com`). Patch lives in `scripts/dev/patch-coredns-wsl.sh` (idempotent, opt-out via `SKIP_COREDNS_PATCH=1`); Windows/WSL+Kind operators run it manually against their cluster after bootstrap. - Gold views migration (`20260422000000_gold-views.sql`) references bronze tables from optional connectors (jira, m365, zoom). When the corresponding bronze table does not yet exist (no real connector data ingested yet), `scripts/create-bronze-placeholders.sh` creates empty placeholder tables with a minimal compatible schema so the gold-views migration succeeds on a partial install. - **Placeholder handoff caveat**: Airbyte ClickHouse destination v2.0.8+ throws an error on the first sync if the target bronze table exists with a schema that does not match the destination's expected schema for that stream. The placeholder schemas in `create-bronze-placeholders.sh` are intentionally minimal (only the columns referenced by gold views) — they are **not** a drop-in replacement for a native Airbyte-generated table. Before enabling a previously-placeholdered connector, the operator should manually `DROP TABLE` the placeholder(s) in ClickHouse so Airbyte can create them fresh on its first sync. - - Script runs automatically via `src/ingestion/run-init.sh` before migrations. + - Runs automatically inside the `clickhouse-migrate` Helm Hook Job, immediately before the gold-view migrations (see §4.4.1). - All credentials managed via Kubernetes Secrets (see §4.1.1). - Service access via Ingress (PR #224): the umbrella chart configures `ingress-nginx` routes for Frontend, API Gateway, Airbyte UI and Argo UI; on a local Kubernetes cluster the Kind config maps host ports 80/443 to the ingress controller. Direct port-forwards (`kubectl port-forward`) remain available for debugging. @@ -698,7 +698,7 @@ All Insight components share the release namespace (default `insight`); override **Resolution order** (all scripts): 1. Read from K8s Secret — sole credential source for all environments 2. If Secret missing → skip connector with error (no inline fallback) -3. ClickHouse password: inside the StatefulSet pod, the `clickhouse` container picks up `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD` env vars from `insight-db-creds`; ingestion scripts run `clickhouse-client` via `kubectl exec` and inherit those vars without passing `--user` / `--password` explicitly +3. ClickHouse password: the `clickhouse-migrate` Hook Job and ingestion workflows read `clickhouse-password` from `insight-db-creds` and authenticate to the external ClickHouse over its HTTP interface (ClickHouse `X-ClickHouse-User` / `X-ClickHouse-Key` headers — the password is never placed on the command line) **Connector credentials** use label-based discovery: `app.kubernetes.io/part-of: insight` label + `insight.cyberfabric.com/connector` annotation. See [ADR-0003](ADR/0003-k8s-secrets-credentials.md) for details. @@ -713,11 +713,11 @@ All Insight components share the release namespace (default `insight`); override 3. **No inline credential fallback.** Scripts do not fall back to reading credentials from tenant YAML. If a Secret is missing, the connector is skipped with an error. -4. **Destination password sync.** `airbyte-toolkit/connect.sh` always updates the ClickHouse destination password from the K8s Secret on every run. This ensures password rotation takes effect without recreating connections. +4. **Destination password sync.** The reconcile loop (`reconcile-connectors/main.sh`) resolves the Airbyte bronze destination from `insight-db-creds` on every run (ADR-0012), so a rotated ClickHouse password takes effect without recreating connections. -5. **Password rotation procedure.** Update Secret → apply to cluster → restart the ClickHouse StatefulSet (`kubectl rollout restart statefulset/insight-clickhouse -n "${INSIGHT_NAMESPACE:-insight}"`) → run `airbyte-toolkit/connect.sh` (which honours `INSIGHT_NAMESPACE`) to sync the Airbyte destination password. +5. **Password rotation procedure.** Update `insight-db-creds` → apply to cluster → rotate the credential on the external ClickHouse L2 release in `insight-infra` (owned by that release). The reconcile loop (`reconcile-connectors/main.sh`, run by the reconcile CronWorkflow) then re-syncs the Airbyte bronze-destination password from `insight-db-creds` on its next tick (ADR-0012). -6. **Destination sync modes.** `connect.sh` assigns destination sync modes based on source stream capabilities: +6. **Destination sync modes.** The reconcile loop assigns destination sync modes based on source stream capabilities: - `full_refresh` streams → `overwrite` (each sync replaces all data — no duplicate accumulation) - `incremental` streams → `append_dedup` (appends new records, deduplicates by primary key) @@ -745,13 +745,13 @@ the [gitops SPEC](../../components/deployment/gitops/README.md). | Script | Purpose | |--------|---------| | `cd deploy/gitops && make deploy ENV=local` | Brings up the full local stack: bootstrap (L0), Airbyte + Argo Workflows + the L2 system services, then the Insight umbrella chart (L3). Idempotent — re-running reconciles. Honours `$KUBECONFIG`. | -| `src/ingestion/run-init.sh` | Post-deploy init: verifies secrets, runs ClickHouse migrations, registers connectors, applies Airbyte connections, syncs Argo flows. (MariaDB schema is applied per-service by each backend service's own sea-orm `Migrator` at startup — see §4.4 and [ADR-0006](ADR/0006-service-owned-migrations.md). Per-service databases beyond the umbrella default are provisioned by `charts/insight/templates/mariadb-initdb-scripts.yaml` on the first MariaDB pod boot.) | +| `src/ingestion/run-init.sh` | Post-deploy init: verifies secrets, registers connectors, applies Airbyte connections, syncs Argo flows. (ClickHouse migrations are applied separately by the `clickhouse-migrate` Helm Hook Job on every install/upgrade — see §4.4.1. MariaDB schema is applied per-service by each backend service's own sea-orm `Migrator` at startup — see §4.4 and [ADR-0006](ADR/0006-service-owned-migrations.md). Per-service MariaDB databases are provisioned by the `mariadb-init-svcdbs` Helm Hook Job against the external `mariadb.host`.) | | `src/ingestion/sync-all.sh` | Trigger Airbyte sync for all connections. Reads connection IDs from state, calls Airbyte API. Use after `run-init.sh` to start first data load, or anytime to re-sync all sources. | **First-time setup**: 1. Copy connector secret examples → fill credentials (`src/ingestion/secrets/connectors/`) 2. `cd deploy/gitops && make deploy ENV=local` — full stack deployment (answer the wizard prompts on first run) -3. `./src/ingestion/run-init.sh` — databases, connectors, connections +3. `./src/ingestion/run-init.sh` — connectors, connections, Argo flows (ClickHouse migrations run automatically via the `clickhouse-migrate` Hook Job during step 2's helm install) 4. `cd src/ingestion && ./sync-all.sh` — trigger first Airbyte sync for all connections **Re-running**: `make deploy ENV=local` is idempotent — re-running on a converged cluster reconciles the stack in place. @@ -775,9 +775,9 @@ See [Airbyte Connector DESIGN](../../connector/specs/DESIGN.md) for detailed deb ### 4.4 Schema Migrations The project persists data in two stores with two different migration -mechanisms. **ClickHouse** schema is file-based and invoked from -`init.sh` (cluster bootstrap path); **MariaDB** schema is service- -owned — each backend service carries its own embedded `Migrator` and +mechanisms. **ClickHouse** schema is file-based and applied by the +`clickhouse-migrate` Helm Hook Job on every install/upgrade; **MariaDB** +schema is service-owned — each backend service carries its own embedded `Migrator` and applies its migrations at startup (see [ADR-0006](ADR/0006-service-owned-migrations.md)). The two paths diverge on bookkeeping because of different evolution patterns. @@ -787,11 +787,16 @@ patterns. - **Location**: `src/ingestion/scripts/migrations/*.sql` - **Naming**: `YYYYMMDDHHMMSS_.sql` — timestamp prefix enforces chronological order under lexicographic glob -- **Runner**: inline loop in `init.sh`, applied via - `kubectl exec -i -n "${INSIGHT_NAMESPACE:-insight}" statefulset/insight-clickhouse -- clickhouse-client --multiquery` - (override the namespace and pod selector via `INSIGHT_NAMESPACE` / `CLICKHOUSE_POD`) -- **Bookkeeping**: none — every migration is re-run on every `init.sh` - invocation and therefore must be written idempotently +- **Runner**: the `clickhouse-migrate` Helm Hook Job + (`post-install,post-upgrade`, `charts/insight/templates/clickhouse-migrate-job.yaml`) + runs `src/ingestion/scripts/apply-ch-migrations.sh` in the toolbox image. + It creates the `staging`/`silver`/app databases and the ADR-0007 + placeholders, then applies each migration in glob order against the + external ClickHouse over its HTTP interface (`lib/ch-exec.sh`). (The + legacy `init.sh` loop that `kubectl exec`'d into a bundled CH StatefulSet + was retired with the StatefulSet in #1428.) +- **Bookkeeping**: none — every migration is re-run on every helm + install/upgrade and therefore must be written idempotently (`CREATE ... IF NOT EXISTS`, `ALTER ... IF NOT EXISTS`, etc.) - **Rationale**: analytics schema is mostly `CREATE TABLE` / `CREATE VIEW` statements where idempotency guards are free, so a @@ -825,16 +830,15 @@ listener. This provides: - **Database isolation**: each service lives in its own MariaDB database; cross-service table access is an explicit architectural decision, not an accident of shared-schema layout. -- **No ordering coupling between services**: no single `init.sh` step +- **No ordering coupling between services**: no central bootstrap step must run before any service starts. -Infra responsibility (umbrella chart) stays minimal: the `mariadb` -subchart provisions the MariaDB instance, and the -`mariadb.primary.initdbScriptsConfigMap` (rendered from -`charts/insight/templates/mariadb-initdb-scripts.yaml`) creates the +Infra responsibility (umbrella chart) stays minimal: MariaDB runs as an +external L2 release, and a Helm Hook Job +(`charts/insight/templates/mariadb-init-svcdbs-job.yaml`, +`pre-install,pre-upgrade`) dials `mariadb.host` and creates the per-service databases (e.g. `CREATE DATABASE IF NOT EXISTS identity`) -and grants the app user access — bitnami runs the scripts on the -first MariaDB pod boot only, the data persists in the PVC, and +and grants the app user access — idempotent (`IF NOT EXISTS`), so helm upgrades are no-ops on this front. **Schema inside each database** is the owning service's job. diff --git a/src/ingestion/run-init.sh b/src/ingestion/run-init.sh index 7a34be303..1603aeefd 100755 --- a/src/ingestion/run-init.sh +++ b/src/ingestion/run-init.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash -# Initialize the ingestion stack: validate the umbrella install, run dbt -# database setup + ClickHouse migrations, adopt any pre-existing Airbyte -# resources, then drive the cluster to the descriptor-declared state via -# the single reconcile entrypoint. +# Initialize the ingestion stack: validate the umbrella install, sync Argo +# workflows, adopt any pre-existing Airbyte resources, then drive the cluster +# to the descriptor-declared state via the single reconcile entrypoint. +# +# ClickHouse migrations are NOT run from here — they are owned by the +# clickhouse-migrate Helm Hook Job (applied on every helm install/upgrade). # # Runs from the host machine (requires kubectl, curl, python3). # Run AFTER: helm install of the umbrella chart + ./secrets/apply.sh @@ -21,18 +23,13 @@ INSIGHT_NS="${INSIGHT_NAMESPACE}" # --- Verify the umbrella is installed --- echo "=== Verifying umbrella install ===" -if ! kubectl get -n "$INSIGHT_NS" statefulset/insight-clickhouse >/dev/null 2>&1; then - echo "ERROR: insight-clickhouse StatefulSet not found in namespace '$INSIGHT_NS'" >&2 - echo " Run: ./dev-compose.sh up (or: cd deploy/gitops && make deploy ENV=local)" >&2 - exit 1 -fi if ! kubectl get -n "$INSIGHT_NS" secret insight-db-creds >/dev/null 2>&1; then echo "ERROR: insight-db-creds Secret not found in namespace '$INSIGHT_NS'" >&2 echo " The umbrella chart should have created it on install." >&2 exit 1 fi -# --- Migrations + dbt databases (still managed by scripts/init.sh) --- +# --- Sync Argo workflows (scripts/init.sh) --- source ./scripts/init.sh # --- Single declarative reconcile chain --- diff --git a/src/ingestion/scripts/apply-ch-migrations.sh b/src/ingestion/scripts/apply-ch-migrations.sh new file mode 100755 index 000000000..1fe60a678 --- /dev/null +++ b/src/ingestion/scripts/apply-ch-migrations.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Apply the ClickHouse gold-view migrations against an EXTERNAL ClickHouse. +# +# This is the in-cluster, network-mode counterpart to the ClickHouse half +# of scripts/init.sh. init.sh `kubectl exec`s into a bundled CH StatefulSet +# (retired in #1428 when the umbrella stopped bundling L2 infra), so it +# cannot reach an external CH. This script talks to CH over its HTTP +# interface via lib/ch-exec.sh (selected by CLICKHOUSE_URL) and is invoked +# by the clickhouse-migrate Helm Hook Job (post-install,post-upgrade). +# +# Steps (same order and contract as init.sh): +# 1. Create the core databases (staging, silver, app db). +# 2. Run create-bronze-placeholders.sh — minimum-viable bronze/silver +# stubs so gold-view CREATE VIEW type-checks on a fresh cluster +# (CH validates referenced tables at parse time). See ADR-0007. +# 3. Apply migrations/*.sql in lexicographic order. +# +# Bookkeeping: none — every migration is re-run on every invocation and +# MUST stay idempotent/re-runnable (CREATE OR REPLACE / IF NOT EXISTS). +# This matches the existing init.sh contract (see ingestion DESIGN §migrations). +# +# Required env (set by the Hook Job from chart values + insight-db-creds): +# CLICKHOUSE_URL e.g. http://ch-host:8123 (selects the HTTP backend) +# CLICKHOUSE_USER, CLICKHOUSE_PASSWORD +# CLICKHOUSE_DATABASE the Insight app database +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +: "${CLICKHOUSE_URL:?CLICKHOUSE_URL must be set (e.g. http://ch-host:8123)}" +: "${CLICKHOUSE_DATABASE:?CLICKHOUSE_DATABASE must be set (the Insight app database)}" + +source "$SCRIPT_DIR/lib/ch-exec.sh" + +echo "=== Creating core databases (staging, silver, ${CLICKHOUSE_DATABASE}) ===" +run_ch </dev/null || echo "0") - [[ "$result" == "1" ]] -} +# ClickHouse access helpers (run_ch, ch_table_exists) that talk to the +# external ClickHouse over HTTP. Requires CLICKHOUSE_URL/USER/PASSWORD in +# the env — set by the clickhouse-migrate Hook Job. See lib/ch-exec.sh. +source "$SCRIPT_DIR/lib/ch-exec.sh" echo "=== Placeholders (for missing connectors / unbuilt silver) ===" diff --git a/src/ingestion/scripts/init.sh b/src/ingestion/scripts/init.sh index f3d64fc9b..d3f589dda 100755 --- a/src/ingestion/scripts/init.sh +++ b/src/ingestion/scripts/init.sh @@ -16,81 +16,25 @@ export RECONCILE_DIR="${SCRIPT_DIR}/../reconcile-connectors" # running `reconcile-connectors/main.sh` from the host install yq / jq # themselves; the toolbox image ships them pre-installed for cron pods. -# Single-namespace umbrella (PR #224). All Insight components — including the -# bundled ClickHouse StatefulSet — live in the release namespace, default -# `insight`. Exported so child scripts (airbyte-toolkit/*.sh, sync-flows.sh) -# inherit the value. +# Single-namespace umbrella (PR #224). Exported so child scripts +# (reconcile-connectors/*.sh, sync-flows.sh) inherit the value. : "${INSIGHT_NAMESPACE:?INSIGHT_NAMESPACE must be set, e.g. insight}" export INSIGHT_NAMESPACE -INSIGHT_NS="$INSIGHT_NAMESPACE" -CH_POD="${CLICKHOUSE_POD:-statefulset/insight-clickhouse}" # RULE-DEFAULTS-OK: bundled umbrella deploys this exact StatefulSet name; override only for non-bundled CH -# clickhouse-client inside the StatefulSet pod inherits CLICKHOUSE_USER / -# CLICKHOUSE_PASSWORD from the container env (set by the chart from -# auth.existingSecret), so we do not pass --user / --password. -ch_exec() { - kubectl exec -n "$INSIGHT_NS" "$CH_POD" -- clickhouse-client "$@" -} -ch_exec_stdin() { - kubectl exec -i -n "$INSIGHT_NS" "$CH_POD" -- clickhouse-client "$@" -} - -echo "=== Verifying ClickHouse pod ===" -if ! kubectl get -n "$INSIGHT_NS" "$CH_POD" >/dev/null 2>&1; then - echo "ERROR: ClickHouse not found at -n $INSIGHT_NS $CH_POD" >&2 - echo " Ensure the umbrella chart is installed with clickhouse.deploy=true" >&2 - echo " (helm list -n $INSIGHT_NS | grep insight)" >&2 - exit 1 -fi - -# Resolve the configured ClickHouse database name from the -# `insight-platform` ConfigMap (or CLICKHOUSE_DATABASE env override). The -# umbrella chart's `clickhouse.database` value drives both the bitnami -# subchart's CREATE DATABASE on first boot AND every consumer (Airbyte -# destination, analytics-api DSN, …) — keeping this loop in lock-step -# means a non-default `clickhouse.database` no longer breaks first-run -# init by silently creating the wrong DB. `staging` and `silver` are -# project-internal dbt schemas, those names are stable. +# ClickHouse migrations + dbt databases (staging/silver/app) + bronze/silver +# placeholders are NOT applied here anymore. They moved to the +# clickhouse-migrate Helm Hook Job +# (charts/insight/templates/clickhouse-migrate-job.yaml), which applies them +# over HTTP against the external ClickHouse on every install/upgrade. The +# bundled-CH `kubectl exec` path this script used died with the +# insight-clickhouse StatefulSet in #1428. # -# Fail-fast: no silent default. If neither env var nor ConfigMap key is -# set, abort with a clear message instead of guessing `insight` and -# creating a database the rest of the platform won't use. -CH_DB="${CLICKHOUSE_DATABASE:-}" # RULE-DEFAULTS-OK: empty sentinel; resolved from ConfigMap on next line, then asserted non-empty -if [[ -z "$CH_DB" ]]; then - CH_DB=$(kubectl get configmap -n "$INSIGHT_NS" insight-platform \ - -o jsonpath='{.data.CLICKHOUSE_DATABASE}') -fi -: "${CH_DB:?CLICKHOUSE_DATABASE not resolvable: set the env var explicitly, or ensure the umbrella chart is installed and the insight-platform ConfigMap has CLICKHOUSE_DATABASE populated (mirrors clickhouse.database in chart values).}" - -echo "=== Creating dbt databases (namespace=$INSIGHT_NS, app db=$CH_DB) ===" -for db in staging silver "$CH_DB"; do - if ! ch_exec --query "CREATE DATABASE IF NOT EXISTS $db"; then - echo "ERROR: failed to create $db database (namespace=$INSIGHT_NS)" >&2 - exit 1 - fi -done - -echo "=== Creating bronze placeholders for missing connectors (namespace=$INSIGHT_NS) ===" -"$SCRIPT_DIR/create-bronze-placeholders.sh" - -echo "=== Running ClickHouse migrations (namespace=$INSIGHT_NS) ===" -for migration in "$SCRIPT_DIR/migrations"/*.sql; do - [ -f "$migration" ] || continue - echo " $(basename "$migration")" - # `sed` instead of `grep -v` so a comment-only migration (matching every - # line) does not return exit 1 and abort the pipeline under `set -o pipefail`. - sed -E '/^[[:space:]]*--/d' "$migration" | ch_exec_stdin --multiquery -done - -# MariaDB migrations: each backend service now owns and applies its own -# migrations at startup (SeaORM Migrator::up). See ADR-0006. +# MariaDB migrations: each backend service owns and applies its own at +# startup (SeaORM Migrator::up). See ADR-0006. # -# NOTE: connector registration + connection apply are now handled by -# ../reconcile-connectors/main.sh (called from ../run-init.sh after this -# script finishes the migrations + dbt-database setup above). Do NOT add -# new `register.sh`/`connect.sh`-style invocations here — they were -# removed along with the legacy fan of scripts in the version-driven- -# reconcile refactor (ADR-0001). +# Connector registration + connection apply: ../reconcile-connectors/main.sh +# (called from ../run-init.sh). Do NOT add register.sh/connect.sh-style +# invocations here — removed in the version-driven-reconcile refactor (ADR-0001). echo "=== Syncing workflows ===" ./scripts/sync-flows.sh --all diff --git a/src/ingestion/scripts/lib/ch-exec.sh b/src/ingestion/scripts/lib/ch-exec.sh new file mode 100755 index 000000000..b771a0a71 --- /dev/null +++ b/src/ingestion/scripts/lib/ch-exec.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Shared ClickHouse access helpers for the in-cluster migration Job, which +# applies DDL against the EXTERNAL ClickHouse over its HTTP interface. +# +# ClickHouse is always external (both clusters install the L2 infra as +# separate releases via `make deploy` — the umbrella stopped bundling CH in +# #1428), so there is no in-cluster pod to `kubectl exec` into. HTTP is the +# only path. +# +# Sourced by apply-ch-migrations.sh and create-bronze-placeholders.sh. +# Exposes: +# run_ch — execute a (multi-statement) SQL block read from stdin. +# ch_table_exists — `ch_table_exists `; exit 0 if present. +# +# Required env: +# CLICKHOUSE_URL e.g. http://ch-host:8123 +# CLICKHOUSE_USER, CLICKHOUSE_PASSWORD + +# Idempotent source guard. +[[ -n "${__CH_EXEC_SH:-}" ]] && return 0 +__CH_EXEC_SH=1 + +: "${CLICKHOUSE_URL:?CLICKHOUSE_URL must be set (e.g. http://ch-host:8123)}" +: "${CLICKHOUSE_USER:?CLICKHOUSE_USER must be set}" +: "${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD must be set}" + +# Execute the SQL piped on stdin as one statement over the HTTP interface. +# The body is sent verbatim as the POST payload (--data-binary), mirroring +# clickhouse-init-svcdbs-job.yaml — form encoding would mangle the SQL into +# `query=CREATE+...` (Code 62). +# +# Credentials go via ClickHouse's native auth headers, NOT `-u`: the +# username header is harmless in argv, but the password is fed through a +# header file (process substitution) so it never lands in curl's argv (and +# thus /proc//cmdline, visible to any process in the pod). +_ch_http_query() { + curl -sS --fail-with-body \ + -H "X-ClickHouse-User: ${CLICKHOUSE_USER}" \ + -H @<(printf 'X-ClickHouse-Key: %s' "${CLICKHOUSE_PASSWORD}") \ + --data-binary @- \ + "${CLICKHOUSE_URL%/}/" +} + +# The HTTP interface runs one statement per request, so a multi-statement +# heredoc/file is fanned out statement-by-statement. We first drop full-line +# `--` comments (mirrors the init.sh sed pass + silver.py _split_statements), +# then split on `;`. +# +# DDL-ONLY invariant: splitting on `;` assumes no `;` inside string literals +# or /* */ blocks, and no inline `-- ...; ...` trailer. Every migration + +# placeholder honours this (same simplification silver.py relies on). A +# future migration with an in-string `;` MUST not use this path unguarded. +run_ch() { + local sql stmt + sql="$(sed -E '/^[[:space:]]*--/d')" + while IFS= read -r -d ';' stmt; do + # Skip whitespace-only segments (e.g. the tail after the last `;`). CH + # tolerates leading/trailing whitespace, so non-empty segments are sent + # as-is — no fragile per-line trim needed. + [[ "$stmt" =~ [^[:space:]] ]] || continue + printf '%s' "$stmt" | _ch_http_query + done < <(printf '%s;' "$sql") +} + +# NB: callers use `if ! ch_table_exists ...`, which disables `set -e` for +# this body — a probe failure (auth/transient/DNS) reads as "absent" and the +# caller falls through to CREATE ... IF NOT EXISTS, which is idempotent. +ch_table_exists() { + local db="$1" tbl="$2" result + result="$(printf "SELECT count() FROM system.tables WHERE database='%s' AND name='%s'" \ + "$db" "$tbl" | _ch_http_query | tr -d '[:space:]')" + [[ "$result" == "1" ]] +} diff --git a/src/ingestion/tests/e2e/e2e_lib/migration_applier.py b/src/ingestion/tests/e2e/e2e_lib/migration_applier.py index 248dcea73..409859b88 100644 --- a/src/ingestion/tests/e2e/e2e_lib/migration_applier.py +++ b/src/ingestion/tests/e2e/e2e_lib/migration_applier.py @@ -3,7 +3,7 @@ Migrations CREATE VIEW objects that reference bronze_*, silver, and staging databases. ClickHouse 24.x validates these references at CREATE-time, so we must materialize the bronze placeholder schemas BEFORE running migrations — -mirroring the prod order from src/ingestion/scripts/init.sh: +mirroring the prod order from src/ingestion/scripts/apply-ch-migrations.sh: 1. CREATE DATABASE staging | silver | insight 2. Run src/ingestion/scripts/create-bronze-placeholders.sh @@ -80,8 +80,8 @@ def reapply_migrations(cfg: SessionConfig) -> int: def apply_bronze_placeholders(cfg: SessionConfig) -> int: """Parse `create-bronze-placeholders.sh` heredocs and run the SQL. - The prod script invokes `kubectl exec` to talk to the in-cluster CH; we - extract just the SQL between `run_ch <<'SQL'` ... `SQL` markers and run + The prod script talks to the external CH over HTTP (via lib/ch-exec.sh); + we extract just the SQL between `run_ch <<'SQL'` ... `SQL` markers and run it via our HTTP client. """ script = cfg.repo_root / "src/ingestion/scripts/create-bronze-placeholders.sh"