diff --git a/.gitignore b/.gitignore index 8d4711089..53bce45ad 100644 --- a/.gitignore +++ b/.gitignore @@ -22,13 +22,13 @@ uv.lock local access -# Exceptions for the gitops-sample starter under docs/deploy — `local` +# Exceptions for the real gitops deployment under deploy/gitops — `local` # is a valid env name in the committed template and the `local/` # subdirectories must ship. -!docs/deploy/bootstrap/local/ -!docs/deploy/bootstrap/local/** -!docs/deploy/environments/local/ -!docs/deploy/environments/local/** +!deploy/gitops/bootstrap/local/ +!deploy/gitops/bootstrap/local/** +!deploy/gitops/environments/local/ +!deploy/gitops/environments/local/** # IDE .vs/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 21bb4a369..34f4400cd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,16 +98,18 @@ If you keep them elsewhere, set `INSIGHT_FRONT_PATH` in `.env.compose`. We are currently in transition. Both paths exist; **the compose path is the preferred one going forward**. -| Path | Driver | Use it when | -| ----------- | -------------------- | ---------------------------------------------------- | -| **compose** | `dev-compose.sh up` | Day-to-day backend / frontend work. Default. | -| k8s/helm | `dev-up.sh` (Kind) | Testing helm charts; ingestion (Airbyte) work; | -| | | anything that needs Argo Workflows or a real | -| | | cluster shape. | +| Path | Driver | Use it when | +| ----------- | ------------------------------------------- | ---------------------------------------------------- | +| **compose** | `dev-compose.sh up` | Day-to-day backend / frontend work. Default. | +| k8s/helm | `cd deploy/gitops && make deploy ENV=local` | Testing the published umbrella; ingestion (Airbyte) work; anything that needs Argo Workflows or the real cluster shape. | The compose path does **not** ship Airbyte or Argo Workflows — see [Beyond compose](#beyond-compose) below. +Both first-run paths share a single wizard at +[`compose/insight-init.sh`](compose/insight-init.sh), so the questions +(MariaDB / ClickHouse / tenant / dev email) are identical across them. + --- ## What's in the compose stack @@ -557,7 +559,7 @@ No rebuild, no compose bounce. To revert, undo the edits. The compose stack ships **9-ish services** but **does NOT include**: -- **Airbyte** — needs k8s. Use `./dev-up.sh ingestion`. +- **Airbyte** — needs k8s. - **Argo Workflows** — k8s controller; same deal. - **dbt scheduling** that depends on Argo Workflows. @@ -568,8 +570,31 @@ To run these, install **one** of: - kind (`brew install kind`) - minikube (`brew install minikube`) -…and use the existing `dev-up.sh` path. The k8s and compose stacks can -coexist — they use disjoint host ports by default. +…provision a cluster, then bring up Insight on k8s with the shared +first-run wizard: + +```bash +cd deploy/gitops +make deploy ENV=local +# or, if your kubeconfig lives outside ~/.kube/config: +KUBECONFIG=/path/to/config.yaml make deploy ENV=local +``` + +`kubectl`, `helm`, and `kubeseal` all honour `$KUBECONFIG`; the wizard +prints which file it's using at startup so you can abort and retry with +the right path if the context list looks wrong. + +On the first run the wizard generates `environments/local/inventory.yaml`, +`secrets-store.yaml`, and — when Airbyte is enabled — +`environments/local/.env.local` (carries the post-install setup creds the +`make system-airbyte` step needs). Subsequent runs skip the wizard and +just reconcile the stack. The wizard asks the same MariaDB/ClickHouse/ +tenant questions as the compose path, plus kube-context and which L2 +services to install (Airbyte/Argo/redpanda-console/observability). + +The k8s and compose stacks can coexist — they use disjoint host ports +by default. Demo-data seeding on the k8s path is manual — see the wizard +output for the port-forward + `compose/seed/` commands. --- diff --git a/charts/insight/values.yaml b/charts/insight/values.yaml index 12de58a0a..67eab0ecc 100644 --- a/charts/insight/values.yaml +++ b/charts/insight/values.yaml @@ -136,7 +136,7 @@ global: # # The umbrella NEVER installs the collector stack — like Airbyte and Argo # it is separate releases, driven by the deploy pipeline (inventory -# toggles; see docs/deploy/system/README.md). This block only governs what +# toggles; see deploy/gitops/system/README.md). This block only governs what # services EMIT. The values are published into the `{release}-platform` # ConfigMap as INSIGHT_LOG_LEVEL / OTEL_* env vars, so every pod that does # `envFrom` picks them up uniformly — no per-service wiring. diff --git a/compose/insight-init.sh b/compose/insight-init.sh new file mode 100755 index 000000000..c351c825d --- /dev/null +++ b/compose/insight-init.sh @@ -0,0 +1,812 @@ +#!/usr/bin/env bash +# Insight — shared first-run wizard. +# +# Generates the env-specific config file for one of two bring-up paths: +# +# --target=compose Writes .env.compose at the insight repo root. +# Invoked by `./dev-compose.sh up` when .env.compose +# is missing. +# +# --target=k8s-local Writes deploy/gitops/environments/local/inventory.yaml +# and populates deploy/gitops/secrets-store.yaml. +# Invoked by `make deploy ENV=local` when inventory.yaml +# is missing. +# +# Behavior: +# - Common questions (MariaDB / ClickHouse / tenant / dev email) ask once. +# - Target-specific extras follow. +# - Errors out hard on non-TTY stdin: the wizard is interactive only. +# - Errors out hard if the target output file already exists: delete it +# first to re-run. +# +# Why this lives in compose/ rather than its own top-level directory: +# The wizard descends from dev-compose.sh's first-run prompts. Keeping it +# alongside the compose stack avoids a third top-level dir; the k8s caller +# reaches it via `../../compose/insight-init.sh` from deploy/gitops/. + +set -euo pipefail + +# ────────────────────────────────────────────────────────────────────── +# Resolve repo root from this script's location, regardless of CWD. +# Layout: /compose/insight-init.sh +# ────────────────────────────────────────────────────────────────────── +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +usage() { + cat <<'EOF' +usage: insight-init.sh --target=compose|k8s-local + +Targets: + --target=compose Generate .env.compose for the docker-compose stack. + --target=k8s-local Generate deploy/gitops/environments/local/ + inventory.yaml + populate secrets-store.yaml for the + k8s gitops stack. + +The wizard is interactive only. Run from a terminal. +EOF +} + +TARGET="" +while [[ $# -gt 0 ]]; do + case "$1" in + --target=*) TARGET="${1#*=}"; shift ;; + --target) + TARGET="${2:-}" + shift + [[ $# -gt 0 ]] && shift + ;; + -h|--help) usage; exit 0 ;; + *) echo "ERROR: unknown arg: $1" >&2; usage >&2; exit 2 ;; + esac +done + +case "$TARGET" in + compose|k8s-local) ;; + "") echo "ERROR: --target is required" >&2; usage >&2; exit 2 ;; + *) echo "ERROR: unknown target: $TARGET" >&2; usage >&2; exit 2 ;; +esac + +if [[ ! -t 0 ]]; then + echo "ERROR: insight-init.sh needs an interactive shell (stdin is not a TTY)." >&2 + echo " Run from a terminal — there is no non-interactive fallback." >&2 + exit 1 +fi + +# ────────────────────────────────────────────────────────────────────── +# IO helpers — lifted verbatim from dev-compose.sh so behavior matches. +# ────────────────────────────────────────────────────────────────────── + +# ask — print prompt, read one line, echo answer (or +# default on empty input). Prompts go to stderr so the captured stdout +# stays clean. +ask() { + local prompt="$1" default="${2:-}" answer + if [[ -n "$default" ]]; then + printf '%s [%s]: ' "$prompt" "$default" >&2 + else + printf '%s: ' "$prompt" >&2 + fi + read -r answer + [[ -z "$answer" ]] && answer="$default" + printf '%s' "$answer" +} + +# ask_secret — read a password without echoing it. No default. +ask_secret() { + local prompt="$1" answer + printf '%s: ' "$prompt" >&2 + read -rs answer + printf '\n' >&2 + printf '%s' "$answer" +} + +# ask_yes_no — loop until a yes/no answer; return +# 0 for yes, 1 for no. Default is taken when the user hits Enter. +ask_yes_no() { + local prompt="$1" default="${2:-y}" answer hint + if [[ "$default" == "y" ]]; then hint="Y/n"; else hint="y/N"; fi + while true; do + printf '%s [%s]: ' "$prompt" "$hint" >&2 + read -r answer + [[ -z "$answer" ]] && answer="$default" + case "$(printf '%s' "$answer" | tr '[:upper:]' '[:lower:]')" in + y|yes) return 0 ;; + n|no) return 1 ;; + *) echo " Please answer y or n." >&2 ;; + esac + done +} + +# update_env_var — replace `KEY=...` in , or +# append a new line if the key doesn't exist. Portable across BSD (mac) +# and GNU sed. +update_env_var() { + local file="$1" key="$2" value="$3" escaped tmp + escaped=$(printf '%s' "$value" | sed -e 's/[\\&|]/\\&/g') + if grep -qE "^[[:space:]]*${key}=" "$file" 2>/dev/null; then + tmp=$(mktemp) + sed -E "s|^[[:space:]]*${key}=.*|${key}=${escaped}|" "$file" > "$tmp" + mv "$tmp" "$file" + else + printf '%s=%s\n' "$key" "$value" >> "$file" + fi +} + +# Warn loudly when the user pastes localhost as an "external" DB host — +# inside a container, that points at the container itself. +warn_localhost_host() { + local host="$1" label="$2" + case "$host" in + localhost|127.0.0.1|::1) + echo " WARN: '$host' resolves to the container itself, not your host." >&2 + echo " For a $label running on the docker host, use" >&2 + echo " host.docker.internal (Mac/Windows) or your LAN IP." >&2 + ;; + esac +} + +# validate_mariadb host port user pass — `mariadb -e "SELECT 1"` via a +# transient mariadb container. Returns 0 on success. +validate_mariadb() { + local host="$1" port="$2" user="$3" pass="$4" + echo " Probing MariaDB at ${host}:${port}..." >&2 + if docker run --rm mariadb:11.4 mariadb \ + -h "$host" -P "$port" -u "$user" "--password=$pass" \ + -e "SELECT 1" >/dev/null 2>&1; then + echo " MariaDB OK." >&2 + return 0 + fi + echo " ERROR: could not connect to MariaDB at ${host}:${port} as ${user}." >&2 + return 1 +} + +# validate_clickhouse host http_port user pass db — SELECT 1 via the HTTP +# interface using host-side curl. Returns 0 on success. +validate_clickhouse() { + local host="$1" port="$2" user="$3" pass="$4" db="$5" + echo " Probing ClickHouse at ${host}:${port}..." >&2 + if curl -sf -u "${user}:${pass}" \ + --data-urlencode "query=SELECT 1" \ + --data-urlencode "database=${db}" \ + "http://${host}:${port}/" >/dev/null 2>&1; then + echo " ClickHouse OK." >&2 + return 0 + fi + echo " ERROR: could not connect to ClickHouse at ${host}:${port} as ${user}." >&2 + return 1 +} + +# ────────────────────────────────────────────────────────────────────── +# Cross-OS tooling preflight. +# ────────────────────────────────────────────────────────────────────── + +_pkg_hint_macos=() +_pkg_hint_linux=() +_pkg_hint_url=() + +# require [macos-hint] [linux-hint] [url] +require() { + local cmd="$1" mh="${2:-brew install $1}" lh="${3:-sudo apt-get install $1}" url="${4:-}" + command -v "$cmd" >/dev/null 2>&1 && return 0 + echo " MISSING: $cmd" >&2 + case "$(uname -s)" in + Darwin) echo " install: $mh" >&2 ;; + Linux) + if grep -qi microsoft /proc/version 2>/dev/null; then + echo " install (WSL): $lh" >&2 + else + echo " install: $lh # or your distro's equivalent" >&2 + fi ;; + *) [[ -n "$url" ]] && echo " see: $url" >&2 || echo " install via your OS package manager" >&2 ;; + esac + return 1 +} + +preflight_compose() { + echo "--- Tooling preflight (compose) ---" >&2 + local missing=0 + require docker "brew install --cask docker" "sudo apt-get install docker.io" "https://docs.docker.com/engine/install/" || missing=$((missing+1)) + if ! docker compose version >/dev/null 2>&1; then + echo " MISSING: docker compose (v2 plugin)" >&2 + echo " upgrade Docker Desktop / install the docker-compose-plugin package" >&2 + missing=$((missing+1)) + fi + [[ $missing -gt 0 ]] && { echo " Resolve the missing tools and re-run." >&2; exit 1; } + echo " OK." >&2 + echo "" >&2 +} + +preflight_k8s() { + echo "--- Tooling preflight (k8s-local) ---" >&2 + local missing=0 + require kubectl "brew install kubectl" "sudo apt-get install kubectl" "https://kubernetes.io/docs/tasks/tools/" || missing=$((missing+1)) + require helm "brew install helm" "sudo apt-get install helm" "https://helm.sh/docs/intro/install/" || missing=$((missing+1)) + require kubeseal "brew install kubeseal" "see kubeseal release page" "https://github.com/bitnami-labs/sealed-secrets/releases" || missing=$((missing+1)) + require yq "brew install yq" "sudo snap install yq" "https://github.com/mikefarah/yq#install" || missing=$((missing+1)) + require jq "brew install jq" "sudo apt-get install jq" "https://jqlang.github.io/jq/download/" || missing=$((missing+1)) + + # Block helm v4.2.1 — known --wait regression that hangs the full + # --timeout on every fast hook-resource deletion. Trips bootstrap-* + # and system-* steps that use `before-hook-creation` lifecycle hooks + # (ingress-nginx admission, cert-manager startupapicheck, etc.). + # See helm/helm#32214; #32230 is the proposed revert. Pin to v4.2.0 + # or v3.x until v4.2.2+ ships. + if command -v helm >/dev/null 2>&1; then + local helm_ver + helm_ver="$(helm version --short 2>/dev/null | sed 's/+.*//; s/^v//')" + if [[ "$helm_ver" == "4.2.1" ]]; then + echo " BAD: helm $helm_ver has the --wait hook-deletion regression" >&2 + echo " https://github.com/helm/helm/issues/32214" >&2 + echo " Pin to v4.2.0 (https://get.helm.sh/) or v3.21.x until v4.2.2+ ships." >&2 + missing=$((missing+1)) + fi + fi + + [[ $missing -gt 0 ]] && { echo " Resolve the missing tools and re-run." >&2; exit 1; } + echo " OK." >&2 + echo "" >&2 +} + +# ────────────────────────────────────────────────────────────────────── +# Shared prompts — same questions for both targets. +# ────────────────────────────────────────────────────────────────────── +# +# Outputs assigned to shell variables for the target writers to consume. +# (Bash 3.2 lacks associative arrays, so plain globals it is.) + +# Sentinel UUID used by Insight when a default tenant is needed. +TENANT_DEFAULT_UUID="00000000-df51-5b42-9538-d2b56b7ee953" + +ask_shared() { + cat >&2 <&2 + if ask_yes_no "Use the local MariaDB (in-stack)?" "y"; then + MARIADB_EXTERNAL=false + case "$TARGET" in + compose) MARIADB_HOST=mariadb ;; + k8s-local) MARIADB_HOST=mariadb.insight-infra.svc.cluster.local ;; + esac + MARIADB_PORT=3306 + MARIADB_USER=insight + MARIADB_PASSWORD=insight-local + MARIADB_ROOT_PASSWORD=root-local + else + MARIADB_EXTERNAL=true + MARIADB_HOST=$(ask " External MariaDB host" "") + [[ -z "$MARIADB_HOST" ]] && { echo " ERROR: host is required." >&2; exit 1; } + [[ "$TARGET" == "compose" ]] && warn_localhost_host "$MARIADB_HOST" "MariaDB" + MARIADB_PORT=$(ask " External MariaDB port" "3306") + MARIADB_USER=$(ask " MariaDB user" "insight") + MARIADB_PASSWORD=$(ask_secret " MariaDB password") + # For compose, the root password is unused once MARIADB_EXTERNAL=true + # (nothing in the stack authenticates as root against an external DB). + # For k8s-local, the wizard writes the root password into the + # insight-db-creds Secret which the chart-side init job consumes — + # so we must collect the real value, not a placeholder. + if [[ "$TARGET" == "k8s-local" ]]; then + MARIADB_ROOT_PASSWORD=$(ask_secret " MariaDB root password (used by cluster init jobs)") + [[ -z "$MARIADB_ROOT_PASSWORD" ]] && { echo " ERROR: root password is required for k8s-local external MariaDB." >&2; exit 1; } + else + MARIADB_ROOT_PASSWORD=root-local + fi + # Connectivity probe — same for both targets, requires docker. + if command -v docker >/dev/null 2>&1; then + validate_mariadb "$MARIADB_HOST" "$MARIADB_PORT" "$MARIADB_USER" "$MARIADB_PASSWORD" || exit 1 + else + echo " (skipping connectivity probe — docker not available)" >&2 + fi + fi + echo "" >&2 + + # ── ClickHouse ──────────────────────────────────────────────────── + echo "--- ClickHouse ---" >&2 + if ask_yes_no "Use the local ClickHouse (in-stack)?" "y"; then + CLICKHOUSE_EXTERNAL=false + case "$TARGET" in + compose) CLICKHOUSE_HOST=clickhouse ;; + k8s-local) CLICKHOUSE_HOST=clickhouse.insight-infra.svc.cluster.local ;; + esac + CLICKHOUSE_HTTP_PORT=8123 + CLICKHOUSE_DATABASE=insight + CLICKHOUSE_USER=insight + CLICKHOUSE_PASSWORD=insight-local + else + CLICKHOUSE_EXTERNAL=true + CLICKHOUSE_HOST=$(ask " External ClickHouse host" "") + [[ -z "$CLICKHOUSE_HOST" ]] && { echo " ERROR: host is required." >&2; exit 1; } + [[ "$TARGET" == "compose" ]] && warn_localhost_host "$CLICKHOUSE_HOST" "ClickHouse" + CLICKHOUSE_HTTP_PORT=$(ask " External ClickHouse HTTP port" "8123") + CLICKHOUSE_DATABASE=$(ask " ClickHouse database" "insight") + CLICKHOUSE_USER=$(ask " ClickHouse user" "insight") + CLICKHOUSE_PASSWORD=$(ask_secret " ClickHouse password") + if command -v curl >/dev/null 2>&1; then + validate_clickhouse "$CLICKHOUSE_HOST" "$CLICKHOUSE_HTTP_PORT" "$CLICKHOUSE_USER" "$CLICKHOUSE_PASSWORD" "$CLICKHOUSE_DATABASE" || exit 1 + else + echo " (skipping connectivity probe — curl not available)" >&2 + fi + fi + echo "" >&2 + + # ── Tenant ID ───────────────────────────────────────────────────── + if [[ "$MARIADB_EXTERNAL" == "true" || "$CLICKHOUSE_EXTERNAL" == "true" ]]; then + echo "--- Tenant ID ---" >&2 + echo " External DBs already contain data tied to a specific tenant." >&2 + echo " Enter the UUID present in persons.insight_tenant_id." >&2 + TENANT_DEFAULT_ID=$(ask " TENANT_DEFAULT_ID" "") + if [[ -z "$TENANT_DEFAULT_ID" ]]; then + echo " ERROR: tenant ID is required when using external DBs." >&2 + exit 1 + fi + echo "" >&2 + else + TENANT_DEFAULT_ID="$TENANT_DEFAULT_UUID" + fi + + # ── Dev impersonation email ─────────────────────────────────────── + echo "--- Dev impersonation ---" >&2 + DEV_USER_EMAIL=$(ask "VITE_DEV_USER_EMAIL" "dev@company.nonpresent") + echo "" >&2 +} + +# ────────────────────────────────────────────────────────────────────── +# Compose target +# ────────────────────────────────────────────────────────────────────── + +write_compose() { + local env_file="$ROOT_DIR/.env.compose" + local example="$ROOT_DIR/.env.compose.example" + + if [[ -e "$env_file" ]]; then + echo "ERROR: $env_file already exists — delete it first to re-run the wizard." >&2 + exit 1 + fi + if [[ ! -f "$example" ]]; then + echo "ERROR: $example is missing — can't bootstrap .env.compose." >&2 + exit 1 + fi + + preflight_compose + ask_shared + + # ── Frontend mode (compose-only) ────────────────────────────────── + echo "--- Frontend ---" >&2 + local fe_mode fe_path default_fe_path="../insight-front" + echo " How should the frontend run?" >&2 + echo " 1) ghcr — pull the pre-built image (no source needed)" >&2 + echo " 2) local — Vite + HMR against an existing insight-front checkout" >&2 + echo " 3) clone — git clone insight-front, then run Vite + HMR" >&2 + local fe_choice + while true; do + fe_choice=$(ask " Choice" "1") + case "$fe_choice" in + 1|ghcr) + fe_mode="ghcr" + fe_path="$default_fe_path" + break ;; + 2|local|dev) + fe_mode="dev" + fe_path=$(ask " Path to insight-front checkout" "$default_fe_path") + if [[ -z "$fe_path" || ! -d "$ROOT_DIR/$fe_path" && ! -d "$fe_path" ]]; then + echo " ERROR: '$fe_path' does not exist. Pick option 3 to clone." >&2 + exit 1 + fi + break ;; + 3|clone) + if ! command -v git >/dev/null 2>&1; then + echo " ERROR: git is not installed; pick 1 or 2." >&2 + continue + fi + fe_path=$(ask " Clone insight-front into" "$default_fe_path") + # Resolve relative paths against the repo root. + local clone_target + if [[ "$fe_path" = /* ]]; then clone_target="$fe_path" + else clone_target="$ROOT_DIR/$fe_path"; fi + if [[ -e "$clone_target" ]]; then + echo " ERROR: '$clone_target' already exists; refusing to clone over it." >&2 + echo " Remove it first, or pick 2 to reuse the existing checkout." >&2 + exit 1 + fi + if ! git clone https://github.com/constructorfabric/insight-front.git "$clone_target" >&2; then + echo " ERROR: clone failed." >&2 + exit 1 + fi + fe_mode="dev" + break ;; + *) echo " Please answer 1, 2, or 3." >&2 ;; + esac + done + echo "" >&2 + + # ── Seeding decision for external DBs ───────────────────────────── + local seed_external=false + if [[ "$MARIADB_EXTERNAL" == "true" || "$CLICKHOUSE_EXTERNAL" == "true" ]]; then + echo "--- Test data ---" >&2 + echo " Local DBs are always seeded on first up. For external DBs the" >&2 + echo " wizard leaves them alone unless you opt in here." >&2 + if ask_yes_no " Seed test data into your external DB(s)?" "n"; then + seed_external=true + fi + echo "" >&2 + fi + + # ── Write .env.compose ──────────────────────────────────────────── + cp "$example" "$env_file" + update_env_var "$env_file" MARIADB_EXTERNAL "$MARIADB_EXTERNAL" + update_env_var "$env_file" MARIADB_HOST "$MARIADB_HOST" + update_env_var "$env_file" MARIADB_INTERNAL_PORT "$MARIADB_PORT" + update_env_var "$env_file" MARIADB_USER "$MARIADB_USER" + update_env_var "$env_file" MARIADB_PASSWORD "$MARIADB_PASSWORD" + update_env_var "$env_file" MARIADB_ROOT_PASSWORD "$MARIADB_ROOT_PASSWORD" + update_env_var "$env_file" CLICKHOUSE_EXTERNAL "$CLICKHOUSE_EXTERNAL" + update_env_var "$env_file" CLICKHOUSE_HOST "$CLICKHOUSE_HOST" + update_env_var "$env_file" CLICKHOUSE_INTERNAL_HTTP_PORT "$CLICKHOUSE_HTTP_PORT" + update_env_var "$env_file" CLICKHOUSE_DATABASE "$CLICKHOUSE_DATABASE" + update_env_var "$env_file" CLICKHOUSE_USER "$CLICKHOUSE_USER" + update_env_var "$env_file" CLICKHOUSE_PASSWORD "$CLICKHOUSE_PASSWORD" + update_env_var "$env_file" TENANT_DEFAULT_ID "$TENANT_DEFAULT_ID" + update_env_var "$env_file" VITE_DEV_USER_EMAIL "$DEV_USER_EMAIL" + update_env_var "$env_file" FRONTEND_MODE "$fe_mode" + update_env_var "$env_file" INSIGHT_FRONT_PATH "$fe_path" + + # SEEDED_LOCAL_* gates the first-run auto-seed in dev-compose.sh. + if [[ "$MARIADB_EXTERNAL" == "true" && "$seed_external" != "true" ]]; then + update_env_var "$env_file" SEEDED_LOCAL_MARIA true + fi + if [[ "$CLICKHOUSE_EXTERNAL" == "true" && "$seed_external" != "true" ]]; then + update_env_var "$env_file" SEEDED_LOCAL_CH true + fi + + echo "Wrote $env_file." >&2 +} + +# ────────────────────────────────────────────────────────────────────── +# K8s-local target — writes the gitops env files for `ENV=local`. +# +# Outputs (under deploy/gitops/): +# environments/local/inventory.yaml (concrete, gitignored) +# environments/local/.env.local (chain-sourced env vars: airbyte +# setup creds. gitignored.) +# secrets-store.yaml (cleartext, gitignored — read by +# scripts/secret-fetch.sh during seal) +# +# Does NOT provision the cluster, fetch the kubeseal pub-cert, run +# `helm install`, or write values.yaml. Those happen in subsequent +# Makefile chain steps (bootstrap → fetch-cert → seal → system → deploy). +# ────────────────────────────────────────────────────────────────────── + +# Auto-detect cluster type from kube-context name, returns one of +# kind | k3d | k3s | orbstack | colima | minikube | remote (best-effort). +detect_cluster_type() { + local ctx="$1" + case "$ctx" in + kind-*) echo "kind" ;; + k3d-*) echo "k3d" ;; + orbstack|orbstack-*) echo "orbstack" ;; + colima|colima-*) echo "colima" ;; + minikube|*minikube*) echo "minikube" ;; + *k3s*|insight-local) echo "k3s" ;; + *) echo "remote" ;; + esac +} + +write_k8s_local() { + local gitops_dir="$ROOT_DIR/deploy/gitops" + local inventory_out="$gitops_dir/environments/local/inventory.yaml" + local inventory_tmpl="$gitops_dir/environments/local/inventory.yaml.template" + local env_local_out="$gitops_dir/environments/local/.env.local" + local secrets_store_out="$gitops_dir/secrets-store.yaml" + local secrets_store_tmpl="$gitops_dir/secrets-store.yaml.template" + + if [[ -e "$inventory_out" ]]; then + echo "ERROR: $inventory_out already exists — delete it first to re-run the wizard." >&2 + exit 1 + fi + if [[ ! -f "$inventory_tmpl" ]]; then + echo "ERROR: $inventory_tmpl is missing — can't bootstrap inventory.yaml." >&2 + exit 1 + fi + if [[ ! -f "$secrets_store_tmpl" ]]; then + echo "ERROR: $secrets_store_tmpl is missing — can't bootstrap secrets-store.yaml." >&2 + exit 1 + fi + if [[ -e "$secrets_store_out" ]]; then + echo "ERROR: $secrets_store_out already exists — delete it first to re-run the wizard." >&2 + echo " (Contains cleartext passwords; verify you're not overwriting real state.)" >&2 + exit 1 + fi + + preflight_k8s + ask_shared + + # ── Kube-context ───────────────────────────────────────────────── + # + # kubectl/helm/kubeseal read from $KUBECONFIG (or ~/.kube/config if + # unset). If the cluster you want isn't listed, abort, re-run with the + # right kubeconfig: + # KUBECONFIG=/path/to/config.yaml make deploy ENV=local + echo "--- Kubernetes cluster ---" >&2 + if [[ -n "${KUBECONFIG:-}" ]]; then + echo " Using KUBECONFIG=$KUBECONFIG" >&2 + else + echo " Using ~/.kube/config (KUBECONFIG not set)" >&2 + fi + local available current default_ctx kube_ctx + available=$(kubectl config get-contexts -o name 2>/dev/null || true) + current=$(kubectl config current-context 2>/dev/null || true) + default_ctx="${current:-insight-local}" + if [[ -z "$available" ]]; then + echo " No kube-contexts found in your kubeconfig." >&2 + echo " Either provision a cluster (kind / k3d / OrbStack / …) and re-run," >&2 + echo " or point at a different kubeconfig:" >&2 + echo " KUBECONFIG=/path/to/config.yaml make deploy ENV=local" >&2 + exit 1 + fi + echo " Available contexts:" >&2 + printf '%s\n' "$available" | while IFS= read -r ctx; do + [[ -n "$ctx" ]] && printf ' - %s\n' "$ctx" >&2 + done + kube_ctx=$(ask " Kube context" "$default_ctx") + if ! printf '%s\n' "$available" | grep -qFx -- "$kube_ctx"; then + echo " ERROR: '$kube_ctx' is not in your kubeconfig." >&2 + exit 1 + fi + echo " Probing cluster '${kube_ctx}'..." >&2 + if ! kubectl --context "$kube_ctx" --request-timeout=5s cluster-info >/dev/null 2>&1; then + echo " ERROR: cannot reach cluster '$kube_ctx'." >&2 + echo " Bring it up (kind create cluster / k3d cluster create / OrbStack …)" >&2 + echo " and re-run. The wizard does not provision clusters." >&2 + exit 1 + fi + local cluster_type + cluster_type=$(detect_cluster_type "$kube_ctx") + echo " OK (cluster type: $cluster_type)" >&2 + echo "" >&2 + + # ── L0 cluster prereqs pre-flight ──────────────────────────────── + # + # bootstrap-* targets call `helm upgrade --install`, which fails if a + # matching resource already exists but isn't Helm-managed (OrbStack + # ships its own ingress-nginx, k3s sometimes ships traefik+klipper, a + # shared sandbox cluster might have cert-manager from another stack, + # etc). For each controller, probe the cluster, show what's there, + # and let the operator decide whether to skip our install. No silent + # reconfig — if the operator wants to install anyway and it fails, + # the Makefile aborts with helm's own error and they can clean up. + echo "--- L0 cluster prereqs (preflight) ---" >&2 + local l0_ingress_nginx=true l0_cert_manager=true l0_sealed_secrets=true + _check_l0_controller() { + local label="$1" ns="$2" release="$3" probe_kind="$4" probe_name="$5" + local found_present=false found_helm=false + if kubectl --context "$kube_ctx" get namespace "$ns" >/dev/null 2>&1; then + if kubectl --context "$kube_ctx" -n "$ns" get "$probe_kind" "$probe_name" >/dev/null 2>&1; then + found_present=true + if helm --kube-context "$kube_ctx" list -n "$ns" -q 2>/dev/null | grep -qFx -- "$release"; then + found_helm=true + fi + fi + fi + if [[ "$found_present" == "false" ]]; then + echo " $label: not installed → bootstrap will install." >&2 + return 0 + fi + if [[ "$found_helm" == "true" ]]; then + echo " $label: already Helm-managed (release '$release' in '$ns') → bootstrap will upgrade in place." >&2 + return 0 + fi + echo " $label: $probe_kind '$probe_name' exists in '$ns' but isn't Helm-managed." >&2 + echo " Installing via bootstrap will fail (ownership conflict)." >&2 + if ask_yes_no " Skip installing $label via bootstrap?" "y"; then + return 1 + fi + echo " OK — bootstrap will proceed; clean up the existing install manually if helm refuses." >&2 + return 0 + } + _check_l0_controller "ingress-nginx" ingress-nginx ingress-nginx sa ingress-nginx || l0_ingress_nginx=false + _check_l0_controller "cert-manager" cert-manager cert-manager deploy cert-manager || l0_cert_manager=false + _check_l0_controller "sealed-secrets" kube-system sealed-secrets-controller deploy sealed-secrets-controller || l0_sealed_secrets=false + echo "" >&2 + + # ── L2 services to install ─────────────────────────────────────── + echo "--- L2 services ---" >&2 + echo " MariaDB / ClickHouse / Redis / Redpanda are required (always on)." >&2 + echo " Pick the optional services to install:" >&2 + local sys_airbyte sys_argo sys_redpanda_console sys_obs airbyte_email airbyte_org + if ask_yes_no " Install Airbyte (data ingestion)?" "y"; then + sys_airbyte=true + airbyte_email=$(ask " Airbyte setup admin email" "admin@example.com") + airbyte_org=$(ask " Airbyte setup workspace name" "Insight") + else + sys_airbyte=false + fi + if ask_yes_no " Install Argo Workflows (job orchestration)?" "y"; then + sys_argo=true + else + sys_argo=false + fi + if ask_yes_no " Install redpanda-console (Kafka UI)?" "n"; then + sys_redpanda_console=true + else + sys_redpanda_console=false + fi + if ask_yes_no " Install observability stack (Loki + Alloy + Grafana)?" "n"; then + sys_obs=true + else + sys_obs=false + fi + echo "" >&2 + + # ── Write inventory.yaml ───────────────────────────────────────── + cp "$inventory_tmpl" "$inventory_out" + yq -i ".kubeContext = \"$kube_ctx\"" "$inventory_out" + yq -i ".bootstrap.ingressNginx = $l0_ingress_nginx" "$inventory_out" + yq -i ".bootstrap.certManager = $l0_cert_manager" "$inventory_out" + yq -i ".bootstrap.sealedSecrets = $l0_sealed_secrets" "$inventory_out" + yq -i ".system.airbyte = $sys_airbyte" "$inventory_out" + yq -i ".system.argoWorkflows = $sys_argo" "$inventory_out" + yq -i ".system.redpandaConsole = $sys_redpanda_console" "$inventory_out" + yq -i ".system.loki = $sys_obs" "$inventory_out" + yq -i ".system.alloy = $sys_obs" "$inventory_out" + yq -i ".system.grafana = $sys_obs" "$inventory_out" + echo "Wrote $inventory_out." >&2 + + # ── Write .env.local (airbyte chain creds) ─────────────────────── + # The chain sources this file via `set -a; . ./.env.local; set +a`, so + # the values must be safe to re-parse by a sourced bash. `printf %q` + # quotes whitespace + shell metacharacters; even though the wizard's + # defaults are tame, an org name like "Acme & Co" or an email with a + # `$` would otherwise corrupt the sourced env. + if [[ "$sys_airbyte" == "true" ]]; then + { + echo "# Generated by compose/insight-init.sh on first \`make deploy ENV=local\`." + echo "# Sourced by the local-chain target before invoking \`make system-airbyte\`." + echo "# Gitignored; safe to delete and regenerate." + printf 'AIRBYTE_SETUP_EMAIL=%q\n' "$airbyte_email" + printf 'AIRBYTE_SETUP_ORG=%q\n' "$airbyte_org" + } > "$env_local_out" + echo "Wrote $env_local_out." >&2 + fi + + # ── Write secrets-store.yaml ───────────────────────────────────── + # The template has 5 top-level keys: insight-local-{mariadb,clickhouse,redis}-creds + # under insight-infra and insight-local-insight-{db-creds,oidc} under insight. + # The wizard fills the cleartext from collected passwords; the oidc entry + # is left commented (sandbox runs with authDisabled=true). + # + # YAML-escape passwords before interpolating — operator-supplied external + # DB passwords can contain `"` or `\`, which would otherwise produce + # invalid manifests and break the seal step. Defaults (insight-local / + # root-local) are safe but we don't want the path to diverge per input. + yaml_escape() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } + local mdb_root_esc mdb_pass_esc ch_pass_esc + mdb_root_esc=$(yaml_escape "$MARIADB_ROOT_PASSWORD") + mdb_pass_esc=$(yaml_escape "$MARIADB_PASSWORD") + ch_pass_esc=$(yaml_escape "$CLICKHOUSE_PASSWORD") + cat > "$secrets_store_out" <). +## Values are cleartext Kubernetes Secret manifests; make seal-secret +## pipes them through kubeseal into committable sealed manifests. +## + +insight-local-mariadb-creds: | + apiVersion: v1 + kind: Secret + metadata: + name: mariadb-creds + namespace: insight-infra + type: Opaque + stringData: + mariadb-root-password: "$mdb_root_esc" + mariadb-password: "$mdb_pass_esc" + +insight-local-clickhouse-creds: | + apiVersion: v1 + kind: Secret + metadata: + name: clickhouse-creds + namespace: insight-infra + type: Opaque + stringData: + admin-password: "$ch_pass_esc" + +insight-local-redis-creds: | + apiVersion: v1 + kind: Secret + metadata: + name: redis-creds + namespace: insight-infra + type: Opaque + stringData: + redis-password: "redis-local" + +insight-local-insight-db-creds: | + apiVersion: v1 + kind: Secret + metadata: + name: insight-db-creds + namespace: insight + type: Opaque + stringData: + mariadb-root-password: "$mdb_root_esc" + mariadb-password: "$mdb_pass_esc" + clickhouse-password: "$ch_pass_esc" + redis-password: "redis-local" + +## insight-oidc — required only when authDisabled=false. The shipped +## local overlay disables auth, so this is left commented. Uncomment + +## fill in for envs with a real IdP, then add insight-oidc to +## inventory.secrets.services. +# insight-local-insight-oidc: | +# apiVersion: v1 +# kind: Secret +# metadata: +# name: insight-oidc +# namespace: insight +# type: Opaque +# stringData: +# APP__gears__oidc-authn-plugin__config__issuer_url: "https:///..." +# APP__gears__oidc-authn-plugin__config__audience: "" +# APP__gears__oidc-authn-plugin__config__jwks_url: "https:///.../jwks.json" +# APP__gears__auth-info__config__issuer_url: "https:///..." +# APP__gears__auth-info__config__client_id: "" +# APP__gears__auth-info__config__redirect_uri: "https:///callback" +# APP__gears__auth-info__config__scopes: "openid profile email" +EOF + echo "Wrote $secrets_store_out." >&2 + echo "" >&2 + + # ── tenant_default_id wiring into the umbrella overlay ─────────── + # values.yaml carries .global.tenantDefaultId; mirror the wizard's + # collected value unconditionally. Re-running the wizard after + # switching from external DBs back to local would otherwise leave a + # stale tenant id in values.yaml. + local values_file="$gitops_dir/environments/local/values.yaml" + if [[ -f "$values_file" ]]; then + yq -i ".global.tenantDefaultId = \"$TENANT_DEFAULT_ID\"" "$values_file" + echo "Updated .global.tenantDefaultId in $values_file." >&2 + fi + + cat >&2 </sealed-secrets/ +# are the only safe-to-commit representation). +secrets-store.yaml + +# The wizard writes a concrete inventory.yaml from inventory.yaml.template +# on first `make deploy ENV=local`. Each env's inventory.yaml IS committed +# (it's the source of truth for that cluster's topology), so this ignore +# is intentionally narrow — only the sandbox local copy is gitignored, +# because the wizard regenerates it from the template. +environments/local/inventory.yaml + +# pub-cert is fetched from the cluster via `make fetch-cert`; the +# corresponding generated *.pem is committed, but never the controller's +# private key. +*.key +*.pkcs8 diff --git a/docs/deploy/.gitleaks.toml b/deploy/gitops/.gitleaks.toml similarity index 100% rename from docs/deploy/.gitleaks.toml rename to deploy/gitops/.gitleaks.toml diff --git a/docs/deploy/.insight-version b/deploy/gitops/.insight-version similarity index 100% rename from docs/deploy/.insight-version rename to deploy/gitops/.insight-version diff --git a/docs/deploy/Brewfile b/deploy/gitops/Brewfile similarity index 100% rename from docs/deploy/Brewfile rename to deploy/gitops/Brewfile diff --git a/docs/deploy/Makefile b/deploy/gitops/Makefile similarity index 86% rename from docs/deploy/Makefile rename to deploy/gitops/Makefile index 48355e769..694a8e94b 100644 --- a/docs/deploy/Makefile +++ b/deploy/gitops/Makefile @@ -55,17 +55,20 @@ VALUES ?= environments/$(ENV)/values.yaml TIMEOUT ?= 10m RENDER_DIR := .deploy -# Extra flags appended to umbrella-related helm invocations (show chart, -# template, upgrade --install). Default empty; override to pass e.g. -# `--plain-http` when CHART points at a local HTTP OCI registry such as -# `oci://localhost:5001/charts/insight` for offline testing. +# Extra flags appended to helm invocations. Default empty. # -# HELM_EXTRA_FLAGS: passed to ALL helm calls (show / template / upgrade). -# Use for flags valid on every helm command, e.g. -# `--plain-http` when pulling from a local OCI registry. -# HELM_UPGRADE_FLAGS: passed ONLY to `helm upgrade --install`. Use for -# apply-time flags like `--force-conflicts` (SSA -# ownership override) that error out on +# HELM_EXTRA_FLAGS: passed to ALL helm calls (bootstrap-*, system-*, +# umbrella deploy, show/template/list/status/rollback). +# Use for flags valid on every helm command: +# `--plain-http` when pulling from a local HTTP OCI +# registry (e.g. oci://localhost:5001/...) +# `--debug` when a step is hanging and you want +# helm to print every kubectl API call it makes +# (verbose; great for diagnosing a stuck install). +# Example: HELM_EXTRA_FLAGS=--debug make deploy ENV=local +# HELM_UPGRADE_FLAGS: passed ONLY to the umbrella `helm upgrade --install`. +# Use for apply-time flags like `--force-conflicts` +# (SSA ownership override) that error out on # read-only `helm show` / `helm template`. HELM_EXTRA_FLAGS ?= HELM_UPGRADE_FLAGS ?= @@ -102,7 +105,7 @@ AIRBYTE_VERSION ?= 1.9.2 ARGO_RELEASE ?= argo-workflows ARGO_VERSION ?= 1.0.13 -ARGO_RBAC_TMPL ?= bootstrap/argo-rbac.yaml.tmpl +ARGO_RBAC_TMPL ?= bootstrap/argo-rbac.yaml.template ARGO_WORKFLOW_SA ?= argo-workflow # Observability stack (LGTM — logs first). Installed only when the bundled @@ -221,9 +224,11 @@ sync-clean: inventory-present: @test -f $(INVENTORY) \ || { echo "$(C_RED)ERROR$(C_RST): inventory not found at $(INVENTORY)"; \ - echo " Copy environments/local/inventory.yaml to a new env directory,"; \ - echo " adjust kubeContext / namespaces / bootstrap / system / secrets,"; \ - echo " then re-run."; \ + echo " For ENV=local: run \`make deploy ENV=local\` and the first-run wizard"; \ + echo " will generate it interactively."; \ + echo " For other envs: copy environments/local/inventory.yaml.template to"; \ + echo " environments//inventory.yaml, adjust kubeContext / namespaces /"; \ + echo " bootstrap / system / secrets, then re-run."; \ exit 1; } .PHONY: kube-ctx-required @@ -277,14 +282,49 @@ diff: sync-clean values-present chart-present @bash scripts/render-diff.sh "$(ENV)" "$(NAMESPACE)" "$(RELEASE)" \ "$(CHART)" "$(VALUES)" "$(RENDER_DIR)" "$(INSIGHT_VERSION)" -# Top-level deploy targets the L3 app only. L2 system services are -# deliberately not chained — each cluster picks which services it -# self-hosts vs. swaps for managed endpoints. Engineers run the L2 -# subset they need (system-mariadb / system-clickhouse / system-redis / -# system-redpanda / system-redpanda-console / system-airbyte / -# system-argo) before this target. +# Top-level deploy. For ENV=local (sandbox), `deploy` chains the full +# bring-up: wizard → bootstrap → fetch-cert → seal → system → deploy-app. +# For any other env, `deploy` keeps the narrow behavior (just upgrade +# the L3 umbrella); each cluster picks which L2 services it self-hosts +# vs. swaps for managed endpoints, so chaining `system` would be wrong. .PHONY: deploy +ifeq ($(ENV),local) +deploy: local-up +else deploy: deploy-app +endif + +# Local sandbox bring-up. Runs the wizard if inventory.yaml is missing, +# then chains the L0/L2/L3 install. Idempotent — re-running is a no-op +# apart from helm-upgrade noise. Each sub-make re-evaluates the +# inventory, so the wizard-written inventory.yaml is visible to the +# chained targets even though this make process started without it. +.PHONY: local-up +local-up: local-wizard + @set -a; [ -f environments/local/.env.local ] && . ./environments/local/.env.local; set +a; \ + CTX=$$(yq -r '.kubeContext' $(INVENTORY)); \ + if [ -z "$$CTX" ] || [ "$$CTX" = "null" ]; then \ + echo "$(C_RED)ERROR$(C_RST): inventory.yaml missing .kubeContext after wizard"; exit 1; \ + fi; \ + $(MAKE) --no-print-directory bootstrap ENV=local KUBE_CTX=$$CTX && \ + $(MAKE) --no-print-directory fetch-cert ENV=local KUBE_CTX=$$CTX && \ + $(MAKE) --no-print-directory seal ENV=local KUBE_CTX=$$CTX && \ + $(MAKE) --no-print-directory system ENV=local KUBE_CTX=$$CTX && \ + $(MAKE) --no-print-directory deploy-app ENV=local KUBE_CTX=$$CTX + +# Run the wizard only when inventory.yaml is missing. The wizard itself +# enforces non-TTY rejection. If the file is present (re-run on the +# same machine), skip silently. +.PHONY: local-wizard +local-wizard: + @if [ -f $(INVENTORY) ]; then \ + exit 0; \ + fi; \ + WIZARD="$$(cd ../.. && pwd)/compose/insight-init.sh"; \ + if [ ! -f "$$WIZARD" ]; then \ + echo "$(C_RED)ERROR$(C_RST): wizard not found at $$WIZARD"; exit 1; \ + fi; \ + bash "$$WIZARD" --target=k8s-local .PHONY: deploy-app deploy-app: deploy-insight @@ -426,7 +466,7 @@ system-mariadb: vpn-up kube-ctx $(call _require_system_creds,mariadb) helm upgrade --install $(MARIADB_RELEASE) $(MARIADB_CHART) \ --namespace $(NS_INFRA) --create-namespace \ - --version $(MARIADB_VERSION) \ + --version $(MARIADB_VERSION) $(HELM_EXTRA_FLAGS) \ $(call _system_values_args,mariadb) \ --wait --timeout 10m @@ -435,7 +475,7 @@ system-clickhouse: vpn-up kube-ctx $(call _require_system_creds,clickhouse) helm upgrade --install $(CLICKHOUSE_RELEASE) $(CLICKHOUSE_CHART) \ --namespace $(NS_INFRA) --create-namespace \ - --version $(CLICKHOUSE_VERSION) \ + --version $(CLICKHOUSE_VERSION) $(HELM_EXTRA_FLAGS) \ $(call _system_values_args,clickhouse) \ --wait --timeout 10m @@ -444,7 +484,7 @@ system-redis: vpn-up kube-ctx $(call _require_system_creds,redis) helm upgrade --install $(REDIS_RELEASE) $(REDIS_CHART) \ --namespace $(NS_INFRA) --create-namespace \ - --version $(REDIS_VERSION) \ + --version $(REDIS_VERSION) $(HELM_EXTRA_FLAGS) \ $(call _system_values_args,redis) \ --wait --timeout 5m @@ -454,7 +494,7 @@ system-redpanda: vpn-up kube-ctx $(call _apply_optional_system_secrets,redpanda) helm upgrade --install $(REDPANDA_RELEASE) $(REDPANDA_CHART) \ --namespace $(NS_INFRA) --create-namespace \ - --version $(REDPANDA_VERSION) \ + --version $(REDPANDA_VERSION) $(HELM_EXTRA_FLAGS) \ $(call _system_values_args,redpanda) \ --wait --timeout 10m @@ -464,7 +504,7 @@ system-redpanda-console: vpn-up kube-ctx $(call _apply_optional_system_secrets,redpanda-console) helm upgrade --install $(REDPANDA_CONSOLE_RELEASE) $(REDPANDA_CONSOLE_CHART) \ --namespace $(NS_INFRA) --create-namespace \ - --version $(REDPANDA_CONSOLE_VERSION) \ + --version $(REDPANDA_CONSOLE_VERSION) $(HELM_EXTRA_FLAGS) \ $(call _system_values_args,redpanda-console) \ --wait --timeout 5m @@ -480,7 +520,7 @@ system-loki: vpn-up kube-ctx $(call _apply_optional_system_secrets,loki) helm upgrade --install $(LOKI_RELEASE) $(LOKI_CHART) \ --namespace $(NS_INFRA) --create-namespace \ - --version $(LOKI_VERSION) \ + --version $(LOKI_VERSION) $(HELM_EXTRA_FLAGS) \ $(call _system_values_args,loki) \ --wait --timeout 10m @@ -490,7 +530,7 @@ system-alloy: vpn-up kube-ctx $(call _apply_optional_system_secrets,alloy) helm upgrade --install $(ALLOY_RELEASE) $(ALLOY_CHART) \ --namespace $(NS_INFRA) --create-namespace \ - --version $(ALLOY_VERSION) \ + --version $(ALLOY_VERSION) $(HELM_EXTRA_FLAGS) \ $(call _system_values_args,alloy) \ --wait --timeout 5m @@ -500,7 +540,7 @@ system-grafana: vpn-up kube-ctx $(call _apply_optional_system_secrets,grafana) helm upgrade --install $(GRAFANA_RELEASE) $(GRAFANA_CHART) \ --namespace $(NS_INFRA) --create-namespace \ - --version $(GRAFANA_VERSION) \ + --version $(GRAFANA_VERSION) $(HELM_EXTRA_FLAGS) \ $(call _system_values_args,grafana) \ --wait --timeout 5m @@ -527,7 +567,7 @@ system-airbyte: vpn-up kube-ctx $(call _apply_optional_system_secrets,airbyte) helm upgrade --install $(AIRBYTE_RELEASE) airbyte/airbyte \ --namespace $(NS_INFRA) --create-namespace \ - --version $(AIRBYTE_VERSION) \ + --version $(AIRBYTE_VERSION) $(HELM_EXTRA_FLAGS) \ $(call _system_values_args,airbyte) \ --wait --timeout 15m @NAMESPACE=$(NS_INFRA) AIRBYTE_RELEASE=$(AIRBYTE_RELEASE) \ @@ -544,7 +584,7 @@ system-argo: vpn-up kube-ctx $(call _apply_optional_system_secrets,argo-workflows) helm upgrade --install $(ARGO_RELEASE) argo/argo-workflows \ --namespace $(NS_INFRA) --create-namespace \ - --version $(ARGO_VERSION) \ + --version $(ARGO_VERSION) $(HELM_EXTRA_FLAGS) \ $(call _system_values_args,argo-workflows) \ --set controller.workflowNamespaces[0]=$(NS_INFRA) \ --set controller.instanceID.enabled=true \ @@ -559,19 +599,19 @@ system-argo: vpn-up kube-ctx .PHONY: system-status system-status: vpn-up kube-ctx @echo "=== Helm releases in $(NS_INFRA) ===" - @helm list -n $(NS_INFRA) 2>/dev/null || echo "(namespace $(NS_INFRA) absent — run 'make bootstrap ENV=$(ENV)')" + @helm list -n $(NS_INFRA) $(HELM_EXTRA_FLAGS) 2>/dev/null || echo "(namespace $(NS_INFRA) absent — run 'make bootstrap ENV=$(ENV)')" @echo @echo "=== Pods in $(NS_INFRA) ===" @kubectl -n $(NS_INFRA) get pods -o wide 2>/dev/null || true .PHONY: rollback rollback: vpn-up kube-ctx - @helm rollback $(RELEASE) --namespace $(NAMESPACE) - @helm history $(RELEASE) --namespace $(NAMESPACE) --max 5 + @helm rollback $(RELEASE) --namespace $(NAMESPACE) $(HELM_EXTRA_FLAGS) + @helm history $(RELEASE) --namespace $(NAMESPACE) --max 5 $(HELM_EXTRA_FLAGS) .PHONY: status status: vpn-up kube-ctx - @helm status $(RELEASE) --namespace $(NAMESPACE) --show-resources || true + @helm status $(RELEASE) --namespace $(NAMESPACE) --show-resources $(HELM_EXTRA_FLAGS) || true @echo "---" @kubectl -n $(NAMESPACE) rollout status deploy --timeout=30s || true @@ -590,9 +630,10 @@ tag: sync-clean # through kubeseal. # # `scripts/secret-fetch.sh` ships as a STUB: it reads from a local YAML -# file (secrets-store.yaml.sample) keyed by the resource name. Replace -# it with your own password-manager / vault / KMS integration — -# anything that prints a Kubernetes Secret manifest to stdout will work. +# file (secrets-store.yaml.template → copy to secrets-store.yaml) keyed +# by the resource name. Replace it with your own password-manager / +# vault / KMS integration — anything that prints a Kubernetes Secret +# manifest to stdout will work. # # Override the resource name with SECRET_NAME=… Or skip the script # entirely with VALUE_FILE=… (one-off bootstrapping; the cleartext file @@ -605,8 +646,9 @@ PUB_CERT := environments/$(ENV)/pub-cert.pem secret-fetch-present: @test -x scripts/secret-fetch.sh \ || { echo "$(C_RED)ERROR$(C_RST): scripts/secret-fetch.sh not found or not executable"; \ - echo " This sample ships a stub that reads from secrets-store.yaml."; \ - echo " Replace it with an integration against your password manager."; \ + echo " The shipped stub reads from secrets-store.yaml (copy from"; \ + echo " secrets-store.yaml.template). Replace it with an integration"; \ + echo " against your password manager for non-sandbox envs."; \ exit 1; } .PHONY: seal @@ -744,7 +786,7 @@ bootstrap-ingress-nginx: vpn-up kube-ctx fi; \ helm upgrade --install $(INGRESS_NGINX_RELEASE) ingress-nginx/ingress-nginx \ --namespace $(INGRESS_NGINX_NAMESPACE) --create-namespace \ - --version $(INGRESS_NGINX_VERSION) \ + --version $(INGRESS_NGINX_VERSION) $(HELM_EXTRA_FLAGS) \ $$VALUES_ARGS \ --wait --timeout 5m @@ -758,7 +800,7 @@ bootstrap-cert-manager: vpn-up kube-ctx fi; \ helm upgrade --install $(CERT_MANAGER_RELEASE) jetstack/cert-manager \ --namespace $(CERT_MANAGER_NAMESPACE) --create-namespace \ - --version $(CERT_MANAGER_VERSION) \ + --version $(CERT_MANAGER_VERSION) $(HELM_EXTRA_FLAGS) \ --set crds.enabled=true \ $$VALUES_ARGS \ --wait --timeout 5m @@ -771,7 +813,16 @@ bootstrap-cert-manager: vpn-up kube-ctx .PHONY: bootstrap-sealed-secrets bootstrap-sealed-secrets: vpn-up kube-ctx - @helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets >/dev/null 2>&1 || true + @# The old bitnami-labs.github.io/sealed-secrets host now returns + @# 404. Bitnami consolidated the index to bitnami.github.io (note + @# the -labs is gone). If the repo was registered under the old + @# URL on this machine, drop it first so `repo add` re-registers + @# the new host instead of silently keeping the dead one. + @OLD_REPOS=$$(helm repo list 2>/dev/null | awk '$$1=="sealed-secrets" && $$2!="https://bitnami.github.io/sealed-secrets" {print $$1}'); \ + if [ -n "$$OLD_REPOS" ]; then \ + printf '%s\n' "$$OLD_REPOS" | xargs -n1 helm repo remove >/dev/null 2>&1 || true; \ + fi + @helm repo add sealed-secrets https://bitnami.github.io/sealed-secrets >/dev/null 2>&1 || true @helm repo update sealed-secrets >/dev/null @VALUES_ARGS=""; \ if [ -s $(BOOTSTRAP_DIR)/sealed-secrets-values.yaml ]; then \ @@ -779,7 +830,7 @@ bootstrap-sealed-secrets: vpn-up kube-ctx fi; \ helm upgrade --install $(SEALED_SECRETS_RELEASE) sealed-secrets/sealed-secrets \ --namespace $(SEALED_SECRETS_NAMESPACE) \ - --version $(SEALED_SECRETS_VERSION) \ + --version $(SEALED_SECRETS_VERSION) $(HELM_EXTRA_FLAGS) \ $$VALUES_ARGS \ --wait --timeout 3m diff --git a/docs/deploy/README.md b/deploy/gitops/README.md similarity index 59% rename from docs/deploy/README.md rename to deploy/gitops/README.md index 294970cfd..abd8b277c 100644 --- a/docs/deploy/README.md +++ b/deploy/gitops/README.md @@ -1,64 +1,71 @@ -# Insight GitOps — Sample Starter +# Insight GitOps -A clone-and-adapt template for deploying [Insight](https://github.com/constructorfabric/insight) -into a Kubernetes cluster. The umbrella Helm chart is published from the -public Insight repo to `oci://ghcr.io/constructorfabric/charts/insight` per +The real deployment surface for [Insight](https://github.com/constructorfabric/insight) +on Kubernetes. The umbrella Helm chart is published from the public +Insight repo to `oci://ghcr.io/constructorfabric/charts/insight` per merge to `main`; this directory holds everything else — values overlays, sealed-secret manifests, the Makefile, and a few helper scripts. -**This is a sample.** Copy it into a private repo of your own, swap the -sample `local` env for one or more of your real clusters, replace the -secret-fetch stub with your password manager, and you have a working -gitops setup. +The bundled `local` env (sandbox) doubles as a starter template for +new envs: copy `environments/local/inventory.yaml.template` into a new +env directory, fill in `kubeContext` + the rest, swap the +`scripts/secret-fetch.sh` stub for your password-manager integration +when you go past sandbox, and you have a working gitops setup. -> The reference design lives in [`../components/deployment/`](../components/deployment/). +> The reference design lives in [`../../docs/components/deployment/`](../../docs/components/deployment/). > Below is the operator-facing summary; the linked docs go deeper into > rationale (DESIGN, PRD, ADR). ## What's in this directory -``` -docs/deploy/ -├── README.md # this file -├── Makefile # engineer entry point (bootstrap / system-* / deploy / seal …) -├── Brewfile # required tooling -├── .insight-version # one line: the umbrella chart semver pinned for this repo +```text +deploy/gitops/ +├── README.md # this file +├── Makefile # engineer entry point (bootstrap / system-* / deploy / seal …) +├── Brewfile # required tooling (macOS — Linux uses your package manager) +├── .insight-version # one line: the umbrella chart semver pinned for this repo ├── .gitignore -├── .gitleaks.toml # pre-commit secret-scanning rules -├── secrets-store.yaml.sample # template for the sample secret store +├── .gitleaks.toml # pre-commit secret-scanning rules +├── secrets-store.yaml.template # template for the sample secret store; copy to secrets-store.yaml and fill in ├── bootstrap/ -│ ├── argo-rbac.yaml.tmpl # supplemental Argo RBAC; templated, applied by Makefile -│ └── local/ # per-cluster L0 prereqs (one dir per env) +│ ├── argo-rbac.yaml.template # supplemental Argo RBAC; rendered + applied by Makefile +│ └── local/ # per-cluster L0 prereqs (one dir per env) │ ├── ingress-nginx-values.yaml │ ├── cert-manager-values.yaml │ ├── sealed-secrets-values.yaml │ └── selfsigned-issuer.yaml -├── system/ # L2 base values, one dir per service -│ ├── README.md # services table + secret layout -│ ├── mariadb/ # values.yaml + SECRETS.md -│ ├── clickhouse/ # values.yaml + SECRETS.md -│ ├── redis/ # values.yaml + SECRETS.md -│ ├── redpanda/ # values.yaml -│ ├── redpanda-console/ # values.yaml -│ ├── airbyte/ # values.yaml -│ └── argo-workflows/ # values.yaml +├── system/ # L2 base values, one dir per service +│ ├── README.md # services table + secret layout +│ ├── mariadb/ # values.yaml + SECRETS.md +│ ├── clickhouse/ # values.yaml + SECRETS.md +│ ├── redis/ # values.yaml + SECRETS.md +│ ├── redpanda/ # values.yaml +│ ├── redpanda-console/ # values.yaml +│ ├── airbyte/ # values.yaml +│ └── argo-workflows/ # values.yaml ├── environments/ -│ └── local/ # sample sandbox env (start here) -│ ├── inventory.yaml # what this cluster has (drives bootstrap / system / seal / deploy) -│ ├── values.yaml # umbrella overlay (L3) -│ ├── pub-cert.pem.sample +│ └── local/ # sandbox env (also the starter template for new envs) +│ ├── inventory.yaml.template # what this cluster has (drives bootstrap / system / seal / deploy) +│ ├── values.yaml # umbrella overlay (L3) │ └── sealed-secrets/ -│ ├── insight-infra/*.yaml.sample # L2 sealed secrets (one folder per Kubernetes namespace) -│ └── insight/*.yaml.sample # L3 sealed secrets +│ ├── insight-infra/*.yaml.template # L2 sealed-secret shape (one folder per Kubernetes namespace) +│ └── insight/*.yaml.template # L3 sealed-secret shape └── scripts/ - ├── doctor.sh # invoked by `make doctor` - ├── render-diff.sh # invoked by `make diff` - ├── secret-fetch.sh # password-manager stub for `make seal-secret` - ├── compose-app-secrets.sh# derives insight-{analytics-api,identity}-config from insight-db-creds - └── airbyte-setup.sh # post-install Airbyte setup-wizard automation + ├── doctor.sh # invoked by `make doctor` + ├── render-diff.sh # invoked by `make diff` + ├── secret-fetch.sh # password-manager stub for `make seal-secret` + ├── compose-app-secrets.sh # derives insight-{analytics-api,identity}-config from insight-db-creds + └── airbyte-setup.sh # post-install Airbyte setup-wizard automation ``` +The wizard at `../../compose/insight-init.sh` is shared with the +docker-compose stack. For `ENV=local`, `make deploy ENV=local` +auto-invokes it whenever `environments/local/inventory.yaml` is +missing, then chains `bootstrap → fetch-cert → seal → system → +deploy-app`. For other envs the operator copies the template manually +and runs each target individually. + ## Layer model | Layer | What | Namespace | Driven by | @@ -84,7 +91,19 @@ Before running any `make` target against a cluster: fail fast with `cannot reach cluster ''` if `kubectl cluster-info` errors. -3. **Kube-context named `insight-`** — the Makefile expects the +3. **Kubeconfig** — `kubectl`, `helm`, and `kubeseal` all read from + `$KUBECONFIG` (or `~/.kube/config` if unset). If your cluster's + kubeconfig lives elsewhere, pass it on the make invocation: + + ```bash + KUBECONFIG=/path/to/config.yaml make deploy ENV=local + ``` + + The wizard prints which kubeconfig it's reading at startup; if the + context list is empty or wrong, abort and re-invoke with the right + `KUBECONFIG=` prefix. + +4. **Kube-context named `insight-`** — the Makefile expects the context for env `` to be called `insight-` (e.g. ENV=local → context `insight-local`). If your kubeconfig uses a different name, either rename: @@ -111,25 +130,45 @@ which L2 services to install, which secrets to seal, and whether per-secret targets (`make system-mariadb`, `make seal-secret …`) remain available for one-off / rotation work. -Skim `environments/local/inventory.yaml` for the schema; it's the -shortest path to understanding what each env can declare. +Skim `environments/local/inventory.yaml.template` for the schema; it's +the shortest path to understanding what each env can declare. The +wizard generates the concrete `environments/local/inventory.yaml` from +it on the first `make deploy ENV=local`. -## Quick start (local k3s sandbox) +## Quick start (local sandbox — kind / k3d / OrbStack) + +For `ENV=local`, one command does it all: ```bash -brew bundle install -make doctor # verify tooling +make deploy ENV=local +``` + +On the first run, when `environments/local/inventory.yaml` is missing, +this auto-invokes the first-run wizard (shared with the docker-compose +stack) which prompts for kube-context, L2 toggles, passwords, and the +tenant ID, then writes: + +- `environments/local/inventory.yaml` +- `secrets-store.yaml` (gitignored cleartext) with the entries the + wizard collected + +After the wizard, the same `make deploy ENV=local` continues with the +full chain: `bootstrap → fetch-cert → seal → system → deploy-app`. +Subsequent `make deploy ENV=local` calls skip the wizard (inventory +already exists) and re-run the chain idempotently. -# L0 — install ingress-nginx, cert-manager, sealed-secrets-controller -# plus the insight-infra + insight namespaces. Driven by -# inventory.bootstrap.*. Idempotent. -make bootstrap ENV=local +If you'd rather run the steps manually: + +```bash +brew bundle install # macOS — Linux uses your package manager +make doctor # verify tooling +make bootstrap ENV=local # L0 make fetch-cert ENV=local # capture the controller's pub cert for `make seal*` # Stage cleartext Secret manifests in the sample secret store. (Copy -# the sample, fill in real passwords. NEVER COMMIT the populated file — +# the template, fill in real passwords. NEVER COMMIT the populated file — # it's gitignored.) -cp secrets-store.yaml.sample secrets-store.yaml +cp secrets-store.yaml.template secrets-store.yaml $EDITOR secrets-store.yaml # Seal everything listed in inventory.secrets. Cleartext is streamed @@ -142,26 +181,26 @@ AIRBYTE_SETUP_EMAIL=admin@example.com AIRBYTE_SETUP_ORG=Sandbox \ make system ENV=local make system-status ENV=local # what's installed in insight-infra -# L3 — the umbrella app. `make deploy` is an alias for `make deploy-app` -# and only touches the `insight` namespace. It applies every L3 sealed -# manifest, waits for `insight-db-creds` to materialise, composes the -# derived `insight-analytics-api-config` + `insight-identity-config` -# Secrets, then helm-upgrades. -# -# Before the first `make deploy`: edit `environments/local/values.yaml` -# and replace the `REPLACE_WITH_LATEST_*_TAG` placeholders with concrete -# image tags. The umbrella chart's appVersion is the canonical source — -# check the tag on `oci://ghcr.io/constructorfabric/charts/insight:<.insight-version>` -# or peek at GHCR directly. +# L3 — the umbrella app. Only touches the `insight` namespace. Applies +# every L3 sealed manifest, waits for `insight-db-creds` to materialise, +# composes the derived `insight-analytics-api-config` + +# `insight-identity-config` Secrets, then helm-upgrades. Image tags are +# inherited from the umbrella chart's appVersion — no per-service tag +# overrides are needed in values.yaml for the sandbox path. make diff ENV=local # inspect what would change make deploy ENV=local ``` ## Adding a new environment +The shared wizard only writes the `local` env. For new envs, copy from +the templates: + ```bash -# 1. Copy the sample env. -cp -r environments/local environments/ +# 1. Bootstrap a new env directory from the local templates. +mkdir -p environments/ +cp environments/local/inventory.yaml.template environments//inventory.yaml +cp environments/local/values.yaml environments//values.yaml # 2. Edit environments//inventory.yaml — kube-context, which L0 # controllers / L2 services / secrets this env wants, whether it's @@ -175,8 +214,7 @@ cp -r environments/local environments/ # values. (The bootstrap// dir is read by the bootstrap-* # sub-targets; missing = chart defaults.) -# 5. Bootstrap + fetch cert + seal + L2 + L3 — same as the quick start -# above, just with ENV=. +# 5. Bootstrap + fetch cert + seal + L2 + L3, individually. make bootstrap ENV= make fetch-cert ENV= make seal ENV= @@ -188,7 +226,7 @@ The `local` env disables OIDC for sandbox convenience. For production or staging envs, set `apiGateway.authDisabled: false`, configure an OIDC IdP (Okta, Entra, Auth0, Keycloak, …), and seal a corresponding `insight-oidc` Secret — see -[`environments/local/sealed-secrets/insight/insight-oidc-sealedsecret.yaml.sample`](environments/local/sealed-secrets/insight/insight-oidc-sealedsecret.yaml.sample) +[`environments/local/sealed-secrets/insight/insight-oidc-sealedsecret.yaml.template`](environments/local/sealed-secrets/insight/insight-oidc-sealedsecret.yaml.template) for the seven required keys. ## Secret management @@ -201,7 +239,7 @@ the cleartext lives in your password manager. `make seal-secret` calls `scripts/secret-fetch.sh ` under the hood. The shipped stub reads from a local -`secrets-store.yaml` file (see `secrets-store.yaml.sample` for the +`secrets-store.yaml` file (see `secrets-store.yaml.template` for the format). **Replace this stub before you go to production.** Plug in whichever password manager / vault / KMS you use: @@ -219,22 +257,22 @@ Exit non-zero on lookup failure. Per-service key shapes are in [`system//SECRETS.md`](system/). -### Sealed-secret samples in this directory +### Sealed-secret templates in this directory -The committed `*.yaml.sample` files under +The committed `*.yaml.template` files under `environments/local/sealed-secrets/` show the **shape** of a sealed manifest — they intentionally don't contain working ciphertext, because a SealedSecret can only be decrypted by the cluster it was sealed -against. Run the `make seal-secret …` commands in the quick-start -above and you'll get real `*.yaml` siblings beside them, safe to -commit. +against. Run the `make seal-secret …` commands (or `make deploy +ENV=local`, which seals everything in the inventory) and you'll get +real `*.yaml` siblings beside them, safe to commit. ## Chart-pin flow (L3) 1. The public Insight repo's CI publishes umbrella chart versions to `oci://ghcr.io/constructorfabric/charts/insight:` per merge to `main`. See - [`../components/deployment/specs/ADR/0001-chart-publishing-on-merge.md`](../components/deployment/specs/ADR/0001-chart-publishing-on-merge.md) + [`../../docs/components/deployment/specs/ADR/0001-chart-publishing-on-merge.md`](../../docs/components/deployment/specs/ADR/0001-chart-publishing-on-merge.md) for the contract. 2. The `.insight-version` file in this repo pins one semver. Bump it to promote a new chart version. The Makefile reads it as diff --git a/docs/deploy/bootstrap/argo-rbac.yaml.tmpl b/deploy/gitops/bootstrap/argo-rbac.yaml.template similarity index 100% rename from docs/deploy/bootstrap/argo-rbac.yaml.tmpl rename to deploy/gitops/bootstrap/argo-rbac.yaml.template diff --git a/docs/deploy/bootstrap/local/cert-manager-values.yaml b/deploy/gitops/bootstrap/local/cert-manager-values.yaml similarity index 100% rename from docs/deploy/bootstrap/local/cert-manager-values.yaml rename to deploy/gitops/bootstrap/local/cert-manager-values.yaml diff --git a/docs/deploy/bootstrap/local/ingress-nginx-values.yaml b/deploy/gitops/bootstrap/local/ingress-nginx-values.yaml similarity index 100% rename from docs/deploy/bootstrap/local/ingress-nginx-values.yaml rename to deploy/gitops/bootstrap/local/ingress-nginx-values.yaml diff --git a/docs/deploy/bootstrap/local/sealed-secrets-values.yaml b/deploy/gitops/bootstrap/local/sealed-secrets-values.yaml similarity index 100% rename from docs/deploy/bootstrap/local/sealed-secrets-values.yaml rename to deploy/gitops/bootstrap/local/sealed-secrets-values.yaml diff --git a/docs/deploy/bootstrap/local/selfsigned-issuer.yaml b/deploy/gitops/bootstrap/local/selfsigned-issuer.yaml similarity index 100% rename from docs/deploy/bootstrap/local/selfsigned-issuer.yaml rename to deploy/gitops/bootstrap/local/selfsigned-issuer.yaml diff --git a/docs/deploy/environments/local/inventory.yaml b/deploy/gitops/environments/local/inventory.yaml.template similarity index 100% rename from docs/deploy/environments/local/inventory.yaml rename to deploy/gitops/environments/local/inventory.yaml.template diff --git a/docs/deploy/environments/local/sealed-secrets/insight-infra/clickhouse-creds-sealedsecret.yaml.sample b/deploy/gitops/environments/local/sealed-secrets/insight-infra/clickhouse-creds-sealedsecret.yaml.template similarity index 83% rename from docs/deploy/environments/local/sealed-secrets/insight-infra/clickhouse-creds-sealedsecret.yaml.sample rename to deploy/gitops/environments/local/sealed-secrets/insight-infra/clickhouse-creds-sealedsecret.yaml.template index 88970e83f..5cf1b08d4 100644 --- a/docs/deploy/environments/local/sealed-secrets/insight-infra/clickhouse-creds-sealedsecret.yaml.sample +++ b/deploy/gitops/environments/local/sealed-secrets/insight-infra/clickhouse-creds-sealedsecret.yaml.template @@ -1,5 +1,5 @@ --- -# Sample SHAPE — see mariadb-creds-sealedsecret.yaml.sample in this +# Sample SHAPE — see mariadb-creds-sealedsecret.yaml.template in this # directory for the workflow that generates a real, cluster-specific # sealed manifest. apiVersion: bitnami.com/v1alpha1 diff --git a/docs/deploy/environments/local/sealed-secrets/insight-infra/mariadb-creds-sealedsecret.yaml.sample b/deploy/gitops/environments/local/sealed-secrets/insight-infra/mariadb-creds-sealedsecret.yaml.template similarity index 100% rename from docs/deploy/environments/local/sealed-secrets/insight-infra/mariadb-creds-sealedsecret.yaml.sample rename to deploy/gitops/environments/local/sealed-secrets/insight-infra/mariadb-creds-sealedsecret.yaml.template diff --git a/docs/deploy/environments/local/sealed-secrets/insight-infra/redis-creds-sealedsecret.yaml.sample b/deploy/gitops/environments/local/sealed-secrets/insight-infra/redis-creds-sealedsecret.yaml.template similarity index 82% rename from docs/deploy/environments/local/sealed-secrets/insight-infra/redis-creds-sealedsecret.yaml.sample rename to deploy/gitops/environments/local/sealed-secrets/insight-infra/redis-creds-sealedsecret.yaml.template index 9ce3f4f0c..29256aadd 100644 --- a/docs/deploy/environments/local/sealed-secrets/insight-infra/redis-creds-sealedsecret.yaml.sample +++ b/deploy/gitops/environments/local/sealed-secrets/insight-infra/redis-creds-sealedsecret.yaml.template @@ -1,5 +1,5 @@ --- -# Sample SHAPE — see mariadb-creds-sealedsecret.yaml.sample in this +# Sample SHAPE — see mariadb-creds-sealedsecret.yaml.template in this # directory for the workflow that generates a real, cluster-specific # sealed manifest. apiVersion: bitnami.com/v1alpha1 diff --git a/docs/deploy/environments/local/sealed-secrets/insight/insight-db-creds-sealedsecret.yaml.sample b/deploy/gitops/environments/local/sealed-secrets/insight/insight-db-creds-sealedsecret.yaml.template similarity index 91% rename from docs/deploy/environments/local/sealed-secrets/insight/insight-db-creds-sealedsecret.yaml.sample rename to deploy/gitops/environments/local/sealed-secrets/insight/insight-db-creds-sealedsecret.yaml.template index 48e0b98a4..ca4433ff8 100644 --- a/docs/deploy/environments/local/sealed-secrets/insight/insight-db-creds-sealedsecret.yaml.sample +++ b/deploy/gitops/environments/local/sealed-secrets/insight/insight-db-creds-sealedsecret.yaml.template @@ -1,6 +1,6 @@ --- # Sample SHAPE — see -# ../insight-infra/mariadb-creds-sealedsecret.yaml.sample for the +# ../insight-infra/mariadb-creds-sealedsecret.yaml.template for the # workflow that generates a real, cluster-specific sealed manifest. # # `insight-db-creds` lives in the L3 `insight` namespace and is diff --git a/docs/deploy/environments/local/sealed-secrets/insight/insight-oidc-sealedsecret.yaml.sample b/deploy/gitops/environments/local/sealed-secrets/insight/insight-oidc-sealedsecret.yaml.template similarity index 94% rename from docs/deploy/environments/local/sealed-secrets/insight/insight-oidc-sealedsecret.yaml.sample rename to deploy/gitops/environments/local/sealed-secrets/insight/insight-oidc-sealedsecret.yaml.template index d7dae4fef..cd3722ece 100644 --- a/docs/deploy/environments/local/sealed-secrets/insight/insight-oidc-sealedsecret.yaml.sample +++ b/deploy/gitops/environments/local/sealed-secrets/insight/insight-oidc-sealedsecret.yaml.template @@ -1,6 +1,6 @@ --- # Sample SHAPE — see -# ../insight-infra/mariadb-creds-sealedsecret.yaml.sample for the +# ../insight-infra/mariadb-creds-sealedsecret.yaml.template for the # workflow that generates a real, cluster-specific sealed manifest. # # `insight-oidc` is REQUIRED only when `apiGateway.authDisabled: false` diff --git a/docs/deploy/environments/local/values.yaml b/deploy/gitops/environments/local/values.yaml similarity index 82% rename from docs/deploy/environments/local/values.yaml rename to deploy/gitops/environments/local/values.yaml index 3f9c93dd1..ea6f5ce14 100644 --- a/docs/deploy/environments/local/values.yaml +++ b/deploy/gitops/environments/local/values.yaml @@ -74,19 +74,16 @@ ingestion: # ─── App services ────────────────────────────────────────────────────────── # -# Pin `image.tag` to a concrete tag published to GHCR by the upstream -# CI. The published tag format is `YYYY.MM.DD.HH.MM-`; pick -# the latest one from the chart appVersion in -# https://github.com/constructorfabric/insight (or whichever umbrella -# you're tracking) and update on bumps. -# -# The example tags below are illustrative — replace them with whatever -# the umbrella chart pinned in `.insight-version` was built against. +# Image tags are NOT pinned here. The umbrella chart's appVersion (and +# each subchart's image.tag default) drives which image versions get +# deployed — bumping `.insight-version` switches to that chart's image +# defaults atomically. Override `image.tag` per service ONLY when you +# need to roll out a hotfix on one service ahead of a new chart version; +# remove the override on the next chart bump so you don't drift from +# the chart's tested set. apiGateway: replicaCount: 1 - image: - tag: "REPLACE_WITH_LATEST_BACKEND_TAG" # Sandbox: no OIDC. Real envs flip this to false and supply an # `insight-oidc` Secret in the `insight` namespace. authDisabled: true @@ -98,8 +95,6 @@ apiGateway: analyticsApi: replicaCount: 1 - image: - tag: "REPLACE_WITH_LATEST_BACKEND_TAG" resources: requests: { cpu: 50m, memory: 128Mi } limits: { cpu: 500m, memory: 512Mi } @@ -115,9 +110,6 @@ analyticsApi: # from insight-db-creds. identity: deploy: true - image: - repository: ghcr.io/constructorfabric/insight-identity - tag: "REPLACE_WITH_LATEST_BACKEND_TAG" databaseName: "identity" # Optional. When set (must be a valid GUID), identity-resolution # lookups don't 400 with "Tenant not provided" if the caller omits @@ -131,12 +123,5 @@ identity: frontend: replicaCount: 1 - # The frontend image is built from a separate repo (insight-front) - # with its own publish-chart workflow; its tag is decoupled from the - # backend services above. Pin the latest tag from - # https://github.com/orgs/constructorfabric/packages?q=insight-front - # (or whichever upstream you're tracking). - image: - tag: "REPLACE_WITH_LATEST_FRONTEND_TAG" ingress: enabled: false diff --git a/docs/deploy/scripts/airbyte-setup.sh b/deploy/gitops/scripts/airbyte-setup.sh similarity index 100% rename from docs/deploy/scripts/airbyte-setup.sh rename to deploy/gitops/scripts/airbyte-setup.sh diff --git a/docs/deploy/scripts/compose-app-secrets.sh b/deploy/gitops/scripts/compose-app-secrets.sh similarity index 100% rename from docs/deploy/scripts/compose-app-secrets.sh rename to deploy/gitops/scripts/compose-app-secrets.sh diff --git a/docs/deploy/scripts/doctor.sh b/deploy/gitops/scripts/doctor.sh similarity index 72% rename from docs/deploy/scripts/doctor.sh rename to deploy/gitops/scripts/doctor.sh index e0f9d14da..e7d67e9b8 100755 --- a/docs/deploy/scripts/doctor.sh +++ b/deploy/gitops/scripts/doctor.sh @@ -27,6 +27,24 @@ check() { echo "Tooling:" check helm helm "3.14" "brew install helm" 'helm version --short | sed s/^v//' + +# Refuse helm v4.2.1 — known regression where `--wait` hangs the full +# --timeout on fast hook-resource deletions, turning every +# `before-hook-creation` lifecycle into a 5–10 minute stall. Affects +# every bootstrap-* / system-* step. See helm/helm#32214 (regression) +# and helm/helm#32230 (proposed revert). Pin to v4.2.0 or v3.x until a +# v4.2.2+ release lands the fix. +if command -v helm >/dev/null 2>&1; then + _helm_ver="$(helm version --short 2>/dev/null | sed 's/+.*//; s/^v//')" + if [ "$_helm_ver" = "4.2.1" ]; then + echo "${C_RED}BAD${C_RST} helm $_helm_ver — known --wait regression" + echo " https://github.com/helm/helm/issues/32214" + echo " Pin to v4.2.0: curl -fL https://get.helm.sh/helm-v4.2.0-\$(uname -s | tr A-Z a-z)-\$(uname -m | sed s/x86_64/amd64/).tar.gz | tar -xz" + echo " Or v3.21.1: brew install helm@3 (and brew link --overwrite helm@3)" + _fail=$((_fail + 1)) + fi +fi + check kubectl kubectl "1.27" "brew install kubectl" 'kubectl version --client -o json | jq -r .clientVersion.gitVersion' check kubeseal kubeseal "0.27" "brew install kubeseal" 'kubeseal --version' check skopeo skopeo "1.14" "brew install skopeo" 'skopeo --version' diff --git a/docs/deploy/scripts/push-deploy-log.sh b/deploy/gitops/scripts/push-deploy-log.sh similarity index 100% rename from docs/deploy/scripts/push-deploy-log.sh rename to deploy/gitops/scripts/push-deploy-log.sh diff --git a/docs/deploy/scripts/render-diff.sh b/deploy/gitops/scripts/render-diff.sh similarity index 100% rename from docs/deploy/scripts/render-diff.sh rename to deploy/gitops/scripts/render-diff.sh diff --git a/docs/deploy/scripts/secret-fetch.sh b/deploy/gitops/scripts/secret-fetch.sh similarity index 94% rename from docs/deploy/scripts/secret-fetch.sh rename to deploy/gitops/scripts/secret-fetch.sh index 73a57a019..ab7d08bdd 100755 --- a/docs/deploy/scripts/secret-fetch.sh +++ b/deploy/gitops/scripts/secret-fetch.sh @@ -13,7 +13,7 @@ # This stub reads from a local YAML file (default: # `secrets-store.yaml` at the repo root) whose top-level keys are # resource names and values are cleartext Secret manifests. See -# `secrets-store.yaml.sample` for the format. +# `secrets-store.yaml.template` for the format. # # IMPORTANT — replace this stub before going to production. The flat # YAML file is only convenient for sandbox / first-time-walkthrough @@ -37,7 +37,7 @@ STORE="${SECRET_STORE_FILE:-$SCRIPT_DIR/../secrets-store.yaml}" if [ ! -f "$STORE" ]; then echo "ERROR: secret-store file not found at $STORE" >&2 - echo " Copy secrets-store.yaml.sample to secrets-store.yaml," >&2 + echo " Copy secrets-store.yaml.template to secrets-store.yaml," >&2 echo " fill in the cleartext Secret manifests, and re-run." >&2 echo " (Or override the path with SECRET_STORE_FILE=...)" >&2 exit 1 diff --git a/docs/deploy/secrets-store.yaml.sample b/deploy/gitops/secrets-store.yaml.template similarity index 100% rename from docs/deploy/secrets-store.yaml.sample rename to deploy/gitops/secrets-store.yaml.template diff --git a/docs/deploy/system/README.md b/deploy/gitops/system/README.md similarity index 100% rename from docs/deploy/system/README.md rename to deploy/gitops/system/README.md diff --git a/docs/deploy/system/airbyte/values.yaml b/deploy/gitops/system/airbyte/values.yaml similarity index 100% rename from docs/deploy/system/airbyte/values.yaml rename to deploy/gitops/system/airbyte/values.yaml diff --git a/docs/deploy/system/alloy/values.yaml b/deploy/gitops/system/alloy/values.yaml similarity index 100% rename from docs/deploy/system/alloy/values.yaml rename to deploy/gitops/system/alloy/values.yaml diff --git a/docs/deploy/system/argo-workflows/values.yaml b/deploy/gitops/system/argo-workflows/values.yaml similarity index 100% rename from docs/deploy/system/argo-workflows/values.yaml rename to deploy/gitops/system/argo-workflows/values.yaml diff --git a/docs/deploy/system/clickhouse/SECRETS.md b/deploy/gitops/system/clickhouse/SECRETS.md similarity index 100% rename from docs/deploy/system/clickhouse/SECRETS.md rename to deploy/gitops/system/clickhouse/SECRETS.md diff --git a/docs/deploy/system/clickhouse/values.yaml b/deploy/gitops/system/clickhouse/values.yaml similarity index 100% rename from docs/deploy/system/clickhouse/values.yaml rename to deploy/gitops/system/clickhouse/values.yaml diff --git a/docs/deploy/system/grafana/values.yaml b/deploy/gitops/system/grafana/values.yaml similarity index 100% rename from docs/deploy/system/grafana/values.yaml rename to deploy/gitops/system/grafana/values.yaml diff --git a/docs/deploy/system/loki/values.yaml b/deploy/gitops/system/loki/values.yaml similarity index 100% rename from docs/deploy/system/loki/values.yaml rename to deploy/gitops/system/loki/values.yaml diff --git a/docs/deploy/system/mariadb/SECRETS.md b/deploy/gitops/system/mariadb/SECRETS.md similarity index 100% rename from docs/deploy/system/mariadb/SECRETS.md rename to deploy/gitops/system/mariadb/SECRETS.md diff --git a/docs/deploy/system/mariadb/values.yaml b/deploy/gitops/system/mariadb/values.yaml similarity index 100% rename from docs/deploy/system/mariadb/values.yaml rename to deploy/gitops/system/mariadb/values.yaml diff --git a/docs/deploy/system/redis/SECRETS.md b/deploy/gitops/system/redis/SECRETS.md similarity index 100% rename from docs/deploy/system/redis/SECRETS.md rename to deploy/gitops/system/redis/SECRETS.md diff --git a/docs/deploy/system/redis/values.yaml b/deploy/gitops/system/redis/values.yaml similarity index 100% rename from docs/deploy/system/redis/values.yaml rename to deploy/gitops/system/redis/values.yaml diff --git a/docs/deploy/system/redpanda-console/values.yaml b/deploy/gitops/system/redpanda-console/values.yaml similarity index 100% rename from docs/deploy/system/redpanda-console/values.yaml rename to deploy/gitops/system/redpanda-console/values.yaml diff --git a/docs/deploy/system/redpanda/values.yaml b/deploy/gitops/system/redpanda/values.yaml similarity index 100% rename from docs/deploy/system/redpanda/values.yaml rename to deploy/gitops/system/redpanda/values.yaml diff --git a/dev-compose.sh b/dev-compose.sh index 71f9f1387..9c0988321 100755 --- a/dev-compose.sh +++ b/dev-compose.sh @@ -47,38 +47,13 @@ resolve_env_file() { } # ────────────────────────────────────────────────────────────────────── -# First-run wizard helpers +# Helpers that survived the wizard extraction # -# Used by cmd_up to generate .env.compose interactively when it's -# missing. Everything here is io-helpers and validation — the wizard -# proper is `bootstrap_env_wizard` below. +# The first-run wizard moved to compose/insight-init.sh (shared with the +# k8s-local bring-up). These two helpers stay because non-wizard +# subcommands here (prune, cmd_up's seed-gate flip) still use them. # ────────────────────────────────────────────────────────────────────── -# ask — print prompt, read one line, echo answer (or -# default on empty input). Prompts go to stderr so the captured stdout -# stays clean. -ask() { - local prompt="$1" default="${2:-}" answer - if [[ -n "$default" ]]; then - printf '%s [%s]: ' "$prompt" "$default" >&2 - else - printf '%s: ' "$prompt" >&2 - fi - read -r answer - [[ -z "$answer" ]] && answer="$default" - printf '%s' "$answer" -} - -# ask_secret — read a password without echoing it. No default -# (passwords shouldn't have defaults you can't see). -ask_secret() { - local prompt="$1" answer - printf '%s: ' "$prompt" >&2 - read -rs answer - printf '\n' >&2 - printf '%s' "$answer" -} - # ask_yes_no — loops until a yes/no answer; return # 0 for yes, 1 for no. Default is taken when the user hits Enter. ask_yes_no() { @@ -101,7 +76,6 @@ ask_yes_no() { # and GNU sed by writing through a temp file. update_env_var() { local file="$1" key="$2" value="$3" escaped tmp - # Escape sed replacement-side metacharacters: \, &, and our delimiter |. escaped=$(printf '%s' "$value" | sed -e 's/[\\&|]/\\&/g') if grep -qE "^[[:space:]]*${key}=" "$file" 2>/dev/null; then tmp=$(mktemp) @@ -112,238 +86,6 @@ update_env_var() { fi } -# Warn loudly when the user pastes localhost as an "external" DB host — -# inside the container, that points at the container itself. Common -# gotcha; the wizard can't fix it but should flag it. -warn_localhost_host() { - local host="$1" label="$2" - case "$host" in - localhost|127.0.0.1|::1) - echo " WARN: '$host' resolves to the container itself, not your host." >&2 - echo " For a $label running on the docker host, use" >&2 - echo " host.docker.internal (Mac/Windows) or your LAN IP." >&2 - ;; - esac -} - -# validate_mariadb host port user pass — runs `mariadb -e "SELECT 1"` -# inside a transient mariadb container. Returns 0 on success. -validate_mariadb() { - local host="$1" port="$2" user="$3" pass="$4" - echo " Probing MariaDB at ${host}:${port}..." >&2 - if docker run --rm mariadb:11.4 mariadb \ - -h "$host" -P "$port" -u "$user" "--password=$pass" \ - -e "SELECT 1" >/dev/null 2>&1; then - echo " MariaDB OK." >&2 - return 0 - fi - echo " ERROR: could not connect to MariaDB at ${host}:${port} as ${user}." >&2 - return 1 -} - -# validate_clickhouse host http_port user pass db — issues SELECT 1 via -# the HTTP interface using host-side curl. Returns 0 on success. -validate_clickhouse() { - local host="$1" port="$2" user="$3" pass="$4" db="$5" - echo " Probing ClickHouse at ${host}:${port}..." >&2 - if curl -sf -u "${user}:${pass}" \ - --data-urlencode "query=SELECT 1" \ - --data-urlencode "database=${db}" \ - "http://${host}:${port}/" >/dev/null 2>&1; then - echo " ClickHouse OK." >&2 - return 0 - fi - echo " ERROR: could not connect to ClickHouse at ${host}:${port} as ${user}." >&2 - return 1 -} - -# Walk the user through the questions needed to populate .env.compose -# the first time. Caller (cmd_up) has already verified that -# .env.compose is missing AND the env-file path wasn't overridden. -bootstrap_env_wizard() { - if [[ ! -t 0 ]]; then - echo "ERROR: .env.compose is missing and stdin is not a TTY." >&2 - echo " The first-run wizard needs an interactive shell." >&2 - echo " Copy .env.compose.example to .env.compose and edit," >&2 - echo " or re-run ./dev-compose.sh up from a terminal." >&2 - return 1 - fi - if [[ ! -f .env.compose.example ]]; then - echo "ERROR: .env.compose.example is missing — can't bootstrap." >&2 - return 1 - fi - - cat >&2 <<'EOF' - -=== First-run wizard: configuring .env.compose === - -You're bringing up the Insight dev stack for the first time. This -wizard collects the few values needed to generate .env.compose, then -hands off to `docker compose up`. Press Enter to accept defaults shown -in [brackets]. - -EOF - - # ── MariaDB ───────────────────────────────────────────────────────── - local mariadb_external maria_host maria_port maria_user maria_pass - local maria_root_pass=root-local - echo "--- MariaDB ---" >&2 - if ask_yes_no "Use the local MariaDB in docker compose?" "y"; then - mariadb_external=false - maria_host=mariadb - maria_port=3306 - maria_user=insight - maria_pass=insight-local - else - mariadb_external=true - maria_host=$(ask " External MariaDB host" "") - [[ -z "$maria_host" ]] && { echo " ERROR: host is required." >&2; return 1; } - warn_localhost_host "$maria_host" "MariaDB" - maria_port=$(ask " External MariaDB port" "3306") - maria_user=$(ask " MariaDB user" "insight") - maria_pass=$(ask_secret " MariaDB password") - validate_mariadb "$maria_host" "$maria_port" "$maria_user" "$maria_pass" || return 1 - fi - echo "" >&2 - - # ── ClickHouse ────────────────────────────────────────────────────── - local ch_external ch_host ch_port ch_db ch_user ch_pass - echo "--- ClickHouse ---" >&2 - if ask_yes_no "Use the local ClickHouse in docker compose?" "y"; then - ch_external=false - ch_host=clickhouse - ch_port=8123 - ch_db=insight - ch_user=insight - ch_pass=insight-local - else - ch_external=true - ch_host=$(ask " External ClickHouse host" "") - [[ -z "$ch_host" ]] && { echo " ERROR: host is required." >&2; return 1; } - warn_localhost_host "$ch_host" "ClickHouse" - ch_port=$(ask " External ClickHouse HTTP port" "8123") - ch_db=$(ask " ClickHouse database" "insight") - ch_user=$(ask " ClickHouse user" "insight") - ch_pass=$(ask_secret " ClickHouse password") - validate_clickhouse "$ch_host" "$ch_port" "$ch_user" "$ch_pass" "$ch_db" || return 1 - fi - echo "" >&2 - - # ── Tenant ID (only meaningful when binding to existing data) ─────── - local tenant_id - if [[ "$mariadb_external" == "true" || "$ch_external" == "true" ]]; then - echo "--- Tenant ID ---" >&2 - echo " External DBs already contain data tied to a specific tenant." >&2 - echo " Enter the UUID present in persons.insight_tenant_id." >&2 - tenant_id=$(ask " TENANT_DEFAULT_ID" "") - if [[ -z "$tenant_id" ]]; then - echo " ERROR: tenant ID is required when using external DBs." >&2 - return 1 - fi - echo "" >&2 - else - tenant_id="00000000-df51-5b42-9538-d2b56b7ee953" - fi - - # ── Email ─────────────────────────────────────────────────────────── - echo "--- Dev impersonation ---" >&2 - local dev_email - dev_email=$(ask "VITE_DEV_USER_EMAIL" "dev@company.nonpresent") - echo "" >&2 - - # ── Frontend ──────────────────────────────────────────────────────── - echo "--- Frontend ---" >&2 - local fe_mode fe_path default_fe_path="../insight-front" - echo " How should the frontend run?" >&2 - echo " 1) ghcr — pull the pre-built image (no source needed)" >&2 - echo " 2) local — Vite + HMR against an existing insight-front checkout" >&2 - echo " 3) clone — git clone insight-front, then run Vite + HMR" >&2 - local fe_choice - while true; do - fe_choice=$(ask " Choice" "1") - case "$fe_choice" in - 1|ghcr) - fe_mode="ghcr" - fe_path="$default_fe_path" - break ;; - 2|local|dev) - fe_mode="dev" - fe_path=$(ask " Path to insight-front checkout" "$default_fe_path") - if [[ -z "$fe_path" || ! -d "$fe_path" ]]; then - echo " ERROR: '$fe_path' does not exist. Pick option 3 to clone." >&2 - return 1 - fi - break ;; - 3|clone) - if ! command -v git >/dev/null 2>&1; then - echo " ERROR: git is not installed; pick 1 or 2." >&2 - continue - fi - fe_path=$(ask " Clone insight-front into" "$default_fe_path") - if [[ -e "$fe_path" ]]; then - echo " ERROR: '$fe_path' already exists; refusing to clone over it." >&2 - echo " Remove it first, or pick 2 to reuse the existing checkout." >&2 - return 1 - fi - if ! git clone https://github.com/constructorfabric/insight-front.git "$fe_path" >&2; then - echo " ERROR: clone failed." >&2 - return 1 - fi - fe_mode="dev" - break ;; - *) - echo " Please answer 1, 2, or 3." >&2 ;; - esac - done - echo "" >&2 - - # ── Seeding decision for external DBs ─────────────────────────────── - local seed_external=false - if [[ "$mariadb_external" == "true" || "$ch_external" == "true" ]]; then - echo "--- Test data ---" >&2 - echo " Local DBs are always seeded on first up. For external DBs the" >&2 - echo " wizard leaves them alone unless you opt in here." >&2 - if ask_yes_no " Seed test data into your external DB(s)?" "n"; then - seed_external=true - fi - echo "" >&2 - fi - - # ── Write .env.compose ────────────────────────────────────────────── - cp .env.compose.example .env.compose - update_env_var .env.compose MARIADB_EXTERNAL "$mariadb_external" - update_env_var .env.compose MARIADB_HOST "$maria_host" - update_env_var .env.compose MARIADB_INTERNAL_PORT "$maria_port" - update_env_var .env.compose MARIADB_USER "$maria_user" - update_env_var .env.compose MARIADB_PASSWORD "$maria_pass" - update_env_var .env.compose MARIADB_ROOT_PASSWORD "$maria_root_pass" - update_env_var .env.compose CLICKHOUSE_EXTERNAL "$ch_external" - update_env_var .env.compose CLICKHOUSE_HOST "$ch_host" - update_env_var .env.compose CLICKHOUSE_INTERNAL_HTTP_PORT "$ch_port" - update_env_var .env.compose CLICKHOUSE_DATABASE "$ch_db" - update_env_var .env.compose CLICKHOUSE_USER "$ch_user" - update_env_var .env.compose CLICKHOUSE_PASSWORD "$ch_pass" - update_env_var .env.compose TENANT_DEFAULT_ID "$tenant_id" - update_env_var .env.compose VITE_DEV_USER_EMAIL "$dev_email" - update_env_var .env.compose FRONTEND_MODE "$fe_mode" - update_env_var .env.compose INSIGHT_FRONT_PATH "$fe_path" - - # SEEDED_LOCAL_* gates the first-run auto-seed in cmd_up. - # - empty/false → seed will run. - # - true → seed will be skipped. - # For external DBs the user explicitly declined, we pre-mark them - # seeded so cmd_up doesn't touch them. - if [[ "$mariadb_external" == "true" && "$seed_external" != "true" ]]; then - update_env_var .env.compose SEEDED_LOCAL_MARIA true - fi - if [[ "$ch_external" == "true" && "$seed_external" != "true" ]]; then - update_env_var .env.compose SEEDED_LOCAL_CH true - fi - - echo "Wrote .env.compose. Continuing with up..." >&2 - echo "" >&2 -} - # ────────────────────────────────────────────────────────────────────── # up # ────────────────────────────────────────────────────────────────────── @@ -370,8 +112,9 @@ Options: Out-of-scope: --start-airbyte / --start-argo - Both need k8s and are not shipped by this compose stack. - The script exits with a pointer to CONTRIBUTING.md if you pass them. + Both need k8s and are not shipped by this compose stack. For a + k8s-local bring-up that includes Airbyte and Argo Workflows, run + `make deploy ENV=local` from deploy/gitops/. EOF } @@ -397,8 +140,9 @@ cmd_up() { --no-frontend) no_frontend=true; shift ;; --start-airbyte|--start-argo) echo "ERROR: $1 is not supported by the compose stack." >&2 - echo " Both need k8s. Install orbstack/k3d/kind and use the" >&2 - echo " existing ./dev-up.sh path. See CONTRIBUTING.md." >&2 + echo " Both need k8s. Bring up a kind/k3d/OrbStack cluster, then:" >&2 + echo " cd deploy/gitops && make deploy ENV=local" >&2 + echo " The first-run wizard prompts for which L2 services to install." >&2 return 2 ;; -h|--help) cmd_up_help; return 0 ;; *) echo "ERROR: unknown arg: $1" >&2; cmd_up_help; return 2 ;; @@ -407,8 +151,10 @@ cmd_up() { # First-run wizard: only when the user is using the default env file # and it doesn't exist yet. A custom --env-file path is left alone. + # The wizard itself lives in compose/insight-init.sh, shared with the + # k8s-local bring-up. if [[ "$env_file" == ".env.compose" && ! -f "$env_file" ]]; then - bootstrap_env_wizard || return $? + bash "$ROOT_DIR/compose/insight-init.sh" --target=compose || return $? fi env_file="$(resolve_env_file "$env_file")" diff --git a/docs/deploy/.gitignore b/docs/deploy/.gitignore deleted file mode 100644 index 0a8855a0c..000000000 --- a/docs/deploy/.gitignore +++ /dev/null @@ -1,27 +0,0 @@ -# Local deploy artefacts (rendered manifests, deploy logs) -.deploy/ - -# Editor scratch -.idea/ -.vscode/ -*.swp - -# macOS -.DS_Store - -# Anything that smells like a raw secret -*.env -*.env.local -secrets.local.* -*-plain.yaml - -# The sample secret store consumed by scripts/secret-fetch.sh. The -# .sample file is committed as the template; the populated copy must -# never be committed (the sealed manifests in environments// -# sealed-secrets/ are the only safe-to-commit representation). -secrets-store.yaml - -# pub-cert placeholder is committed; the corresponding generated *.pem is too, -# but never the controller's private key -*.key -*.pkcs8 diff --git a/docs/deploy/environments/local/pub-cert.pem.sample b/docs/deploy/environments/local/pub-cert.pem.sample deleted file mode 100644 index 62fbeb646..000000000 --- a/docs/deploy/environments/local/pub-cert.pem.sample +++ /dev/null @@ -1,14 +0,0 @@ -# Placeholder for the sealed-secrets-controller PUBLIC certificate. -# -# After `make bootstrap-sealed-secrets ENV=local` has installed the -# controller in your cluster, run: -# -# make fetch-cert ENV=local -# -# which writes the real cert to environments/local/pub-cert.pem. That -# real .pem IS safe to commit — it's the public half. The controller's -# PRIVATE key never leaves the cluster. -# -# Once pub-cert.pem exists for this env, `make seal-secret …` and the -# `seal-from-*` fallbacks can encrypt cleartext Secret manifests -# locally without needing kubectl access to the cluster. diff --git a/src/backend/services/identity/helm/templates/deployment.yaml b/src/backend/services/identity/helm/templates/deployment.yaml index d7e047f08..f031153fa 100644 --- a/src/backend/services/identity/helm/templates/deployment.yaml +++ b/src/backend/services/identity/helm/templates/deployment.yaml @@ -37,13 +37,20 @@ spec: - name: wait-for-mariadb image: "{{ .Values.waitForMariadb.image.repository }}:{{ .Values.waitForMariadb.image.tag }}" imagePullPolicy: IfNotPresent + envFrom: + # The umbrella publishes MARIADB_HOST + MARIADB_PORT here via + # `insight.mariadb.host` / `.port` helpers — works for both + # mariadb.deploy=true and =false. The chart is published only + # as part of the umbrella, so this ref is hardcoded. + - configMapRef: + name: {{ .Release.Name }}-platform command: - sh - -c - | set -eu - host="{{ default (printf "%s-mariadb" .Release.Name) .Values.waitForMariadb.host }}" - port="{{ .Values.waitForMariadb.port }}" + host="$MARIADB_HOST" + port="${MARIADB_PORT:-3306}" timeout={{ .Values.waitForMariadb.timeoutSeconds }} elapsed=0 echo "wait-for-mariadb: ${host}:${port} (timeout=${timeout}s)" diff --git a/src/backend/services/identity/helm/values.yaml b/src/backend/services/identity/helm/values.yaml index a7d55cec2..8f2bca261 100644 --- a/src/backend/services/identity/helm/values.yaml +++ b/src/backend/services/identity/helm/values.yaml @@ -35,8 +35,6 @@ waitForMariadb: image: repository: busybox tag: "1.36" - host: "" # empty → "{{ .Release.Name }}-mariadb"; set for external MariaDB - port: 3306 timeoutSeconds: 180 livenessProbe: