diff --git a/.claude/skills/connector-status.md b/.claude/skills/connector-status.md index fc0868752..92ca34311 100644 --- a/.claude/skills/connector-status.md +++ b/.claude/skills/connector-status.md @@ -10,7 +10,7 @@ Collect the current state of all connectors across Argo, Airbyte, and ClickHouse ## Prerequisites -- `KUBECONFIG` set to target cluster (e.g. `access/virtuozzo/cyber-insight-k8s.kubeconfig`) +- `KUBECONFIG` set to target cluster - Airbyte API accessible via port-forward on `localhost:8001` (or already forwarded) - `kubectl` access to namespaces: `argo`, `airbyte`, `data` diff --git a/.env.virtuozzo.example b/.env.remote.example similarity index 84% rename from .env.virtuozzo.example rename to .env.remote.example index 35ada0215..4e4f13a09 100644 --- a/.env.virtuozzo.example +++ b/.env.remote.example @@ -1,24 +1,24 @@ -# Insight — Virtuozzo (remote OpenStack-Magnum cluster). +# Insight — remote cluster with pre-existing infrastructure. # -# Copy to .env.virtuozzo (gitignored) and fill in real values. +# Copy to .env.remote (gitignored) and fill in real values. # # One-time setup: # 1. Images are built and pushed by CI on merge to main (see .github/workflows/build-images.yml). # Tags follow YYYY.MM.DD.HH.mm- format. Set IMAGE_TAG below to pin a version. # For manual builds: re-run with BUILD_AND_PUSH=true. # 2. Create Okta SPA application (PKCE, no client_secret), register redirect URI: -# http:///callback e.g. http://192.168.44.36/callback +# http:///callback # 3. Apply insight-oidc Secret (see src/backend/services/api-gateway/secrets/oidc.yaml.example) # -# Typical deploy — додеплой недостающего без переразвертывания существующего: -# ./up.sh --env virtuozzo app +# Typical deploy — install missing components without redeploying existing ones: +# ./dev-up.sh --env remote app # This deploys: ingress-nginx, redis, api-gateway, analytics-api, identity, frontend. # It does NOT touch existing: mariadb, clickhouse, airbyte, argo, metabase. # ── Cluster ──────────────────────────────────────────────────── CLUSTER_MODE=remote NAMESPACE=insight -KUBECONFIG=access/virtuozzo/cyber-insight/cyber-insight-k8s-31_03_2026_18_03.kubeconfig +KUBECONFIG=path/to/cluster.kubeconfig # ── Images ───────────────────────────────────────────────────── IMAGE_REGISTRY=ghcr.io/constructorfabric @@ -29,9 +29,9 @@ IMAGE_PULL_POLICY=Always # ANALYTICS_API_IMAGE_TAG=2026.04.21.14.30-abc1234 # IDENTITY_IMAGE_TAG=2026.04.21.14.30-abc1234 # TOOLBOX_IMAGE_TAG=2026.04.21.14.30-abc1234 -# Target platform for the cluster. On an Apple Silicon workstation this -# triggers buildx + QEMU emulation (slow but needed because Virtuozzo nodes -# are amd64). When IMAGE_PLATFORM is set, buildx pushes directly (no local load). +# Target platform for the cluster. On an Apple Silicon workstation building +# for amd64 nodes this triggers buildx + QEMU emulation (slow but required). +# When IMAGE_PLATFORM is set, buildx pushes directly (no local load). IMAGE_PLATFORM=linux/amd64 # For routine deploys when image is already published, set both to false. @@ -50,7 +50,7 @@ INGRESS_HTTPS_PORT=443 INGRESS_ENABLED=true # turn on chart-level Ingress resources INGRESS_CLASS=nginx -# ── Infra: додеплой недостающего ─────────────────────────────── +# ── Infra: deploy only what is missing ───────────────────────── # mariadb, clickhouse, airbyte, argo already exist — DO NOT re-deploy. # Redis is missing in the cluster but required by analytics-api. DEPLOY_REDIS=true diff --git a/DEVLOG.md b/DEVLOG.md index 504755304..42feb0b72 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,7 +2,7 @@ **Purpose**: full narrative of the first-run debugging of `./dev-up.sh` on a clean Kind cluster, after the umbrella / canonical-installer refactor (PR #224). -**Worktree**: `/Users/roman/alemira/insight/.claude/worktrees/laughing-feistel-fe1e16` +**Worktree**: `.claude/worktrees/laughing-feistel-fe1e16` **Branch**: `claude/laughing-feistel-fe1e16` **Start**: 2026-04-23 diff --git a/dev-restart.sh b/dev-restart.sh index 067f496c4..751cdd440 100644 --- a/dev-restart.sh +++ b/dev-restart.sh @@ -20,7 +20,7 @@ # # Usage: # ./dev-restart.sh # default: --env local -# ./dev-restart.sh --env virtuozzo # remote env (just verify connectivity) +# ./dev-restart.sh --env remote # remote env (just verify connectivity) set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/dev-up.sh b/dev-up.sh index 36e1066ac..ebcd724a8 100755 --- a/dev-up.sh +++ b/dev-up.sh @@ -2,7 +2,7 @@ # Insight platform — DEV bring-up from source. # # Use this when you work on the codebase: builds Docker images from src/, -# creates a local Kind cluster (or targets a dev-owned remote like virtuozzo), +# creates a local Kind cluster (or targets a dev-owned remote cluster), # loads images into the cluster, and deploys all services. # # NOT for end-user installations. For customers / production-like installs @@ -21,8 +21,8 @@ # # Usage: # ./dev-up.sh # --env local, all components -# ./dev-up.sh --env virtuozzo # remote cluster, all components -# ./dev-up.sh --env virtuozzo app # backend + frontend only +# ./dev-up.sh --env remote # remote cluster, all components +# ./dev-up.sh --env remote app # backend + frontend only # ./dev-up.sh ingestion # only ingestion (default env=local) # # Valid components: all | ingestion | app | backend | frontend @@ -565,7 +565,7 @@ if [[ "$CLUSTER_MODE" == "local" || "${SHOW_CREDS:-}" == "1" ]]; then [[ -n "$MDB_ROOT" ]] && echo " MariaDB root / $MDB_ROOT" [[ -n "$REDIS_PASS" ]] && echo " Redis default / $REDIS_PASS" else - # For remote envs (virtuozzo, prod-like) — DO NOT dump passwords into + # For remote (prod-like) envs — DO NOT dump passwords into # the developer's terminal scrollback. Print only how to fetch them. echo " (remote cluster — passwords not printed; pass SHOW_CREDS=1 to override)" echo " Fetch with:" diff --git a/docs/components/airbyte-toolkit/specs/ADR/0001-version-driven-reconcile.md b/docs/components/airbyte-toolkit/specs/ADR/0001-version-driven-reconcile.md index bc1f5be99..f4c9248d0 100644 --- a/docs/components/airbyte-toolkit/specs/ADR/0001-version-driven-reconcile.md +++ b/docs/components/airbyte-toolkit/specs/ADR/0001-version-driven-reconcile.md @@ -37,7 +37,7 @@ How do we represent the *intended* connector version durably and unambiguously, - **No recreate on version bump**: when the operator bumps the version, sources and connections must NOT be deleted and recreated by default — sync state (Airbyte cursors per stream) is precious. - **Human-editable**: the version anchor must live in version-controlled source code so PR review and `git blame` apply. - **Low operational overhead**: no extra storage system to provision, monitor, back up, or recover. -- **Drift-resistant**: the anchor must not depend on a parallel local state that can desynchronize from cluster reality (this is the failure mode of `state.yaml` and `airbyte-state` ConfigMap on the virtuozzo cluster as of 2026-05-04). +- **Drift-resistant**: the anchor must not depend on a parallel local state that can desynchronize from cluster reality (the observed failure mode of `state.yaml` and the `airbyte-state` ConfigMap in production). ## Considered Options @@ -76,7 +76,7 @@ A YAML file checked into the repo, containing `applied_version` per connector al - Good, because zero new infrastructure. - Neutral, because requires a strict contract between writer (CI/CD) and reader (operators). - Bad, because the writer must commit and push for the file to reflect reality; manual recovery edits or out-of-band Airbyte changes leave the file inconsistent. -- Bad, because in observed production (virtuozzo cluster) the file already drifted heavily from Airbyte — UUIDs in the file no longer existed in the cluster. +- Bad, because in observed production the file already drifted heavily from Airbyte — UUIDs in the file no longer existed in the cluster. ### Option B — Cluster-side `ConfigMap airbyte-state` @@ -115,7 +115,6 @@ A strict semver `MAJOR.MINOR.PATCH` string in `descriptor.yaml`; on publish, the - `cpt-insightspec-adr-cdk-prebuilt-images` (ADR-0011) — CDK image identity carried verbatim in `descriptor.cdk_image`; reconcile splits and applies; never builds at runtime. - `cpt-insightspec-adr-enrich-image-in-descriptor` (ADR-0014) — same single-source-of-truth principle extended to enrich sidecar images (jira-enrich, future youtrack-enrich). - `cpt-insightspec-adr-semver-and-full-refresh` (ADR-0015) — strict semver `MAJOR.MINOR.PATCH` format; any bump triggers catalog re-discover so new streams/fields are auto-enabled; major bump additionally dispatches a one-shot `dbt --full-refresh` for that connector's `dbt_select` scope. No cross-connector cascade. -- Background — original investigation against virtuozzo cluster (2026-05-04): `state.yaml` `definitions.{connector}.id` did not match any source's `sourceDefinitionId` for 8 of 9 connectors; clear evidence the parallel-store approach had failed. ## Traceability diff --git a/docs/components/airbyte-toolkit/specs/ADR/0002-adoption-of-existing-resources.md b/docs/components/airbyte-toolkit/specs/ADR/0002-adoption-of-existing-resources.md index fab1350c1..8919114ed 100644 --- a/docs/components/airbyte-toolkit/specs/ADR/0002-adoption-of-existing-resources.md +++ b/docs/components/airbyte-toolkit/specs/ADR/0002-adoption-of-existing-resources.md @@ -27,7 +27,7 @@ decision-makers: platform-engineering **ID**: `cpt-insightspec-adr-adoption-of-existing-resources` ## Context and Problem Statement -Legacy clusters (e.g., virtuozzo, observed 2026-05-04) already have running Airbyte sources and connections that were created by the pre-refactor `register.sh` and `connect.sh` scripts. Each connection has accumulated Airbyte sync state (per-stream cursors). The new reconcile engine introduced by ADR-0001 expects each connection to carry an `insight` membership tag plus a `cfg-hash:` tag, and each definition to carry the descriptor version in its `description` field — none of which exist on legacy resources. +Clusters provisioned before this refactor already have running Airbyte sources and connections that were created by the pre-refactor `register.sh` and `connect.sh` scripts. Each connection has accumulated Airbyte sync state (per-stream cursors). The new reconcile engine introduced by ADR-0001 expects each connection to carry an `insight` membership tag plus a `cfg-hash:` tag, and each definition to carry the descriptor version in its `description` field — none of which exist on legacy resources. How do we bring legacy resources under the new declarative model **without** recreating any source or connection — i.e., without losing sync state and without forcing a full historical re-fetch? @@ -81,7 +81,7 @@ Delete every existing source and connection; let the new reconcile pass create t ### Option B — Rename to canonical pattern with version-encoded names -Rename existing sources and connections to a name pattern that encodes the descriptor version, e.g., `bamboohr-bamboohr-main-virtuozzo-v2026.05.04`. +Rename existing sources and connections to a name pattern that encodes the descriptor version, e.g., `bamboohr-bamboohr-main-acme-v2026.05.04`. - Good, because metadata stays inside the resource itself. - Neutral, because Airbyte does not strongly index by name — querying "all our resources" becomes a substring scan. @@ -102,7 +102,7 @@ Rename existing sources and connections to a name pattern that encodes the descr ## More Information - Sequence: `cpt-insightspec-seq-adopt-one-shot` in `DESIGN.md` §3.6 details the API call order. -- Tag verification (2026-05-04, virtuozzo): `POST /api/public/v1/tags` to create the `insight` tag, `PATCH /api/public/v1/connections/{id}` to attach, `GET /api/public/v1/connections?tagIds=` to query — all three round-tripped successfully. +- Tag verification against a live Airbyte instance: `POST /api/public/v1/tags` to create the `insight` tag, `PATCH /api/public/v1/connections/{id}` to attach, `GET /api/public/v1/connections?tagIds=` to query — all three round-tripped successfully. - Related decisions: - `cpt-insightspec-adr-version-driven-reconcile` (ADR-0001) — provides the version anchor that adoption sets. - `cpt-insightspec-adr-credential-rotation-no-env` (ADR-0003) — explains why `cfg-hash` is computed from K8s Secret, not from Airbyte's masked credential view. diff --git a/docs/components/airbyte-toolkit/specs/ADR/0003-credential-rotation-no-env.md b/docs/components/airbyte-toolkit/specs/ADR/0003-credential-rotation-no-env.md index 62f290cf2..6e3582d62 100644 --- a/docs/components/airbyte-toolkit/specs/ADR/0003-credential-rotation-no-env.md +++ b/docs/components/airbyte-toolkit/specs/ADR/0003-credential-rotation-no-env.md @@ -53,7 +53,7 @@ Chosen option: **Option C — `sources/update` on cfg-hash mismatch**. ### Consequences -- **Good**, because no infrastructure addition: works on virtuozzo and any future cluster as-is. +- **Good**, because no infrastructure addition: works on any cluster as-is. - **Good**, because state is preserved: `sources/update` is documented to keep the source's `sourceId` and the connection's `connectionId`. - **Good**, because rotation cost is one API call per rotated Secret, not a full reconcile cycle. - **Good**, because the rotation event is observable via Airbyte's audit logs (`sources/update` is a discrete, logged operation). diff --git a/docs/components/airbyte-toolkit/specs/ADR/0004-cluster-config-via-configmap.md b/docs/components/airbyte-toolkit/specs/ADR/0004-cluster-config-via-configmap.md index 24c48449a..c2a380be3 100644 --- a/docs/components/airbyte-toolkit/specs/ADR/0004-cluster-config-via-configmap.md +++ b/docs/components/airbyte-toolkit/specs/ADR/0004-cluster-config-via-configmap.md @@ -63,7 +63,7 @@ Chosen option: **Option C — `ConfigMap insight-config` with env override**. ### Confirmation -- `kubectl -n "${INSIGHT_NAMESPACE}" get configmap insight-config -o jsonpath='{.data.tenant_id}'` returns the cluster's tenant ID; reconcile uses the value without falling back. `INSIGHT_NAMESPACE` matches the Helm release namespace (e.g. `data` on virtuozzo, `insight` on dev) and is supplied to the runtime by the chart via the pod env; no namespace literal is hard-coded by reconcile. +- `kubectl -n "${INSIGHT_NAMESPACE}" get configmap insight-config -o jsonpath='{.data.tenant_id}'` returns the cluster's tenant ID; reconcile uses the value without falling back. `INSIGHT_NAMESPACE` matches the Helm release namespace (e.g. `insight` on dev; customer clusters may differ) and is supplied to the runtime by the chart via the pod env; no namespace literal is hard-coded by reconcile. - Setting `INSIGHT_TENANT_ID=test-tenant` and invoking `reconcile-connectors.sh --dry-run` resolves the env value, ignoring any ConfigMap content (precedence test). - Removing the ConfigMap and unsetting the env var causes `reconcile-connectors.sh` to abort with a clear message ("`tenant_id` not configured: set `INSIGHT_TENANT_ID` or create `ConfigMap insight-config`"). @@ -71,7 +71,7 @@ Chosen option: **Option C — `ConfigMap insight-config` with env override**. ### Option A — `connections/.yaml` in repo -A YAML file per cluster (e.g., `connections/virtuozzo.yaml`) with `tenant_id: virtuozzo`. The toolkit reads from a default-named file or via `--tenant `. +A YAML file per cluster (e.g., `connections/acme.yaml`) with `tenant_id: acme`. The toolkit reads from a default-named file or via `--tenant `. - Good, because `tenant_id` is version-controlled and reviewable. - Neutral, because the file's content (only `tenant_id`) is sparse — historically also held destination overrides that are now redundant. @@ -112,7 +112,7 @@ Cluster-level ConfigMap in the Insight Helm release namespace (resolved at runti name: insight-config namespace: data: - tenant_id: virtuozzo + tenant_id: acme ``` - Resolution precedence (toolkit): `INSIGHT_TENANT_ID` env var (if set and non-empty) → `ConfigMap insight-config.data.tenant_id` (looked up in `${INSIGHT_NAMESPACE}`) → abort. - Related decisions: diff --git a/docs/components/airbyte-toolkit/specs/DESIGN.md b/docs/components/airbyte-toolkit/specs/DESIGN.md index cb9b05324..4ef63193d 100644 --- a/docs/components/airbyte-toolkit/specs/DESIGN.md +++ b/docs/components/airbyte-toolkit/specs/DESIGN.md @@ -893,8 +893,8 @@ The reconcile engine identifies resources by deterministic conventions, not by s | Airbyte `connection.tags` | membership + config hash | `["insight", "cfg-hash:"]` | Membership marker + per-instance config drift detector | | K8s Secret label | membership | `app.kubernetes.io/part-of=insight` | Discovery filter | | K8s Secret annotations | identity | `insight.cyberfabric.com/connector=`, `insight.cyberfabric.com/source-id=` | Pair Secret with `connectors//descriptor.yaml` and Airbyte source name | -| Airbyte `source.name` | composed | `{connector_name}-{source_id}-{tenant_id}` | Stable lookup pattern (e.g., `bamboohr-bamboohr-main-virtuozzo`) | -| Airbyte `connection.name` | composed | `{connector_name}-{source_id}-to-clickhouse-{tenant_id}` | Stable lookup pattern (e.g., `bamboohr-bamboohr-main-to-clickhouse-virtuozzo`) | +| Airbyte `source.name` | composed | `{connector_name}-{source_id}-{tenant_id}` | Stable lookup pattern (e.g., `bamboohr-bamboohr-main-acme`) | +| Airbyte `connection.name` | composed | `{connector_name}-{source_id}-to-clickhouse-{tenant_id}` | Stable lookup pattern (e.g., `bamboohr-bamboohr-main-to-clickhouse-acme`) | | Airbyte `connection.namespaceFormat` | bronze database | `bronze_{connector_name_underscored}` | Per-connector ClickHouse Bronze database | > **Tenant resolution**: `tenant_id` comes from cluster-level `ConfigMap insight-config` (data field `tenant_id`) or env var `INSIGHT_TENANT_ID` as fallback. Per-tenant `connections/.yaml` files are removed (Decision #6). diff --git a/docs/components/connectors/collaboration/zulip-proxy/REPRODUCIBILITY-LOG.md b/docs/components/connectors/collaboration/zulip-proxy/REPRODUCIBILITY-LOG.md index f0cb07c63..8c0435be2 100644 --- a/docs/components/connectors/collaboration/zulip-proxy/REPRODUCIBILITY-LOG.md +++ b/docs/components/connectors/collaboration/zulip-proxy/REPRODUCIBILITY-LOG.md @@ -18,7 +18,7 @@ Conventions: ## 0. Inputs and ground truth - **Reference manifest** (Airbyte declarative source v0.57.0, incompatible with current repo): - `/Users/roman/alemira/insight/zulip_proxy.yaml` — Bearer auth against a proxy that aggregates + `zulip_proxy.yaml` (local workspace copy, not in repo) — Bearer auth against a proxy that aggregates Zulip data. Streams `users` (offset-paginated) and `messages` (cursor-paginated, incremental on `created_at`). - **Same-data sibling spec** (existing Zulip Basic-Auth connector docs in the repo): diff --git a/docs/components/connectors/collaboration/zulip-proxy/specs/FEATURE.md b/docs/components/connectors/collaboration/zulip-proxy/specs/FEATURE.md index 519a95bb5..eb1a2cac2 100644 --- a/docs/components/connectors/collaboration/zulip-proxy/specs/FEATURE.md +++ b/docs/components/connectors/collaboration/zulip-proxy/specs/FEATURE.md @@ -81,7 +81,7 @@ contribution. - **Skill workflows**: `cypilot/.core/skills/connector/workflows/create.md`, `cypilot/.core/skills/connector/workflows/test.md`, `cypilot/.core/skills/connector/workflows/validate.md` -- **Reference manifest** (incompatible 0.57.0): `/Users/roman/alemira/insight/zulip_proxy.yaml` +- **Reference manifest** (incompatible 0.57.0): `zulip_proxy.yaml` (local workspace copy, not in repo) - **Closest existing connector for dbt patterns**: `src/ingestion/connectors/collaboration/zoom/` - **Dependencies**: `promote_bronze_to_rmt`, `identity_inputs_from_history`, `snapshot`, `fields_history` macros (already exist in `src/ingestion/dbt/macros/`). diff --git a/docs/components/deployment/gitops/README.md b/docs/components/deployment/gitops/README.md index 000af4a82..a5a11c5fa 100644 --- a/docs/components/deployment/gitops/README.md +++ b/docs/components/deployment/gitops/README.md @@ -88,7 +88,7 @@ The deployment system has five explicit goals: | | | | Kubernetes (VPN) | | | dev / stage / | | - | virtuozzo / … | | + | customers / … | | +-------------------+ | | +-------------------+ reads | @@ -218,7 +218,7 @@ The deployment artifact is the umbrella Helm chart, published per merge to `oci: | Subchart `version` | same `Chart.yaml` | semver | PR author, only when subchart templates change. | | Umbrella `version` | `charts/insight/Chart.yaml` | semver, patch per publish, minor on shape change | CI per merge to `main`. | | Umbrella `appVersion` | same `Chart.yaml` | image tag of the publishing CI run | CI per merge to `main`. Display only. | -| Gitops pin | `infra/insight-gitops/.insight-version` | one line, umbrella semver, e.g. `0.1.47` | poller (auto for `dev`); engineer MR (for any non-dev env, e.g. `stage`, `virtuozzo`, `constructor`, `acronis`). | +| Gitops pin | `infra/insight-gitops/.insight-version` | one line, umbrella semver, e.g. `0.1.47` | poller (auto for `dev`); engineer MR (for any non-dev env, e.g. `stage`, `acme`, `globex`). | Rules that follow from this: @@ -301,7 +301,7 @@ Behaviour: - Strict semver regex on the tag listing. Pre-release tags or anything off-format is ignored. - One commit per poll run when the pin moves; nothing committed when no new version exists. - Commits are authored by a service account (`infra-poller@cyberfabric.local`) with a deploy key scoped to push to `main` of `infra/insight-gitops` only. -- The poller acts only on environments listed in `auto_envs` of `.poller.yaml`. `dev` is included; non-dev envs (the internal `stage`/`test` clusters and every customer-named production cluster — `virtuozzo`, `constructor`, `acronis`, …) are **not** auto-polled, those bumps are PR'd by an engineer, see [§3.4](#34-engineer-pulls-and-deploys). +- The poller acts only on environments listed in `auto_envs` of `.poller.yaml`. `dev` is included; non-dev envs (the internal `stage`/`test` clusters and every customer-named production cluster — `acme`, `globex`, …) are **not** auto-polled, those bumps are PR'd by an engineer, see [§3.6](#36-engineer-deploys-the-app-l3). - The poller does not write to per-environment values files. Image tags in env values are expected to be empty (the chart's per-subchart `appVersion` flows through). Hotfix-style explicit `image.tag` overrides are an engineer-authored MR, never a poller action. - A failed `git push` (e.g. someone else pushed a manual change in the same hour) retries with `git pull --rebase` once; on a second failure it leaves the repo dirty and surfaces a CI failure. @@ -361,11 +361,11 @@ The app deploy is the routine hands-off step — a deploy is always initiated by `make deploy` is an alias for `make deploy-app` and only touches the L3 layer. The L0 bootstrap and L2 system services are not chained — they are explicit prior steps with their own engineer-approved moments. This is by design: an app upgrade should never be able to migrate a database. -For every non-`dev` environment — both the internal `test` and `stage` clusters and every customer-named production cluster (`virtuozzo`, `constructor`, `acronis`, …; one entry per customer install, no generic "prod"): +For every non-`dev` environment — both the internal `test` and `stage` clusters and every customer-named production cluster (`acme`, `globex`, …; one entry per customer install, no generic "prod"): - The poller does not auto-bump the chart pin. An engineer opens a merge request that bumps `environments//.insight-version` (or the umbrella values file) to the desired version (typically the one currently green on `dev`). - After review and merge, the engineer runs `make deploy ENV=` from their workstation. -- For environments listed in the Makefile's `PROTECTED_ENVS` (every customer cluster; internal `test`/`stage` are at the team's discretion), `make deploy` requires an additional `CONFIRM=yes-deploy-` flag — e.g. `CONFIRM=yes-deploy-virtuozzo` — so a typo on a sleepy morning does not push to a customer cluster. See [§6.2](#62-public-targets) and [§6.3](#63-pre-flight-safety-checks) for the safety check. +- For environments listed in the Makefile's `PROTECTED_ENVS` (every customer cluster; internal `test`/`stage` are at the team's discretion), `make deploy` requires an additional `CONFIRM=yes-deploy-` flag — e.g. `CONFIRM=yes-deploy-acme` — so a typo on a sleepy morning does not push to a customer cluster. See [§6.2](#62-public-targets) and [§6.3](#63-pre-flight-safety-checks) for the safety check. ## 4. Security Implementation @@ -391,7 +391,7 @@ There is no path that puts a raw secret on disk in cleartext between Passbolt an - Authoritative store for raw passwords, OIDC client secrets, database passwords, GHCR pull secrets, TLS keys. - **Storage convention**: one Passbolt resource per Kubernetes Secret per environment. The resource's **password field carries the entire cleartext Kubernetes Secret manifest as a single-line JSON object**, ready to be piped to `kubeseal` without further composition. JSON (not YAML) because Passbolt's password field is single-line in the UI and silently strips newlines on save; `kubeseal` accepts JSON and YAML interchangeably. The resource's URI/username/description fields are documentation only (e.g. `kubectl-namespace=insight`, `kubectl-name=insight-oidc`). Example payload (paste verbatim, with the password substituted): `{"apiVersion":"v1","kind":"Secret","metadata":{"name":"","namespace":""},"type":"Opaque","stringData":{"":""}}`. -- **Naming**: `insight--` (e.g. `insight-dev-oidc`, `insight-virtuozzo-db-creds`). The Makefile defaults `PASSBOLT_NAME` to this expression so the engineer rarely passes it explicitly. +- **Naming**: `insight--` (e.g. `insight-dev-oidc`, `insight-acme-db-creds`). The Makefile defaults `PASSBOLT_NAME` to this expression so the engineer rarely passes it explicitly. - **Authentication**: each engineer's Passbolt account is bound to their personal GPG keypair. `passbolt configure` is run once per workstation to register the server URL, the user, and the private key; subsequent `passbolt get resource --json --id ` decrypts the resource via the local GPG agent (passphrase cached in the OS keychain). CI never authenticates to Passbolt — the sealing step is a human action. - The `passbolt` CLI (community: [`go-passbolt-cli`](https://github.com/passbolt/go-passbolt-cli)) is the only sanctioned way to read a secret. Browser-extension copy/paste, screenshots, or pasting into chat are explicitly not. @@ -476,7 +476,7 @@ A `Brewfile` at the repo root captures these dependencies; `make doctor` runs `b - **GitHub** — read access to `constructorfabric/insight` (public) is enough for clones; pushes are protected and only the CI workflow's `GITHUB_TOKEN` can publish images. - **GitLab** — engineer authenticates with SSH key (`gitlab.cyberfabric.internal`); the deploy-key for the poller is separate and lives only in the GitLab CI variables store. - **GHCR** — pulls are public; the cluster's image-pull secret is only needed if the team flips an image to private later. The pull secret itself is a sealed secret in the repo. -- **Kubernetes** — engineer's kubeconfig is generated by the corporate IdP; per-cluster contexts follow `insight-` (e.g. `insight-dev`, `insight-stage`, `insight-virtuozzo`, `insight-constructor`). The Makefile checks `kubectl config current-context` against the requested `ENV` before any apply. +- **Kubernetes** — engineer's kubeconfig is generated by the corporate IdP; per-cluster contexts follow `insight-` (e.g. `insight-dev`, `insight-stage`, `insight-acme`). The Makefile checks `kubectl config current-context` against the requested `ENV` before any apply. - **Passbolt** — `passbolt configure` is run once per workstation: it asks for the server URL, the user's private GPG key file, and the key passphrase. Subsequent `passbolt get resource` invocations decrypt via the local GPG agent; the passphrase is cached in the OS keychain for the agent's TTL. ### 5.3 VPN and Cluster Access @@ -543,7 +543,7 @@ CERT_MANAGER_VERSION ?= v1.18.0 SEALED_SECRETS_VERSION ?= 2.17.4 ``` -- All variables are overridable on the command line (`make deploy ENV=stage`, `make system-airbyte ENV=virtuozzo AIRBYTE_VERSION=1.9.0`). +- All variables are overridable on the command line (`make deploy ENV=stage`, `make system-airbyte ENV=acme AIRBYTE_VERSION=1.9.0`). - `ENV` selects which values file, which kube-context, which sealed-secrets directory, and which app namespace. - `NS_APP` (= `insight-$(ENV)`) is the L3 target namespace; `NS_INFRA` (= `insight-infra`, cluster-shared) is the L2 target namespace. Both are created by `make bootstrap` (see [§3.4](#34-engineer-bootstraps-a-cluster-l0)). - `INSIGHT_VERSION` defaults to the contents of `.insight-version` at the repo root — the umbrella semver currently pinned for this repo. Override only for ad-hoc one-off renders (`make diff INSIGHT_VERSION=0.1.42`). @@ -630,8 +630,8 @@ kube-ctx: # PROTECTED_ENVS is the list of customer-named production clusters # (and any internal env the team wants gated). Add new entries as -# customers come online — virtuozzo, constructor, acronis, … -PROTECTED_ENVS := virtuozzo +# customers come online — acme, globex, … +PROTECTED_ENVS := acme .PHONY: confirm confirm: @@ -653,7 +653,7 @@ Rationale, one line per check: - `sync-clean` rejects ambiguous state. The cluster must reflect a known commit. - `vpn-up` makes "wrong network" a clean error rather than a 10-minute Helm timeout. - `kube-ctx` prevents the worst class of accident: deploying customer values into the wrong cluster (or `dev` values into a customer cluster) because the context was left selected from a previous task. -- `confirm` is a deliberately ugly flag, scoped per env. If you can type `CONFIRM=yes-deploy-virtuozzo`, you have looked at it. Each customer cluster requires its own token (`yes-deploy-constructor`, `yes-deploy-acronis`, …) so muscle memory does not carry across customers. +- `confirm` is a deliberately ugly flag, scoped per env. If you can type `CONFIRM=yes-deploy-acme`, you have looked at it. Each customer cluster requires its own token (`yes-deploy-globex`, …) so muscle memory does not carry across customers. - `passbolt-configured` is checked at the start of `seal-secret` rather than inside the pipe so the failure message is clear. ### 6.4 Deploy Logic @@ -824,9 +824,9 @@ infra/insight-gitops/ │ │ └── insight-db-creds-sealedsecret.yaml │ ├── stage/ # internal — promote-by-MR (optional) │ │ └── … # same shape as dev/ -│ ├── virtuozzo/ # customer prod — promote-by-MR + confirm token +│ ├── acme/ # customer prod — promote-by-MR + confirm token │ │ └── … # same shape; sealed-secrets/insight/ for L3 -│ └── / # one dir per customer install (constructor, acronis, …) +│ └── / # one dir per customer install (globex, …) │ └── … # same shape │ └── scripts/ @@ -851,7 +851,7 @@ Conventions: These are accepted gaps that do not block the MVP but must be tracked. - **Public certificate rotation.** The sealed-secrets-controller rotates its keypair periodically; when it does, the committed `pub-cert.pem` files go stale and previously sealed secrets continue to decrypt (old keys are kept), but new ones must be sealed against the new cert. Procedure: `kubeseal --fetch-cert > environments//pub-cert.pem`, commit, re-seal any in-flight changes. A scheduled monthly check is appropriate; not yet automated. -- **Promotion-MR poller for non-`dev` envs.** Currently only `dev` is auto-bumped. For internal `stage`/`test` and every customer-named cluster (`virtuozzo`, `constructor`, `acronis`, …), the team may want a "dry-run" poller that opens a merge request rather than committing to `main`. Captured but not yet designed. +- **Promotion-MR poller for non-`dev` envs.** Currently only `dev` is auto-bumped. For internal `stage`/`test` and every customer-named cluster (`acme`, `globex`, …), the team may want a "dry-run" poller that opens a merge request rather than committing to `main`. Captured but not yet designed. - **Migration to in-cluster ArgoCD.** The Makefile-driven manual deploy is an MVP shortcut. Once a managed ArgoCD instance is provisioned inside the corporate network, the same `infra/insight-gitops` repo becomes its source. The contract (one `values.yaml` per environment, sealed secrets per namespace) is designed to survive that migration unchanged; only the trigger mechanism changes from `make deploy` to ArgoCD reconciliation. - **Artifact signing (images + chart).** Neither GHCR images nor the umbrella Helm chart at `oci://ghcr.io/constructorfabric/charts/insight` are signed today. The deploy admits any image tag the poller resolves and any chart version `.insight-version` pins. Follow-up: cosign-sign both at publish time, have `make chart-present` verify the chart signature before allowing deploy, and add `cosign verify` to the cluster admission policy for images. - **Audit log of deploys.** `make deploy` writes a local log file; there is no central audit. A trivial follow-up posts the log to a `#deploys` Slack channel via the poller's bot token; deferred until the team needs it. @@ -859,6 +859,6 @@ These are accepted gaps that do not block the MVP but must be tracked. - **Cross-namespace defaults in the umbrella.** The umbrella keeps its infra subcharts gated by `.deploy: true|false` (see [§1.5 dual-purpose umbrella](#15-layer-model)). For the gitops production case (`.deploy: false`), the app's connection helpers must default the host to `.insight-infra.svc.cluster.local` when no explicit `.host` is supplied — so a values file that only says `.deploy: false` "just works" against `insight-infra`. Verify the helpers do this; if not, a small chart-template change is needed. Also document the dual-purpose intent in `charts/insight/README.md` so external chart consumers understand the toggle. - **dev-up.sh Airbyte/Argo namespace.** `dev-up.sh` installs Airbyte and Argo Workflows into the same namespace as the umbrella (`insight` for local). Production gitops puts them in `insight-infra`. The chart values surface for both is identical (Airbyte API URL, Argo SA name) — confirm by render. If anything still hard-codes the `insight` namespace in templates, parameterise it. - **L2 chart-pin policy.** System service chart versions (`MARIADB_VERSION`, `CLICKHOUSE_VERSION`, etc.) are Makefile constants today; bumping is a deliberate PR. A future enhancement: split each service's pin into its own `system//.version` file (mirroring `.insight-version`) so a poller could pre-flight version compatibility against published Bitnami / Redpanda / Airbyte releases. Out of scope for v0. -- **Per-cluster L2 inventory.** When a cluster swaps a self-hosted service for a managed endpoint (e.g. virtuozzo uses RDS instead of `system/mariadb`), the gitops repo currently has no machine-readable record of "this cluster runs MariaDB on-cluster vs. external." A small `environments//inventory.yaml` listing which `system-*` targets to run on this cluster would make `make doctor` able to validate that the cluster matches the expected inventory, and would make it possible to render a per-customer install runbook from the repo. Captured. +- **Per-cluster L2 inventory.** When a cluster swaps a self-hosted service for a managed endpoint (e.g. a managed database instead of `system/mariadb`), the gitops repo currently has no machine-readable record of "this cluster runs MariaDB on-cluster vs. external." A small `environments//inventory.yaml` listing which `system-*` targets to run on this cluster would make `make doctor` able to validate that the cluster matches the expected inventory, and would make it possible to render a per-customer install runbook from the repo. Captured. - **Per-service database provisioning works only when `mariadb.deploy=true`.** The umbrella chart's `templates/mariadb-initdb-scripts.yaml` ConfigMap creates per-service databases (currently `identity` per ADR-0006; future services as they land) via Bitnami's `primary.initdbScriptsConfigMap` — which the bundled MariaDB only runs on **first pod boot**. In the gitops layered model (`mariadb.deploy=false` + external L2 MariaDB), the ConfigMap is not emitted at all, leaving the gitops repo with a choice between (a) mirroring the chart's per-service DB list into `system/mariadb/values.yaml` — couples gitops to chart, breaks every time a new per-service DB lands in the chart — or (b) the engineer creates each DB manually. Neither is good. **Proper fix**: emit a Helm pre-install/pre-upgrade hook Job from the umbrella that runs against `mariadb.host:mariadb.port` (bundled or external) using `mariadb-root-password` from `insight-db-creds`, executing `CREATE DATABASE IF NOT EXISTS` + `GRANT` for every per-service DB the chart's values declare. Engineers in gitops mode pre-create `insight-db-creds` with `mariadb-root-password` included (currently they only seal `mariadb-password` + `clickhouse-password` + `redis-password`). The chart's previous `identity-db-init-job.yaml` had this shape and was rolled back over a credential-timing bug with a regular-template Secret — fix the hook ordering and bring it back, parameterised on the per-service DB list. diff --git a/docs/components/deployment/specs/DESIGN.md b/docs/components/deployment/specs/DESIGN.md index 41cf95db6..2b60b39a9 100644 --- a/docs/components/deployment/specs/DESIGN.md +++ b/docs/components/deployment/specs/DESIGN.md @@ -175,7 +175,7 @@ For gitops production, L2 system services (stateful infra in `insight-infra`) an - [ ] `p2` - **ID**: `cpt-insightspec-principle-dep-customer-named-envs` -The gitops repo has no generic "prod". Internal envs (`dev`, `test`, `stage`) and customer-named envs (`virtuozzo`, `constructor`, `acronis`, …) each get their own directory, their own `.insight-version` history (via merge requests for non-auto envs), and their own per-env `CONFIRM=yes-deploy-` token. The chart itself is environment-agnostic; the env model lives in the gitops Makefile. +The gitops repo has no generic "prod". Internal envs (`dev`, `test`, `stage`) and customer-named envs (`acme`, `globex`, …) each get their own directory, their own `.insight-version` history (via merge requests for non-auto envs), and their own per-env `CONFIRM=yes-deploy-` token. The chart itself is environment-agnostic; the env model lives in the gitops Makefile. **ADRs**: none. @@ -757,7 +757,7 @@ Not applicable. The Deployment subsystem stores no data; it produces a chart art - **Airbyte cross-namespace URL.** The chart's `insight.airbyte.url` helper currently uses `.Release.Namespace` (`insight`). In the layered model Airbyte runs in `insight-infra`, so analytics-api would 404 on real ingestion calls. Parameterise `airbyte.namespace` (or compute the FQDN from a values field). Tracked as a chart-side SPEC §8 follow-up. - **Cross-namespace host defaults for L2 services.** When `.deploy: false`, the chart still `required`-s `.host`. Default it to `.insight-infra.svc.cluster.local` (the layered-model convention) so env values stay minimal. Tracked. - **Artifact signing (images + chart).** Neither images nor the chart are cosign-signed today. Plan: sign at publish time, `chart-present` verifies the chart signature before allowing deploy, image admission policy verifies signatures. See the [gitops SPEC §8 open items](../gitops/README.md#8-open-items). -- **GHCR retention for old umbrella tags.** Long-lived production pins (`virtuozzo`, `constructor`, …) should mirror to a self-hosted registry against GHCR retention deleting the tagged artifact. Documented in the gitops SPEC §8. +- **GHCR retention for old umbrella tags.** Long-lived production pins (customer envs) should mirror to a self-hosted registry against GHCR retention deleting the tagged artifact. Documented in the gitops SPEC §8. - **Migration to in-cluster ArgoCD.** The Makefile-driven manual deploy is an MVP shortcut for Cyberfabric SRE. The contract (one `values.yaml` per env, sealed secrets per namespace, `.insight-version` pin) is designed to survive a switch to an in-cluster ArgoCD picking up the same gitops repo; only the trigger mechanism changes from `make deploy` to ArgoCD reconciliation. **Open questions**: diff --git a/docs/components/deployment/specs/PRD.md b/docs/components/deployment/specs/PRD.md index 8b4c65d5b..536dccdbf 100644 --- a/docs/components/deployment/specs/PRD.md +++ b/docs/components/deployment/specs/PRD.md @@ -47,7 +47,7 @@ date: 2026-05-12 ### 1.1 Purpose -The Deployment subsystem produces **one releasable artifact** for the Insight platform — the `insight` umbrella Helm chart, published per-merge to `oci://ghcr.io/constructorfabric/charts/insight:`. That single artifact is consumed by two distinct paths: the private `infra/insight-gitops` repository, which drives every Cyberfabric-operated cluster (internal `dev` / `test` / `stage` and each customer-named production cluster — `virtuozzo`, `constructor`, `acronis`, …), and any external Helm-aware consumer that wants to run Insight on their own terms (helm, ArgoCD, Flux, kustomize render). The same chart also powers the developer bring-up wrapper `dev-up.sh`, which targets a local Kind/OrbStack cluster. +The Deployment subsystem produces **one releasable artifact** for the Insight platform — the `insight` umbrella Helm chart, published per-merge to `oci://ghcr.io/constructorfabric/charts/insight:`. That single artifact is consumed by two distinct paths: the private `infra/insight-gitops` repository, which drives every Cyberfabric-operated cluster (internal `dev` / `test` / `stage` and each customer-named production cluster — `acme`, `globex`, …), and any external Helm-aware consumer that wants to run Insight on their own terms (helm, ArgoCD, Flux, kustomize render). The same chart also powers the developer bring-up wrapper `dev-up.sh`, which targets a local Kind/OrbStack cluster. The subsystem does not ship product functionality on its own — it composes the application services (API Gateway, Analytics API, Frontend, optional Identity Resolution) with their required infrastructure (ClickHouse, MariaDB, Redis, Redpanda, Airbyte, Argo Workflows) into a versioned chart, enforces the contracts between them (single-namespace dev mode, layered L0/L2/L3 gitops mode, external-mode infra contracts, fail-fast validation, mandatory OIDC in production) and provides the dev wrapper. Orchestration of *customer* installs that are not Cyberfabric-operated is explicitly out of scope: external chart consumers pick their own tooling. @@ -76,7 +76,7 @@ The third driver is reproducibility for the development team itself: a developer | Dev wrapper | `dev-up.sh` (and `dev-down.sh`) — bring-up scripts that build images from source, create a local Kind/OrbStack cluster, and install the same umbrella chart with all `.deploy: true` for a single-namespace fat install. | | Gitops repo | Private `infra/insight-gitops` settings-only repository on internal GitLab that drives every Cyberfabric-operated cluster. Pins exactly one umbrella semver per environment via `.insight-version` and pulls the chart from OCI at deploy time; does **not** vendor the chart. | | L0 / L2 / L3 | Three deploy layers used by gitops production: **L0 Bootstrap** (cluster prerequisites — sealed-secrets-controller, ingress-nginx, cert-manager — cluster-scoped, runs once per cluster); **L2 System** (shared stateful infra — MariaDB, ClickHouse, Redis, Redpanda + Console, Airbyte, Argo Workflows — one Helm release per service in the `insight-infra` namespace, each replaceable by a managed external endpoint); **L3 App** (the umbrella chart, app services only, in the `insight` namespace). There is no L1 — that number is reserved for cluster provisioning, which is out of scope. | -| Customer-named env | A production environment named after the customer that owns it (`virtuozzo`, `constructor`, `acronis`, …). The gitops repo has no generic "prod"; each customer install is its own env directory and its own `kubectl` context (`insight-`). | +| Customer-named env | A production environment named after the customer that owns it (`acme`, `globex`, …). The gitops repo has no generic "prod"; each customer install is its own env directory and its own `kubectl` context (`insight-`). | | Dual-purpose umbrella | One chart, two install shapes selected by `.deploy` toggles. `dev-up.sh` flips them all `true` (single fat `insight` namespace with bundled infra); gitops production flips them all `false` (app services only in `insight`, infra elsewhere). | | External mode | State of an infra dependency where `.deploy: false`. The umbrella does not run the bundled subchart; consumers read the same flat `.host`, `.port` and `.passwordSecret` fields and the Secret is provided by the operator (or platform). The same shape used by gitops production cross-namespace wiring (`.insight-infra.svc.cluster.local`) and by Constructor Platform tenant installs (platform-issued endpoints). | | Constructor Platform | Shared multi-product infrastructure fabric operated by the vendor. It provides ClickHouse, MariaDB, Redpanda and identity services that tenant products consume via external-mode contracts. | @@ -106,7 +106,7 @@ The third driver is reproducibility for the development team itself: a developer **ID**: `cpt-insightspec-actor-cyberfabric-sre` -**Role**: Internal operator running deploys against Cyberfabric-operated clusters (`dev`, `test`, `stage`, and every customer-named production cluster — `virtuozzo`, `constructor`, `acronis`, …). Works from the private `infra/insight-gitops` settings repo through its `Makefile`, on a workstation with VPN + kubeconfig. +**Role**: Internal operator running deploys against Cyberfabric-operated clusters (`dev`, `test`, `stage`, and every customer-named production cluster — `acme`, `globex`, …). Works from the private `infra/insight-gitops` settings repo through its `Makefile`, on a workstation with VPN + kubeconfig. **Needs**: One Makefile-driven workflow that covers L0 bootstrap, L2 system service installs and L3 umbrella deploys against any env; a one-file promotion mechanism (`.insight-version` bump); per-customer deploy gating (`PROTECTED_ENVS` + `CONFIRM=yes-deploy-` token) so muscle memory does not push the wrong values to the wrong cluster; sealed-secrets workflow fed from Passbolt. #### Platform Developer @@ -358,7 +358,7 @@ An L3 upgrade **MUST NOT** re-roll an L2 service; an L2 service upgrade **MUST N - [ ] `p1` - **ID**: `cpt-insightspec-fr-dep-customer-named-envs` -The gitops repo **MUST** name every production environment after the customer that owns it (`virtuozzo`, `constructor`, `acronis`, …) with no generic "prod" alias. Each customer install **MUST** live in its own `environments//` directory and be addressed via its own kube-context (`insight-`). The Makefile **MUST** enforce a `PROTECTED_ENVS` allow-list for every customer-named env and **MUST** require an explicit `CONFIRM=yes-deploy-` token on `make deploy` for envs in that list — each customer cluster requires its own per-env token (`yes-deploy-constructor`, `yes-deploy-acronis`, …) so muscle memory does not carry across customers. +The gitops repo **MUST** name every production environment after the customer that owns it (`acme`, `globex`, …) with no generic "prod" alias. Each customer install **MUST** live in its own `environments//` directory and be addressed via its own kube-context (`insight-`). The Makefile **MUST** enforce a `PROTECTED_ENVS` allow-list for every customer-named env and **MUST** require an explicit `CONFIRM=yes-deploy-` token on `make deploy` for envs in that list — each customer cluster requires its own per-env token (`yes-deploy-acme`, `yes-deploy-globex`, …) so muscle memory does not carry across customers. **Rationale**: "prod" is ambiguous when there are five of them. Customer-named envs plus per-customer confirm tokens make wrong-cluster deploys structurally unlikely; the token has to be typed deliberately for each customer. diff --git a/docs/components/deployment/specs/sop/connector-image-rebuild.md b/docs/components/deployment/specs/sop/connector-image-rebuild.md index 83446a5b3..0f6ebc70a 100644 --- a/docs/components/deployment/specs/sop/connector-image-rebuild.md +++ b/docs/components/deployment/specs/sop/connector-image-rebuild.md @@ -117,9 +117,9 @@ No build identity lives in the workflow YAML. Renaming a GHCR image, moving a Do - Check insight-gitops GitLab CI: most recent `chart-poller` run should have committed an update to `.insight-version`. If it didn't, look at the job log for skopeo/auth errors. - Manual recovery: in insight-gitops, run the `chart-poller` job from the Pipelines UI ("Run pipeline" → manual trigger). -### "Virtuozzo / customer install needs the new image" -- Open `environments/virtuozzo/values.yaml` in insight-gitops. Bump `ingestion.toolboxImage` to the new tag AND `.insight-version` to the new umbrella semver. One MR; deploy manually via `make deploy ENV=virtuozzo` from a workstation (per inventory.protected). -- Connector image refs (`images.cdk.image` / `images.enrich.image`) travel INSIDE the toolbox image — no need to pin them per-connector at the virtuozzo overlay. +### "Customer install needs the new image" +- Open `environments//values.yaml` in insight-gitops. Bump `ingestion.toolboxImage` to the new tag AND `.insight-version` to the new umbrella semver. One MR; deploy manually via `make deploy ENV=` from a workstation (per inventory.protected). +- Connector image refs (`images.cdk.image` / `images.enrich.image`) travel INSIDE the toolbox image — no need to pin them per-connector at the customer overlay. ## References diff --git a/docs/domain/identity-resolution/specs/DESIGN.md b/docs/domain/identity-resolution/specs/DESIGN.md index ca186f6f4..2db2705cb 100644 --- a/docs/domain/identity-resolution/specs/DESIGN.md +++ b/docs/domain/identity-resolution/specs/DESIGN.md @@ -24,7 +24,7 @@ - [4.3 Merge and Split Operations](#43-merge-and-split-operations) - [4.4 ClickHouse Integration Patterns](#44-clickhouse-integration-patterns) - [4.5 End-to-End Walkthrough: Anna Ivanova](#45-end-to-end-walkthrough-anna-ivanova) - - [4.6 End-to-End Walkthrough: Alexei Vavilov (Min-Propagation)](#46-end-to-end-walkthrough-alexei-vavilov-min-propagation) + - [4.6 End-to-End Walkthrough: Andrei Sokolov (Min-Propagation)](#46-end-to-end-walkthrough-andrei-sokolov-min-propagation) - [4.7 Deployment](#47-deployment) - [4.8 Operational Considerations](#48-operational-considerations) - [5. Implementation Recommendations](#5-implementation-recommendations) @@ -1173,10 +1173,10 @@ The MatchingEngine (`cpt-insightspec-ir-component-matching-engine`) evaluates ru **Email normalization pipeline**: ``` -Input: "John.Doe+test@Constructor.TECH" - 1. lowercase → "john.doe+test@constructor.tech" - 2. trim whitespace → "john.doe+test@constructor.tech" - 3. remove plus tags → "john.doe@constructor.tech" +Input: "John.Doe+test@Acme.COM" + 1. lowercase → "john.doe+test@acme.com" + 2. trim whitespace → "john.doe+test@acme.com" + 3. remove plus tags → "john.doe@acme.com" 4. domain alias → also matches "john.doe@constructor.dev" ``` @@ -1314,29 +1314,29 @@ Both BambooHR and AD claim `email` alias for `p-1001` but with different values --- -### 4.6 End-to-End Walkthrough: Alexei Vavilov (Min-Propagation) +### 4.6 End-to-End Walkthrough: Andrei Sokolov (Min-Propagation) > Source: `inbox/IDENTITY_RESOLUTION.md` This walkthrough demonstrates the min-propagation algorithm (§4.1) as a verification mechanism. -**Sources**: BambooHR (`source_account_id: b1`, `display_name: alexei vavilov`, `email: Alexei.Vavilov@alemira.com`), Git commits (`source_account_id: c1–c3`, `username: he4et`, various personal emails), YouTrack (`source_account_id: y1`, `display_name: Alexey Vavilov`, `email: a.vavilov@constructor.tech`). +**Sources**: BambooHR (`source_account_id: b1`, `display_name: andrei sokolov`, `email: Andrei.Sokolov@acme.io`), Git commits (`source_account_id: c1–c3`, `username: sokol`, various personal emails), YouTrack (`source_account_id: y1`, `display_name: Andrey Sokolov`, `email: a.sokolov@acme.com`). **Token extraction from `identity_inputs`**: | `insight_source_type` | `source_account_id` | `value_type` | `value` (token) | |---|---|---|---| -| `bamboohr` | `b1` | `display_name` | `alexei vavilov` | -| `bamboohr` | `b1` | `email` | `alexei.vavilov@alemira.com` | -| `git` | `c1` | `username` | `he4et` | -| `git` | `c2` | `email` | `he4et@gmail.com` | -| `git` | `c3` | `email` | `a.vavilov@gmail.com` | -| `youtrack` | `y1` | `display_name` | `alexey vavilov` | -| `youtrack` | `y1` | `email` | `a.vavilov@constructor.tech` | +| `bamboohr` | `b1` | `display_name` | `andrei sokolov` | +| `bamboohr` | `b1` | `email` | `andrei.sokolov@acme.io` | +| `git` | `c1` | `username` | `sokol` | +| `git` | `c2` | `email` | `sokol@gmail.com` | +| `git` | `c3` | `email` | `a.sokolov@gmail.com` | +| `youtrack` | `y1` | `display_name` | `andrey sokolov` | +| `youtrack` | `y1` | `email` | `a.sokolov@acme.com` | -**After name alias enrichment** (`alexei` <-> `alexey`): BambooHR and YouTrack get synthetic tokens for each other's name spelling. +**After name alias enrichment** (`andrei` <-> `andrey`): BambooHR and YouTrack get synthetic tokens for each other's name spelling. -**After domain alias enrichment** (`gmail.com`, `alemira.com`, `constructor.tech` grouped): Git c3 and YouTrack share `a.vavilov@gmail.com`. +**After domain alias enrichment** (`gmail.com`, `acme.io`, `acme.com` grouped): Git c3 and YouTrack share `a.sokolov@gmail.com`. **Min-propagation result**: `rid(b1)`, `rid(c1)`, `rid(c2)`, `rid(c3)`, `rid(y1)` → all converge to same minimum group ID → `profile_group_id = 1`. diff --git a/docs/domain/ingestion-data-flow/specs/ADR/0001-rmt-with-version-and-unique-key.md b/docs/domain/ingestion-data-flow/specs/ADR/0001-rmt-with-version-and-unique-key.md index dbd104f99..6ab8bf6c2 100644 --- a/docs/domain/ingestion-data-flow/specs/ADR/0001-rmt-with-version-and-unique-key.md +++ b/docs/domain/ingestion-data-flow/specs/ADR/0001-rmt-with-version-and-unique-key.md @@ -76,8 +76,8 @@ For all other models (event/append semantics): use `RMT(_version)` + `incrementa ## Update (2026-06-03): silver moved to `delete+insert` **Trigger.** A misconfigured Airbyte sync ran `full_refresh | append` (instead of -`incremental`) across all virtuozzo connectors for a period, re-appending every -source row. Bronze accumulated many duplicates per key. RMT only collapses on +`incremental`) across all connectors of a deployment for a period, re-appending +every source row. Bronze accumulated many duplicates per key. RMT only collapses on background merge, so the duplicates were still live at query time and propagated through staging into silver. Because the read-time-`FINAL` discipline of option A is unenforced, a consumer that forgot `FINAL` — the gold view diff --git a/docs/domain/ingestion/specs/DESIGN.md b/docs/domain/ingestion/specs/DESIGN.md index bc2fbf2d5..56d9a2aca 100644 --- a/docs/domain/ingestion/specs/DESIGN.md +++ b/docs/domain/ingestion/specs/DESIGN.md @@ -736,7 +736,7 @@ KUBECONFIG: `~/.kube/insight.kubeconfig` | Script | Purpose | |--------|---------| -| `./dev-up.sh` | Dev wrapper: builds Docker images from `src/`, creates a local Kind cluster (or targets a dev-owned remote like virtuozzo), loads images into the cluster, and delegates to `deploy/scripts/install*.sh` to install Airbyte, Argo Workflows, and the Insight umbrella chart. Uses `.env.` for configuration. Idempotent. | +| `./dev-up.sh` | Dev wrapper: builds Docker images from `src/`, creates a local Kind cluster (or targets a dev-owned remote cluster), loads images into the cluster, and delegates to `deploy/scripts/install*.sh` to install Airbyte, Argo Workflows, and the Insight umbrella chart. Uses `.env.` for configuration. Idempotent. | | `./dev-down.sh` | Graceful stop: scales all `insight`-namespace deployments to 0, stops the Kind container. Data preserved — `./dev-restart.sh` brings everything back. | | `./dev-restart.sh` | Quick restart after WSL/Docker crash or `./dev-down.sh`: restarts the Kind container, scales `insight`-namespace pods back to 1, re-patches CoreDNS, restores port-forwards. Falls back to `./dev-up.sh` if the cluster is gone. Lightweight — no image builds, no helm upgrade. | | `deploy/scripts/install.sh` | Production-style installer (canonical path): chains `install-airbyte.sh` → `install-argo.sh` → `install-insight.sh` against the current kubeconfig. Used by `dev-up.sh` and end-user installs from published chart artifacts. | diff --git a/inbox/IDENTITY_RESOLUTION.md b/inbox/IDENTITY_RESOLUTION.md index 634353be9..e1d067cea 100644 --- a/inbox/IDENTITY_RESOLUTION.md +++ b/inbox/IDENTITY_RESOLUTION.md @@ -29,7 +29,7 @@ Each source normalizes raw data into `(source, source_id, token)` tuples. A sing ## Walkthrough Example -The following example traces one person — Alexei Vavilov — through the entire pipeline. He appears across three source systems under different usernames and emails. +The following example traces one person — Andrei Sokolov — through the entire pipeline. He appears across three source systems under different usernames and emails. ### Source Data @@ -37,21 +37,21 @@ The following example traces one person — Alexei Vavilov — through the entir | id | first_name | last_name | work_email | |---|---|---|---| -| b1 | Alexei | Vavilov | Alexei.Vavilov@alemira.com | +| b1 | Andrei | Sokolov | Andrei.Sokolov@acme.io | **Git commits:** | hash | author | email | |---|---|---| -| c1 | he4et | he4ethb1u@gmail.com | -| c2 | he4et | he4et@oddsquat.org | -| c3 | he4et | a.vavilov@gmail.com | +| c1 | sokol | sokol91@gmail.com | +| c2 | sokol | sokol@example.org | +| c3 | sokol | a.sokolov@gmail.com | **YouTrack:** | id | username | email | |---|---|---| -| y1 | Alexey Vavilov | a.vavilov@constructor.tech | +| y1 | Andrey Sokolov | a.sokolov@acme.com | ### Step 1 — Unified References @@ -59,22 +59,22 @@ Each source record is decomposed into `(source, source_id, token, rid, meta)` tu | source | source_id | token | rid | meta | |---|---|---|---|---| -| bamboo | b1 | alexei vavilov | h1 | | -| bamboo | b1 | alexei.vavilov@alemira.com | h1 | | -| git | c1 | he4et | h2 | | -| git | c1 | he4ethb1u@gmail.com | h2 | | -| git | c2 | he4et | h3 | | -| git | c2 | he4et@oddsquat.org | h3 | | -| git | c3 | he4et | h4 | | -| git | c3 | a.vavilov@gmail.com | h4 | | -| youtrack | y1 | alexey vavilov | h5 | | -| youtrack | y1 | a.vavilov@constructor.tech | h5 | | +| bamboo | b1 | andrei sokolov | h1 | | +| bamboo | b1 | andrei.sokolov@acme.io | h1 | | +| git | c1 | sokol | h2 | | +| git | c1 | sokol91@gmail.com | h2 | | +| git | c2 | sokol | h3 | | +| git | c2 | sokol@example.org | h3 | | +| git | c3 | sokol | h4 | | +| git | c3 | a.sokolov@gmail.com | h4 | | +| youtrack | y1 | andrey sokolov | h5 | | +| youtrack | y1 | a.sokolov@acme.com | h5 | | At this point the algorithm can already group some records. Running min-propagation on this data produces **3 groups**: | Group | Members | Connected by | |---|---|---| -| 1 | h2, h3, h4 | shared token `he4et` | +| 1 | h2, h3, h4 | shared token `sokol` | | 2 | h1 | isolated — no shared tokens | | 3 | h5 | isolated — no shared tokens | @@ -86,8 +86,8 @@ Injects **synthetic bridge records** from a seed table. Each pair forces two acc This step does not apply to our example, but a pair like: -``` -v.samun@examus.net <-> vsamun@examus.net +```text +p.jones@example.net <-> pjones@example.net ``` would create synthetic records that bridge two otherwise unrelated accounts. @@ -104,17 +104,17 @@ Alias seed data (excerpt): | first_name | alias | |---|---| -| alexei | alexey | -| alexey | alexei | +| andrei | andrey | +| andrey | andrei | -Applied to our example — the bamboo record has username "alexei vavilov" which contains whole word "alexei", and the youtrack record has "alexey vavilov" which contains "alexey": +Applied to our example — the bamboo record has username "andrei sokolov" which contains whole word "andrei", and the youtrack record has "andrey sokolov" which contains "andrey": | source | source_id | token | rid | meta | |---|---|---|---|---| -| bamboo | b1 | alexey vavilov | h1 | alias: a1 | -| youtrack | y1 | alexei vavilov | h5 | alias: a2 | +| bamboo | b1 | andrey sokolov | h1 | alias: a1 | +| youtrack | y1 | andrei sokolov | h5 | alias: a2 | -Now `h1` and `h5` share the token "alexey vavilov" (and also "alexei vavilov") — they are connected. +Now `h1` and `h5` share the token "andrey sokolov" (and also "andrei sokolov") — they are connected. ### Step 3b — Email Domain Aliases @@ -125,23 +125,23 @@ Domain alias seed data (excerpt): | domain | |---| | gmail.com | -| alemira.com | -| constructor.tech | +| acme.io | +| acme.com | Applied to our example — every email on a listed domain gets variants for the other domains: | source | source_id | token | rid | meta | |---|---|---|---|---| -| bamboo | b1 | alexei.vavilov@gmail.com | h1 | domain: d1 | -| bamboo | b1 | alexei.vavilov@constructor.tech | h1 | domain: d3 | -| git | c1 | he4ethb1u@alemira.com | h2 | domain: d2 | -| git | c1 | he4ethb1u@constructor.tech | h2 | domain: d3 | -| git | c3 | a.vavilov@alemira.com | h4 | domain: d2 | -| git | c3 | a.vavilov@constructor.tech | h4 | domain: d3 | -| youtrack | y1 | a.vavilov@alemira.com | h5 | domain: d2 | -| youtrack | y1 | a.vavilov@gmail.com | h5 | domain: d1 | +| bamboo | b1 | andrei.sokolov@gmail.com | h1 | domain: d1 | +| bamboo | b1 | andrei.sokolov@acme.com | h1 | domain: d3 | +| git | c1 | sokol91@acme.io | h2 | domain: d2 | +| git | c1 | sokol91@acme.com | h2 | domain: d3 | +| git | c3 | a.sokolov@acme.io | h4 | domain: d2 | +| git | c3 | a.sokolov@acme.com | h4 | domain: d3 | +| youtrack | y1 | a.sokolov@acme.io | h5 | domain: d2 | +| youtrack | y1 | a.sokolov@gmail.com | h5 | domain: d1 | -Now `h4` (git c3) and `h5` (youtrack y1) share the token "a.vavilov@gmail.com" — another connection. +Now `h4` (git c3) and `h5` (youtrack y1) share the token "a.sokolov@gmail.com" — another connection. ### Full Token Table After Enrichment @@ -149,26 +149,26 @@ Combining all real and synthetic records: | source | source_id | token | rid | meta | |---|---|---|---|---| -| bamboo | b1 | alexei vavilov | h1 | | -| bamboo | b1 | alexei.vavilov@alemira.com | h1 | | -| git | c1 | he4et | h2 | | -| git | c1 | he4ethb1u@gmail.com | h2 | | -| git | c2 | he4et | h3 | | -| git | c2 | he4et@oddsquat.org | h3 | | -| git | c3 | he4et | h4 | | -| git | c3 | a.vavilov@gmail.com | h4 | | -| youtrack | y1 | alexey vavilov | h5 | | -| youtrack | y1 | a.vavilov@constructor.tech | h5 | | -| bamboo | b1 | alexey vavilov | h1 | alias: a1 | -| youtrack | y1 | alexei vavilov | h5 | alias: a2 | -| bamboo | b1 | alexei.vavilov@gmail.com | h1 | domain: d1 | -| bamboo | b1 | alexei.vavilov@constructor.tech | h1 | domain: d3 | -| git | c1 | he4ethb1u@alemira.com | h2 | domain: d2 | -| git | c1 | he4ethb1u@constructor.tech | h2 | domain: d3 | -| git | c3 | a.vavilov@alemira.com | h4 | domain: d2 | -| git | c3 | a.vavilov@constructor.tech | h4 | domain: d3 | -| youtrack | y1 | a.vavilov@alemira.com | h5 | domain: d2 | -| youtrack | y1 | a.vavilov@gmail.com | h5 | domain: d1 | +| bamboo | b1 | andrei sokolov | h1 | | +| bamboo | b1 | andrei.sokolov@acme.io | h1 | | +| git | c1 | sokol | h2 | | +| git | c1 | sokol91@gmail.com | h2 | | +| git | c2 | sokol | h3 | | +| git | c2 | sokol@example.org | h3 | | +| git | c3 | sokol | h4 | | +| git | c3 | a.sokolov@gmail.com | h4 | | +| youtrack | y1 | andrey sokolov | h5 | | +| youtrack | y1 | a.sokolov@acme.com | h5 | | +| bamboo | b1 | andrey sokolov | h1 | alias: a1 | +| youtrack | y1 | andrei sokolov | h5 | alias: a2 | +| bamboo | b1 | andrei.sokolov@gmail.com | h1 | domain: d1 | +| bamboo | b1 | andrei.sokolov@acme.com | h1 | domain: d3 | +| git | c1 | sokol91@acme.io | h2 | domain: d2 | +| git | c1 | sokol91@acme.com | h2 | domain: d3 | +| git | c3 | a.sokolov@acme.io | h4 | domain: d2 | +| git | c3 | a.sokolov@acme.com | h4 | domain: d3 | +| youtrack | y1 | a.sokolov@acme.io | h5 | domain: d2 | +| youtrack | y1 | a.sokolov@gmail.com | h5 | domain: d1 | ### Step 4 — Identity Resolution (Min-Propagation) @@ -178,18 +178,18 @@ The algorithm groups records by shared tokens. Each `(token, rid)` pair is a seg | Shared token | Connects | |---|---| -| `he4et` | h2, h3, h4 | -| `alexey vavilov` | h1 (alias), h5 | -| `alexei vavilov` | h1, h5 (alias) | -| `a.vavilov@gmail.com` | h4, h5 (domain) | +| `sokol` | h2, h3, h4 | +| `andrey sokolov` | h1 (alias), h5 | +| `andrei sokolov` | h1, h5 (alias) | +| `a.sokolov@gmail.com` | h4, h5 (domain) | **Transitive closure:** -``` -h2 ←— "he4et" —→ h3 -h2 ←— "he4et" —→ h4 -h4 ←— "a.vavilov@gmail.com" —→ h5 -h5 ←— "alexey vavilov" —→ h1 +```text +h2 ←— "sokol" —→ h3 +h2 ←— "sokol" —→ h4 +h4 ←— "a.sokolov@gmail.com" —→ h5 +h5 ←— "andrey sokolov" —→ h1 Result: h1, h2, h3, h4, h5 → profile_group_id = 1 ``` @@ -244,7 +244,7 @@ The input is a table of `(token, rid)` pairs. The algorithm finds connected comp 3. **Converge** — after enough iterations, all transitively connected rids share the same minimum group ID. 4. **Rank** — `dense_rank()` converts raw group IDs into sequential `profile_group_id` values. -Matching is always on **full token values** — token "he4et" matches token "he4et", not a substring of "he4et123". +Matching is always on **full token values** — token "sokol" matches token "sokol", not a substring of "sokol123". ### Blacklist @@ -262,7 +262,7 @@ This section collects approaches for discovering identity connections that the c ### Fuzzy Last Name Search in Email Local Parts -**Problem:** A person's email local part may contain a misspelled or abbreviated last name that exact matching will never catch. For example, `avavilov@gmail.com` (missing letter) won't match the username "alexei vavilov" — there is no shared token. +**Problem:** A person's email local part may contain a misspelled or abbreviated last name that exact matching will never catch. For example, `asokolov@gmail.com` (missing letter) won't match the username "andrei sokolov" — there is no shared token. **Idea:** Extract last names (or rare name parts) from all sources and use them as search queries against email local parts across all unresolved records. Rank results by character-level similarity: @@ -273,17 +273,17 @@ This section collects approaches for discovering identity connections that the c **Example:** -Search term: `vavilov` (extracted last name) +Search term: `sokolov` (extracted last name) Candidates are email local parts split into words by non-letter characters. The best-matching word determines the score: | candidate (raw) | words | best match | matched letters | length diff | score | |---|---|---|---|---|---| -| a.vavilov | a, vavilov | vavilov | 7/7 | 0 | high | -| avavilov | avavilov | avavilov | 7/7 | +1 | high | -| vavilov123 | vavilov | vavilov | 7/7 | 0 | high | -| vavlov | vavlov | vavlov | 3/7 | -1 | low | -| vavilova | vavilova | vavilova | 7/7 | +1 | high | +| a.sokolov | a, sokolov | sokolov | 7/7 | 0 | high | +| asokolov | asokolov | asokolov | 7/7 | +1 | high | +| sokolov123 | sokolov | sokolov | 7/7 | 0 | high | +| soklov | soklov | soklov | 3/7 | -1 | low | +| sokolova | sokolova | sokolova | 7/7 | +1 | high | | ivanov | ivanov | ivanov | 2/7 | -1 | low | This kind of search cannot be used for automatic merging — the false positive rate is too high. Instead, it should produce a **review list** of candidate pairs for manual verification. Confirmed pairs are then added to the `identity_pairs` seed table. diff --git a/inbox/architecture/CONNECTOR_AUTOMATION.md b/inbox/architecture/CONNECTOR_AUTOMATION.md index f0aa271b4..9a83e1892 100644 --- a/inbox/architecture/CONNECTOR_AUTOMATION.md +++ b/inbox/architecture/CONNECTOR_AUTOMATION.md @@ -232,7 +232,7 @@ Four columns appear in every Bronze table across the entire connector fleet. The - **`collected_at DateTime64(3)`** — timestamp of the collection run that produced the row. Populated by the base framework at write time. - **`data_source String`** — the connector's canonical identifier (e.g. `insight_youtrack`, `insight_hubspot`). Populated from the `connector.id` field in the manifest. -- **`insight_source_id String`** — the specific instance being collected from (e.g. `youtrack-acme-prod`, `jira-virtuozzo-prod`). Populated from the connector's runtime configuration. +- **`insight_source_id String`** — the specific instance being collected from (e.g. `youtrack-acme-prod`, `jira-globex-prod`). Populated from the connector's runtime configuration. - **`_version UInt64`** — epoch milliseconds at collection time. Used as the deduplication sort key in ClickHouse ReplacingMergeTree tables. These columns are structural infrastructure. Any connector that declares them explicitly is doing unnecessary work; the base framework should own them entirely. @@ -464,7 +464,7 @@ No equivalent constraint exists for YouTrack, Jira, HubSpot, or Salesforce — t The platform supports multiple instances of the same connector type in parallel — five Jenkins instances, two Jira instances, two Zabbix instances, multiple YouTrack deployments across different customer tenants. Every Bronze row carries a `insight_source_id` string that identifies which instance produced it. The `insight_source_id` value for each instance must be: -- **Human-readable**: `jira-virtuozzo-prod` and `jira-osystems-prod`, not `instance-1` and `instance-2` +- **Human-readable**: `jira-acme-prod` and `jira-globex-prod`, not `instance-1` and `instance-2` - **Stable**: changing a `insight_source_id` after historical data has been collected breaks all historical joins — every existing row that referenced the old value becomes unresolvable - **Unique across all instances of the same type**: two Jira instances cannot share an ID, even if they are on different customer tenants - **Meaningful in context**: `youtrack-acme-prod` tells an analyst at a glance which system they are querying; a UUID does not diff --git a/inbox/architecture/IDENTITY_RESOLUTION_V2.md b/inbox/architecture/IDENTITY_RESOLUTION_V2.md index 257ef6720..0c79f9cdd 100644 --- a/inbox/architecture/IDENTITY_RESOLUTION_V2.md +++ b/inbox/architecture/IDENTITY_RESOLUTION_V2.md @@ -299,7 +299,7 @@ INSERT INTO match_rule (name, description, rule_type, weight, phase, condition_t ('hr_id_match', 'HR system employee ID match', 'exact', 1.00, 'B1', 'hr_id_exact', '{}', 2), ('username_same_sys', 'Same username in same system type', 'exact', 0.95, 'B1', 'username_exact', '{}', 3), ('email_case_norm', 'Email match ignoring case', 'normalization', 0.95, 'B2', 'email_normalize', '{"transform": "lowercase"}', 10), - ('email_domain_alias','Same local part, known domain alias', 'normalization', 0.92, 'B2', 'email_domain', '{"aliases": {"constructor.tech": ["constructor.dev"]}}', 11), + ('email_domain_alias','Same local part, known domain alias', 'normalization', 0.92, 'B2', 'email_domain', '{"aliases": {"acme.com": ["acme.dev"]}}', 11), ('email_plus_tag', 'Email match ignoring +tag', 'normalization', 0.93, 'B2', 'email_normalize', '{"transform": "remove_plus_tags"}', 12), ('username_cross_sys','Same username in related systems (Git<->Jira)', 'cross_system', 0.85, 'B2', 'username_cross', '{"system_pairs": [["gitlab","github"],["gitlab","jira"]]}', 20), ('email_to_username', 'Email local part matches username in other sys', 'cross_system', 0.72, 'B2', 'email_username', '{}', 21), @@ -540,7 +540,7 @@ FOR each record in raw_people WHERE _synced_at > last_bootstrap_run: - **Bootstrap resolves the "unmapped purgatory."** When a new employee appears in HR, the bootstrap creates their email aliases. Any previously unmapped aliases from Git or Jira that match those emails are auto-resolved. -- **Multiple HR sources.** If two HR connectors write to `raw_people` (e.g., BambooHR for Constructor, Workday for Acronis), the bootstrap processes both. Different `source_system` values prevent collisions. +- **Multiple HR sources.** If two HR connectors write to `raw_people` (e.g., BambooHR for one company, Workday for another), the bootstrap processes both. Different `source_system` values prevent collisions. --- @@ -664,11 +664,11 @@ Email normalization and cross-system username correlation. **Email normalization pipeline:** ``` -Input: "John.Doe+test@Constructor.TECH" - 1. lowercase -> "john.doe+test@constructor.tech" - 2. trim whitespace -> "john.doe+test@constructor.tech" - 3. remove plus tags -> "john.doe@constructor.tech" - 4. apply domain alias -> "john.doe@constructor.dev" (also matches) +Input: "John.Doe+test@Acme.COM" + 1. lowercase -> "john.doe+test@acme.com" + 2. trim whitespace -> "john.doe+test@acme.com" + 3. remove plus tags -> "john.doe@acme.com" + 4. apply domain alias -> "john.doe@acme.dev" (also matches) ``` **Domain alias configuration** (stored in `match_rule.config` JSON): @@ -676,8 +676,8 @@ Input: "John.Doe+test@Constructor.TECH" ```json { "aliases": { - "constructor.tech": ["constructor.dev", "constructor.io"], - "acronis.com": ["acronis.work"] + "acme.com": ["acme.dev", "acme.io"], + "globex.com": ["globex.work"] } } ``` diff --git a/inbox/architecture/PRODUCT_SPECIFICATION.md b/inbox/architecture/PRODUCT_SPECIFICATION.md index be0bf569f..572e0c377 100644 --- a/inbox/architecture/PRODUCT_SPECIFICATION.md +++ b/inbox/architecture/PRODUCT_SPECIFICATION.md @@ -36,7 +36,7 @@ Insight's architecture is designed for **50M+ rows over 3 years** based on the f - **Total with planned:** ~19,000 events/employee/year **Deployment Phases:** -- **Year 1:** 300 employees (Constructor) → 5.1M rows +- **Year 1:** 300 employees (first company) → 5.1M rows - **Year 3:** 1,000 employees (portfolio) → 33M cumulative rows - **Year 5:** 1,000 employees + 5,000 students → 120M cumulative rows @@ -72,7 +72,7 @@ Insight's architecture is designed for **50M+ rows over 3 years** based on the f | Marketing Leaders | Campaign activity, lead generation, and funnel contribution | | Operations Leaders | Cross-functional workflow efficiency and throughput | -> **Principle: Measure all employees, not just engineers.** Engineering headcount is typically a minority of the total workforce — for example, at Acronis, engineers are ~26% of headcount and ~15% of costs. Limiting measurement to engineering roles means the majority of the organization remains a blind spot. Insight is designed to measure productivity signals across all employee functions using domain-appropriate metrics for each role. +> **Principle: Measure all employees, not just engineers.** Engineering headcount is typically a minority of the total workforce — often a quarter or less of headcount and a smaller share of costs. Limiting measurement to engineering roles means the majority of the organization remains a blind spot. Insight is designed to measure productivity signals across all employee functions using domain-appropriate metrics for each role. ### Education Segment @@ -609,9 +609,9 @@ Alerts highlight anomalies and correlations, prompting investigation rather than Insight is domain-agnostic by design. Any dataset can be integrated with a connector and semantic description. -### Currently Integrated Sources (Phase 1: Constructor) +### Currently Integrated Sources (Phase 1) -Active integrations for the initial deployment at Constructor (~300 employees): +Active integrations for the initial deployment (~300 employees): | Category | Source | Status | Est. Events/Employee/Year | |----------|--------|--------|---------------------------| @@ -630,10 +630,10 @@ Active integrations for the initial deployment at Constructor (~300 employees): | Source | Target Use Case | Target Company | Est. Additional Events/Year | |--------|-----------------|----------------|----------------------------| -| Bitbucket | PR analytics, reviewer metrics | Acronis | +500 | +| Bitbucket | PR analytics, reviewer metrics | Portfolio company | +500 | | Confluence | Wiki pages, comments, knowledge base | All portfolio companies | +1,000 | -| Jira | Project management (alternative to YouTrack) | Virtuozzo | ~5,000 (replaces YouTrack) | -| Workday | Enterprise HR system | Acronis | ~50 (replaces BambooHR) | +| Jira | Project management (alternative to YouTrack) | Portfolio company | ~5,000 (replaces YouTrack) | +| Workday | Enterprise HR system | Portfolio company | ~50 (replaces BambooHR) | **Mid-term (Phase 3 - Education Segment):** @@ -849,21 +849,21 @@ An analyst receives a request for a new metric: ## Implementation Phases -### Phase 1: Metrics-First Validation (Constructor) +### Phase 1: Metrics-First Validation (internal) Focus on proving semantic governance on internal data: - **Sources**: Git, YouTrack, Zulip, M365, Office Attendance + HR (BambooHR) - **Goal**: Prove AI-agent correctly generates SQL from semantic descriptions - **Visualization**: Custom React dashboards -- **Outcome**: Working prototype on Constructor's ~300 employees +- **Outcome**: Working prototype on the vendor's own ~300 employees -### Phase 2: Portfolio Expansion (Virtuozzo, Acronis) +### Phase 2: Portfolio Expansion Normalize additional companies into the same semantic model: -- **Virtuozzo**: Jira for project management -- **Acronis**: Jira, Workday, Bitbucket for global productivity +- **Second company**: Jira for project management +- **Third company**: Jira, Workday, Bitbucket for global productivity - **Goal**: Cross-company benchmarking with unified definitions ### Phase 3: Education Scale (Universities, LPE) diff --git a/inbox/architecture/STORAGE_TECHNOLOGY_EVALUATION.md b/inbox/architecture/STORAGE_TECHNOLOGY_EVALUATION.md index 24f5d826d..fc758a543 100644 --- a/inbox/architecture/STORAGE_TECHNOLOGY_EVALUATION.md +++ b/inbox/architecture/STORAGE_TECHNOLOGY_EVALUATION.md @@ -57,27 +57,27 @@ Data volume estimates based on integrated data sources from [Product Specificati | **Total (current sources)** | **52-119** | **13,000 - 29,750** | | **Total (with planned sources)** | **57-129** | **14,250 - 32,250** | -**Conservative estimate:** 17,000 events/employee/year (based on Phase 1 Constructor deployment) +**Conservative estimate:** 17,000 events/employee/year (based on the Phase 1 deployment) **Aggressive estimate:** 30,000 events/employee/year (power users with all integrations) --- -**Year 1 (Constructor only):** +**Year 1 (first company only):** - **Employees:** 300 - **Sources:** Git, MCP, YouTrack, Zulip, M365, BambooHR, Office Attendance (7 sources) - **Events/employee/year:** ~17,000 (based on actual source table) - **Annual volume:** 300 × 17,000 = **5.1M rows/year** - **Cumulative Year 1:** **5.1M rows** -**Year 2 (Add Virtuozzo):** -- **Employees:** 300 (Constructor) + 200 (Virtuozzo) = 500 -- **Sources:** Jira replaces YouTrack for Virtuozzo +**Year 2 (add second company):** +- **Employees:** 300 (first company) + 200 (second company) = 500 +- **Sources:** Jira replaces YouTrack for the second company - **Events/employee/year:** ~17,000 (same source mix) - **Annual volume:** 500 × 17,000 = **8.5M rows/year** - **Cumulative Year 2:** 5.1M + 8.5M = **13.6M rows** -**Year 3 (Add Acronis):** -- **Employees:** 500 + 500 (Acronis) = 1,000 +**Year 3 (add third company):** +- **Employees:** 500 + 500 (third company) = 1,000 - **Sources:** Add Bitbucket, Confluence, Workday - **Events/employee/year:** ~19,000 (additional Bitbucket + Confluence) - **Annual volume:** 1,000 × 19,000 = **19M rows/year** @@ -96,7 +96,7 @@ Data volume estimates based on integrated data sources from [Product Specificati | Timeframe | Cumulative Rows | Primary Use Case | |-----------|----------------|------------------| -| **Year 1** | 5.1M | Single company (Constructor) | +| **Year 1** | 5.1M | Single company | | **Year 3** | 32.6M | Portfolio (3 companies) | | **Year 5** | 120M | Portfolio + Education segment | @@ -435,7 +435,7 @@ Data volume estimates based on integrated data sources from [Product Specificati - Strong ecosystem and community support - Standard SQL reduces vendor lock-in risk - Better than MariaDB ColumnStore for JSON workloads -- Viable **only for Phase 1-2** (Constructor + Virtuozzo, <15M rows) +- Viable **only for Phase 1-2** (first two companies, <15M rows) - **Not viable for Phase 3+** without enterprise license or migration --- diff --git a/src/backend/services/identity/src/Insight.Identity.Infrastructure/Migrations/004_persons_relax_constraints.sql b/src/backend/services/identity/src/Insight.Identity.Infrastructure/Migrations/004_persons_relax_constraints.sql index a06a68714..5c15f6cb7 100644 --- a/src/backend/services/identity/src/Insight.Identity.Infrastructure/Migrations/004_persons_relax_constraints.sql +++ b/src/backend/services/identity/src/Insight.Identity.Infrastructure/Migrations/004_persons_relax_constraints.sql @@ -21,8 +21,8 @@ -- B) Switch value_id collation from utf8mb4_bin (case-sensitive) to -- utf8mb4_unicode_ci (case-insensitive), matching value_full_text. -- The original utf8mb4_bin choice made every value_id comparison --- case-sensitive: a lookup for `roman.mitasov@...` would not match --- a stored `Roman.Mitasov@...` even though both refer to the same +-- case-sensitive: a lookup for `jane.doe@...` would not match +-- a stored `Jane.Doe@...` even though both refer to the same -- person. The service-side ToLowerInvariant() partially mitigated -- this for the GET /v1/persons/{email} happy path but did nothing -- for raw callers, for any other value_type (id / username / diff --git a/src/backend/services/identity/tests/Insight.Identity.Tests.Integration/JwtCallerResolveTests.cs b/src/backend/services/identity/tests/Insight.Identity.Tests.Integration/JwtCallerResolveTests.cs index be39456a9..23c4bdc88 100644 --- a/src/backend/services/identity/tests/Insight.Identity.Tests.Integration/JwtCallerResolveTests.cs +++ b/src/backend/services/identity/tests/Insight.Identity.Tests.Integration/JwtCallerResolveTests.cs @@ -34,7 +34,7 @@ public sealed class JwtCallerResolveTests : IAsyncLifetime private const string OidValue = "22c303cd-1364-4165-8fb8-7b412f0e3bb6"; private const string SubValue = "ZfBGhYsNQWyvDWPNRGfd-zefLAdrC0x8g_vJe2veeEI"; - private const string EmailValue = "grigoriy.gogin@constructor.tech"; + private const string EmailValue = "john.doe@example.com"; private readonly MariaDbFixture _fixture; diff --git a/src/backend/services/identity/tests/Insight.Identity.Tests.Integration/PersonsSchemaTests.cs b/src/backend/services/identity/tests/Insight.Identity.Tests.Integration/PersonsSchemaTests.cs index af8e974e4..63de4a02e 100644 --- a/src/backend/services/identity/tests/Insight.Identity.Tests.Integration/PersonsSchemaTests.cs +++ b/src/backend/services/identity/tests/Insight.Identity.Tests.Integration/PersonsSchemaTests.cs @@ -16,7 +16,7 @@ namespace Insight.Identity.Tests.Integration; /// * `value_id` collation is `utf8mb4_unicode_ci` so all value-column /// comparisons (email, source-native ids, parent_email, etc.) are /// case-insensitive at the storage layer — a lookup for -/// `roman.mitasov@...` finds a stored `Roman.Mitasov@...`. +/// `jane.doe@...` finds a stored `Jane.Doe@...`. /// /// If a future migration accidentally reintroduces the value_hash /// UNIQUE or reverts the collation, the tests below fail loudly. @@ -88,11 +88,11 @@ public async Task Persons_value_id_comparison_is_case_insensitive_for_emails() // Store the email with mixed case (as BambooHR may emit it), // then look it up lowercase. With utf8mb4_unicode_ci, the // comparison treats them as equal. - await InsertEmailAsync("Roman.Mitasov@constructor.tech").ConfigureAwait(false); + await InsertEmailAsync("Jane.Doe@example.com").ConfigureAwait(false); - var personIdLowercase = await ResolveByEmailAsync("roman.mitasov@constructor.tech").ConfigureAwait(false); - var personIdMixedCase = await ResolveByEmailAsync("ROMAN.MITASOV@CONSTRUCTOR.TECH").ConfigureAwait(false); - var personIdOriginal = await ResolveByEmailAsync("Roman.Mitasov@constructor.tech").ConfigureAwait(false); + var personIdLowercase = await ResolveByEmailAsync("jane.doe@example.com").ConfigureAwait(false); + var personIdMixedCase = await ResolveByEmailAsync("JANE.DOE@EXAMPLE.COM").ConfigureAwait(false); + var personIdOriginal = await ResolveByEmailAsync("Jane.Doe@example.com").ConfigureAwait(false); personIdLowercase.Should().Be(PersonId); personIdMixedCase.Should().Be(PersonId); diff --git a/src/ingestion/connectors/ai/claude-admin/dbt/claude_admin__ai_dev_usage.sql b/src/ingestion/connectors/ai/claude-admin/dbt/claude_admin__ai_dev_usage.sql index 3ca9a5084..7b4e82e06 100644 --- a/src/ingestion/connectors/ai/claude-admin/dbt/claude_admin__ai_dev_usage.sql +++ b/src/ingestion/connectors/ai/claude-admin/dbt/claude_admin__ai_dev_usage.sql @@ -7,14 +7,13 @@ -- • actor_type='user' → actor_identifier = user email -- email column populated, api_key_id NULL -- • actor_type='api_actor' → actor_identifier = API key *name* --- (e.g. "karsten-claude-code-key"). JOIN with +-- (e.g. "alice-claude-code-key"). JOIN with -- claude_admin_api_keys on name to resolve api_key_id. -- email column NULL, api_key_id populated. --- Observed in real Virtuozzo data: 100% of activity --- is api_actor — org admins provision named per- --- developer keys rather than letting users auth --- directly. Silver Step 2 (Identity Resolution) --- maps api_key_id → person_id. +-- Orgs whose admins provision named per-developer +-- keys (rather than letting users auth directly) +-- report all activity as api_actor. Silver Step 2 +-- (Identity Resolution) maps api_key_id → person_id. -- -- Note: Claude Code usage is also present in Enterprise data -- (claude_enterprise_users.code_*). For orgs on the Enterprise subscription, @@ -26,9 +25,9 @@ -- silver:class_ai_api_usage (token-level metrics Enterprise does not publish). -- -- Why prefer Enterprise here: --- • user-grain attribution out of the box (user_email) — Admin reports --- api_actor for 100% of activity in real-world data, requiring an extra --- api_key_name → api_key_id → person_id resolution step. +-- • user-grain attribution out of the box (user_email) — for orgs on +-- per-developer API keys Admin reports everything as api_actor, requiring +-- an extra api_key_name → api_key_id → person_id resolution step. -- • Enterprise also exposes adjacent surfaces (chat, cowork, office) -- that Admin's code_usage endpoint does not. -- diff --git a/src/ingestion/connectors/ai/claude-admin/dbt/schema.yml b/src/ingestion/connectors/ai/claude-admin/dbt/schema.yml index 776d54e04..19933f3d8 100644 --- a/src/ingestion/connectors/ai/claude-admin/dbt/schema.yml +++ b/src/ingestion/connectors/ai/claude-admin/dbt/schema.yml @@ -99,12 +99,6 @@ models: orphan-identity rows in Silver — Silver Step 2 (Identity Resolution) then maps email / api_key_id → person_id. - Note: the initial PRD §5.7 required actor_type='user' only. Real - Virtuozzo data showed 100% of activity arrives as api_actor (the org - provisions named per-developer API keys), so strict user-only - filtering would yield zero rows in production. Dual-actor support - is the intended behaviour; the PRD clause is to be updated. - For orgs on the Enterprise subscription, Claude Code is sourced from claude_enterprise__ai_dev_usage instead (Enterprise reports per-user directly, avoiding api_key_name resolution). This model remains available diff --git a/src/ingestion/connectors/collaboration/zoom/dbt/zoom__collab_meeting_activity.sql b/src/ingestion/connectors/collaboration/zoom/dbt/zoom__collab_meeting_activity.sql index ecbce91f7..0021d6acb 100644 --- a/src/ingestion/connectors/collaboration/zoom/dbt/zoom__collab_meeting_activity.sql +++ b/src/ingestion/connectors/collaboration/zoom/dbt/zoom__collab_meeting_activity.sql @@ -83,7 +83,7 @@ SELECT )) AS unique_key, p.email AS user_id, -- Pick one display name when the same email surfaces under multiple - -- spellings (e.g., "Karolis Dambrava" vs "karolisdambrava"). Without + -- spellings (e.g., "Jane Doe" vs "janedoe"). Without -- this, GROUP BY would split them and produce two rows with identical -- unique_key — the staging model's `unique_key` is keyed on -- (tenant, source, lower(email), date), so user_name is non-keying. diff --git a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql index a45bb16b4..4dfdbc431 100644 --- a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql +++ b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql @@ -33,12 +33,13 @@ WITH src AS ( properties_dealname AS name, properties_hs_manual_forecast_category AS forecast_category, properties_dealstage AS stage, - -- `amount` is the raw deal-currency line-item value (Constructor's - -- school subscriptions are often quoted monthly here). `amount_home` + -- `amount` is the raw deal-currency line-item value and may be + -- quoted per billing period rather than annualized. `amount_home` -- is HubSpot's home-currency conversion — use this for cross-rep -- aggregates. `hs_acv` / `hs_tcv` / `hs_arr` are HubSpot-computed - -- contract-level rollups from line items; populated on ~10% of - -- Constructor's deals, so expose them as nullable for selective use. + -- contract-level rollups from line items; typically populated on + -- only a minority of deals, so expose them as nullable for + -- selective use. toFloat64OrNull(properties_amount) AS amount, toFloat64OrNull(properties_amount_in_home_currency) AS amount_home, toFloat64OrNull(properties_hs_acv) AS acv, diff --git a/src/ingestion/dbt/audit_rmt_read_dedup.py b/src/ingestion/dbt/audit_rmt_read_dedup.py index 27165037a..496b24d6c 100644 --- a/src/ingestion/dbt/audit_rmt_read_dedup.py +++ b/src/ingestion/dbt/audit_rmt_read_dedup.py @@ -8,8 +8,8 @@ ReplacingMergeTree(_version)), and bronze tables are promoted to RMT too. RMT only collapses duplicates during background merges — never guaranteed at query time. If an upstream table holds transient pre-merge duplicates (e.g. - an erroneous Airbyte full_refresh|append re-appending every row, as happened - across all virtuozzo connectors), a plain SELECT leaks them downstream and + an erroneous Airbyte full_refresh|append re-appending every row on each + sync), a plain SELECT leaks them downstream and inflates metrics. Per ADR-0001 every read of an RMT relation MUST dedup at read time: FINAL / argMax / QUALIFY ROW_NUMBER / LIMIT 1 BY. diff --git a/src/ingestion/dbt/macros/union_by_tag.sql b/src/ingestion/dbt/macros/union_by_tag.sql index c02aca920..fa1775cbd 100644 --- a/src/ingestion/dbt/macros/union_by_tag.sql +++ b/src/ingestion/dbt/macros/union_by_tag.sql @@ -5,12 +5,12 @@ Builds a UNION ALL of every model tagged `tag_name` and deduplicates the result at read time to exactly one row per `unique_key`. - Why the read-time dedup (added 2026-06): + Why the read-time dedup: Every staging/silver table is ReplacingMergeTree, which only collapses duplicates during background merges — never guaranteed at query time. If an upstream table holds transient pre-merge duplicates (e.g. an erroneous - Airbyte `full_refresh|append` re-appending every row, as happened across - all virtuozzo connectors), a plain `SELECT * FROM staging` leaks those + Airbyte `full_refresh|append` re-appending every row on each sync), a + plain `SELECT * FROM staging` leaks those duplicates straight into silver and inflates metrics. Deduping here makes silver duplicate-free regardless of merge timing. See ADR-0001. diff --git a/src/ingestion/scripts/migrations/20260422000000_gold-views.sql b/src/ingestion/scripts/migrations/20260422000000_gold-views.sql index a0363ea4a..f3e733952 100644 --- a/src/ingestion/scripts/migrations/20260422000000_gold-views.sql +++ b/src/ingestion/scripts/migrations/20260422000000_gold-views.sql @@ -148,7 +148,7 @@ AS SELECT toDate(parseDateTimeBestEffortOrNull(assumeNotNull(date))) AS metric_date, count() AS commits FROM bronze_bitbucket_cloud.commits -WHERE (author_email IS NOT NULL) AND (author_email LIKE '%@virtuozzo.com') AND (date IS NOT NULL) +WHERE (author_email IS NOT NULL) AND (date IS NOT NULL) GROUP BY person_id, metric_date diff --git a/src/ingestion/scripts/migrations/20260427120000_views-from-silver.sql b/src/ingestion/scripts/migrations/20260427120000_views-from-silver.sql index 8ee08687c..70f4c543e 100644 --- a/src/ingestion/scripts/migrations/20260427120000_views-from-silver.sql +++ b/src/ingestion/scripts/migrations/20260427120000_views-from-silver.sql @@ -198,7 +198,7 @@ GROUP BY person_id, metric_date; -- --------------------------------------------------------------------- -- The previous definition referenced a non-existent column `issue_type` -- (renamed/removed during a bronze schema migration), so the view was --- broken on virtuozzo. silver.class_task_field_history would be the +-- broken at query time. silver.class_task_field_history would be the -- right source long-term, but it stores per-field events (not per-issue -- snapshots), so reproducing the gold contract needs a new -- silver.class_task_daily — out of scope for this PR.