feat(seed): run the demo seeder on Kubernetes stands

.env.compose.example

@@ -166,7 +166,8 @@ AUTHENTICATOR_IDENTITY_URL=http://identity-resolution:8082
 # ── Local login identity ──────────────────────────────────────────────
 # The Keycloak realm anchors its dev-lead persona on this person, and the
 # seeder creates the matching development-team lead in Identity. Keep them
-# in sync or re-seed after changing this. See deploy/seed/profiles.py.
+# in sync or re-seed after changing this. See
+# src/ingestion/tools/seed/insight_seed/profiles.py.
 DEV_USER_EMAIL=dev@company.nonpresent
 
 # ── Frontend security headers ─────────────────────────────────────────

.github/dependabot.yml

@@ -36,7 +36,7 @@ updates:
         patterns: ["*"]
     ignore:
       # dbt's dependency chain (dbt-common -> mashumaro) does not import on
-      # Python 3.14 yet; deploy/seed stays on 3.13 until it does.
+      # Python 3.14 yet, and the toolbox image the seeder ships in is 3.12.
       - dependency-name: python
         versions: [">=3.14"]
 
@@ -60,7 +60,7 @@ updates:
         versions: [">=0.13"]
 
   - package-ecosystem: pip
-    directory: /deploy/seed
+    directory: /src/ingestion/tools/seed
     schedule:
       interval: daily
       time: "21:30"

.github/workflows/authenticator.yml

@@ -16,9 +16,10 @@ on:
       - "src/backend/services/authenticator/**"
       - "src/backend/libs/authenticator-sdk/**"
       # The e2e stack imports the generated compose realm (run-e2e.sh), which
-      # gen-realm.py builds from the seed roster.
+      # `insight-seed-realm` builds from the seed roster.
       - "deploy/compose/keycloak/**"
-      - "deploy/seed/profiles.py"
+      - "src/ingestion/tools/seed/insight_seed/profiles.py"
+      - "src/ingestion/tools/seed/insight_seed/keycloak_realm.py"
       # A workspace-wide dependency bump can break the authenticator build
       # without touching its sources (reqwest 0.12 -> 0.13 did, via
       # openidconnect's pinned HTTP-client impls).
@@ -61,6 +62,13 @@ jobs:
           workspaces: src/backend -> target
           key: authenticator-e2e
 
+      # run-e2e.sh generates the realm through `uv run --project`: the
+      # generator is a console script of the seed package, and uv resolves and
+      # installs it on first use. The runner has python3 but not uv.
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          cache-suffix: authenticator-e2e
+
       - name: Run the authenticator e2e suite
         # run-e2e.sh builds the authenticator (release), boots the stack
         # (Redis + Keycloak with the imported generated realm), runs every

.github/workflows/build-images.yml

@@ -83,6 +83,7 @@ jobs:
       identity_resolution: ${{ steps.filter.outputs.identity_resolution }}
       frontend: ${{ steps.filter.outputs.frontend }}
       toolbox: ${{ steps.filter.outputs.toolbox }}
+      seed: ${{ steps.filter.outputs.seed }}
       ui_tests: ${{ steps.filter.outputs.ui_tests }}
       umbrella: ${{ steps.filter.outputs.umbrella }}
       build_tag: ${{ steps.tag.outputs.build_tag }}
@@ -139,6 +140,14 @@ jobs:
               - 'src/frontend/**'
             toolbox:
               - 'src/ingestion/**'
+            # The sample-data seeder's own image. It COPYs a subset of the
+            # ingestion tree (scripts, dbt, silver, gold, connectors, the tool
+            # itself), but the filter stays as wide as the toolbox's on
+            # purpose: a narrower list would have to be kept in step with the
+            # Dockerfile's COPY lines, and the failure mode of forgetting is a
+            # silently stale published image.
+            seed:
+              - 'src/ingestion/**'
             # The browser runner for tests/stand (deploy/compose/ui-tests.Dockerfile).
             # It COPYs tests/lib + tests/stand and installs from the committed
             # tests/uv.lock, so all four are baked in and any of them changing
@@ -1101,6 +1110,162 @@ jobs:
         run: |
           echo '- `${{ env.IMAGE_PREFIX }}/insight-toolbox:${{ needs.changes.outputs.build_tag }}`' >> "$GITHUB_STEP_SUMMARY"
 
+  # ─── Sample-data seeder build (with import smoke) ──────────────────────────
+  # The seeder's own image, separate from the toolbox on purpose: the toolbox
+  # runs migrations against real stands and a demo-data generator has no
+  # business being installed there. Same build context (src/ingestion), a
+  # narrower COPY set, and `insight-seed` as the entry point — see
+  # src/ingestion/tools/seed/Dockerfile.
+  #
+  # `seed-stand.sh` runs this image as a one-shot Job; the ref it discovers is
+  # `ingestion.seedImage`, pinned by publish-chart below.
+  seed:
+    needs: changes
+    if: |
+      needs.changes.outputs.seed == 'true'
+      || (github.event_name == 'workflow_dispatch' && !inputs.frontend_only)
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-latest
+            platform: linux/amd64
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+            platform: linux/arm64
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
+
+      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        if: needs.changes.outputs.should_push == 'true'
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Build → validate (amd64 leg only) → push by digest, exactly as the
+      # toolbox above: the content is arch-agnostic, so one validate is enough
+      # and the push-build reuses the buildx cache.
+      - name: Build seed image (load locally, amd64 only)
+        if: matrix.arch == 'amd64'
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: src/ingestion
+          file: src/ingestion/tools/seed/Dockerfile
+          platforms: linux/amd64
+          load: true
+          push: false
+          tags: insight-seed:ci-validate
+          cache-from: type=gha,scope=seed-amd64
+          cache-to: ${{ needs.changes.outputs.should_push == 'true' && 'type=gha,mode=max,scope=seed-amd64,ignore-error=true' || '' }}
+
+      - name: Validate the seeder (entry point, imports, tests, dbt)
+        if: matrix.arch == 'amd64'
+        # The failure this catches is the one that made the seeder unrunnable
+        # on a stand in the first place: a module whose dependency is not
+        # installed in the image. Importing every step module exercises the
+        # whole dependency set (MariaDB and ClickHouse drivers included)
+        # without touching a database, `--help` proves the console script is on
+        # PATH, and `dbt --version` proves the silver step's subprocess
+        # environment exists.
+        #
+        # The package's own tests run here rather than in ci.yml's Python
+        # matrix: they are stdlib `unittest` and touch no database, so the
+        # shipped image can run them with nothing installed on top — and this
+        # way what is tested is exactly what is published.
+        run: |
+          docker run --rm insight-seed:ci-validate --help
+          docker run --rm --entrypoint python insight-seed:ci-validate -c \
+            'import insight_seed.identity, insight_seed.silver, insight_seed.analytics, insight_seed.preflight, insight_seed.keycloak_realm'
+          docker run --rm --entrypoint python insight-seed:ci-validate -m unittest discover -s tests -t .
+          docker run --rm --entrypoint dbt insight-seed:ci-validate --version
+
+      - name: Build and push seed image (by digest)
+        id: build
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: src/ingestion
+          file: src/ingestion/tools/seed/Dockerfile
+          platforms: ${{ matrix.platform }}
+          outputs: type=image,name=${{ env.IMAGE_PREFIX }}/insight-seed,push-by-digest=true,name-canonical=true,push=${{ needs.changes.outputs.should_push == 'true' }}
+          cache-from: type=gha,scope=seed-${{ matrix.arch }}
+          cache-to: ${{ needs.changes.outputs.should_push == 'true' && format('type=gha,mode=max,scope=seed-{0},ignore-error=true', matrix.arch) || '' }}
+
+      - name: Export digest
+        if: needs.changes.outputs.should_push == 'true'
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+      - name: Upload digest
+        if: needs.changes.outputs.should_push == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: digests-seed-${{ matrix.arch }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge-seed:
+    needs: [changes, seed]
+    if: |
+      always()
+      && needs.seed.result != 'skipped'
+      && needs.changes.outputs.should_push == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require the platform builds to have passed
+        if: needs.seed.result != 'success'
+        run: |
+          echo "::error::seed finished '${{ needs.seed.result }}' — nothing to merge."
+          exit 1
+      - name: Download digests
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: /tmp/digests
+          pattern: digests-seed-*
+          merge-multiple: true
+      - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
+      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
+        id: meta
+        with:
+          images: ${{ env.IMAGE_PREFIX }}/insight-seed
+          tags: |
+            type=raw,value=${{ needs.changes.outputs.build_tag }}
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+      - name: Create multi-arch manifest and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create \
+            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.IMAGE_PREFIX }}/insight-seed@sha256:%s ' *)
+      - name: Inspect manifest digest
+        id: inspect
+        run: |
+          DIGEST=$(docker buildx imagetools inspect \
+            ${{ env.IMAGE_PREFIX }}/insight-seed:${{ needs.changes.outputs.build_tag }} \
+            --format '{{json .Manifest}}' | jq -r .digest)
+          echo "digest=$DIGEST" >> "$GITHUB_OUTPUT"
+      - name: Attest build provenance
+        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
+        with:
+          subject-name: ${{ env.IMAGE_PREFIX }}/insight-seed
+          subject-digest: ${{ steps.inspect.outputs.digest }}
+          push-to-registry: true
+      # Copy-pastable pushed ref on the run's Summary page (#1994).
+      - name: Report pushed image
+        run: |
+          echo '- `${{ env.IMAGE_PREFIX }}/insight-seed:${{ needs.changes.outputs.build_tag }}`' >> "$GITHUB_STEP_SUMMARY"
+
   # ─── ui-tests browser runner (deployed-stand journeys) ─────────────────────
   # The Playwright image e2e-stand.yml's `ui-journeys` check pulls. It bakes in
   # tests/lib + tests/stand and installs from the committed tests/uv.lock, so
@@ -1588,6 +1753,7 @@ jobs:
       - merge-identity-resolution
       - merge-frontend
       - merge-toolbox
+      - merge-seed
       - merge-image
       - bump-descriptors
     if: |
@@ -1601,6 +1767,7 @@ jobs:
       && (needs.merge-identity-resolution.result == 'success' || needs.merge-identity-resolution.result == 'skipped')
       && (needs.merge-frontend.result      == 'success' || needs.merge-frontend.result      == 'skipped')
       && (needs.merge-toolbox.result       == 'success' || needs.merge-toolbox.result       == 'skipped')
+      && (needs.merge-seed.result          == 'success' || needs.merge-seed.result          == 'skipped')
       && (needs.merge-image.result         == 'success' || needs.merge-image.result         == 'skipped')
       && (needs.bump-descriptors.result    == 'success' || needs.bump-descriptors.result    == 'skipped')
       && needs.bump-descriptors.outputs.committed != 'true'
@@ -1611,6 +1778,7 @@ jobs:
         || needs.changes.outputs.identity_resolution == 'true'
         || needs.changes.outputs.frontend     == 'true'
         || needs.changes.outputs.toolbox      == 'true'
+        || needs.changes.outputs.seed         == 'true'
         || needs.changes.outputs.umbrella     == 'true'
       )
     runs-on: ubuntu-latest
@@ -1688,14 +1856,15 @@ jobs:
           FRONTEND: ${{ needs.changes.outputs.frontend }}
         run: .github/workflows/scripts/bump-service-appversions.sh
 
-      - name: Bump ingestion toolbox image ref
+      - name: Bump ingestion image refs
         env:
           BUILD_TAG: ${{ needs.changes.outputs.build_tag }}
           TOOLBOX: ${{ needs.changes.outputs.toolbox }}
+          SEED: ${{ needs.changes.outputs.seed }}
           IMAGE_PREFIX: ${{ env.IMAGE_PREFIX }}
         run: |
           set -euo pipefail
-          # toolbox is the only ingestion image still pinned in
+          # toolbox and seed are the only ingestion images pinned in
           # charts/insight/values.yaml — connector image refs travel inside
           # the toolbox image's baked descriptors (ADR-0016). Connector
           # descriptors are patched by `bump-descriptors` in a separate
@@ -1705,6 +1874,14 @@ jobs:
             echo "Setting ingestion.toolboxImage → $REF"
             yq -i ".ingestion.toolboxImage = \"$REF\"" charts/insight/values.yaml
           fi
+          # seedImage is not deployed by the chart — nothing renders it into a
+          # pod. It is published state: `seed-stand.sh` reads it back off an
+          # installed release to learn which seeder image matches that stand.
+          if [ "$SEED" = "true" ]; then
+            REF="$IMAGE_PREFIX/insight-seed:$BUILD_TAG"
+            echo "Setting ingestion.seedImage → $REF"
+            yq -i ".ingestion.seedImage = \"$REF\"" charts/insight/values.yaml
+          fi
 
       # Umbrella `version` always patch-bumps (semver, per-publish contract).
       # Umbrella `appVersion` is DISPLAY-ONLY in the deploy path — the actual
@@ -1850,7 +2027,7 @@ jobs:
           # push is expected to fast-forward. Do NOT rebase the bump commit on
           # failure: the only way the tip moves under us now is a direct human
           # push to ${GITHUB_REF_NAME}, and rebasing our bump onto it conflicts
-          # on the values.yaml lines we just edited (toolboxImage) —
+          # on the values.yaml lines we just edited (toolboxImage, seedImage) —
           # unresolvable in CI. Fail loud instead; a plain job re-run is
           # idempotent (it recomputes against the refreshed tip, see
           # "Re-anchor" above).

.github/workflows/e2e-stand.yml

@@ -117,7 +117,7 @@ jobs:
           fi
           changed="$(git diff --name-only "$BASE" "$HEAD")"
           echo "$changed" | sed 's/^/  /'
-          if echo "$changed" | grep -qE '^(tests/(stand|lib)/|tests/pyproject\.toml|tests/uv\.lock|deploy/(seed|compose)/|docker-compose\.yml|dev-compose\.sh|src/backend/|src/frontend/helm/|docs/components/backend/.*/openapi\.json|\.github/workflows/e2e-stand\.yml|\.github/workflows/scripts/redact-playwright-trace\.py)'; then
+          if echo "$changed" | grep -qE '^(tests/(stand|lib)/|tests/pyproject\.toml|tests/uv\.lock|deploy/compose/|src/ingestion/tools/seed/|docker-compose\.yml|dev-compose\.sh|src/backend/|src/frontend/helm/|docs/components/backend/.*/openapi\.json|\.github/workflows/e2e-stand\.yml|\.github/workflows/scripts/redact-playwright-trace\.py)'; then
             echo "relevant=true" >> "$GITHUB_OUTPUT"
           else
             echo "relevant=false" >> "$GITHUB_OUTPUT"
@@ -301,6 +301,14 @@ jobs:
         with:
           python-version: "3.12" # runs the redaction script only — stdlib alone
 
+      # `test-stand up` forces Keycloak, and the realm is generated by the seed
+      # package's own entry point (`insight-seed-realm`), which dev-compose.sh
+      # runs through uv. The API job above installs uv for the suite; this job
+      # needs it for the bring-up itself.
+      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          cache-suffix: stand-ui
+
       - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
         with:
           registry: ghcr.io

.github/workflows/trivy-images.yml

@@ -44,6 +44,7 @@ jobs:
           - insight-frontend
           # Python tooling
           - insight-toolbox
+          - insight-seed
           - insight-jira-enrich
           # Python connectors
           - source-active-directory-insight

.gitignore

@@ -10,10 +10,10 @@ up-orb.sh
 /deploy/compose/build/
 /deploy/compose/override.generated.yml
 /deploy/compose/keycloak/realm-insight.generated.json
-/deploy/seed/.venv/
-/deploy/seed/__pycache__/
-/deploy/seed/.mypy_cache/
-/deploy/seed/.ruff_cache/
+/src/ingestion/tools/seed/.venv/
+/src/ingestion/tools/seed/__pycache__/
+/src/ingestion/tools/seed/.mypy_cache/
+/src/ingestion/tools/seed/.ruff_cache/
 __pycache__/
 *.pyc
 .venv/
@@ -582,7 +582,7 @@ trivy-image.sarif
 trufflehog-findings.jsonl
 
 # Seed manifest — per-stand runtime output (PROFILE.md is the committed view).
-deploy/seed/manifest.json
+src/ingestion/tools/seed/manifest.json
 
 # pytest-playwright's artefact directory: traces, screenshots and video from a
 # stand run. Written by `./dev-compose.sh test-stand test`, uploaded by CI.