diff --git a/.github/workflows/connectors-ddl-reminder.yml b/.github/workflows/connectors-ddl-reminder.yml deleted file mode 100644 index 56bffc08f..000000000 --- a/.github/workflows/connectors-ddl-reminder.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Reminds contributors to regenerate the committed connectors-ddl snapshot when a -# PR touches src/ingestion. The snapshot (scripts/connectors-ddl/*.sql) is -# regenerated MANUALLY (see src/ingestion/scripts/bootstrap-db/), so this workflow -# is only a nudge — it does NOT run the regeneration and pushes nothing. -# -# Uses pull_request_target so the token can comment on fork PRs too (a plain -# pull_request from a fork gets a read-only token). This is safe here because the -# job never checks out or executes PR code and uses no secrets — it only reads the -# changed-file list (via the paths filter) and posts a single sticky comment. -# -# NB: pull_request_target always uses the workflow file from the base branch, so -# this takes effect only once merged to main. -name: connectors-ddl reminder - -on: - pull_request_target: - types: [opened, synchronize, reopened] - paths: - - "src/ingestion/**" - -permissions: - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - remind: - name: Remind to regenerate connectors-ddl - runs-on: ubuntu-latest - steps: - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 - with: - script: | - // Sticky marker: post the reminder at most once per PR (not per push). - const marker = ''; - const { owner, repo } = context.repo; - const issue_number = context.payload.pull_request.number; - const existing = await github.paginate(github.rest.issues.listComments, { - owner, repo, issue_number, per_page: 100, - }); - if (existing.some((c) => c.body && c.body.includes(marker))) return; - const body = [ - marker, - '### ⚠️ Regenerate the connectors-ddl snapshot', - '', - 'This PR changes `src/ingestion/**`. If your change affects any', - 'bronze / silver / gold schema, regenerate the committed DDL snapshot', - 'and include it in this PR.', - '', - 'Prerequisites (details: `src/ingestion/scripts/bootstrap-db/README.md`):', - '- docker + a fresh throwaway ClickHouse 25.7.5 (README "Local ClickHouse for testing")', - '- `.env` from `.env.bootstrap.example` pointing at it; use the host LAN IP,', - ' reachable from both the host and connector containers', - ' (`host.docker.internal` does not resolve on the macOS host itself)', - '- python3.12 or python3.11 on PATH (pinned dbt venv)', - '- HubSpot + Salesforce credentials in `.env` — their `discover` calls the', - ' live APIs; without them, apply `../connectors-ddl/{hubspot,salesforce}.sql`', - ' (relative to bootstrap-db/) to seed their bronze, then run the dbt step', - '', - '```bash', - 'cd src/ingestion/scripts/bootstrap-db', - 'set -a; source pins.env; source .env; set +a', - './bootstrap-db.sh connectors-config.yaml # fresh ClickHouse 25.7.5', - './dump-ddl.sh # writes scripts/connectors-ddl/*.sql', - '```', - '', - 'Commit the resulting `scripts/connectors-ddl/*.sql` diff. If nothing', - 'changed, no snapshot update is needed. (Regeneration is manual for now.)', - ].join('\n'); - await github.rest.issues.createComment({ owner, repo, issue_number, body }); diff --git a/.github/workflows/connectors-ddl.yml b/.github/workflows/connectors-ddl.yml index fa991cfc3..53d072a2b 100644 --- a/.github/workflows/connectors-ddl.yml +++ b/.github/workflows/connectors-ddl.yml @@ -28,8 +28,10 @@ name: connectors-ddl # never `pull_request_target` — that would run fork code with access to those # secrets. Pushes always carry secrets, so they need no such guard. # -# The job validates; it never commits. A stale snapshot is the author's to -# regenerate (see the failure message). +# The gate job validates; it never commits. On PR drift the separate regen-pr +# job delivers the regenerated snapshot as a reviewable stacked PR against the +# PR's own branch — see its header for the trust boundary. Drift on a main push +# stays a red run: there is no PR branch to stack onto. on: pull_request: @@ -55,6 +57,10 @@ jobs: if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest timeout-minutes: 60 + outputs: + # Written by the drift step before its exit 1 — step outputs survive the + # step's own failure, which is how regen-pr below learns it has work. + drift: ${{ steps.drift.outputs.drift }} env: BOOTSTRAP_DIR: src/ingestion/scripts/bootstrap-db CLICKHOUSE_CONTAINER: connectors-ddl-clickhouse @@ -132,6 +138,7 @@ jobs: "${BOOTSTRAP_DIR}/bootstrap-db.sh" "${BOOTSTRAP_DIR}/connectors-config.yaml" - name: Re-dump the snapshot and fail on drift + id: drift run: | set -euo pipefail "${BOOTSTRAP_DIR}/dump-ddl.sh" @@ -140,7 +147,8 @@ jobs: git add --intent-to-add --all -- src/ingestion/scripts/connectors-ddl git diff -- src/ingestion/scripts/connectors-ddl > "$RUNNER_TEMP/connectors-ddl.diff" if [[ -s "$RUNNER_TEMP/connectors-ddl.diff" ]]; then - echo "::error title=connectors-ddl snapshot is stale::Regenerate it: ${BOOTSTRAP_DIR}/bootstrap-db.sh ${BOOTSTRAP_DIR}/connectors-config.yaml && ${BOOTSTRAP_DIR}/dump-ddl.sh — then commit the result. Full diff in the connectors-ddl-drift artifact." + echo "drift=true" >> "$GITHUB_OUTPUT" + echo "::error title=connectors-ddl snapshot is stale::The regen-pr job opens a pull request against this branch with the regenerated snapshot — review its DDL diff and merge it. Full diff also in the connectors-ddl-drift artifact." git diff --stat -- src/ingestion/scripts/connectors-ddl head -n 200 "$RUNNER_TEMP/connectors-ddl.diff" exit 1 @@ -155,6 +163,15 @@ jobs: path: ${{ runner.temp }}/connectors-ddl.diff if-no-files-found: ignore + - name: Upload the regenerated snapshot + # Input for the regen-pr job below: the freshly dumped *.sql files, not + # the diff — the diff may not apply if the author pushes meanwhile. + if: failure() && steps.drift.outputs.drift == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: connectors-ddl-regenerated + path: src/ingestion/scripts/connectors-ddl/ + - name: Field parity # Runs even when the snapshot drifted: the two gates are independent, and a # PR that breaks both should learn both in one run instead of one per push. @@ -176,3 +193,92 @@ jobs: - name: ClickHouse logs if: failure() run: docker logs --tail 200 "${CLICKHOUSE_CONTAINER}" 2>&1 || true + + # On snapshot drift in a PR, deliver the regenerated snapshot as a REVIEWABLE + # stacked pull request against the PR's own branch, instead of telling the + # author to regenerate locally — which requires the HubSpot/Salesforce + # credentials most contributors do not have. The author reviews the DDL diff + # and merges it; that merge is a human push, so the required checks re-run + # normally (a bot push with GITHUB_TOKEN would not trigger them). + # + # This job never executes code from the PR: it checks out the head branch only + # as a git base and replaces the snapshot directory with the artifact the gate + # job produced. That separation is why THIS job may hold write tokens while + # the gate job runs PR code with contents: read. + # + # Convergence terminates: the re-dump on an unchanged tree is byte-identical, + # so once the regen commit is merged the next run finds no drift. + regen-pr: + name: Open a regen PR on snapshot drift + needs: connectors-ddl + if: >- + always() && github.event_name == 'pull_request' && needs.connectors-ddl.result == 'failure' && needs.connectors-ddl.outputs.drift == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + env: + HEAD_REF: ${{ github.event.pull_request.head.ref }} + REGEN_BRANCH: regen-ddl/${{ github.event.pull_request.head.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + DDL_DIR: src/ingestion/scripts/connectors-ddl + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + ref: ${{ github.event.pull_request.head.ref }} + # This checkout keeps credentials on purpose: the push below uses them. + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: connectors-ddl-regenerated + path: ${{ runner.temp }}/regenerated + + - name: Commit the snapshot and push the regen branch + id: commit + run: | + set -euo pipefail + # Full replace, not overlay: a relation the code no longer produces + # must disappear from the snapshot, and cp alone cannot delete. + rm -f "${DDL_DIR}"/*.sql + cp "${RUNNER_TEMP}/regenerated"/*.sql "${DDL_DIR}/" + git add --all -- "${DDL_DIR}" + if git diff --cached --quiet; then + # The author pushed a regeneration while the gate was running. + echo "changed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit --signoff -m "chore(connectors-ddl): regenerate the snapshot for ${HEAD_REF}" \ + -m "Produced by the connectors-ddl gate from a full bootstrap of this branch: real connector discover, destination-clickhouse, dbt, migrations, then dump-ddl.sh." + # Force: this branch is bot-owned and rebuilt from scratch on every + # drifting run, so its only history is the latest regeneration. + git push --force origin "HEAD:refs/heads/${REGEN_BRANCH}" + echo "changed=true" >> "$GITHUB_OUTPUT" + + - name: Open or reuse the stacked PR + sticky comment + if: steps.commit.outputs.changed == 'true' + run: | + set -euo pipefail + url="$(gh pr list --head "${REGEN_BRANCH}" --base "${HEAD_REF}" --state open --json url -q '.[0].url // empty')" + if [[ -z "${url}" ]]; then + url="$(gh pr create --base "${HEAD_REF}" --head "${REGEN_BRANCH}" \ + --title "chore(connectors-ddl): regenerate the snapshot for ${HEAD_REF}" \ + --body "The connectors-ddl gate on #${PR_NUMBER} found the committed snapshot stale against what this branch's connectors + dbt + migrations actually produce. This PR carries the regenerated snapshot — review the DDL diff and merge it into \`${HEAD_REF}\`; the gate then re-runs and goes green. Closes automatically if #${PR_NUMBER} is merged without it (the base branch disappears).")" + fi + echo "regen PR: ${url}" + + marker='' + body="$(printf '%s\n### :robot: connectors-ddl snapshot drift\n\n%s\n\n%s' \ + "${marker}" \ + "The committed snapshot does not match what this branch actually produces. The regenerated snapshot is waiting in ${url} — review the DDL diff there and merge it into this branch; the gate re-runs on your merge." \ + "_Refreshed on every drifting gate run (the regen branch is force-pushed), so it reflects this branch as of the last completed run._")" + existing="$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + -q "[.[] | select(.body | startswith(\"${marker}\"))][0].id // empty")" + if [[ -z "${existing}" ]]; then + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -f body="${body}" > /dev/null + else + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${existing}" -f body="${body}" > /dev/null + fi diff --git a/src/ingestion/scripts/bootstrap-db/README.md b/src/ingestion/scripts/bootstrap-db/README.md index 72b89c44c..ea91d3256 100644 --- a/src/ingestion/scripts/bootstrap-db/README.md +++ b/src/ingestion/scripts/bootstrap-db/README.md @@ -161,7 +161,7 @@ Contributors whose physical table is not owned by dbt are covered too. `jira__ta | `bootstrap-db.sh ` | Sources `pins.env` and `.env` (if present), runs `seed-connectors.sh`, runs all dbt models, runs `../apply-ch-migrations.sh`. | | `run-dbt.sh [dbt args]` | Helper: generates a profiles.yml from the `CLICKHOUSE_*` variables and runs `dbt run` in `src/ingestion/dbt`. | | `check-field-parity.py [--manifest PATH]` | Audits every staging contributor against its silver union target (column set, positional order, exact type) plus manifest-vs-warehouse coverage. Same `CLICKHOUSE_*` env contract as the other scripts. Non-zero exit on any finding. | -| `dump-ddl.sh` | Dumps `SHOW CREATE` for every `bronze_*` table, the `person`/`identity`/`silver`/`insight` databases (tables and views), and the gold-referenced `staging` tables into `../connectors-ddl/*.sql` — the committed snapshot that `../create-bronze-placeholders.sh` applies on fresh clusters. **Run it manually** after `bootstrap-db.sh` (see step above) whenever a schema changes, and commit the diff. `.github/workflows/connectors-ddl.yml` re-runs the whole pipeline on every same-repository PR and on every commit to `main`, and fails when the committed snapshot no longer matches — it validates, it never commits. | +| `dump-ddl.sh` | Dumps `SHOW CREATE` for every `bronze_*` table, the `person`/`identity`/`silver`/`insight` databases (tables and views), and the gold-referenced `staging` tables into `../connectors-ddl/*.sql` — the committed snapshot that `../create-bronze-placeholders.sh` applies on fresh clusters. **Run it manually** after `bootstrap-db.sh` (see step above) whenever a schema changes, and commit the diff. `.github/workflows/connectors-ddl.yml` re-runs the whole pipeline on every same-repository PR and on every commit to `main`, and fails when the committed snapshot no longer matches. On PR drift its `regen-pr` job opens a stacked PR against the PR's branch with the regenerated snapshot (and links it in a sticky comment) — review the DDL diff there and merge, no local regeneration or CRM credentials needed. Drift on `main` stays a red run. | ## Image pins (pins.env)