From 20ecffe2832d081f9d4203aacce99ab7e02be8c8 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Mon, 8 Jun 2026 01:07:56 +0300 Subject: [PATCH] feat(e2e): add WIF shim and PR authorization gate for CI (#1604) Introduce e2e_shim.yml (pull_request_target) calling trusted e2e.yml@main via workflow_call, with member/collaborator or fresh ok-to-test authorization, GCP WIF auth bound to job_workflow_ref, and hardcoded defaults for shared e2e infra. Partial progress toward secretless e2e: WIF replaces long-lived GCP keys; canonical project, SA, and WIF provider are workflow defaults (overridable via optional secrets). pull_request_target lets fork PRs inherit secrets/OIDC while the shim stays safe (no PR code checkout). Signed-off-by: Barak Korren Co-authored-by: Cursor Co-authored-by: Cursor --- .../check-e2e-authorization/action.yml | 98 ++++++ .github/workflows/e2e.yml | 84 +++-- .github/workflows/e2e_shim.yml | 47 +++ CLAUDE.md | 1 + Makefile | 3 +- ...010-stored-session-for-e2e-browser-auth.md | 2 + .../0040-org-pool-for-parallel-e2e-tests.md | 1 + ...-e2e-wif-shim-and-pr-authorization-gate.md | 166 ++++++++++ docs/guides/README.md | 2 + docs/guides/dev/cli-internals.md | 1 + docs/guides/dev/e2e-testing.md | 106 +++++++ docs/guides/infrastructure/e2e-gcp-setup.md | 290 ++++++++++++++++++ .../infrastructure-reference.md | 50 +++ e2e/admin/testutil.go | 7 + scripts/check-e2e-authorization.sh | 105 +++++++ scripts/check-e2e-authorization_test.sh | 172 +++++++++++ 16 files changed, 1113 insertions(+), 22 deletions(-) create mode 100644 .github/actions/check-e2e-authorization/action.yml create mode 100644 .github/workflows/e2e_shim.yml create mode 100644 docs/ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md create mode 100644 docs/guides/dev/e2e-testing.md create mode 100644 docs/guides/infrastructure/e2e-gcp-setup.md create mode 100755 scripts/check-e2e-authorization.sh create mode 100755 scripts/check-e2e-authorization_test.sh diff --git a/.github/actions/check-e2e-authorization/action.yml b/.github/actions/check-e2e-authorization/action.yml new file mode 100644 index 0000000000..c1ba571dac --- /dev/null +++ b/.github/actions/check-e2e-authorization/action.yml @@ -0,0 +1,98 @@ +name: Check E2E Authorization +description: >- + Authorize PR-triggered e2e runs for trusted authors or a fresh ok-to-test label. + Removes stale ok-to-test labels and posts a sticky PR comment when unauthorized. + +inputs: + pr_number: + description: Pull request number to authorize + required: true + repository: + description: Repository in owner/name form + required: true + docs_url: + description: Link to e2e testing documentation + required: false + default: https://github.com/fullsend-ai/fullsend/blob/main/docs/guides/dev/e2e-testing.md + +outputs: + authorized: + description: Whether e2e tests may run for this PR + value: ${{ steps.check.outputs.authorized }} + reason: + description: Authorization outcome reason code + value: ${{ steps.check.outputs.reason }} + label_removed: + description: Whether a stale ok-to-test label was removed + value: ${{ steps.check.outputs.label_removed }} + +runs: + using: composite + steps: + - name: Check authorization + id: check + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ inputs.pr_number }} + REPOSITORY: ${{ inputs.repository }} + run: bash "${GITHUB_WORKSPACE}/scripts/check-e2e-authorization.sh" "${PR_NUMBER}" "${REPOSITORY}" + + - name: Post gate comment + if: steps.check.outputs.authorized != 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ inputs.repository }} + PR_NUMBER: ${{ inputs.pr_number }} + REASON: ${{ steps.check.outputs.reason }} + LABEL_REMOVED: ${{ steps.check.outputs.label_removed }} + DOCS_URL: ${{ inputs.docs_url }} + run: | + set -euo pipefail + + marker='' + owner="${REPOSITORY%%/*}" + repo="${REPOSITORY#*/}" + + body="${marker} + ## E2E tests did not run + + " + + case "${REASON}" in + stale_ok_to_test) + body+="The \`ok-to-test\` label was removed because new commits landed after it was applied. A maintainer must re-apply \`ok-to-test\` after reviewing the latest changes. + " + ;; + error) + body+="The authorization check failed (GitHub API error). Re-run the workflow when the API is available; if this persists, contact a maintainer. + " + ;; + *) + body+="E2E tests run automatically for org/repo **members and collaborators** on pull requests. + + For other contributors, a maintainer must add the \`ok-to-test\` label **after** the latest push. + " + ;; + esac + + if [[ "${LABEL_REMOVED}" == "true" ]]; then + body+=" + > **Note:** \`ok-to-test\` was cleared due to new commits. + " + fi + + body+=" + See [E2E testing guide](${DOCS_URL}) for details." + + existing_id="$(gh api "repos/${owner}/${repo}/issues/${PR_NUMBER}/comments" --paginate \ + --jq ".[] | select(.body | contains(\"${marker}\")) | .id" | head -n1 || true)" + + if [[ -n "${existing_id}" ]]; then + jq -n --arg body "${body}" '{body: $body}' | \ + gh api -X PATCH "repos/${owner}/${repo}/issues/comments/${existing_id}" --input - >/dev/null + else + jq -n --arg body "${body}" '{body: $body}' | \ + gh api -X POST "repos/${owner}/${repo}/issues/${PR_NUMBER}/comments" --input - >/dev/null + fi diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f7ba8ea7de..a075f7e936 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,10 +1,24 @@ name: E2E Tests -permissions: - contents: read - id-token: write +# PRs invoke this workflow via e2e_shim.yml (workflow_call @ refs/heads/main). +# See docs/ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md on: + workflow_call: + inputs: + pr_number: + description: Pull request number to authorize and test + required: true + type: number + head_sha: + description: PR head SHA to checkout for tests + required: true + type: string + pr_author: + description: PR author login (informational; auth uses API) + required: false + type: string + default: "" push: branches: [main] paths: @@ -18,30 +32,58 @@ on: - 'internal/sentencetoken/english.json' - 'Makefile' - '.github/workflows/e2e.yml' - pull_request: - paths: - - '**/*.go' - - 'go.mod' - - 'go.sum' - - 'e2e/**' - - 'internal/scaffold/fullsend-repo/**' - - 'internal/security/hooks/**' - - 'internal/dispatch/gcf/mintsrc/**' - - 'internal/sentencetoken/english.json' - - 'Makefile' - - '.github/workflows/e2e.yml' + - '.github/workflows/e2e_shim.yml' + - '.github/actions/check-e2e-authorization/**' + - 'scripts/check-e2e-authorization.sh' + - 'scripts/check-e2e-authorization_test.sh' workflow_dispatch: +permissions: {} + concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: e2e-${{ github.event_name == 'workflow_call' && inputs.pr_number || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: + gate: + if: github.event_name == 'workflow_call' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + outputs: + authorized: ${{ steps.auth.outputs.authorized }} + steps: + - uses: actions/checkout@v4 + with: + ref: refs/heads/main + + - name: Check PR authorization + id: auth + uses: ./.github/actions/check-e2e-authorization + with: + pr_number: ${{ inputs.pr_number }} + repository: ${{ github.repository }} + e2e: + needs: gate + if: >- + !cancelled() && + (github.event_name != 'workflow_call' || needs.gate.result == 'skipped' || needs.gate.outputs.authorized == 'true') runs-on: ubuntu-latest timeout-minutes: 30 + permissions: + contents: read + id-token: write + env: + # Shared e2e GCP defaults when repository secrets are unset (see ADR 0043). + E2E_GCP_PROJECT_ID: ${{ secrets.E2E_GCP_PROJECT_ID != '' && secrets.E2E_GCP_PROJECT_ID || 'it-gcp-konflux-e2e-fullsend' }} + E2E_GCP_SERVICE_ACCOUNT: ${{ secrets.E2E_GCP_SERVICE_ACCOUNT != '' && secrets.E2E_GCP_SERVICE_ACCOUNT || 'fullsend-e2e@it-gcp-konflux-e2e-fullsend.iam.gserviceaccount.com' }} + E2E_GCP_WIF_PROVIDER: ${{ secrets.E2E_GCP_WIF_PROVIDER != '' && secrets.E2E_GCP_WIF_PROVIDER || 'projects/208332380190/locations/global/workloadIdentityPools/fullsend-e2e-pool/providers/github-oidc' }} steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_call' && inputs.head_sha || github.sha }} - uses: actions/setup-go@v5 with: @@ -73,10 +115,11 @@ jobs: - name: Authenticate to GCP if: steps.secrets-check.outputs.available == 'true' - uses: google-github-actions/auth@v2 + uses: google-github-actions/auth@v3 with: - workload_identity_provider: ${{ secrets.E2E_GCP_WIF_PROVIDER }} - service_account: ${{ secrets.E2E_GCP_SERVICE_ACCOUNT }} + workload_identity_provider: ${{ env.E2E_GCP_WIF_PROVIDER }} + service_account: ${{ env.E2E_GCP_SERVICE_ACCOUNT }} + project_id: ${{ env.E2E_GCP_PROJECT_ID }} - name: Run e2e tests if: steps.secrets-check.outputs.available == 'true' @@ -86,13 +129,12 @@ jobs: E2E_GITHUB_PASSWORD: ${{ secrets.E2E_GITHUB_PASSWORD }} E2E_GITHUB_TOTP_SECRET: ${{ secrets.E2E_GITHUB_TOTP_SECRET }} E2E_MINT_URL: ${{ secrets.E2E_MINT_URL }} - E2E_GCP_PROJECT_ID: ${{ secrets.E2E_GCP_PROJECT_ID }} - name: Upload debug screenshots if: always() && steps.secrets-check.outputs.available == 'true' uses: actions/upload-artifact@v4 with: - name: e2e-screenshots + name: e2e-screenshots-${{ github.event_name == 'workflow_call' && inputs.pr_number || github.run_id }} path: ${{ runner.temp }}/e2e-screenshots/ if-no-files-found: ignore retention-days: 5 diff --git a/.github/workflows/e2e_shim.yml b/.github/workflows/e2e_shim.yml new file mode 100644 index 0000000000..4f174ee2be --- /dev/null +++ b/.github/workflows/e2e_shim.yml @@ -0,0 +1,47 @@ +name: E2E Shim + +# Lightweight PR shim: calls trusted e2e.yml@main without checking out PR code. +# See docs/ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md +# +# Security: pull_request_target runs the BASE branch version of this workflow, +# preventing PRs from modifying it to exfiltrate credentials. This shim never +# checks out PR code, so it is not vulnerable to "pwn request" attacks. + +on: + pull_request_target: + types: [opened, synchronize, reopened, labeled] + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + - 'e2e/**' + - 'internal/scaffold/fullsend-repo/**' + - 'internal/security/hooks/**' + - 'internal/dispatch/gcf/mintsrc/**' + - 'internal/sentencetoken/english.json' + - 'Makefile' + - '.github/workflows/e2e.yml' + - '.github/workflows/e2e_shim.yml' + - '.github/actions/check-e2e-authorization/**' + - 'scripts/check-e2e-authorization.sh' + - 'scripts/check-e2e-authorization_test.sh' + +concurrency: + group: e2e-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + call-e2e: + if: >- + github.event.action != 'labeled' || + github.event.label.name == 'ok-to-test' + permissions: + contents: read + id-token: write + pull-requests: write + uses: fullsend-ai/fullsend/.github/workflows/e2e.yml@refs/heads/main + with: + pr_number: ${{ github.event.pull_request.number }} + head_sha: ${{ github.event.pull_request.head.sha }} + pr_author: ${{ github.event.pull_request.user.login }} + secrets: inherit diff --git a/CLAUDE.md b/CLAUDE.md index f78833c6c9..192b702e3e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,6 +13,7 @@ Fullsend is a platform for fully autonomous agentic development for GitHub-hoste - Always run `make lint` before submitting changes and fix any failures. - Use [Conventional Commits](https://www.conventionalcommits.org/) for all commit messages. See [CONTRIBUTING.md](CONTRIBUTING.md#commit-messages) for the full specification. This is critical — GoReleaser uses commit prefixes to generate release notes. - Never commit secrets (tokens, API keys, PEM keys, gcloud credentials) or sensitive data (GCP project names, service account identifiers, Model Armor template names, internal hostnames). Use environment variables with no defaults for sensitive values. +- **Exception — shared e2e test infrastructure:** The dedicated CI e2e GCP project ID (`it-gcp-konflux-e2e-fullsend`), e2e service account name (`fullsend-e2e`), and WIF pool/provider IDs (`fullsend-e2e-pool`, `github-oidc`) may appear in workflow defaults, tests, and docs. Access is bound by WIF attribute conditions (`repository`, `job_workflow_ref`) — not by hiding these identifiers. See [ADR 0043](docs/ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md) and [E2E GCP setup](docs/guides/infrastructure/e2e-gcp-setup.md). ## Go code diff --git a/Makefile b/Makefile index 9939839760..1f97e97879 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ help: @echo " go-vet - Run go vet" @echo " go-tidy - Run go mod tidy" @echo " lint-md-links - Check markdown files for broken in-repo links and anchors" - @echo " script-test - Run shell script tests (post-triage, post-code, post-review, pre-fetch-prior-review, reconcile-repos, validate-output-schema)" + @echo " script-test - Run shell script tests (post-triage, post-code, post-review, pre-fetch-prior-review, reconcile-repos, validate-output-schema, check-e2e-authorization)" @echo " test - Run all checks: lint-all, go-test, script-test" @echo " e2e-test - Run admin e2e tests (requires E2E_GITHUB_SESSION_FILE or E2E_GITHUB_USERNAME + E2E_GITHUB_PASSWORD)" @echo " e2e-export-session - Login to GitHub and export a Playwright session file" @@ -116,6 +116,7 @@ script-test: bash internal/scaffold/fullsend-repo/scripts/pre-fetch-prior-review-test.sh python3 internal/scaffold/fullsend-repo/scripts/process-fix-result-test.py python3 skills/topissues/scripts/topissues_test.py + bash scripts/check-e2e-authorization_test.sh test: lint-all go-test script-test diff --git a/docs/ADRs/0010-stored-session-for-e2e-browser-auth.md b/docs/ADRs/0010-stored-session-for-e2e-browser-auth.md index 2f32446d83..f1f20b0cdc 100644 --- a/docs/ADRs/0010-stored-session-for-e2e-browser-auth.md +++ b/docs/ADRs/0010-stored-session-for-e2e-browser-auth.md @@ -146,3 +146,5 @@ it does expire, a developer runs `make e2e-upload-session` to refresh it. every two weeks (and less often on active repos). - Locally, developers can use username/password directly — `make e2e-test` auto-generates a session file when credentials are available. +- CI credential gating and WIF for GCP access are described in + [ADR 0043](0043-e2e-wif-shim-and-pr-authorization-gate.md). diff --git a/docs/ADRs/0040-org-pool-for-parallel-e2e-tests.md b/docs/ADRs/0040-org-pool-for-parallel-e2e-tests.md index 62870d61b1..218601942e 100644 --- a/docs/ADRs/0040-org-pool-for-parallel-e2e-tests.md +++ b/docs/ADRs/0040-org-pool-for-parallel-e2e-tests.md @@ -72,3 +72,4 @@ slice in the test code. No architectural changes are needed. across all orgs; session export and PAT creation remain per-run. - Pool expansion is an operational task (provision org, update one slice literal), not an architectural change. +- CI authorization and GCP WIF for e2e workflows: [ADR 0043](0043-e2e-wif-shim-and-pr-authorization-gate.md). diff --git a/docs/ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md b/docs/ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md new file mode 100644 index 0000000000..db9c085850 --- /dev/null +++ b/docs/ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md @@ -0,0 +1,166 @@ +--- +title: "43. E2E WIF shim and PR authorization gate" +status: Accepted +relates_to: + - testing-agents +topics: + - e2e + - ci + - security + - wif +--- + +# 43. E2E WIF shim and PR authorization gate + +Date: 2026-06-07 + +## Status + +Accepted + +## Context + +Admin e2e tests run in GitHub Actions against live GitHub orgs and GCP +resources ([ADR 0010](0010-stored-session-for-e2e-browser-auth.md), +[ADR 0040](0040-org-pool-for-parallel-e2e-tests.md)). The workflow needs +short-lived GCP credentials for inference provisioning during install tests, but +PR-triggered workflows must not expose long-lived secrets or GCP access to +untrusted PR authors. + +Issue [#1604](https://github.com/fullsend-ai/fullsend/issues/1604) proposed a +three-workflow shim → gate → test pattern. We simplify to two workflows while +keeping the security boundary at GCP Workload Identity Federation (WIF) bound +to the trusted `e2e.yml` workflow on `main`. + +### Goal: secretless e2e (incremental) + +The long-term goal is for authorized e2e runs to work **without** per-repo +GitHub secrets or variables for GCP and session material. This ADR delivers an +**increment toward** that goal, not the full outcome: + +| Area | This change | Still required today | +|------|-------------|----------------------| +| GCP auth | WIF (short-lived); no SA JSON keys | Optional override secrets only | +| GCP project / WIF / SA | Workflow defaults to shared konflux e2e resources when secrets unset | Optional override secrets only | +| GitHub session | Unchanged | `E2E_GITHUB_SESSION`, password/TOTP secrets | +| Mint URL | Unchanged | `E2E_MINT_URL` secret | +| Fork PRs | `pull_request_target` shim inherits secrets/OIDC | Same session/mint secrets as same-repo PRs when authorized | + +Follow-up work includes eliminating the remaining secrets ([ADR +0010](0010-stored-session-for-e2e-browser-auth.md)). + +## Decision + +### Two-workflow PR path + +1. **`e2e_shim.yml`** — triggers on `pull_request_target` (base-branch workflow + definition; inherits repository secrets and OIDC, including fork PRs). Does + not checkout PR code. Calls + `fullsend-ai/fullsend/.github/workflows/e2e.yml@refs/heads/main` via + `workflow_call`, passing PR number and head SHA. + +2. **`e2e.yml`** — trusted reusable workflow on `main`: + - **Gate job:** authorize PR (org member/collaborator, or fresh `ok-to-test` + label applied after the latest push). Remove stale `ok-to-test` labels. + Post a sticky PR comment when unauthorized. + - **E2e job:** checkout PR head, authenticate to GCP via WIF, run + `make e2e-test`. + +### Trusted paths that skip the gate + +- `push` to `main` (path-filtered) +- `workflow_dispatch` + +### PR authorization + +Authorized when **either**: + +- `author_association` is `OWNER`, `MEMBER`, or `COLLABORATOR`, or +- PR has `ok-to-test` and the most recent label event is **after** the latest + commit on the PR head. + +Stale `ok-to-test` labels are removed automatically. + +### GCP credential boundary + +A dedicated E2E WIF provider validates GitHub OIDC tokens with an attribute +condition scoped to: + +- `repository == fullsend-ai/fullsend` +- `job_workflow_ref` starts with `fullsend-ai/fullsend/.github/workflows/e2e.yml@` + +Only jobs running the trusted workflow definition from `main` can impersonate +the e2e service account. The shim uses `pull_request_target` so PR branches +cannot replace the workflow file that runs on the caller side. + +### Default GCP configuration + +When the corresponding repository secrets are unset, `e2e.yml` uses hardcoded +defaults for the shared konflux e2e infrastructure in [E2E GCP +setup](../guides/infrastructure/e2e-gcp-setup.md#canonical-resource-names). +The WIF provider path includes the GCP project number, which cannot be derived +at runtime without prior GCP credentials: + +| Secret (optional) | Workflow default | +|-------------------|------------------| +| `E2E_GCP_PROJECT_ID` | `it-gcp-konflux-e2e-fullsend` | +| `E2E_GCP_SERVICE_ACCOUNT` | `fullsend-e2e@it-gcp-konflux-e2e-fullsend.iam.gserviceaccount.com` | +| `E2E_GCP_WIF_PROVIDER` | `projects/208332380190/locations/global/workloadIdentityPools/fullsend-e2e-pool/providers/github-oidc` | + +Operators provision infrastructure using those canonical names; override +secrets only when pointing at different resources. + +### Fork PRs + +The shim uses `pull_request_target` (same pattern as the agent +`fullsend.yaml` shim) so fork PRs receive repository secrets and OIDC tokens +in the base-repo workflow context. This is safe because the shim never checks +out or executes PR code — it only forwards PR metadata to `e2e.yml@main`. +PR head code runs only inside the trusted reusable workflow after the +authorization gate. + +## Consequences + +- **Positive:** Progress toward secretless e2e: WIF replaces long-lived GCP SA + keys; default GCP project, WIF provider, and service account reduce required + configuration. +- **Positive:** Untrusted authors cannot run e2e with secrets without maintainer + `ok-to-test`. +- **Positive:** WIF `job_workflow_ref` binding prevents credential theft via + modified shim workflows. +- **Incomplete:** GitHub session and mint URL secrets are still required — full + secretless e2e is follow-up work. +- **Negative:** Workflow-step changes to `e2e.yml` require merge to `main` or + manual `workflow_dispatch` on a branch — test invocation customization + belongs in `Makefile` / Go test code on the PR head. + +## References + +- Operator setup (verify-then-create): [E2E GCP setup guide](../guides/infrastructure/e2e-gcp-setup.md) +- Contributor guide: [E2E testing](../guides/dev/e2e-testing.md) +- Issues: [#1604](https://github.com/fullsend-ai/fullsend/issues/1604), [#817](https://github.com/fullsend-ai/fullsend/issues/817) + +## Architecture + +```mermaid +sequenceDiagram + participant PR as PR_event + participant Shim as e2e_shim.yml + participant E2E as e2e.yml_main + participant GH as GitHub_API + participant GCP as GCP_WIF + + PR->>Shim: pull_request_target or labeled + Note over Shim: No checkout in shim + Shim->>E2E: workflow_call @refs/heads/main + E2E->>GH: Verify PR author / ok-to-test freshness + alt unauthorized + E2E->>GH: Upsert sticky comment + E2E-->>Shim: gate succeeds (authorized=false), tests skipped + else authorized + E2E->>GCP: OIDC exchange via google-github-actions/auth + Note over GCP: attributeCondition checks job_workflow_ref + GCP-->>E2E: short-lived SA token + E2E->>E2E: checkout PR head, run make e2e-test + end +``` diff --git a/docs/guides/README.md b/docs/guides/README.md index 5987135b98..e87483b97d 100644 --- a/docs/guides/README.md +++ b/docs/guides/README.md @@ -17,6 +17,7 @@ Guides for platform operators who deploy and manage the GCP-side infrastructure - [Mint service administration](infrastructure/mint-administration.md) — Deploying and managing the token mint Cloud Function - [Infrastructure reference](infrastructure/infrastructure-reference.md) — Token mint, WIF, and secrets deployment details +- [E2E GCP setup](infrastructure/e2e-gcp-setup.md) — Verify and configure GCP + GitHub secrets for CI e2e tests - [Enabling fullsend on private repositories](infrastructure/private-repositories.md) — Additional guardrails and configuration for private repos ## User guides @@ -35,4 +36,5 @@ Guides for contributors developing and testing fullsend itself. - [Local development](dev/local-dev.md) — Run fullsend agents locally on macOS and Linux (amd64 + arm64) - [CLI internals](dev/cli-internals.md) — Command structure, installation pipeline, and sandbox runtime +- [E2E testing](dev/e2e-testing.md) — Local and CI admin e2e tests, ok-to-test, troubleshooting - [Testing workflow changes](dev/testing-workflows.md) — Point a live GitHub org at a branch to test workflow, action, and agent changes before release diff --git a/docs/guides/dev/cli-internals.md b/docs/guides/dev/cli-internals.md index 0723a15ca8..942a4be41b 100644 --- a/docs/guides/dev/cli-internals.md +++ b/docs/guides/dev/cli-internals.md @@ -516,4 +516,5 @@ var executableFiles = map[string]struct{}{ - [Setting up with pre-provisioned infrastructure](../getting-started/github-setup.md) — GitHub-only setup guide - [Mint service administration](../infrastructure/mint-administration.md) — Deploying and managing the token mint - [Infrastructure Reference](../infrastructure/infrastructure-reference.md) — Infrastructure details +- [E2E testing](e2e-testing.md) — Admin e2e tests locally and in CI - [Customizing Agents](../user/customizing-agents.md) — User customization guide diff --git a/docs/guides/dev/e2e-testing.md b/docs/guides/dev/e2e-testing.md new file mode 100644 index 0000000000..ecffe0a9f4 --- /dev/null +++ b/docs/guides/dev/e2e-testing.md @@ -0,0 +1,106 @@ +# E2E Testing + +Guide for running and debugging fullsend admin e2e tests locally and in CI. + +Related ADRs: [0010](../../ADRs/0010-stored-session-for-e2e-browser-auth.md) (browser +session), [0039](../../ADRs/0039-totp-automation-for-e2e-2fa.md) (2FA), +[0040](../../ADRs/0040-org-pool-for-parallel-e2e-tests.md) (org pool), +[0043](../../ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md) (CI gate + WIF). + +Operators: see [E2E GCP setup](../infrastructure/e2e-gcp-setup.md). + +## Goal: toward secretless e2e + +Issue [#1604](https://github.com/fullsend-ai/fullsend/issues/1604) aims to let +authorized e2e runs proceed **without** defining repository secrets or +variables. That work is **not complete**. This change is one increment: + +- **Done here:** WIF for GCP auth (no SA JSON keys); PR authorization gate; + workflow defaults for GCP project, WIF provider, and service account (see + [E2E GCP setup](../infrastructure/e2e-gcp-setup.md#canonical-resource-names)). +- **Still required:** `E2E_GITHUB_SESSION`, `E2E_MINT_URL`, password/TOTP + secrets. + +See [ADR 0043](../../ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md) for the +full delivered vs. outstanding breakdown. + +## Local runs + +```bash +# Export a Playwright session (once per session expiry) +make e2e-export-session + +# Run tests (uses E2E_GITHUB_SESSION_FILE or credentials from env) +make e2e-test + +# Upload session to GitHub repo secret (maintainers) +make e2e-upload-session +``` + +Required environment variables are documented in the `Makefile` help (`make help`). + +Tests acquire an exclusive lock on one org from the pool (`halfsend-01` … +`halfsend-06`) — see [ADR 0040](../../ADRs/0040-org-pool-for-parallel-e2e-tests.md). + +## CI architecture + +Pull requests trigger **E2E Shim** (`.github/workflows/e2e_shim.yml`) via +`pull_request_target` (base-branch workflow; no PR code checkout in the shim), +which calls the trusted **E2E Tests** workflow on `main` via `workflow_call`: + +1. **Gate** — authorize the PR author or a fresh `ok-to-test` label +2. **GCP WIF auth** — short-lived credentials via `google-github-actions/auth` +3. **Tests** — checkout PR head, `make e2e-test` (GCP project, WIF provider, + and service account default to the shared konflux e2e resources documented + in [E2E GCP setup](../infrastructure/e2e-gcp-setup.md#canonical-resource-names) + unless override secrets are set) + +Pushes to `main` and `workflow_dispatch` run **E2E Tests** directly (gate +skipped). + +## ok-to-test (same-repo PRs) + +E2e runs automatically when the PR author is an org/repo **member or +collaborator**. + +For other same-repo contributors, a maintainer must add the **`ok-to-test`** +label **after** reviewing the latest push. If new commits land after the label +was applied, CI removes the stale label and skips tests until it is re-applied. + +Create the label if missing — see +[E2E GCP setup](../infrastructure/e2e-gcp-setup.md#4-github-repository-secrets-and-labels). + +## Fork pull requests + +Fork PRs use the same shim and gate as same-repo PRs. The shim runs from +`main` (`pull_request_target`) and inherits repository secrets, so authorized +fork PRs can run e2e when path filters match. First-time contributors may still +need maintainer approval before GitHub Actions runs on the PR. + +## CI readiness checks + +### Contributors + +- Read the sticky **E2E tests did not run** comment (marker ``) + on unauthorized PRs. +- **Secrets unavailable** warning — investigate repo secret configuration; not + expected for authorized fork PRs once the shim is on `main`. +- Member/collaborator PRs should show **E2E Shim** → **E2E Tests** in the + Actions tab when path filters match. + +### Maintainers + +- Run the [quick health check](../infrastructure/e2e-gcp-setup.md#quick-health-check) + for GCP and GitHub secrets. +- `gh secret list --repo fullsend-ai/fullsend | grep E2E_` — session and mint + secrets are still required; GCP project/WIF/SA secrets are optional (workflow + defaults documented in [E2E GCP setup](../infrastructure/e2e-gcp-setup.md#canonical-resource-names)). +- Failed **Authenticate to GCP** — compare workflow log with WIF provider + attribute condition in the setup guide. +- Test workflow changes on a branch via **workflow_dispatch** on **E2E Tests** + before merging `e2e.yml` changes (PR shim always calls `@refs/heads/main`). + +## Path filters + +E2e CI runs when PRs touch Go code, e2e tests, scaffold, mint sources, Makefile, +or the e2e workflow/action files listed in `e2e_shim.yml`. diff --git a/docs/guides/infrastructure/e2e-gcp-setup.md b/docs/guides/infrastructure/e2e-gcp-setup.md new file mode 100644 index 0000000000..36e719395a --- /dev/null +++ b/docs/guides/infrastructure/e2e-gcp-setup.md @@ -0,0 +1,290 @@ +# E2E GCP Setup + +Operator guide for the dedicated GCP project and GitHub secrets used by CI e2e +tests. Follows a **verify-then-create** pattern: inspect existing state first, +create or update only what is missing or wrong. + +The **long-term goal** is secretless e2e (no per-repo secrets/variables for GCP +or sessions). This guide covers infrastructure for the **current** workflow, +which still requires several secrets — see [ADR 0043](../../ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md) +for what is delivered vs. outstanding. + +For architecture context see ADR 0043. For contributor CI behavior see +[E2E testing](../dev/e2e-testing.md). + +## Canonical resource names + +Provision and inspect resources using these names so they match **`e2e.yml` +workflow defaults** when repository secrets are unset: + +| Resource | Canonical name | +|----------|----------------| +| GCP project ID | `it-gcp-konflux-e2e-fullsend` (workflow default when secret unset) | +| GCP project number | `208332380190` (workflow default when secret unset) | +| Service account ID | `fullsend-e2e` | +| Service account email | `fullsend-e2e@${PROJECT_ID}.iam.gserviceaccount.com` | +| WIF pool ID | `fullsend-e2e-pool` | +| WIF provider ID | `github-oidc` | +| WIF provider resource | `projects/208332380190/locations/global/workloadIdentityPools/fullsend-e2e-pool/providers/github-oidc` | + +Override via `E2E_GCP_PROJECT_ID`, `E2E_GCP_SERVICE_ACCOUNT`, or +`E2E_GCP_WIF_PROVIDER` secrets only when pointing at different infrastructure. +When overriding the project, set all three secrets — the workflow does not +derive the project number or WIF provider path at runtime. + +## Quick health check + +Run these from a machine with `gcloud` and `gh` authenticated. + +```bash +PROJECT_ID="it-gcp-konflux-e2e-fullsend" +PROJECT_NUMBER="$(gcloud projects describe "${PROJECT_ID}" --format='value(projectNumber)')" +SA_EMAIL="fullsend-e2e@${PROJECT_ID}.iam.gserviceaccount.com" +POOL_ID="fullsend-e2e-pool" +PROVIDER_ID="github-oidc" + +gcloud projects describe "${PROJECT_ID}" +gcloud services list --enabled --project="${PROJECT_ID}" | grep -E 'iam|sts|aiplatform|secretmanager|cloudfunctions|run' +gcloud iam service-accounts describe "${SA_EMAIL}" --project="${PROJECT_ID}" +gcloud iam workload-identity-pools describe "${POOL_ID}" --location=global --project="${PROJECT_ID}" +gcloud iam workload-identity-pools providers describe "${PROVIDER_ID}" \ + --workload-identity-pool="${POOL_ID}" --location=global --project="${PROJECT_ID}" +gh secret list --repo fullsend-ai/fullsend | grep E2E_ +gh label list --repo fullsend-ai/fullsend | grep ok-to-test +``` + +All commands should succeed and show expected values described in the sections +below. + +## 1. Project and APIs + +### Check + +| Item | Command | Expected when OK | +|------|---------|----------------| +| Project exists | `gcloud projects describe PROJECT_ID` | `lifecycleState: ACTIVE` | +| Project number | `gcloud projects describe PROJECT_ID --format='value(projectNumber)'` | Numeric ID for WIF resource names | +| APIs enabled | `gcloud services list --enabled --project=PROJECT_ID` | Required APIs listed below | + +Required APIs: + +- `iam.googleapis.com` +- `iamcredentials.googleapis.com` +- `sts.googleapis.com` +- `cloudresourcemanager.googleapis.com` +- `aiplatform.googleapis.com` +- `secretmanager.googleapis.com` +- `cloudfunctions.googleapis.com` +- `run.googleapis.com` +- `cloudbuild.googleapis.com` +- `artifactregistry.googleapis.com` + +### Create / enable if missing + +```bash +gcloud projects create "${PROJECT_ID}" --name="Fullsend E2E" +gcloud services enable iam.googleapis.com iamcredentials.googleapis.com sts.googleapis.com \ + cloudresourcemanager.googleapis.com aiplatform.googleapis.com secretmanager.googleapis.com \ + cloudfunctions.googleapis.com run.googleapis.com cloudbuild.googleapis.com \ + artifactregistry.googleapis.com --project="${PROJECT_ID}" +``` + +## 2. E2E workflow service account + +CI impersonates this account via WIF (`E2E_GCP_SERVICE_ACCOUNT`). + +### Check + +```bash +gcloud iam service-accounts describe "${SA_EMAIL}" --project="${PROJECT_ID}" + +gcloud projects get-iam-policy "${PROJECT_ID}" \ + --flatten='bindings[].members' \ + --filter="bindings.members:serviceAccount:${SA_EMAIL}" \ + --format='table(bindings.role)' +``` + +### Roles + +Grant only roles that are absent from the policy output. + +**Required now (inference during admin install tests):** + +| Role | Purpose | +|------|---------| +| `roles/aiplatform.user` | Vertex / Agent Platform API calls | + +**Required later ([#817](https://github.com/fullsend-ai/fullsend/issues/817) mint/install tests):** + +| Role | Purpose | +|------|---------| +| `roles/iam.workloadIdentityPoolAdmin` | WIF pool/provider provisioning | +| `roles/secretmanager.admin` | PEM secret storage | +| `roles/cloudfunctions.admin` | Mint Cloud Function deploy | +| `roles/run.admin` | Cloud Run (2nd gen functions) | +| `roles/iam.serviceAccountAdmin` | SA creation during install | +| `roles/iam.serviceAccountUser` | SA impersonation during install | +| `roles/resourcemanager.projectIamAdmin` | IAM bindings during install | + +### Create if missing + +```bash +gcloud iam service-accounts create fullsend-e2e \ + --project="${PROJECT_ID}" \ + --display-name="Fullsend E2E CI" + +gcloud projects add-iam-policy-binding "${PROJECT_ID}" \ + --member="serviceAccount:${SA_EMAIL}" \ + --role="roles/aiplatform.user" +``` + +Repeat `add-iam-policy-binding` for each additional role when #817 tests land. + +## 3. WIF pool and provider + +E2E CI uses a **repo-scoped** provider (distinct from org mint/inference pools +created by `fullsend admin install`). See +[infrastructure-reference.md](infrastructure-reference.md). + +### Check + +```bash +PROJECT_NUMBER="$(gcloud projects describe "${PROJECT_ID}" --format='value(projectNumber)')" + +gcloud iam workload-identity-pools describe "${POOL_ID}" \ + --location=global --project="${PROJECT_ID}" + +gcloud iam workload-identity-pools providers describe "${PROVIDER_ID}" \ + --workload-identity-pool="${POOL_ID}" --location=global --project="${PROJECT_ID}" + +gcloud iam service-accounts get-iam-policy "${SA_EMAIL}" --project="${PROJECT_ID}" +``` + +**Expected provider attribute condition:** + +``` +assertion.repository == 'fullsend-ai/fullsend' && +assertion.job_workflow_ref.startsWith('fullsend-ai/fullsend/.github/workflows/e2e.yml@') +``` + +**Expected OIDC issuer:** `https://token.actions.githubusercontent.com` + +**Expected IAM binding on the service account:** principal set for the WIF pool +with `roles/iam.workloadIdentityUser`. + +Provider resource name (matches workflow default when secret is unset): + +``` +projects/208332380190/locations/global/workloadIdentityPools/fullsend-e2e-pool/providers/github-oidc +``` + +Look up `${PROJECT_NUMBER}` with `gcloud projects describe "${PROJECT_ID}" --format='value(projectNumber)'` +when provisioning or rotating infrastructure. The workflow hardcodes this value +for the canonical project because WIF auth cannot bootstrap it without prior +GCP credentials. + +### Create if missing + +```bash +gcloud iam workload-identity-pools create "${POOL_ID}" \ + --project="${PROJECT_ID}" \ + --location=global \ + --display-name="Fullsend E2E GitHub OIDC Pool" + +gcloud iam workload-identity-pools providers create-oidc "${PROVIDER_ID}" \ + --project="${PROJECT_ID}" \ + --location=global \ + --workload-identity-pool="${POOL_ID}" \ + --display-name="GitHub OIDC for E2E" \ + --issuer-uri="https://token.actions.githubusercontent.com" \ + --attribute-mapping="google.subject=assertion.sub,attribute.repository=assertion.repository,attribute.job_workflow_ref=assertion.job_workflow_ref" \ + --attribute-condition="assertion.repository == 'fullsend-ai/fullsend' && assertion.job_workflow_ref.startsWith('fullsend-ai/fullsend/.github/workflows/e2e.yml@')" + +gcloud iam service-accounts add-iam-policy-binding "${SA_EMAIL}" \ + --project="${PROJECT_ID}" \ + --role="roles/iam.workloadIdentityUser" \ + --member="principalSet://iam.googleapis.com/projects/${PROJECT_NUMBER}/locations/global/workloadIdentityPools/${POOL_ID}/attribute.repository/fullsend-ai/fullsend" +``` + +### Update if attribute condition is wrong + +Do not delete an existing pool — update the provider: + +```bash +gcloud iam workload-identity-pools providers update-oidc "${PROVIDER_ID}" \ + --project="${PROJECT_ID}" \ + --location=global \ + --workload-identity-pool="${POOL_ID}" \ + --attribute-condition="assertion.repository == 'fullsend-ai/fullsend' && assertion.job_workflow_ref.startsWith('fullsend-ai/fullsend/.github/workflows/e2e.yml@')" +``` + +## 4. GitHub repository secrets and labels + +Secret **values** cannot be read back from GitHub. Verify names exist, then +confirm values match GCP inspect output when rotating. + +### Check + +```bash +gh secret list --repo fullsend-ai/fullsend | grep '^E2E_' +gh label list --repo fullsend-ai/fullsend | grep ok-to-test +``` + +### Secrets and variables + +| Name | Required? | Value source | +|------|-----------|----------------| +| `E2E_GCP_WIF_PROVIDER` | No | Defaults to canonical provider resource above; set to override | +| `E2E_GCP_SERVICE_ACCOUNT` | No | Defaults to `${SA_EMAIL}`; set to override | +| `E2E_GCP_PROJECT_ID` | No | Defaults to `it-gcp-konflux-e2e-fullsend`; set to override | +| `E2E_GITHUB_SESSION` | **Yes** | Base64 Playwright session ([ADR 0010](../../ADRs/0010-stored-session-for-e2e-browser-auth.md)) | +| `E2E_GITHUB_PASSWORD` | **Yes** | Test account password | +| `E2E_GITHUB_TOTP_SECRET` | **Yes** | Test account TOTP secret ([ADR 0039](../../ADRs/0039-totp-automation-for-e2e-2fa.md)) | +| `E2E_MINT_URL` | **Yes** | Shared mint URL for install tests | + +Full secretless e2e (eliminating the required rows) is follow-up work tracked +under [ADR 0043](../../ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md). + +### Set or rotate + +Optional — omit all three to use workflow defaults from [Canonical resource names](#canonical-resource-names): + +```bash +# gh secret set E2E_GCP_WIF_PROVIDER --repo fullsend-ai/fullsend \ +# --body "projects/${PROJECT_NUMBER}/locations/global/workloadIdentityPools/${POOL_ID}/providers/${PROVIDER_ID}" +# gh secret set E2E_GCP_SERVICE_ACCOUNT --repo fullsend-ai/fullsend --body "${SA_EMAIL}" +# gh secret set E2E_GCP_PROJECT_ID --repo fullsend-ai/fullsend --body "${PROJECT_ID}" +``` + +### Create ok-to-test label if missing + +```bash +gh label create ok-to-test --repo fullsend-ai/fullsend \ + --color fbca04 \ + --description "Allow e2e CI to run after maintainer review (must be re-applied after each push)" +``` + +## 5. End-to-end verification + +1. Run **E2E Tests** via `workflow_dispatch` on `main`. +2. Confirm the **Authenticate to GCP** step succeeds. +3. Open a same-repo PR from a member account — **E2E Shim** should call the gate + and run tests when authorized. + +### Troubleshooting + +| Symptom | Inspect | +|---------|---------| +| `google-github-actions/auth` permission denied | Provider attribute condition and SA `workloadIdentityUser` binding (section 3) | +| WIF provider not found | `E2E_GCP_WIF_PROVIDER` secret vs provider resource name | +| E2e skipped — secrets unavailable | Fork PR or missing `E2E_GITHUB_SESSION` ([e2e-testing.md](../dev/e2e-testing.md)) | +| Gate comment not posted | Caller job needs `pull-requests: write` in `e2e_shim.yml` | +| Tests skip after label added | Label must be applied **after** latest push; stale labels are removed | + +## Comparison with install-time WIF + +| Use case | Pool scope | Provisioned by | +|----------|------------|----------------| +| Token mint | Org allowlist | `fullsend mint deploy` | +| Inference (agents) | Org or repo | `fullsend inference provision` | +| E2E CI | `fullsend-ai/fullsend` repo + `e2e.yml@main` only | Manual (this guide) | diff --git a/docs/guides/infrastructure/infrastructure-reference.md b/docs/guides/infrastructure/infrastructure-reference.md index 385eea4780..a60572db6d 100644 --- a/docs/guides/infrastructure/infrastructure-reference.md +++ b/docs/guides/infrastructure/infrastructure-reference.md @@ -183,6 +183,54 @@ During installation, the GCF provisioner creates: --- +## E2E CI WIF + +> Manual setup: [E2E GCP setup guide](e2e-gcp-setup.md) (verify-then-create). Architecture: [ADR 0043](../../ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md). + +CI e2e tests use a **third** WIF use case, separate from org mint and inference +pools. A dedicated provider binds credentials to the trusted +`fullsend-ai/fullsend/.github/workflows/e2e.yml@refs/heads/main` workflow via +`job_workflow_ref` in the attribute condition. + +This is **incremental** progress toward secretless e2e ([ADR +0043](../../ADRs/0043-e2e-wif-shim-and-pr-authorization-gate.md)): WIF removes +long-lived GCP keys, but session/mint secrets remain required today. The +workflow defaults to the shared konflux e2e project, WIF provider, and service +account when the corresponding secrets are unset — see [E2E GCP +setup](e2e-gcp-setup.md#canonical-resource-names). Defaults are hardcoded in +`e2e.yml` (including the GCP project number in the WIF provider path). + +### Inspect existing E2E WIF + +```bash +PROJECT_ID="it-gcp-konflux-e2e-fullsend" +PROJECT_NUMBER="$(gcloud projects describe "${PROJECT_ID}" --format='value(projectNumber)')" +SA_EMAIL="fullsend-e2e@${PROJECT_ID}.iam.gserviceaccount.com" +POOL_ID="fullsend-e2e-pool" +PROVIDER_ID="github-oidc" + +gcloud iam workload-identity-pools describe "${POOL_ID}" \ + --location=global --project="${PROJECT_ID}" + +gcloud iam workload-identity-pools providers describe "${PROVIDER_ID}" \ + --workload-identity-pool="${POOL_ID}" --location=global --project="${PROJECT_ID}" \ + --format='yaml(name,oidc.issuerUri,attributeCondition)' +``` + +Expected `attributeCondition`: + +``` +assertion.repository == 'fullsend-ai/fullsend' && +assertion.job_workflow_ref.startsWith('fullsend-ai/fullsend/.github/workflows/e2e.yml@') +``` + +GitHub repository secrets `E2E_GCP_WIF_PROVIDER`, `E2E_GCP_SERVICE_ACCOUNT`, and +`E2E_GCP_PROJECT_ID` are optional when infrastructure matches the canonical +names above. Values are not readable from the GitHub API — verify names with +`gh secret list` and match values to the gcloud output when rotating. + +--- + ## GitHub Secrets & Variables Deployment > Individual values can be updated with `fullsend github set `. See [Setting up with pre-provisioned infrastructure](../getting-started/github-setup.md) for the full GitHub management guide. @@ -305,5 +353,7 @@ The GCF provisioner avoids redundant Cloud Function deployments by computing a S - [Installation Guide](../getting-started/installation.md) — Setup instructions (end-user and all-in-one) - [Mint service administration](mint-administration.md) — Deploying and managing the token mint +- [E2E GCP setup](e2e-gcp-setup.md) — CI e2e GCP and GitHub secrets - [Setting up with pre-provisioned infrastructure](../getting-started/github-setup.md) — GitHub-only setup guide +- [E2E testing](../dev/e2e-testing.md) — Contributor e2e guide - [Local Development](../dev/local-dev.md) — Developer setup diff --git a/e2e/admin/testutil.go b/e2e/admin/testutil.go index d4e3bdbf8e..866d6911d1 100644 --- a/e2e/admin/testutil.go +++ b/e2e/admin/testutil.go @@ -46,6 +46,10 @@ const ( staleLockTimeout = 15 * time.Minute ) +// defaultE2EGCPProjectID is the shared CI inference project when E2E_GCP_PROJECT_ID +// is unset. Matches the workflow default in .github/workflows/e2e.yml (ADR 0043). +const defaultE2EGCPProjectID = "it-gcp-konflux-e2e-fullsend" + // orgPool is the set of GitHub orgs available for parallel e2e test runs. // Each run acquires a lock on one org before proceeding. var orgPool = []string{ @@ -184,6 +188,9 @@ func loadEnvConfig(t *testing.T) envConfig { } gcpProjectID := os.Getenv("E2E_GCP_PROJECT_ID") + if gcpProjectID == "" { + gcpProjectID = defaultE2EGCPProjectID + } lockTimeout := defaultLockTimeout if v := os.Getenv("E2E_LOCK_TIMEOUT"); v != "" { diff --git a/scripts/check-e2e-authorization.sh b/scripts/check-e2e-authorization.sh new file mode 100755 index 0000000000..045bdaa850 --- /dev/null +++ b/scripts/check-e2e-authorization.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# check-e2e-authorization.sh — Decide whether a PR may run e2e tests in CI. +# +# Authorized when the PR author is OWNER/MEMBER/COLLABORATOR, or when a fresh +# ok-to-test label was applied after the latest commit on the PR head. +# Push time uses GitHub issue timeline committed/force-push event timestamps +# (server-side), falling back to committer.date only when timeline is empty. +# Removes stale ok-to-test labels (applied at or before the latest push). +# +# Usage: check-e2e-authorization.sh PR_NUMBER OWNER/REPO +# +# Writes authorized, reason, and label_removed to GITHUB_OUTPUT when set. +# Exits 0 always; callers inspect outputs. + +set -euo pipefail + +PR_NUMBER="${1:?PR number required}" +REPOSITORY="${2:?repository (owner/repo) required}" + +TRUSTED_ASSOCIATIONS="OWNER MEMBER COLLABORATOR" +OK_TO_TEST_LABEL="ok-to-test" + +write_error_output() { + echo "check-e2e-authorization: API or script error (see log above)" >&2 + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + { + echo "authorized=false" + echo "reason=error" + echo "label_removed=false" + } >>"${GITHUB_OUTPUT}" + fi + printf 'authorized=false reason=error label_removed=false\n' +} + +trap 'write_error_output; exit 0' ERR + +is_trusted_author() { + local assoc="$1" + case " ${TRUSTED_ASSOCIATIONS} " in + *" ${assoc} "*) return 0 ;; + *) return 1 ;; + esac +} + +pr_json="$(gh api "repos/${REPOSITORY}/pulls/${PR_NUMBER}")" +author_association="$(jq -r '.author_association' <<<"${pr_json}")" +has_ok_label="$(jq -r --arg label "${OK_TO_TEST_LABEL}" '[.labels[].name] | index($label) != null' <<<"${pr_json}")" + +timeline_json="$(gh api "repos/${REPOSITORY}/issues/${PR_NUMBER}/timeline" --paginate \ + -H "Accept: application/vnd.github+json")" +last_push_at="$(jq -r ' + [.[] | select(.event == "committed" or .event == "head_ref_force_pushed") | .created_at] | max // empty +' <<<"${timeline_json}")" +if [[ -z "${last_push_at}" ]]; then + commits_json="$(gh api "repos/${REPOSITORY}/pulls/${PR_NUMBER}/commits" --paginate)" + last_push_at="$(jq -r '[.[] | .commit.committer.date] | max // empty' <<<"${commits_json}")" +fi + +events_json="$(gh api "repos/${REPOSITORY}/issues/${PR_NUMBER}/events" --paginate)" +ok_to_test_at="$(jq -r --arg label "${OK_TO_TEST_LABEL}" ' + [.[] | select(.event == "labeled" and (.label.name // "") == $label) | .created_at] | max // empty +' <<<"${events_json}")" + +is_trusted=false +if is_trusted_author "${author_association}"; then + is_trusted=true +fi + +has_fresh_label=false +label_removed=false +authorized=false +reason="unauthorized" + +if [[ "${has_ok_label}" == "true" && "${is_trusted}" != "true" ]]; then + if [[ -n "${ok_to_test_at}" && -n "${last_push_at}" && "${ok_to_test_at}" > "${last_push_at}" ]]; then + has_fresh_label=true + else + if [[ "${CHECK_E2E_AUTH_DRY_RUN:-}" != "true" ]]; then + gh api -X DELETE "repos/${REPOSITORY}/issues/${PR_NUMBER}/labels/${OK_TO_TEST_LABEL}" >/dev/null + fi + label_removed=true + fi +fi + +if [[ "${is_trusted}" == "true" ]]; then + authorized=true + reason="trusted_author" +elif [[ "${has_fresh_label}" == "true" ]]; then + authorized=true + reason="ok_to_test" +elif [[ "${label_removed}" == "true" ]]; then + reason="stale_ok_to_test" +fi + +trap - ERR + +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + { + echo "authorized=${authorized}" + echo "reason=${reason}" + echo "label_removed=${label_removed}" + } >>"${GITHUB_OUTPUT}" +fi + +printf 'authorized=%s reason=%s label_removed=%s\n' "${authorized}" "${reason}" "${label_removed}" diff --git a/scripts/check-e2e-authorization_test.sh b/scripts/check-e2e-authorization_test.sh new file mode 100755 index 0000000000..5ff6a78f84 --- /dev/null +++ b/scripts/check-e2e-authorization_test.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# check-e2e-authorization_test.sh — Tests for check-e2e-authorization.sh with mock gh. +# +# Run from repo root: bash scripts/check-e2e-authorization_test.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +AUTH_SCRIPT="${SCRIPT_DIR}/check-e2e-authorization.sh" +FAILURES=0 + +TMPDIR="$(mktemp -d)" +trap 'rm -rf "${TMPDIR}"' EXIT + +MOCK_BIN="${TMPDIR}/bin" +mkdir -p "${MOCK_BIN}" + +PR_JSON="${TMPDIR}/pr.json" +COMMITS_JSON="${TMPDIR}/commits.json" +TIMELINE_JSON="${TMPDIR}/timeline.json" +EVENTS_JSON="${TMPDIR}/events.json" +GH_LOG="${TMPDIR}/gh.log" + +write_pr() { + local assoc="$1" + local labels_json="$2" + jq -n --arg assoc "${assoc}" --argjson labels "${labels_json}" \ + '{author_association: $assoc, labels: $labels}' >"${PR_JSON}" +} + +write_commits() { + local dates_json="$1" + jq -n --argjson dates "${dates_json}" \ + '[ $dates[] | {commit: {committer: {date: .}}}]' >"${COMMITS_JSON}" +} + +write_timeline() { + local events_json="$1" + echo "${events_json}" >"${TIMELINE_JSON}" +} + +write_events() { + local events_json="$1" + echo "${events_json}" >"${EVENTS_JSON}" +} + +cat >"${MOCK_BIN}/gh" <> "${GH_LOG}" +case "\$*" in + *"/issues/"*"/timeline"*) + cat "${TIMELINE_JSON}" + ;; + *"/pulls/"*"/commits"*) + cat "${COMMITS_JSON}" + ;; + *"/issues/"*"/events"*) + cat "${EVENTS_JSON}" + ;; + *"/pulls/"*) + cat "${PR_JSON}" + ;; + *DELETE*) + echo "deleted" >> "${GH_LOG}" + ;; + *) + echo "unexpected gh call: \$*" >&2 + exit 1 + ;; +esac +EOF +chmod +x "${MOCK_BIN}/gh" + +export PATH="${MOCK_BIN}:${PATH}" +export GH_TOKEN="test-token" +export CHECK_E2E_AUTH_DRY_RUN="true" + +run_case() { + local name="$1" + local expected_auth="$2" + local expected_reason="$3" + local expected_removed="$4" + + : >"${GH_LOG}" + + local output + output="$("${AUTH_SCRIPT}" 42 "fullsend-ai/fullsend")" + local auth reason removed + auth="$(grep -o 'authorized=[^ ]*' <<<"${output}" | cut -d= -f2)" + reason="$(grep -o 'reason=[^ ]*' <<<"${output}" | cut -d= -f2)" + removed="$(grep -o 'label_removed=[^ ]*' <<<"${output}" | cut -d= -f2)" + + if [[ "${auth}" != "${expected_auth}" || "${reason}" != "${expected_reason}" || "${removed}" != "${expected_removed}" ]]; then + echo "FAIL: ${name}" + echo " expected authorized=${expected_auth} reason=${expected_reason} label_removed=${expected_removed}" + echo " got authorized=${auth} reason=${reason} label_removed=${removed}" + FAILURES=$((FAILURES + 1)) + return + fi + echo "PASS: ${name}" +} + +write_pr "MEMBER" '[]' +write_timeline '[{"event":"committed","created_at":"2026-06-01T10:00:00Z"}]' +write_events '[]' +run_case "trusted member author" "true" "trusted_author" "false" + +write_pr "OWNER" '[]' +write_timeline '[{"event":"committed","created_at":"2026-06-01T10:00:00Z"}]' +write_events '[]' +run_case "trusted owner author" "true" "trusted_author" "false" + +write_pr "COLLABORATOR" '[]' +write_timeline '[{"event":"committed","created_at":"2026-06-01T10:00:00Z"}]' +write_events '[]' +run_case "trusted collaborator author" "true" "trusted_author" "false" + +write_pr "CONTRIBUTOR" '[]' +write_timeline '[{"event":"committed","created_at":"2026-06-01T10:00:00Z"}]' +write_events '[]' +run_case "contributor author denied" "false" "unauthorized" "false" + +write_pr "MEMBER" '[{"name":"ok-to-test"}]' +write_timeline '[{"event":"committed","created_at":"2026-06-01T12:00:00Z"}]' +write_events '[{"event":"labeled","label":{"name":"ok-to-test"},"created_at":"2026-06-01T11:00:00Z"}]' +run_case "trusted member ignores stale ok-to-test label" "true" "trusted_author" "false" + +write_pr "NONE" '[{"name":"ok-to-test"}]' +write_timeline '[{"event":"committed","created_at":"2026-06-01T10:00:00Z"}]' +write_events '[{"event":"labeled","label":{"name":"ok-to-test"},"created_at":"2026-06-01T11:00:00Z"}]' +run_case "fresh ok-to-test label" "true" "ok_to_test" "false" + +write_pr "NONE" '[{"name":"ok-to-test"}]' +write_timeline '[{"event":"committed","created_at":"2026-06-01T12:00:00Z"}]' +write_events '[{"event":"labeled","label":{"name":"ok-to-test"},"created_at":"2026-06-01T11:00:00Z"}]' +run_case "stale ok-to-test label" "false" "stale_ok_to_test" "true" + +write_pr "NONE" '[{"name":"ok-to-test"}]' +write_timeline '[{"event":"committed","created_at":"2026-06-01T12:00:00Z"}]' +write_events '[{"event":"labeled","label":{"name":"ok-to-test"},"created_at":"2026-06-01T12:00:00Z"}]' +run_case "ok-to-test label at push time is stale" "false" "stale_ok_to_test" "true" + +write_pr "NONE" '[{"name":"ok-to-test"}]' +write_timeline '[{"event":"committed","created_at":"2026-06-01T12:00:00Z"}]' +write_events '[{"event":"labeled","label":{"name":"ok-to-test"},"created_at":"2026-06-01T11:00:00Z"}]' +unset CHECK_E2E_AUTH_DRY_RUN +run_case "stale ok-to-test removes label" "false" "stale_ok_to_test" "true" +if ! grep -q DELETE "${GH_LOG}"; then + echo "FAIL: stale ok-to-test removes label (expected gh DELETE call)" + FAILURES=$((FAILURES + 1)) +else + echo "PASS: stale ok-to-test removes label (DELETE exercised)" +fi +export CHECK_E2E_AUTH_DRY_RUN="true" + +write_pr "NONE" '[]' +write_timeline '[{"event":"committed","created_at":"2026-06-01T10:00:00Z"}]' +write_events '[]' +run_case "untrusted author without label" "false" "unauthorized" "false" + +write_pr "NONE" '[{"name":"ok-to-test"}]' +write_commits '["2020-01-01T00:00:00Z"]' +write_timeline '[{"event":"committed","created_at":"2026-06-01T12:00:00Z"}]' +write_events '[{"event":"labeled","label":{"name":"ok-to-test"},"created_at":"2026-06-01T11:00:00Z"}]' +run_case "backdated committer date does not bypass timeline push time" "false" "stale_ok_to_test" "true" + +if [[ "${FAILURES}" -gt 0 ]]; then + echo "${FAILURES} test(s) failed" + exit 1 +fi + +echo "All check-e2e-authorization tests passed."