diff --git a/.github/workflows/pr-metadata.yml b/.github/workflows/pr-metadata.yml new file mode 100644 index 000000000..2d9de6a36 --- /dev/null +++ b/.github/workflows/pr-metadata.yml @@ -0,0 +1,177 @@ +name: PR metadata gates + +on: + push: + branches: [main] + pull_request: + # Description edits and retargets re-run only the metadata gates, using the current PR. + types: [opened, synchronize, reopened, edited] + workflow_dispatch: + +permissions: {} +cache-mode: none + +# Queue edits without replacing a pending run; API reads make freshness independent of queue order. +# github/docs@336b7f546d94, data/reusables/actions/actions-group-concurrency.md. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + queue: max + +jobs: + verdict-review-gate: + # Required check (docs/decisions/2026-09-22-github-automation-closure.md, "verdict-review-gate + # (2026-09-23)"): with one maintainer and zero required approvals, a PR that changes a + # layer-verdict row merges only when the row is consistent with the sealed cross-family lane + # returns its wave registers (the returns are self-attested; see the record's residual). It has + # no path filter so it reports on every pull request and can be required. + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read + steps: + - name: Harden the runner (audit-only network egress) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + - name: Check out repository with full history + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + # Full history: the gate reads the base commit's ledger and wave registry with git show. + - name: Require the current PR base to match this event + if: github.event_name == 'pull_request' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const {data: pullRequest} = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + const eventBase = context.payload.pull_request.base; + if (pullRequest.base.ref !== eventBase.ref) { + core.setFailed('verdict-review-gate: the current PR base differs from the event base; failing closed (push a commit or close and reopen the pull request to rebuild its merge commit).'); + } + - name: Require sealed cross-family review for changed verdict rows + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + # The pull request's base branch (GITHUB_BASE_REF): the gate judges only pull requests into main. + PR_BASE_REF: ${{ github.base_ref }} + # Cross-check only: the base is the checked-out merge commit's first parent. + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + # The checked-out HEAD must be the merge commit whose second parent is this PR head. + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + # The pushed range's previous commit on main. + PUSH_BEFORE_SHA: ${{ github.event.before }} + run: | + set -euo pipefail + case "$EVENT_NAME" in + pull_request) + # A pull request into another branch is not judged against main, and retargeting it to + # main later re-runs this job (the edited type): fail closed rather than pass a base the + # gate does not protect. + if [ "${PR_BASE_REF:-}" != "main" ]; then + echo "::error::verdict-review-gate: the pull request's base branch is '${PR_BASE_REF:-}', not main; the gate judges only pull requests into main (retarget it to main); failing closed" + exit 1 + fi + # actions/checkout checks out the PR merge commit; its first parent is the base-branch + # commit the PR is merged into, which is what the gate must compare with. Assert that + # premise: HEAD has exactly two parents and the second is the payload's PR head, or + # HEAD^1 could be the PR's own previous commit and the gate would judge only the last one. + second_parent="$(git rev-parse --verify --quiet 'HEAD^2^{commit}' || true)" + if [ -z "${PR_HEAD_SHA:-}" ] || [ "$second_parent" != "$PR_HEAD_SHA" ] \ + || git rev-parse --verify --quiet 'HEAD^3^{commit}' >/dev/null; then + echo "::error::verdict-review-gate: HEAD is not the merge commit of the PR head '${PR_HEAD_SHA:-}' (second parent '$second_parent'); failing closed" + exit 1 + fi + first_parent="$(git rev-parse --verify --quiet 'HEAD^1^{commit}' || true)" + payload="${PR_BASE_SHA:-}" + if [ -n "$first_parent" ] && [ -n "$payload" ]; then + if ! git merge-base --is-ancestor "$payload" "$first_parent"; then + echo "::error::verdict-review-gate: the payload base $payload is not an ancestor of the merge commit's first parent $first_parent" + exit 1 + fi + base="$first_parent" + elif [ -n "$first_parent" ]; then + base="$first_parent" + elif [ -n "$payload" ]; then + base="$payload" + else + echo "::error::verdict-review-gate: neither the merge commit's first parent nor the payload base is available" + exit 1 + fi + # The base must be a commit on main itself (full history fetches origin/main), not a + # branch that merely contains main's tip, such as a merge commit left built against an + # earlier base: a retargeted PR is otherwise judged against the branch it left. + if ! git merge-base --is-ancestor "$base" "refs/remotes/origin/$PR_BASE_REF"; then + echo "::error::verdict-review-gate: the base $base is not a commit on origin/$PR_BASE_REF; failing closed (after a retarget the edited run checks out the merge commit built on the old base: close and reopen the pull request, or push a commit, to rebuild it on $PR_BASE_REF)" + exit 1 + fi + ;; + push) + base="${PUSH_BEFORE_SHA:-}" + # A branch-creating push has an all-zero before: there is no base to judge against. + if [ -z "$base" ] || [ -z "${base//0/}" ]; then + echo "::error::verdict-review-gate: the push has no previous commit (before is '$base'); failing closed" + exit 1 + fi + ;; + *) + # workflow_dispatch has no base in its payload: compare with the parent. + base="$(git rev-parse --verify 'HEAD^1^{commit}')" + ;; + esac + # The rules come from the base: run the base commit's gate (and the code it imports) against + # this checkout, so a pull request cannot weaken the gate that judges it. Only the pull + # request that adds the gate (git diff reports it as A against the base) finds none at its + # base and runs its own copy; any other base without the gate fails closed. + gate="scripts/verdict_review_gate.py" + if git cat-file -e "$base:$gate" 2>/dev/null; then + git worktree add --quiet --detach "$RUNNER_TEMP/gate-base" "$base" + gate="$RUNNER_TEMP/gate-base/$gate" + else + added="$(git diff --no-renames --name-status -z "$base...HEAD" -- "$gate" | tr '\0' '\n')" + if [ "$added" != $'A\n'"$gate" ]; then + echo "::error::verdict-review-gate: the base $base has no $gate and this change does not add it; failing closed" + exit 1 + fi + echo "verdict-review-gate: this change adds the gate; running this checkout's copy (bootstrap)" + fi + python3 "$gate" --root "$GITHUB_WORKSPACE" --base "$base" | tee -a "$GITHUB_STEP_SUMMARY" + + sota-sources: + # Top rule (AGENTS.md): every change names the maintained repository or published reference it + # installs or follows. This job fails a PR whose description lacks a non-empty "SOTA sources" section + # (## or ###). It reads the current body through the official actions/github-script + # (github.com/actions/github-script, v9.0.0; docs.github.com/en/rest/pulls/pulls#get-a-pull-request). + # A retrieval error fails the action; no body text reaches a shell. + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 2 + permissions: + pull-requests: read + steps: + - name: Harden the runner (audit-only network egress) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + - name: Require a non-empty "SOTA sources" section in the PR description + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const {data: pullRequest} = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + const body = pullRequest.body || ''; + const match = body.match(/^#{2,3}[ \t]+SOTA sources[ \t]*$([\s\S]*?)(?=^#{2,3}[ \t]|(?![\s\S]))/m); + const content = match ? match[1].replace(//g, '').trim() : ''; + if (!content) { + core.setFailed('The PR description needs a non-empty "SOTA sources" section: the repository, pin and file, or the published reference, behind each change (AGENTS.md top rule).'); + } else { + core.info(`SOTA sources section present (${content.length} characters).`); + } diff --git a/.github/workflows/sota-sources-gate.yml b/.github/workflows/sota-sources-gate.yml index 1ee9a0705..de41f9284 100644 --- a/.github/workflows/sota-sources-gate.yml +++ b/.github/workflows/sota-sources-gate.yml @@ -1,19 +1,20 @@ name: SOTA sources gate -# The reusable form of validate.yml's required sota-sources job, for other repositories: the new-repository scaffold's +# The reusable form of pr-metadata.yml's required sota-sources job, for other repositories: the new-repository scaffold's # .github/workflows/sota-sources.yml (adoption/scaffold/, written by tools/adoption/scaffold_repo.py) calls this file at -# a pinned commit of this repository's main. From its `if:` line on, the job below is validate.yml's job byte for byte -# (tests/test_sota_sources_gate.py); change both together. validate.yml keeps its own copy because .github/ +# a pinned commit of this repository's main. From its `if:` line on, the job below is pr-metadata.yml's job byte for byte +# (tests/test_sota_sources_gate.py); change both together. pr-metadata.yml keeps its own copy because .github/ # main-ruleset.json requires that job's check here. A called workflow runs in its caller's context, so -# github.event_name and the pull_request payload are the caller's, and it can only reduce the caller's token +# github.event_name and the PR number are the caller's, and it can only reduce the caller's token # permissions (docs.github.com/en/actions/reference/workflows-and-actions/reusing-workflow-configurations, "github # context" and "Supported keywords for jobs that call a reusable workflow"). on: workflow_call: -# No token scope and no cache access (docs/decisions/2026-10-04-ci-least-privilege.md): the job reads only the caller's -# pull_request payload. No concurrency group here: in a called workflow ${{ github.workflow }} is the caller's name. +# No default token scope and no cache access (docs/decisions/2026-10-04-ci-least-privilege.md). The job grants only +# pull-requests: read to fetch the current body; callers must grant it too and bump their commit pin to adopt this +# change. No concurrency group here: in a called workflow ${{ github.workflow }} is the caller's name. permissions: {} cache-mode: none @@ -21,10 +22,12 @@ jobs: sota-sources: # Top rule: every change names the maintained repository or published reference it installs or follows. This job # fails a pull request whose description lacks a non-empty "SOTA sources" section (## or ###), reading the body - # from the caller's pull_request payload through actions/github-script; no body text reaches a shell. + # from the current PR through actions/github-script; a retrieval error fails the action and no body text reaches a shell. if: github.event_name == 'pull_request' runs-on: ubuntu-24.04 timeout-minutes: 2 + permissions: + pull-requests: read steps: - name: Harden the runner (audit-only network egress) uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 @@ -34,7 +37,12 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const body = (context.payload.pull_request && context.payload.pull_request.body) || ''; + const {data: pullRequest} = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + const body = pullRequest.body || ''; const match = body.match(/^#{2,3}[ \t]+SOTA sources[ \t]*$([\s\S]*?)(?=^#{2,3}[ \t]|(?![\s\S]))/m); const content = match ? match[1].replace(//g, '').trim() : ''; if (!content) { diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 856f0cafd..2f449aad3 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -4,9 +4,8 @@ on: push: branches: [main] pull_request: - # The default three plus edited, which fires when a pull request's base branch changes: a - # retargeted PR must be judged again against its new base, not keep an earlier green run. - types: [opened, synchronize, reopened, edited] + # Metadata edits are judged by pr-metadata.yml; code changes run this full validation suite. + types: [opened, synchronize, reopened] workflow_dispatch: # No token scope and no cache access by default; each job grants its own (docs/decisions/2026-10-04-ci-least-privilege.md). @@ -268,6 +267,19 @@ jobs: --shard "$VALIDATE_SHARD" --shards 8 \ --report "$RUNNER_TEMP/suite/report.json" --durations 50 \ 2>&1 | tee "$RUNNER_TEMP/suite/unittest-verbose.log" + - name: Report unittest counts + if: always() + run: | + # Parse unittest's native summary as adoption-bootstrap.yml's changed-tests step does. + # An interrupted or missing log has unknown counts; reporting never changes the suite result. + log="$RUNNER_TEMP/suite/unittest-verbose.log" + ran="$(sed -n 's/^Ran \([0-9][0-9]*\) tests\{0,1\} in .*/\1/p' "$log" 2>/dev/null | tail -n 1 || true)" + skipped="$(sed -n -E 's/^(OK|FAILED) \(.*skipped=([0-9]+).*$/\2/p' "$log" 2>/dev/null | tail -n 1 || true)" + summary="$(sed -n -E '/^(OK|FAILED|NO TESTS RAN)( \(.*\))?$/p' "$log" 2>/dev/null | tail -n 1 || true)" + if [ -n "$summary" ]; then + skipped="${skipped:-0}" + fi + printf 'validate: ran %s tests; skipped %s.\n' "${ran:-unknown}" "${skipped:-unknown}" >> "$GITHUB_STEP_SUMMARY" || true - name: Upload the verbose test suite log if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -327,115 +339,6 @@ jobs: include-hidden-files: false retention-days: 30 - verdict-review-gate: - # Required check (docs/decisions/2026-09-22-github-automation-closure.md, "verdict-review-gate - # (2026-09-23)"): with one maintainer and zero required approvals, a PR that changes a - # layer-verdict row merges only when the row is consistent with the sealed cross-family lane - # returns its wave registers (the returns are self-attested; see the record's residual). It has - # no path filter so it reports on every pull request and can be required. - runs-on: ubuntu-24.04 - timeout-minutes: 10 - permissions: - contents: read - steps: - - name: Harden the runner (audit-only network egress) - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - - name: Check out repository with full history - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - # Full history: the gate reads the base commit's ledger and wave registry with git show. - - name: Require sealed cross-family review for changed verdict rows - shell: bash - env: - EVENT_NAME: ${{ github.event_name }} - # The pull request's base branch (GITHUB_BASE_REF): the gate judges only pull requests into main. - PR_BASE_REF: ${{ github.base_ref }} - # Cross-check only: the base is the checked-out merge commit's first parent. - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - # The checked-out HEAD must be the merge commit whose second parent is this PR head. - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - # The pushed range's previous commit on main. - PUSH_BEFORE_SHA: ${{ github.event.before }} - run: | - set -euo pipefail - case "$EVENT_NAME" in - pull_request) - # A pull request into another branch is not judged against main, and retargeting it to - # main later re-runs this job (the edited type): fail closed rather than pass a base the - # gate does not protect. - if [ "${PR_BASE_REF:-}" != "main" ]; then - echo "::error::verdict-review-gate: the pull request's base branch is '${PR_BASE_REF:-}', not main; the gate judges only pull requests into main (retarget it to main); failing closed" - exit 1 - fi - # actions/checkout checks out the PR merge commit; its first parent is the base-branch - # commit the PR is merged into, which is what the gate must compare with. Assert that - # premise: HEAD has exactly two parents and the second is the payload's PR head, or - # HEAD^1 could be the PR's own previous commit and the gate would judge only the last one. - second_parent="$(git rev-parse --verify --quiet 'HEAD^2^{commit}' || true)" - if [ -z "${PR_HEAD_SHA:-}" ] || [ "$second_parent" != "$PR_HEAD_SHA" ] \ - || git rev-parse --verify --quiet 'HEAD^3^{commit}' >/dev/null; then - echo "::error::verdict-review-gate: HEAD is not the merge commit of the PR head '${PR_HEAD_SHA:-}' (second parent '$second_parent'); failing closed" - exit 1 - fi - first_parent="$(git rev-parse --verify --quiet 'HEAD^1^{commit}' || true)" - payload="${PR_BASE_SHA:-}" - if [ -n "$first_parent" ] && [ -n "$payload" ]; then - if ! git merge-base --is-ancestor "$payload" "$first_parent"; then - echo "::error::verdict-review-gate: the payload base $payload is not an ancestor of the merge commit's first parent $first_parent" - exit 1 - fi - base="$first_parent" - elif [ -n "$first_parent" ]; then - base="$first_parent" - elif [ -n "$payload" ]; then - base="$payload" - else - echo "::error::verdict-review-gate: neither the merge commit's first parent nor the payload base is available" - exit 1 - fi - # The base must be a commit on main itself (full history fetches origin/main), not a - # branch that merely contains main's tip, such as a merge commit left built against an - # earlier base: a retargeted PR is otherwise judged against the branch it left. - if ! git merge-base --is-ancestor "$base" "refs/remotes/origin/$PR_BASE_REF"; then - echo "::error::verdict-review-gate: the base $base is not a commit on origin/$PR_BASE_REF; failing closed (after a retarget the edited run checks out the merge commit built on the old base: close and reopen the pull request, or push a commit, to rebuild it on $PR_BASE_REF)" - exit 1 - fi - ;; - push) - base="${PUSH_BEFORE_SHA:-}" - # A branch-creating push has an all-zero before: there is no base to judge against. - if [ -z "$base" ] || [ -z "${base//0/}" ]; then - echo "::error::verdict-review-gate: the push has no previous commit (before is '$base'); failing closed" - exit 1 - fi - ;; - *) - # workflow_dispatch has no base in its payload: compare with the parent. - base="$(git rev-parse --verify 'HEAD^1^{commit}')" - ;; - esac - # The rules come from the base: run the base commit's gate (and the code it imports) against - # this checkout, so a pull request cannot weaken the gate that judges it. Only the pull - # request that adds the gate (git diff reports it as A against the base) finds none at its - # base and runs its own copy; any other base without the gate fails closed. - gate="scripts/verdict_review_gate.py" - if git cat-file -e "$base:$gate" 2>/dev/null; then - git worktree add --quiet --detach "$RUNNER_TEMP/gate-base" "$base" - gate="$RUNNER_TEMP/gate-base/$gate" - else - added="$(git diff --no-renames --name-status -z "$base...HEAD" -- "$gate" | tr '\0' '\n')" - if [ "$added" != $'A\n'"$gate" ]; then - echo "::error::verdict-review-gate: the base $base has no $gate and this change does not add it; failing closed" - exit 1 - fi - echo "verdict-review-gate: this change adds the gate; running this checkout's copy (bootstrap)" - fi - python3 "$gate" --root "$GITHUB_WORKSPACE" --base "$base" | tee -a "$GITHUB_STEP_SUMMARY" - secret-scan: runs-on: ubuntu-24.04 timeout-minutes: 10 @@ -670,29 +573,3 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" exit "$status" - sota-sources: - # Top rule (AGENTS.md): every change names the maintained repository or published reference it - # installs or follows. This job fails a PR whose description lacks a non-empty "SOTA sources" section - # (## or ###). It reads the body from the pull_request payload through the official - # actions/github-script (github.com/actions/github-script, v9.0.0; payload fields per - # docs.github.com/en/webhooks/webhook-events-and-payloads#pull_request); no body text reaches a shell. - if: github.event_name == 'pull_request' - runs-on: ubuntu-24.04 - timeout-minutes: 2 - steps: - - name: Harden the runner (audit-only network egress) - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - - name: Require a non-empty "SOTA sources" section in the PR description - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const body = (context.payload.pull_request && context.payload.pull_request.body) || ''; - const match = body.match(/^#{2,3}[ \t]+SOTA sources[ \t]*$([\s\S]*?)(?=^#{2,3}[ \t]|(?![\s\S]))/m); - const content = match ? match[1].replace(//g, '').trim() : ''; - if (!content) { - core.setFailed('The PR description needs a non-empty "SOTA sources" section: the repository, pin and file, or the published reference, behind each change (AGENTS.md top rule).'); - } else { - core.info(`SOTA sources section present (${content.length} characters).`); - } diff --git a/adoption/bootstrap.md b/adoption/bootstrap.md index 6a60aa1f4..9f6a10eae 100644 --- a/adoption/bootstrap.md +++ b/adoption/bootstrap.md @@ -841,11 +841,12 @@ python3 tools/adoption/scaffold_repo.py --target # writes it; | `.github/workflows/sota-sources.yml` | `sota-sources.yml.template` with `` filled in: it calls this repository's reusable `sota-sources-gate.yml` at that main commit | | `.codex/config.toml` | `adoption/templates/project.codex.config.template.toml`, rendered for this host | -The gate is `validate.yml`'s required `sota-sources` job made reusable (`on: workflow_call`), the same +The gate is `pr-metadata.yml`'s required `sota-sources` job made reusable (`on: workflow_call`), the same check byte for byte (`tests/test_sota_sources_gate.py`); a called workflow runs in its caller's -context, so it reads the new repository's pull request +context, so it reads the new repository's current pull request through the REST API ([reusing workflow configurations](https://docs.github.com/en/actions/reference/workflows-and-actions/reusing-workflow-configurations), -"`github` context"). `` is `--main-sha`, else what `git ls-remote origin refs/heads/main` reports; +"`github` context"). The scaffold caller grants `pull-requests: read`; existing callers must add that +grant when they update their gate pin. `` is `--main-sha`, else what `git ls-remote origin refs/heads/main` reports; when that commit is in this checkout it must carry `.github/workflows/sota-sources-gate.yml`, or nothing is written (exit 2), and a commit this checkout lacks is used as given and reported unchecked. GitHub resolves the reusable workflow when the check runs, so the new repository's check works only once that diff --git a/adoption/scaffold/.github/workflows/sota-sources.yml.template b/adoption/scaffold/.github/workflows/sota-sources.yml.template index e92938743..333bad5f8 100644 --- a/adoption/scaffold/.github/workflows/sota-sources.yml.template +++ b/adoption/scaffold/.github/workflows/sota-sources.yml.template @@ -14,6 +14,7 @@ on: permissions: contents: read + pull-requests: read concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} diff --git a/blueprints/runtime-workers/openhands/resolver.py b/blueprints/runtime-workers/openhands/resolver.py index ab5e99484..3904ea9d6 100644 --- a/blueprints/runtime-workers/openhands/resolver.py +++ b/blueprints/runtime-workers/openhands/resolver.py @@ -453,7 +453,7 @@ def next_branch(harness, number): # -- Unit 5: SOTA sources, the draft PR, one review, one repair, residuals (plan steps 9-12) -# validate.yml:544-545 (the sota-sources job), ported from JavaScript. Line breaks are +# pr-metadata.yml:171-172 (the sota-sources job), ported from JavaScript. Line breaks are # made "\n" first, because JavaScript's multiline ^ and $ also break at \r, U+2028 and # U+2029; JS_TRIM is String.prototype.trim's set (WhiteSpace plus LineTerminator). SOTA_SECTION = re.compile(r"^#{2,3}[ \t]+SOTA sources[ \t]*$([\s\S]*?)(?=^#{2,3}[ \t]|(?![\s\S]))", re.M) diff --git a/docs/decisions/2026-09-22-github-automation-closure.md b/docs/decisions/2026-09-22-github-automation-closure.md index 784c9bad6..26cbedc34 100644 --- a/docs/decisions/2026-09-22-github-automation-closure.md +++ b/docs/decisions/2026-09-22-github-automation-closure.md @@ -1046,7 +1046,7 @@ Hosted and live results after merge. Evidence class: hosted runs and GitHub API - Relying on `validate` alone: `scripts/landscape.py` checks what a row claims against its own files. It does not compare the row with the base, and it does not recompute agreement or winners from the sealed returns. -- **Decision.** The new `verdict-review-gate` job in `validate.yml` runs +- **Decision (2026-09-23; wrapper amended 2026-10-09 below).** The new `verdict-review-gate` job in `validate.yml` runs `scripts/verdict_review_gate.py` on every pull request, with no path filter, and on each push to `main`. Any row that is added or changed outside the grandfathered 20260922 wave needs all of the following: @@ -1073,6 +1073,18 @@ Hosted and live results after merge. Evidence class: hosted runs and GitHub API runs, the verdict tools, `validate.yml`) fails. `.github/main-ruleset.json` adds the check. The coordinator applies the ruleset after merge, and until then the check reports but does not block. +- **Wrapper amendment (2026-10-09, PR #706).** `verdict-review-gate` and `sota-sources` move to + `.github/workflows/pr-metadata.yml`, retaining their required context names and the `opened`, + `synchronize`, `reopened` and `edited` pull-request types, with no path filter. `validate.yml` + drops `edited`, so a description edit no longer starts its eight validation shards. The gate's + `TRUST_PATHS` now names `pr-metadata.yml` instead of `validate.yml`. The new wrapper uses + actions/github-script `v9.0.0` (`3a2844b7e9c422d3c10d287c895573f7108da1b3`) to call + [`GET /repos/{owner}/{repo}/pulls/{pull_number}`](https://docs.github.com/en/rest/pulls/pulls#get-a-pull-request): + `sota-sources` reads the current description, and `verdict-review-gate` fails when the current + base ref differs from the event's base ref before checking the merge commit. Both jobs grant + `pull-requests: read`; the verdict job also keeps `contents: read`. The workflow queues runs + with `queue: max`. These are the PR #706 implementation changes; the dated measurements and + accepted residuals below retain their historical scope. - **Review of the gate (2026-09-23).** An independent review found a decoy-ledger bypass (the manifest could point `scripts/landscape.py` at a copy while the published ledger changed), that the gate ran the PR's own code, that a winner's repository and recipe reference and the published @@ -1500,6 +1512,11 @@ Hosted and live results after merge. Evidence class: hosted runs and GitHub API runs the merged commit's job, so it does not catch a PR that disabled that job. - *Overturn.* The repository moves to an organization with required workflows, or GitHub offers base-defined required checks for personal repositories. +- **Residual amendment (2026-10-09, PR #706).** The PR now supplies the job definition through + `.github/workflows/pr-metadata.yml`, and that file replaces `validate.yml` in the gate's trust + base. Moving the job preserves the accepted residual above: a PR can edit its own wrapper to + stop running the base gate. The current-base REST check in the wrapper amendment above does + not remove that residual. - **Overturn.** A merged PR whose changed verdict row is inconsistent with the sealed evidence its wave registers, while this check was required (the gate checks consistency; the self-attestation residual above bounds what that shows). The other trigger is a second maintainer joining, which would @@ -1885,17 +1902,24 @@ follows. The `gh api` GETs quoted below were taken by the coordinator on - #410 fails `sota-sources` and cannot merge until its sources are fixed. - #205 and #216 had no `sota-sources` run: their last runs predate the job. A new `pull_request` run can report it, and a rebase is not the only way to - start one: `validate.yml` also runs on `reopened` and `edited` + start one: at the time, `validate.yml` also ran on `reopened` and `edited` (`.github/workflows/validate.yml` L6-9), and `docs/lanes.md` ("Hot-file protocol") makes a pushed merge of `main` as valid as a rebase. GitHub starts no `pull_request` run while a pull request has a merge conflict ([events that trigger workflows](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#pull_request)). - Reporting the check is not passing it: the job reads only the pull request's + Reporting the check is not passing it: at the time, the job read only the pull request's description (`.github/workflows/validate.yml` L517-521). Measured 2026-09-29T03:18:04Z (`gh pr view --json mergeable,statusCheckRollup`): both are `CONFLICTING`, with no `sota-sources` run. - #415 and #417 pass it. + **Trigger and description amendment (2026-10-09, PR #706).** The `reopened` and `edited` + metadata runs now belong to `.github/workflows/pr-metadata.yml`; `validate.yml` retains + `reopened` but drops `edited`. Its `sota-sources` job retrieves the current PR description + through `github.rest.pulls.get`, also used by `.github/workflows/sota-sources-gate.yml`, rather + than the event's description. The required context remains `sota-sources`; the 2026-09-29 + observations and workflow line references above describe the implementation measured then. + **Merge-guard mechanics** (moved here from `docs/lanes.md`; each measured or read on 2026-09-29): `gh pr merge --match-head-commit` becomes the `expectedHeadOid` of the `mergePullRequest` mutation (cli/cli v2.101.0 diff --git a/docs/decisions/2026-10-04-ci-least-privilege.md b/docs/decisions/2026-10-04-ci-least-privilege.md index b3a9bd067..d72470b48 100644 --- a/docs/decisions/2026-10-04-ci-least-privilege.md +++ b/docs/decisions/2026-10-04-ci-least-privilege.md @@ -178,16 +178,17 @@ inventories. | `native-service-reboot.yml` | `{}` | `contents: read` on both jobs | | added | | | `native-token-e2e.yml` | `{}` | `contents: read` | `none` | added | | | `practice-references-freshness.yml` | `{}` | `contents: read` | | added | | +| `pr-metadata.yml` (added 2026-10-09, PR #706) | `{}` | `verdict-review-gate`: `contents: read`, `pull-requests: read`; `sota-sources`: `pull-requests: read` | `none` | added, `queue: max` | current PR body and base read through REST | | `publish-catalog.yml` | `{}` | (already job-scoped) | `none` | added, never cancels | | | `receipt-staleness.yml` | `{}` | (already `contents: read`) | `none` | kept | | | `runtime-worker-skills-freshness.yml` | `{}` | `contents: read` | | added | | | `saturation-tracking.yml` | `{}` | (already job-scoped) | | kept | | | `scorecard.yml` | `{}` | (already job-scoped) | | kept | | | `security-scan.yml` | `{}` | (already job-scoped) | `none` | kept | | -| `sota-sources-gate.yml` | `{}` | none (payload only) | `none` | none (reusable) | | +| `sota-sources-gate.yml` | `{}` | `sota-sources`: `pull-requests: read` (PR #706) | `none` | none (reusable) | current PR body read through REST | | `supply-chain.yml` | `{}` | `contents: read` | `none` | kept | | | `token-report.yml` | `{}` | `contents: read` | `none` | kept | | -| `validate.yml` | `{}` | `validate`, `secret-scan`: `contents: read`; `sota-sources`: none | `none` | kept | | +| `validate.yml` | `{}` | `contents: read` on all 4 jobs | `none` | kept | metadata gates move to `pr-metadata.yml` in PR #706 | Tests changed to follow the new layout, without loosening any assertion: `tests/test_workflow_hardening.py` (top-level `{}` instead of `contents: read` in five classes, and `{}` accepted as the one inline permissions form), @@ -204,7 +205,7 @@ Job 004's residual risks, and what this change does about them: (#489) is the control. This layer catches regressions on `main` and in human pull requests, and the required checks still gate merging. 2. **Network-enabled test code can read the checkout and whatever credentials a job holds.** Narrowed: no pull request - job holds a secret or a write scope; tokens are `contents: read` or none and never persisted to disk. Egress + job holds a secret or a write scope; tokens grant read scopes or none and are never persisted to disk. Egress stays audit-only (harden-runner); confidentiality of public-repository source is not at stake. 3. **Caches.** Closed for pull request jobs (`cache-mode: none`, or restore-only with the save step guarded off pull requests), and the attested release restores nothing. A job-level `cache-mode` override on a pull request job fails diff --git a/docs/decisions/2026-10-08-philosophy-only-rules.md b/docs/decisions/2026-10-08-philosophy-only-rules.md index 46f70e957..b240586ec 100644 --- a/docs/decisions/2026-10-08-philosophy-only-rules.md +++ b/docs/decisions/2026-10-08-philosophy-only-rules.md @@ -28,7 +28,7 @@ for the client feature it configures: | Kept line | Where | Evidence | | --- | --- | --- | | The StructuredOutput sentence | Claude user layer only | With it 0 of 30 workflow children made a schema error; without it 5 of 30. The sequential check with the sentence only in the user-level file also counted 0 of 30 ([model-fallback record](2026-09-25-model-fallback-guard.md), lines 108-110). | -| The PR description needs `## SOTA sources` | root `AGENTS.md` | The required `sota-sources` job of `.github/workflows/validate.yml` fails a PR without that section ([required checks](../github-automation.md), lines 29-31). | +| The PR description needs `## SOTA sources` | root `AGENTS.md` | The required `sota-sources` job of `.github/workflows/pr-metadata.yml` (moved from `validate.yml` by PR #706, 2026-10-09) fails a PR without that section ([required checks](../github-automation.md), lines 29-31). | | Run `scripts/validate.py` before committing | root `AGENTS.md` | The required `validate` job runs it, and a stale digest in `manifests/evidence.json` fails it. This change reproduced that before its registry refresh. | | The subtree rule and the trading prerequisite | root `AGENTS.md` | Codex collects `AGENTS.md` only from the project root down to the working directory and never walks past the root (openai/codex `rust-v0.161.0` `codex-rs/core/src/agents_md.rs` lines 7-18). The trading prerequisite is the amendment of the [instruction-core record](2026-10-07-instruction-core.md): paper and broker work runs outside the repository's paths. | | `@AGENTS.md` and `## Compact Instructions` | root `CLAUDE.md` | The memory page keeps the import for sessions that cannot read `AGENTS.md` directly. The costs page (lines 230-237) and the best-practices page (line 392) document compaction instructions in the project `CLAUDE.md`. | diff --git a/docs/github-automation.md b/docs/github-automation.md index 86eb67012..2d242eb63 100644 --- a/docs/github-automation.md +++ b/docs/github-automation.md @@ -1219,12 +1219,14 @@ way this catalog gains evidence; read that chapter for the full flow. The repository has one maintainer and the main ruleset requires 0 approvals, so a PR that changes a layer verdict could merge with no review at all. -`validate.yml`'s `verdict-review-gate` job is the control instead. It runs +`pr-metadata.yml`'s `verdict-review-gate` job is the control instead. It runs [`scripts/verdict_review_gate.py`](../scripts/verdict_review_gate.py) on every pull request (no path filter, so it can be required) and on each push to -`main`. The job has `contents: read`, starts with harden-runner in audit mode, -and checks out full history without persisted credentials. The event values -reach the script only through `env`. On a pull request the job first asserts +`main`. The job has `contents: read` and `pull-requests: read`, starts with +harden-runner in audit mode, and checks out full history without persisted +credentials. On a pull request it first reads the current base branch through +the API and requires its ref to match the event's base ref. The shell step +receives event values only through `env`. It then asserts that the checked-out HEAD is the PR merge commit: it has exactly two parents and the second is the payload's `pull_request.head.sha`, or the job fails (otherwise `HEAD^1` could be the PR's own previous commit and the gate would @@ -1375,11 +1377,16 @@ the gate script, every repository module the gate and its validators import (transitively, which brings in `scripts/validate.py` through `scripts/host_receipts.py`), the rule inputs they read (the lane-provenance registry `tools/sota-convergence/lane-provenance.json`, the host-receipt and -lane-return schemas) and `validate.yml`. A rule input held inside a data file +lane-return schemas) and the gate wrapper `.github/workflows/pr-metadata.yml`. +A rule input held inside a data file counts too: `adoption/manifest.json#/platform_profiles` (which host os/architecture a receipt's platform binds) changed together with verdict data fails the same way (`RULE_INPUT_FIELDS`). The tests derive the list -rather than restate it: one walks the modules' imports with `ast`, and one +rather than restate it: one parses workflow jobs and steps, follows job-level +local reusable-workflow calls in both supported path forms, and requires every +direct gate wrapper and transitive caller to exactly match the workflow entries +in `TRUST_PATHS`. Missing or unparsable local callees fail this check. +Another walks the modules' imports with `ast`, and another records every file opened (a `sys.addaudithook`, in a subprocess) while the gate judges a fixture that reaches every row path and while the validators check this checkout. Every head-side file the gate reads must be a @@ -1398,14 +1405,37 @@ and `build_verdicts.py --check`. Run it locally with: python3 scripts/verdict_review_gate.py --base origin/main ``` -The workflow's `pull_request` trigger adds the `edited` type to the default -three, so a PR whose base branch changes runs again, and the job fails closed -on any `pull_request` event whose base branch (`GITHUB_BASE_REF`, passed -through the step's environment) is not `main`. A PR first judged against -another branch and then retargeted to `main` therefore cannot merge on its -earlier green run. The merge commit's first parent, which the gate compares -with, must also be a commit on `origin/main`, so a merge commit still built on -a branch that merely contains `main`'s tip fails closed. +`pr-metadata.yml` owns `sota-sources` and `verdict-review-gate`, keeping their +required check names. It subscribes to `pull_request` types `opened`, +`synchronize`, `reopened` and `edited`, push to `main` and manual dispatch. +`validate.yml` drops `edited`, so a description edit or retarget starts the +metadata gates without starting or cancelling the full validation suite. +The metadata workflow queues per PR with `queue: max` and no +`cancel-in-progress`. GitHub permits up to 100 pending runs, cancels +additional runs at that limit and does not guarantee dispatch order; the +gates therefore read current metadata rather than depend on queue order +([GitHub concurrency source at `336b7f546d94`](https://github.com/github/docs/blob/336b7f546d94/data/reusables/actions/actions-group-concurrency.md)). + +Both gates use `github.rest.pulls.get` through +[`actions/github-script` v9.0.0, `3a2844b7`](https://github.com/actions/github-script/tree/3a2844b7e9c422d3c10d287c895573f7108da1b3), +with job-local `pull-requests: read`; a retrieval error fails the action. +`sota-sources` checks the current body, keeping description text out of the +shell. Its reusable `sota-sources-gate.yml` job stays byte-identical and the +scaffold caller grants the read permission. Existing callers adopt the +change only when they update their commit pin +([Get a pull request](https://docs.github.com/en/rest/pulls/pulls#get-a-pull-request)). + +`verdict-review-gate` first requires the current base branch ref to match the +event's base ref, failing closed when a queued run's event predates a retarget. +Advancing the same base branch's tip does not fail this metadata check. The +subsequent shell checks fail closed when the event's base branch +(`GITHUB_BASE_REF`, passed through the step's environment) is not `main`. +A PR first judged against another branch and then retargeted to `main` +therefore cannot merge on its earlier green run. The merge commit still must +have exactly two parents with the event's PR head as its second parent. Its +first parent, which the gate compares with, must also be a commit on +`origin/main`, so a merge commit still built on a branch that merely contains +`main`'s tip fails closed. After a retarget, the `edited` run checks out the merge commit still built on the old base. This was measured on 2026-09-23 with throwaway PR #143, and the @@ -1419,7 +1449,7 @@ base has none and this change adds `scripts/verdict_review_gate.py` (the bootstrap PR); a base without the gate otherwise fails closed. One residual is accepted. A pull request runs the job definition from its -own `validate.yml`, so a PR that rewrites this job's step can disable the +own `pr-metadata.yml`, so a PR that rewrites this job's step can disable the check for itself. The tests that pin the job's shape (`tests/test_workflow_hardening.py`) and the gate's rules (`tests/test_verdict_review_gate.py`) are the head's copies too, so such a PR diff --git a/docs/lanes.md b/docs/lanes.md index ec5396b11..16b684b74 100644 --- a/docs/lanes.md +++ b/docs/lanes.md @@ -172,8 +172,9 @@ gh pr merge --squash --match-head-commit and [`aggregate.go` L72-88](https://github.com/cli/cli/blob/v2.101.0/pkg/cmd/pr/checks/aggregate.go#L72-L88) at gh v2.101.0: `CANCELLED` is bucket `cancel`, counted as neither failed nor pending), and with `--json` it exits 0 whatever the state. A description edit - starts a new `validate.yml` run on the same head (its `pull_request` types - include `edited`), so run it after the last edit. `gh` sends `` as + starts a `pr-metadata.yml` run on the same head to read the current PR; + `validate.yml` does not subscribe to `edited`, so run the check read after + the last edit and wait for the metadata gates too. `gh` sends `` as GitHub's `expectedHeadOid` ([gh pr merge](https://cli.github.com/manual/gh_pr_merge): "Commit SHA that the pull request head must match to allow merge"), and GitHub refuses the merge if the head has moved since; the mechanics are in the diff --git a/manifests/evidence.json b/manifests/evidence.json index b306ecf38..d8fdbee4f 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -5149,6 +5149,11 @@ "sha256": "0b73bc55e4329dab4bd69a680769512706c787e4f9dc2e8fb75ab72c2464bb9c", "bytes": 3254 }, + { + "path": ".github/workflows/pr-metadata.yml", + "sha256": "3f0d77d6d28e2c1a15905a277cff15326a75c4960e37f634e2a2c12e2e7bffe6", + "bytes": 9927 + }, { "path": ".github/workflows/practice-references-freshness.yml", "sha256": "edf35611020d9338f3f28db9d247039d25cb1d31e4c001306e395dab2862a7e1", @@ -5186,8 +5191,8 @@ }, { "path": ".github/workflows/sota-sources-gate.yml", - "sha256": "25029eba404feb5bf9fe79c3c8d97c53cc290c6876e72be73c268a7e13d7883b", - "bytes": 2699 + "sha256": "468a1d1b8b765823dc5175f546c236cdf026669de2581e8149c1ef68177274e7", + "bytes": 3020 }, { "path": ".github/workflows/supply-chain.yml", @@ -5201,8 +5206,8 @@ }, { "path": ".github/workflows/validate.yml", - "sha256": "339626f86df3cf5954f23e1365e87eaf3635ff55488b65f0715ed7dbdbd3cacb", - "bytes": 43019 + "sha256": "61a9ed511b422382bcf4433167aa6627e51eb390b6a6a4efd9aa5cc377e99a66", + "bytes": 35456 }, { "path": ".gitignore", @@ -5351,8 +5356,8 @@ }, { "path": "adoption/bootstrap.md", - "sha256": "ca7f66cde7af2396a8e9c728ca1e4067394d9c92dfe5ced3def886ca18d9ff3e", - "bytes": 74554 + "sha256": "0a448d9813a9c74f84ba23f935431eba558c55e8febe42061d4658f6ab8bf9da", + "bytes": 74706 }, { "path": "adoption/credential-inventory.json", @@ -5594,6 +5599,11 @@ "sha256": "10e83b515760e86ffd1a9b6540aaeea769f94af5d569c001954f0031548ca75b", "bytes": 2735 }, + { + "path": "adoption/scaffold/.github/workflows/sota-sources.yml.template", + "sha256": "9d6f5dbd746e9c0589ad26aa103ced6f2b1f1290a39a6beee7711efc3495db3c", + "bytes": 1107 + }, { "path": "adoption/scaffold/AGENTS.md", "sha256": "7bc2fc6160bcc18079850402795324b64632f7cf68d296f5742fadf524181b02", @@ -11342,8 +11352,8 @@ }, { "path": "blueprints/runtime-workers/openhands/resolver.py", - "sha256": "b72dc929e54da878eac6a6ee34cd1e076ac8016f0015153725aadd96861b1a15", - "bytes": 103642 + "sha256": "9ca877e1fce797196a0b24980def08a812fade5366fbcedaa4f01caa7571610f", + "bytes": 103645 }, { "path": "blueprints/runtime-workers/openhands/resolver/gate_reads.py", @@ -17722,8 +17732,8 @@ }, { "path": "docs/decisions/2026-09-22-github-automation-closure.md", - "sha256": "5cd35591a82a8502d17da26adca21019edefeda076c3462fd98cf29dd53c6b66", - "bytes": 159318 + "sha256": "572f707148b14e996441cbe00ec3dc76d5e0f86eb3a247bc0bf3ee9e9fa36a74", + "bytes": 161476 }, { "path": "docs/decisions/2026-09-23-max-effort-default.md", @@ -18127,8 +18137,8 @@ }, { "path": "docs/decisions/2026-10-04-ci-least-privilege.md", - "sha256": "69f7fcd2c2f91412dc478f8d10503820a186efba9788f09f723d3212695cf3d0", - "bytes": 43157 + "sha256": "b4a735ec0834156c89adf66ab7dc3b32c540c638475554621a99b1b702c3252b", + "bytes": 43474 }, { "path": "docs/decisions/2026-10-04-claude-template-holds-out-token-lane-carriers.md", @@ -18452,8 +18462,8 @@ }, { "path": "docs/decisions/2026-10-08-philosophy-only-rules.md", - "sha256": "f4b4f1d9952db662d8c0c9f1f9aec24da40c4666e4655e105c0617a689e6826c", - "bytes": 10028 + "sha256": "b8134a6c28b1f6575d72945bf141ff36de9bc7e268afad23a53c752da4634666", + "bytes": 10082 }, { "path": "docs/decisions/2026-10-08-qmd-shared-mcp.md", @@ -18647,8 +18657,8 @@ }, { "path": "docs/github-automation.md", - "sha256": "28a97e3dea87e15a141bb297e4654fc235bc7445ba54941a23e19de90eaf75c8", - "bytes": 103924 + "sha256": "b5e80791e416eec587e8202253c764aac9454290d5afc6561187a045eee1747b", + "bytes": 105989 }, { "path": "docs/grand-catalog-handbook.md", @@ -18707,8 +18717,8 @@ }, { "path": "docs/lanes.md", - "sha256": "bc464ce7d95542b7a33229de29f50efe2bf52b341bf037772bf91e556468cb8f", - "bytes": 13335 + "sha256": "36a2c7a6bb52252e04fd836ecd0684fc50a412f0b9226ff179c74e46fe4e31b8", + "bytes": 13410 }, { "path": "docs/memory-landscape-maintenance.md", @@ -59460,6 +59470,11 @@ "sha256": "7e6c965fc70147e2753c588376aa92fd6400500beee0fb992e8caac69ee04c21", "bytes": 21933 }, + { + "path": "scripts/verdict_review_gate.py", + "sha256": "daa20fe48f09e64ef3db7814dee3a2860d058031f56cc5be2c21236c16c77fbb", + "bytes": 100220 + }, { "path": "scripts/verify_nautilus_ci.py", "sha256": "e14e3372815747eefa800f3538f30eabfb4fab7025ccc507bb0b946e67d2292f", @@ -59917,8 +59932,8 @@ }, { "path": "tests/test_catalog_freshness_propose.py", - "sha256": "b941755e00c05e08751e64c55eac8ab7f5827a71b4bb8903e6ce229be9c29611", - "bytes": 63830 + "sha256": "6aa868be73a47e3f2347c0728ec4489e59c52ecfb84f115aed051698effc4e6d", + "bytes": 64694 }, { "path": "tests/test_catalog_freshness_runtime.py", @@ -60242,8 +60257,8 @@ }, { "path": "tests/test_install_claude_profile.py", - "sha256": "268aee4e0eff1cf8404068ea3f67d622b962713bb1b0bd874d8ccb07f73f35ba", - "bytes": 144017 + "sha256": "08574857b240aeb89b98aa1b573765442cc860dce7411a8e1d7f2602f68af301", + "bytes": 144020 }, { "path": "tests/test_install_skills.py", @@ -60325,6 +60340,11 @@ "sha256": "4601bbba4a6c1cf04352a484670b1601b468e9aa879c8ac5f55eee6c3fd5dd94", "bytes": 20940 }, + { + "path": "tests/test_local_pages_policy_grants.py", + "sha256": "7ea348554ca4d467b71b18833167858f77715ee962832e1f503ed4682c983585", + "bytes": 19309 + }, { "path": "tests/test_managed_block.py", "sha256": "1509fe2be743a0d318f9e09f0b21c9251774bda2bdcea55b5351e80699621165", @@ -60662,8 +60682,8 @@ }, { "path": "tests/test_runtime_worker_openhands_resolver.py", - "sha256": "b6ce5cf835a00e68494513a79df41238d059abf794e8d8062c2286812cee66b6", - "bytes": 276707 + "sha256": "852f6115d6295c2ea54a3c9aede259a380d45c9c712963f1fcc3363bddaa766c", + "bytes": 276710 }, { "path": "tests/test_runtime_worker_skills.py", @@ -60752,8 +60772,8 @@ }, { "path": "tests/test_sota_sources_gate.py", - "sha256": "35b1e34003658cddd9e9ede37c71bdf8fee7ede6cda5105924792670e3e30d12", - "bytes": 13871 + "sha256": "869038ea7c6261e335e942934b335789e9f3e883ee172835a1c0828ea4afa870", + "bytes": 16230 }, { "path": "tests/test_spy_parity.py", @@ -60860,6 +60880,11 @@ "sha256": "02465e7ca09a3d4ee72e3855084cc5b54ee223b209bb6169cb9deea1c5d9fdab", "bytes": 12669 }, + { + "path": "tests/test_verdict_review_gate.py", + "sha256": "0b829e33823d3d9f3e94295836f63b97342c8cedc7b92ed0d08e3c40553b894b", + "bytes": 116272 + }, { "path": "tests/test_windows_terminal_defaults.py", "sha256": "f4b2c8ebe16f54782204f32fd674debc4b8f3a01f5ca1d0b00b788c175ed0502", @@ -60872,8 +60897,8 @@ }, { "path": "tests/test_workflow_hardening.py", - "sha256": "7f16b2ad9aa5ef3886a24bf2a576f3304c1b9fda354520d89824b8dd7178ec18", - "bytes": 166541 + "sha256": "932313de8026909dad900fcca7da06df65304416cd501da31285e5441050e604", + "bytes": 173584 }, { "path": "tests/test_workflow_policy.py", @@ -60887,8 +60912,8 @@ }, { "path": "tests/test_workflow_security_coverage.py", - "sha256": "804fb4ab24d29bf05becb43facd4d0cf68158ef24ad13101d2eb0d57c69b8ec9", - "bytes": 13367 + "sha256": "4e11c6592c56636c1e557eb93ffb745a2703a4ee87565259568418e51d7bde29", + "bytes": 13398 }, { "path": "tests/test_wsl_memory_maintenance.py", @@ -61125,6 +61150,11 @@ "sha256": "1c88c483bf50c4b12658aa43fec806060c557dda1215178c3e2dff1feb75a86e", "bytes": 5161 }, + { + "path": "tools/local-pages/source_policy.json", + "sha256": "3e65408d5fab88cb3ab7a3167b738ba207d053f37bbe25c5e4890ac6e3ed82b9", + "bytes": 755702 + }, { "path": "tools/omniroute/gateway_record.py", "sha256": "d5cfcfbd61bcfcb1808270deab7e793d4021c8dd0c3079fe5ff71ee723ab46e6", diff --git a/scripts/verdict_review_gate.py b/scripts/verdict_review_gate.py index c4cc9a8bd..73bf35bf6 100644 --- a/scripts/verdict_review_gate.py +++ b/scripts/verdict_review_gate.py @@ -78,8 +78,9 @@ The sealed files are self-attested: the gate checks that a row is consistent with the sealed returns its wave registers and that their declared model families differ, not that a cross-family review -actually ran (the decision record's accepted residual). The gate's own rules are trusted only from the base: CI runs the base commit's copy of this script -against the head checkout (``--root``), and a change to a verdict row, wave or sealed artifact fails +actually ran (the decision record's accepted residual). The gate's own rules are trusted only from +the base: ``.github/workflows/pr-metadata.yml`` runs the base commit's copy of this script against +the head checkout (``--root``), and a change to a verdict row, wave or sealed artifact fails when the same comparison also changes a ``TRUST_PATHS`` file, so a weakening of the rules has to land (and be seen) in its own pull request first. Exit 0 prints one line when nothing of that changed; otherwise every violation is printed with its row key and the exit code is 1 (2 for an unresolvable @@ -176,7 +177,7 @@ # The Claude family's read-boundary audit (round 9, BR9-1). "tools/sota-convergence/transcript_audit.py", "tools/sota-convergence/lane-return.schema.json", "tools/sota-convergence/lane-provenance.json", - "adoption/host-receipt.schema.json", ".github/workflows/validate.yml", + "adoption/host-receipt.schema.json", ".github/workflows/pr-metadata.yml", ) # Any change under these paths runs the repository validators even when no row changed: a PR # that only deletes or rewrites a sealed lane file must still meet scripts/landscape.py. diff --git a/tests/test_catalog_freshness_propose.py b/tests/test_catalog_freshness_propose.py index 606da639d..8e5c58387 100644 --- a/tests/test_catalog_freshness_propose.py +++ b/tests/test_catalog_freshness_propose.py @@ -815,8 +815,17 @@ def test_general_publication_validator_passes_after_the_run(self): artifact_dir = _make_artifact(self.scratch, "artifact-validate") result = _run_propose_cli(self.scratch, artifact_dir, checked_at="2026-09-23T02:00:00Z") self.assertEqual(result.returncode, 0, result.stderr[-3000:]) + # CC measured nested-validator durations (2026-10-10): + # 44.8 s pass (#944): https://github.com/seathatflowsinourveins/native-agent-stack/actions/runs/38016041243 + # 52.2 s pass (#947): https://github.com/seathatflowsinourveins/native-agent-stack/actions/runs/38016041799 + # 57.2 s pass (#948): https://github.com/seathatflowsinourveins/native-agent-stack/actions/runs/38016595551 + # 60.4 s TimeoutExpired (#937, attempt 1): https://github.com/seathatflowsinourveins/native-agent-stack/actions/runs/38016044731/attempts/1 + # On timeout, run() kills/waits for the child and raises TimeoutExpired: + # https://docs.python.org/3.12/library/subprocess.html#subprocess.run + # We choose 300 s for measured margin within the 600-second module budget, + # leaving time for fixture setup and the other tests. checked = subprocess.run( - ["python3", "scripts/validate.py"], cwd=self.scratch, capture_output=True, text=True, timeout=60, + ["python3", "scripts/validate.py"], cwd=self.scratch, capture_output=True, text=True, timeout=300, ) self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) diff --git a/tests/test_install_claude_profile.py b/tests/test_install_claude_profile.py index 6d86a79f9..24525c6f6 100644 --- a/tests/test_install_claude_profile.py +++ b/tests/test_install_claude_profile.py @@ -1911,7 +1911,7 @@ def test_root_claude_md_loads_the_core_through_its_agents_md_import(self): self.assertEqual([phrase for phrase in self.DROPPED if phrase in text], []) # Root AGENTS.md keeps, beside the core, only lines whose removal would cause a shown mistake: the required - # sota-sources check (.github/workflows/validate.yml, job sota-sources; .github/main-ruleset.json), the required + # sota-sources check (.github/workflows/pr-metadata.yml, job sota-sources; .github/main-ruleset.json), the required # validate check over the registered evidence digests (validate.yml runs scripts/validate.py), and the trading # prerequisite that the amendment of docs/decisions/2026-10-07-instruction-core.md keeps for paper operation outside # the repository's paths, with the subtree rule that Codex's root-to-cwd AGENTS.md walk needs. diff --git a/tests/test_local_pages_policy_grants.py b/tests/test_local_pages_policy_grants.py index 825d3026d..fc9b8c945 100644 --- a/tests/test_local_pages_policy_grants.py +++ b/tests/test_local_pages_policy_grants.py @@ -77,7 +77,7 @@ def test_committed_repository_grants_match_current_git_inventory_paths(self): expected = set(grants.inventory_paths([path for path in tracked if path], manifest["catalogs"])) policy = json.loads((ROOT / "tools/local-pages/source_policy.json").read_text()) actual = {row["path"] for row in policy["architecture"]["architecture_inventory"] if row["root"] == "repo"} - self.assertEqual(len(actual), 91) + self.assertEqual(len(actual), 92) self.assertEqual(actual, expected, "Regenerate reviewed repository grants at the landing head; runtime discovery remains names-only.") def test_receipt_documentation_identifies_the_reviewed_union_and_ungranted_proposal(self): diff --git a/tests/test_runtime_worker_openhands_resolver.py b/tests/test_runtime_worker_openhands_resolver.py index f092ecd5d..3598334f4 100644 --- a/tests/test_runtime_worker_openhands_resolver.py +++ b/tests/test_runtime_worker_openhands_resolver.py @@ -2139,7 +2139,7 @@ def loop(self, harness, record, **kwargs): sleep=clock.sleep, **options) def test_sota_check_is_a_port_of_the_ci_job(self): - workflow = (ROOT / ".github/workflows/validate.yml").read_text(encoding="utf-8") + workflow = (ROOT / ".github/workflows/pr-metadata.yml").read_text(encoding="utf-8") self.assertIn(r"/^#{2,3}[ \t]+SOTA sources[ \t]*$([\s\S]*?)(?=^#{2,3}[ \t]|(?![\s\S]))/m", workflow) self.assertIn(r".replace(//g, '').trim()", workflow) content = self.r.sota_section_content diff --git a/tests/test_sota_sources_gate.py b/tests/test_sota_sources_gate.py index 8738fd112..3d90e8be1 100644 --- a/tests/test_sota_sources_gate.py +++ b/tests/test_sota_sources_gate.py @@ -1,12 +1,12 @@ -"""The reusable sota-sources gate and the check it shares with validate.yml. +"""The reusable sota-sources gate and the check it shares with pr-metadata.yml. -.github/workflows/sota-sources-gate.yml is validate.yml's required sota-sources job made callable (`on: +.github/workflows/sota-sources-gate.yml is pr-metadata.yml's required sota-sources job made callable (`on: workflow_call`), so a repository scaffolded by tools/adoption/scaffold_repo.py enforces the same pull-request rule by calling it at a pinned commit. These tests keep the two copies one check: -- from the job's `if:` line to its end the gate's job is validate.yml's byte for byte (only the leading comment may +- from the job's `if:` line to its end the gate's job is pr-metadata.yml's byte for byte (only the leading comment may differ), and a one-character drift is caught; -- the gate is only a reusable workflow, with a read-only token, and validate.yml keeps the job the main ruleset +- the gate is only a reusable workflow, with a read-only token, and pr-metadata.yml keeps the job the main ruleset requires; - run the way actions/github-script runs a script (an AsyncFunction over the named arguments, src/async-function.ts at the pinned v9.0.0 commit 3a2844b7), both copies pass and fail the same descriptions, and @@ -29,7 +29,7 @@ ROOT = Path(__file__).resolve().parents[1] WORKFLOWS = ROOT / ".github/workflows" -VALIDATE = WORKFLOWS / "validate.yml" +VALIDATE = WORKFLOWS / "pr-metadata.yml" GATE = WORKFLOWS / "sota-sources-gate.yml" RULESET = ROOT / ".github/main-ruleset.json" SCAFFOLD = ROOT / "adoption/scaffold" @@ -77,8 +77,8 @@ def sota_job(path: Path) -> str: # actions/github-script v9.0.0 (3a2844b7e9c422d3c10d287c895573f7108da1b3) src/async-function.ts: the script is the -# body of `new AsyncFunction(...Object.keys(args), source)`, called with the argument values. This script reads only -# `context` and `core`, so the harness passes those two. +# body of `new AsyncFunction(...Object.keys(args), source)`, called with the argument values. The fixture supplies +# `context`, `core` and the REST client, and main.ts's handleError turns a rejected script into core.setFailed. HARNESS = r""" const fs = require('fs'); const AsyncFunction = Object.getPrototypeOf(async () => null).constructor; @@ -86,10 +86,21 @@ def sota_job(path: Path) -> str: (async () => { const results = []; for (const payload of payloads) { - const outcome = {failed: null, info: null}; + const outcome = {failed: null, info: null, requests: []}; const core = {setFailed: (message) => { outcome.failed = String(message); }, info: (message) => { outcome.info = String(message); }}; - await new AsyncFunction('context', 'core', script)({payload}, core); + const context = {payload, repo: {owner: 'fixture-owner', repo: 'fixture-repo'}, + issue: {number: payload.pull_request?.number || 17}}; + const github = {rest: {pulls: {get: async (parameters) => { + outcome.requests.push(parameters); + if (payload.retrieval_error) throw new Error(payload.retrieval_error); + return {data: payload.current_pull_request ?? payload.pull_request ?? {}}; + }}}}; + try { + await new AsyncFunction('context', 'core', 'github', script)(context, core, github); + } catch (error) { + core.setFailed(`Unhandled error: ${error}`); + } results.push(outcome); } process.stdout.write(JSON.stringify(results)); @@ -133,13 +144,13 @@ def test_the_gate_is_only_a_reusable_workflow_with_a_read_only_token(self): self.assertEqual(re.findall(r"(?m)^on:.*$", text), ["on:"]) self.assertEqual(re.findall(r"(?m)^ [a-z_]+:", text.split("\non:\n", 1)[1].split("\n\n", 1)[0]), [" workflow_call:"]) - # No token scope and no cache access (docs/decisions/2026-10-04-ci-least-privilege.md); the job adds none. + # No default token scope or cache access; the job grants only the current-PR retrieval scope. self.assertIn("\npermissions: {}\ncache-mode: none\n\n", text) - self.assertNotRegex(text, r"(?m)^ permissions:") + self.assertRegex(text, r"(?m)^ permissions:\n pull-requests: read\n steps:") self.assertNotRegex(text, r"(?m)^[ \t]*[\w-]+:[ \t]*write(?:-all)?[ \t]*(?:#.*)?$") self.assertEqual(list(jobs(text)), [JOB_ID]) - def test_the_gate_runs_validate_ymls_check_byte_for_byte(self): + def test_the_gate_runs_pr_metadata_ymls_check_byte_for_byte(self): self.assertEqual(check_body(sota_job(GATE)), check_body(sota_job(VALIDATE))) self.assertEqual(inline_script(sota_job(GATE)), inline_script(sota_job(VALIDATE))) self.assertIn("core.setFailed(", inline_script(sota_job(GATE))) @@ -154,7 +165,7 @@ def test_a_one_character_drift_in_either_copy_is_caught(self): with self.assertRaises(ValueError): check_body(original.replace(" if: ", " name: x\n if: ", 1)) - def test_validate_yml_keeps_the_job_the_main_ruleset_requires(self): + def test_pr_metadata_yml_keeps_the_job_the_main_ruleset_requires(self): contexts = [check["context"] for rule in json.loads(RULESET.read_text(encoding="utf-8"))["rules"] if rule.get("type") == "required_status_checks" for check in rule["parameters"]["required_status_checks"]] @@ -167,8 +178,8 @@ class GateBehaviourTests(unittest.TestCase): def test_both_copies_pass_and_fail_the_same_descriptions(self): payloads = [payload for _, payload, _ in CASES] outcomes = {name: run_check(inline_script(sota_job(path)), payloads) - for name, path in (("validate.yml", VALIDATE), ("sota-sources-gate.yml", GATE))} - self.assertEqual(outcomes["validate.yml"], outcomes["sota-sources-gate.yml"]) + for name, path in (("pr-metadata.yml", VALIDATE), ("sota-sources-gate.yml", GATE))} + self.assertEqual(outcomes["pr-metadata.yml"], outcomes["sota-sources-gate.yml"]) for (case, _, passes), outcome in zip(CASES, outcomes["sota-sources-gate.yml"]): with self.subTest(case=case): self.assertIs(outcome["failed"] is None, passes, outcome) @@ -177,6 +188,31 @@ def test_both_copies_pass_and_fail_the_same_descriptions(self): else: self.assertIn('non-empty "SOTA sources" section', outcome["failed"]) + def test_both_copies_judge_the_current_body_instead_of_the_event_body(self): + valid = "### SOTA sources\n- https://github.com/o/r at v1.2.3, src/x.py\n" + for path in (VALIDATE, GATE): + with self.subTest(workflow=path.name): + script = inline_script(sota_job(path)) + removed, restored = run_check(script, [ + {**body(valid), "current_pull_request": {"body": "## Summary\nNo sources\n"}}, + {**body(""), "current_pull_request": {"body": valid}}, + ]) + self.assertIsNotNone(removed["failed"]) + self.assertIsNone(restored["failed"], restored) + for outcome in (removed, restored): + self.assertEqual(outcome["requests"], [{"owner": "fixture-owner", "repo": "fixture-repo", + "pull_number": 17}]) + + def test_a_rejected_pulls_get_fails_both_copies_even_with_a_valid_event_body(self): + payload = {**body("### SOTA sources\n- https://example.org/paper\n"), "retrieval_error": "request denied"} + for path in (VALIDATE, GATE): + with self.subTest(workflow=path.name): + (outcome,) = run_check(inline_script(sota_job(path)), [payload]) + self.assertIn("Unhandled error:", outcome["failed"]) + self.assertIn("request denied", outcome["failed"]) + self.assertIsNone(outcome["info"]) + self.assertEqual(len(outcome["requests"]), 1) + def test_both_pull_request_templates_fail_until_the_section_is_filled_in(self): script = inline_script(sota_job(GATE)) for name, path in PR_TEMPLATES.items(): @@ -209,10 +245,11 @@ def zizmor(path: Path, directory: Path) -> tuple: @unittest.skipUnless(ZIZMOR, "native zizmor unavailable; CI installs the pinned analyzer") class GateZizmorTests(unittest.TestCase): - def test_the_gate_has_no_offline_findings(self): - with tempfile.TemporaryDirectory() as temporary: - status, idents = zizmor(GATE, Path(temporary)) - self.assertEqual((status, idents), (0, set())) + def test_both_metadata_workflows_have_no_offline_findings(self): + for path in (GATE, VALIDATE): + with self.subTest(workflow=path.name), tempfile.TemporaryDirectory() as temporary: + status, idents = zizmor(path, Path(temporary)) + self.assertEqual((status, idents), (0, set())) class ScaffoldCallerTests(unittest.TestCase): @@ -220,6 +257,7 @@ def test_the_caller_calls_the_gate_at_a_placeholder_commit(self): text = CALLER_TEMPLATE.read_text(encoding="utf-8") self.assertEqual(re.findall(r"(?m)^ uses: (\S+)$", text), [f"{GATE_REFERENCE}@"]) self.assertIn("\npermissions:\n contents: read\n", text) + self.assertIn("\n pull-requests: read\n", text) self.assertRegex(text, r"(?m)^ pull_request:\n # [^\n]*\n types: \[opened, synchronize, reopened, edited\]$") def test_no_scaffold_file_is_collected_as_a_workflow(self): diff --git a/tests/test_verdict_review_gate.py b/tests/test_verdict_review_gate.py index 2179b425b..d3c236b82 100644 --- a/tests/test_verdict_review_gate.py +++ b/tests/test_verdict_review_gate.py @@ -729,11 +729,22 @@ def test_verdict_row_change_with_a_rules_change_in_the_same_comparison_fails(sel def test_workflow_change_with_a_sealed_artifact_change_fails(self): self.record() self.rebase() - self.write(".github/workflows/validate.yml", b"name: weakened\n") + self.write(".github/workflows/pr-metadata.yml", b"name: weakened\n") path = f"{SEALED}/codex/foundation-beta-{RUN}.json" (self.root / path).unlink() self.registered.discard(path) - self.assertFails(self.report(), "also the gate's trust base (.github/workflows/validate.yml)") + self.assertFails(self.report(), "also the gate's trust base (.github/workflows/pr-metadata.yml)") + + def test_metadata_workflow_change_with_a_verdict_row_change_fails(self): + workflow = ".github/workflows/pr-metadata.yml" + self.write(workflow, b"name: PR metadata gates\n") + self.rebase() + self.record() + self.assertPasses(self.report()) + self.write(workflow, b"name: weakened\n") + report = self.report() + self.assertFails(report, "also the gate's trust base (.github/workflows/pr-metadata.yml)") + self.assertEqual(report["trust_paths_changed"], [workflow]) def test_rules_change_alone_passes(self): self.write("scripts/verdict_review_gate.py", b"# rules change on its own\n") @@ -1636,9 +1647,10 @@ def test_reformatted_newest_wave_manifest_passes(self): class TrustPathDerivationTests(unittest.TestCase): """Review of #123, finding 4, and fourth review G3: TRUST_PATHS covers every repository module the - gate and the validators import (transitively) and every head-side rule input they read, derived - from the modules (an ast walk of the imports) and from what they actually open (a sys.addaudithook - recording every read while the gate judges a fixture and the validators check this checkout).""" + gate and the validators import (transitively), every head-side rule input they read and the + workflow wrappers. Derive them from the modules (an ast walk of the imports), actual reads + (a sys.addaudithook while the gate judges a fixture and the validators check this checkout) + and parsed workflow jobs and steps.""" ROOT = Path(gate.__file__).resolve().parents[1] ENTRY_POINTS = ("scripts/verdict_review_gate.py", "scripts/landscape.py", "scripts/platform_status.py", @@ -1680,6 +1692,119 @@ def test_every_imported_repository_module_is_a_trust_path(self): self.assertIn("scripts/validate.py", seen) # scripts/host_receipts.py imports it self.assertEqual(sorted(seen - set(gate.TRUST_PATHS)), []) + def test_workflow_wrappers_match_the_workflow_trust_paths(self): + from tests.test_workflow_policy import UnsupportedYAML, load_workflow, local_callee, workflow_files + + workflows = {} + for path in workflow_files(self.ROOT / ".github/workflows"): + relative = path.relative_to(self.ROOT).as_posix() + try: + workflows[relative] = load_workflow(path.read_text(encoding="utf-8"))["jobs"] + except (OSError, UnsupportedYAML) as error: + self.fail(f"{relative}: unreadable or unparsable workflow ({type(error).__name__})") + + wrappers, callers = set(), {} + for relative, jobs in workflows.items(): + invokes_gate = any("scripts/verdict_review_gate.py" in step.get("run", "") + for job in jobs.values() for step in job.get("steps", [])) + if "verdict-review-gate" in jobs or invokes_gate: + wrappers.add(relative) + for job in jobs.values(): + callee = local_callee(job.get("uses")) + if callee is not None: + target = (Path(".github/workflows") / callee).as_posix() + self.assertIn(target, workflows, f"{relative}: missing local reusable workflow {target}") + callers.setdefault(target, set()).add(relative) + + pending = list(wrappers) + while pending: + reached = callers.get(pending.pop(), set()) - wrappers + wrappers.update(reached) + pending.extend(reached) + self.assertTrue(wrappers, "no workflow defines or invokes the verdict-review gate") + trusted = {path for path in gate.TRUST_PATHS if path.startswith(".github/workflows/")} + self.assertEqual(wrappers, trusted) + + def workflow_fixture(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + directory = root / ".github/workflows" + directory.mkdir(parents=True) + (directory / "gate.yml").write_text( + "on: workflow_call\njobs:\n verdict-review-gate:\n runs-on: ubuntu-24.04\n" + " steps:\n - run: python3 scripts/verdict_review_gate.py\n", encoding="utf-8") + return root + + @staticmethod + def write_workflow_caller(root, name, callee): + (root / ".github/workflows" / name).write_text( + f"on: workflow_call\njobs:\n review:\n uses: {callee}\n", encoding="utf-8") + + def workflow_guard_result(self, root, trusted): + case = type(self)("test_workflow_wrappers_match_the_workflow_trust_paths") + case.ROOT = root + result = unittest.TestResult() + with mock.patch.object(gate, "TRUST_PATHS", trusted): + case.run(result) + return result + + def test_reusable_workflow_callers_are_trust_paths_for_both_local_forms(self): + from tests.test_workflow_policy import LOCAL_WORKFLOW_PREFIXES + + for prefix in LOCAL_WORKFLOW_PREFIXES: + with self.subTest(prefix=prefix): + root = self.workflow_fixture() + self.write_workflow_caller(root, "caller.yml", prefix + "gate.yml") + omitted = self.workflow_guard_result(root, (".github/workflows/gate.yml",)) + self.assertEqual(omitted.errors, [], omitted.errors) + self.assertTrue(omitted.failures, "the gate caller was omitted from TRUST_PATHS") + registered = self.workflow_guard_result( + root, (".github/workflows/gate.yml", ".github/workflows/caller.yml")) + self.assertTrue(registered.wasSuccessful(), registered.failures + registered.errors) + + def test_transitive_reusable_workflow_callers_are_trust_paths(self): + from tests.test_workflow_policy import LOCAL_WORKFLOW_PREFIXES + + for prefix in LOCAL_WORKFLOW_PREFIXES: + with self.subTest(prefix=prefix): + root = self.workflow_fixture() + self.write_workflow_caller(root, "middle.yml", prefix + "gate.yml") + self.write_workflow_caller(root, "outer.yml", prefix + "middle.yml") + trusted = (".github/workflows/gate.yml", ".github/workflows/middle.yml", + ".github/workflows/outer.yml") + for caller in trusted[1:]: + with self.subTest(omitted=caller): + omitted = self.workflow_guard_result(root, tuple(path for path in trusted if path != caller)) + self.assertEqual(omitted.errors, [], omitted.errors) + self.assertTrue(omitted.failures, "a transitive gate caller was omitted from TRUST_PATHS") + registered = self.workflow_guard_result(root, trusted) + self.assertTrue(registered.wasSuccessful(), registered.failures + registered.errors) + + def test_missing_reusable_workflow_callees_fail_closed(self): + from tests.test_workflow_policy import LOCAL_WORKFLOW_PREFIXES + + for prefix in LOCAL_WORKFLOW_PREFIXES: + with self.subTest(prefix=prefix): + root = self.workflow_fixture() + self.write_workflow_caller(root, "caller.yml", prefix + "missing.yml") + result = self.workflow_guard_result(root, (".github/workflows/gate.yml",)) + self.assertEqual(result.errors, [], result.errors) + self.assertTrue(result.failures, "a missing local callee was silently omitted") + + def test_unparsable_reusable_workflow_callees_fail_closed(self): + from tests.test_workflow_policy import LOCAL_WORKFLOW_PREFIXES + + for prefix in LOCAL_WORKFLOW_PREFIXES: + with self.subTest(prefix=prefix): + root = self.workflow_fixture() + self.write_workflow_caller(root, "caller.yml", prefix + "broken.yml") + (root / ".github/workflows/broken.yml").write_text( + "on: workflow_call\njobs: [broken]\n", encoding="utf-8") + result = self.workflow_guard_result(root, (".github/workflows/gate.yml",)) + self.assertEqual(result.errors, [], result.errors) + self.assertTrue(result.failures, "an unparsable local callee was silently omitted") + # Runs in a subprocess: an audit hook cannot be removed once added. It records every file opened # for reading under the data root and every module imported from the repository. READ_TRACER = r""" diff --git a/tests/test_workflow_hardening.py b/tests/test_workflow_hardening.py index be919c833..a29c12032 100644 --- a/tests/test_workflow_hardening.py +++ b/tests/test_workflow_hardening.py @@ -550,7 +550,7 @@ class VerdictReviewGateTests(unittest.TestCase): """The required verdict-review-gate job (docs/decisions/2026-09-22-github-automation-closure.md, "verdict-review-gate (2026-09-23)").""" - text = (WORKFLOWS / "validate.yml").read_text(encoding="utf-8") + text = (WORKFLOWS / "pr-metadata.yml").read_text(encoding="utf-8") job = jobs(text)["verdict-review-gate"] def test_runs_on_every_pull_request_without_a_path_filter(self): @@ -559,10 +559,10 @@ def test_runs_on_every_pull_request_without_a_path_filter(self): self.assertNotIn("paths", trigger) self.assertNotIn("branches", trigger.split("pull_request:", 1)[1].split("workflow_dispatch:", 1)[0]) self.assertIn("push:", trigger) - self.assertIsNone(block_if(self.job), "a required check must run on every event") + self.assertIsNone(block_if(self.job.split("\n steps:\n", 1)[0]), "a required check must run on every event") def test_read_only_hardened_and_without_persisted_credentials(self): - self.assertEqual(scopes(self.job), [{"contents": "read"}]) + self.assertEqual(scopes(self.job), [{"contents": "read", "pull-requests": "read"}]) step = first_step(self.job) self.assertIn(HARDEN, step) self.assertIn("egress-policy: audit", step) @@ -578,6 +578,47 @@ def test_pull_request_re_runs_when_its_base_branch_changes(self): (types,) = re.findall(r"(?m)^ types: \[([^\]]*)\]$", pull_request) self.assertEqual([item.strip() for item in types.split(",")], ["opened", "synchronize", "reopened", "edited"]) + def test_validate_does_not_run_for_metadata_edits_or_duplicate_the_metadata_jobs(self): + text = (WORKFLOWS / "validate.yml").read_text(encoding="utf-8") + trigger = text.split("\non:\n", 1)[1].split("\n\n", 1)[0] + pull_request = trigger.split("pull_request:", 1)[1].split("workflow_dispatch:", 1)[0] + (types,) = re.findall(r"(?m)^ types: \[([^\]]*)\]$", pull_request) + self.assertEqual([item.strip() for item in types.split(",")], ["opened", "synchronize", "reopened"]) + self.assertNotIn("edited", types) + self.assertTrue({"sota-sources", "verdict-review-gate"}.isdisjoint(jobs(text))) + self.assertEqual(set(jobs(self.text)), {"sota-sources", "verdict-review-gate"}) + + def test_metadata_edits_queue_without_cancelling_and_without_default_permissions_or_cache(self): + self.assertIn("\npermissions: {}\ncache-mode: none\n", self.text) + self.assertIn(" group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}\n", + self.text) + self.assertIn("\n queue: max\n", self.text) + self.assertNotIn("cancel-in-progress:", self.text) + + @unittest.skipUnless(shutil.which("node"), "node unavailable; CI's runner images carry it") + def test_the_current_base_check_allows_sha_drift_and_fails_on_retarget_or_retrieval_error(self): + from tests.test_sota_sources_gate import inline_script, run_check + step = step_block(self.job, "Require the current PR base") + self.assertEqual(block_if(step), "github.event_name == 'pull_request'") + self.assertIn("actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0", step) + self.assertNotIn("continue-on-error", step) + self.assertLess(self.job.index("Require the current PR base"), self.job.index("Require sealed cross-family")) + event_base = {"ref": "main", "sha": "a" * 40} + payload = {"pull_request": {"number": 42, "base": event_base}} + same, retargeted, advanced, unavailable = run_check(inline_script(step), [ + payload, + {**payload, "current_pull_request": {"base": {**event_base, "ref": "develop"}}}, + {**payload, "current_pull_request": {"base": {**event_base, "sha": "b" * 40}}}, + {**payload, "retrieval_error": "request denied"}, + ]) + self.assertIsNone(same["failed"], same) + self.assertIsNone(advanced["failed"], advanced) + self.assertIn("current PR base differs from the event base; failing closed", retargeted["failed"]) + self.assertIn("request denied", unavailable["failed"]) + for outcome in (same, retargeted, advanced, unavailable): + self.assertEqual(outcome["requests"], [{"owner": "fixture-owner", "repo": "fixture-repo", + "pull_number": 42}]) + def run_script(self): return step_block(self.job, "Require sealed cross-family review").split("run: |", 1)[1] @@ -744,7 +785,7 @@ def test_push_to_main_executes_the_gate_and_fails_closed_without_a_previous_comm # Review of #123, finding 2 (accepted residual): the push-to-main run re-checks after merge. trigger = self.text.split("\non:\n", 1)[1].split("\n\n", 1)[0] self.assertRegex(trigger, r"(?m)^ push:\n branches: \[main\]$") - self.assertIsNone(block_if(self.job)) + self.assertIsNone(block_if(self.job.split("\n steps:\n", 1)[0])) self.assertIsNone(block_if(step_block(self.job, "Require sealed cross-family review"))) run = self.run_script() push = run.split("push)", 1)[1].split(";;", 1)[0] @@ -766,7 +807,7 @@ def test_the_gate_that_runs_is_the_base_commits_copy(self): self.assertIn("set -euo pipefail", run.split('python3 "$gate"', 1)[0]) def test_no_dangerous_trigger_is_added_for_the_residual(self): - # The accepted residual (the PR's own validate.yml can disable the job) is not closed with a + # The accepted residual (the PR's own pr-metadata.yml can disable the job) is not closed with a # privileged trigger: zizmor's dangerous-triggers audit runs with --no-config --no-ignores. for workflow in WORKFLOWS.glob("*.yml"): text = workflow.read_text(encoding="utf-8") @@ -1043,6 +1084,45 @@ def test_validate_uploads_its_verbose_log_even_when_the_suite_fails(self): self.assertIn("${{ matrix.shard }}", name, "each matrix cell must retain a distinct log and report") self.assertLess(job.index(self.SUITE_STEPS[self.VALIDATE]), job.index(self.UPLOAD_STEP)) + def test_validate_reports_native_counts_after_the_suite_without_changing_its_result(self): + job = self.jobs[self.VALIDATE] + summary = step_block(job, "Report unittest counts") + self.assertEqual(block_if(summary), "always()") + self.assertNotIn("continue-on-error", summary) + self.assertLess(job.index(self.SUITE_STEPS[self.VALIDATE]), job.index("Report unittest counts")) + self.assertLess(job.index("Report unittest counts"), job.index(self.UPLOAD_STEP)) + cases = [ + ("Ran 12 tests in 0.123s\n\nOK (skipped=2)\n", "ran 12 tests; skipped 2"), + ("Ran 1 test in 0.123s\n\nFAILED (failures=1, skipped=1)\n", "ran 1 tests; skipped 1"), + ("Ran 4 tests in 0.123s\n\nOK\n", "ran 4 tests; skipped 0"), + ("Ran 4 tests in 0.123s\n\nFAILED (errors=1)\n", "ran 4 tests; skipped 0"), + ("Ran 0 tests in 0.123s\n\nNO TESTS RAN\n", "ran 0 tests; skipped 0"), + ("Ran 4 tests in 0.123s\n", "ran 4 tests; skipped unknown"), + ("interrupted before unittest's summary\n", "ran unknown tests; skipped unknown"), + (None, "ran unknown tests; skipped unknown"), + ] + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + log = directory / "suite/unittest-verbose.log" + log.parent.mkdir() + environment = {**os.environ, "RUNNER_TEMP": str(directory), + "GITHUB_STEP_SUMMARY": str(directory / "summary.md")} + for contents, expected in cases: + with self.subTest(log=contents): + if contents is None: + log.unlink(missing_ok=True) + else: + log.write_text(contents, encoding="utf-8") + result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", run_block(summary)], + env=environment, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(expected, (directory / "summary.md").read_text().splitlines()[-1]) + # An unwritable summary destination cannot turn a passing suite into a failing job. + environment["GITHUB_STEP_SUMMARY"] = str(directory) + result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", run_block(summary)], + env=environment, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + def test_validate_uploader_guard_rejects_renamed_or_missing_path_input(self): """Exercise the real guard with valid YAML mutations, not a duplicate path predicate.""" target = "test_validate_uploads_its_verbose_log_even_when_the_suite_fails" @@ -1107,6 +1187,32 @@ def test_final_required_name_runs_after_failed_cancelled_or_skipped_dependencies self.assertEqual(sum(bool(re.search(r"(?m)^ name: validate$", job)) for job in self.jobs.values()), 1) + def test_validate_guard_rejects_failure_suppression_in_every_step(self): + from tests.test_workflow_policy import load_workflow + + def guarded_result(candidate): + case = type(self)("test_eight_distinct_native_shards_keep_every_cell_running") + case.shards = candidate + result = unittest.TestResult() + case.run(result) + return result + + original = guarded_result(self.shards) + self.assertTrue(original.wasSuccessful(), original.failures + original.errors) + steps = load_workflow(self.workflow)["jobs"]["validate"]["steps"] + for index, step in enumerate(steps): + with self.subTest(step=step["name"]): + block = step_block(self.shards, step["name"]) + lines = [line for line in block.splitlines() + if not re.match(r"^ continue-on-error:", line)] + lines.insert(1, " continue-on-error: true") + candidate = self.shards.replace(block, "\n".join(lines), 1) + mutated = load_workflow("on: push\njobs:\n validate:" + candidate) + self.assertEqual(mutated["jobs"]["validate"]["steps"][index]["continue-on-error"], "true") + result = guarded_result(candidate) + self.assertEqual(result.errors, [], result.errors) + self.assertTrue(result.failures, f"the real shard guard accepted suppression in {step['name']}") + def test_final_step_passes_the_real_needs_result_and_attempt_scoped_reports(self): step = step_block(self.final, "Require complete discovery coverage and success from every shard") self.assertEqual(block_if(step), "always()") diff --git a/tests/test_workflow_security_coverage.py b/tests/test_workflow_security_coverage.py index 4f5a590fa..4fa95445a 100644 --- a/tests/test_workflow_security_coverage.py +++ b/tests/test_workflow_security_coverage.py @@ -69,6 +69,7 @@ def test_all_published_workflows_are_listed_and_covered(self): "claude-pr-toolkit-review.yml", # Its offline zizmor pass/fail assertions live in tests/test_sota_sources_gate.py. "sota-sources-gate.yml", + "pr-metadata.yml", } self.assertEqual( actual, expected, diff --git a/tools/local-pages/source_policy.json b/tools/local-pages/source_policy.json index ffabb3fa2..88dbbe242 100644 --- a/tools/local-pages/source_policy.json +++ b/tools/local-pages/source_policy.json @@ -1864,6 +1864,10 @@ "root": "repo", "path": ".github/workflows/native-token-e2e.yml" }, + { + "root": "repo", + "path": ".github/workflows/pr-metadata.yml" + }, { "root": "repo", "path": ".github/workflows/practice-references-freshness.yml"