diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index d352701e9c5..ccd1132e729 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -11,6 +11,9 @@ name: 'Qwen Autofix' # • every 10m → review phase; issue phase only if no PR needs work # • issues:labeled → issue phase when ready label, state, and sender match # • pull_request_review → review phase for submitted feedback on bot PRs +# (open PRs only: reviews on closed/merged PRs +# drop at the route gate; the scheduled scan is +# the backstop for anything missed) # • pull_request:labeled → maintainer applies autofix/takeover → the loop # manages that PR (human-authored included, and # maintainer FORKS too: the fork's author must @@ -231,6 +234,19 @@ env: # everywhere, and wins when both are present. TAKEOVER_LABEL: 'autofix/takeover' SKIP_LABEL: 'autofix/skip' + # Commit-status context stamped PENDING on a PR head when a scan dispatches + # a review-address leg for it, and re-stamped SUCCESS by the leg on + # checkout. It closes the visibility window between dispatch and leg + # materialization: build-cli runs in between, and until the matrix expands + # the leg does not exist in the live-run jobs view, so an overlapping scan + # would re-dispatch the same PR. The scan treats a PENDING marker fresher + # than DISPATCH_STATUS_TTL_MINUTES as busy (a run that dies before the leg + # materializes leaves a marker that expires by age). Commit statuses only — + # the check-run creation API needs a GitHub App, and this workflow + # authenticates with a PAT. Same-repo heads only get a stamp: a fork head + # sha does not exist in this repo's object store. + DISPATCH_STATUS_CONTEXT: 'qwen-autofix/dispatch-pending' + DISPATCH_STATUS_TTL_MINUTES: '30' # Comment-command sugar over TAKEOVER_LABEL ('' applies it, ' # stop' removes it). Matched EXACTLY against the trimmed comment body. TAKEOVER_COMMAND: '@qwen-code /takeover' @@ -289,8 +305,16 @@ jobs: # branch would trim it — fail closed, command must start the comment. # Both commands are prefiltered here (/takeover toggles the label, /retry # re-arms a stranded PR); everything else never starts a job. + # The pull_request_review clause drops reviews on closed/merged PRs at the + # gate: a PR that can no longer receive commits has nothing to address. + # Finding-reply bursts on just-merged PRs otherwise start one no-op run + # per reply (observed 2026-08-16: 24+ reply reviews on merged #9222 and + # 26 runs on merged #9189 within minutes — issue #9296). The fleet never + # loses a legitimate target from this: the schedule scan engages any open + # PR with review context on its next tick, and address-time revalidation + # already drops targets whose PR closed after dispatch. if: |- - ${{ github.repository == 'QwenLM/qwen-code' && (github.event_name != 'issue_comment' || (github.event.issue.pull_request && (startsWith(github.event.comment.body, '@qwen-code /takeover') || startsWith(github.event.comment.body, '@qwen-code /retry')))) && (github.event_name != 'pull_request' || github.event.label.name == 'autofix/takeover') }} + ${{ github.repository == 'QwenLM/qwen-code' && (github.event_name != 'issue_comment' || (github.event.issue.pull_request && (startsWith(github.event.comment.body, '@qwen-code /takeover') || startsWith(github.event.comment.body, '@qwen-code /retry')))) && (github.event_name != 'pull_request' || github.event.label.name == 'autofix/takeover') && (github.event_name != 'pull_request_review' || github.event.pull_request.state == 'open') }} runs-on: 'ubuntu-latest' timeout-minutes: 5 concurrency: @@ -726,7 +750,7 @@ jobs: ${{ always() && needs.route.outputs.do_issue == 'true' && - (github.event_name != 'schedule' || (needs.review-scan.result == 'success' && needs.review-scan.outputs.has_targets != 'true')) + (github.event_name != 'schedule' || (needs.review-scan.result == 'success' && needs.review-scan.outputs.has_targets != 'true' && needs.review-scan.outputs.enum_failed != 'true')) }} # Secret-bearing and executes agent-driven code, but the agent runs inside # the docker sandbox image and only ever writes a new branch as the @@ -766,7 +790,7 @@ jobs: # triage/tmux jobs. concurrency: group: >- - ${{ needs.route.outputs.do_issue == 'true' && needs.route.outputs.dry_run != 'true' && (github.event_name != 'schedule' || (needs.review-scan.result == 'success' && needs.review-scan.outputs.has_targets != 'true')) && format('qwen-autofix-issue-{0}', needs.route.outputs.issue_number || github.event.issue.number || 'scheduled') || format('qwen-autofix-issue-run-{0}', github.run_id) }} + ${{ needs.route.outputs.do_issue == 'true' && needs.route.outputs.dry_run != 'true' && (github.event_name != 'schedule' || (needs.review-scan.result == 'success' && needs.review-scan.outputs.has_targets != 'true' && needs.review-scan.outputs.enum_failed != 'true')) && format('qwen-autofix-issue-{0}', needs.route.outputs.issue_number || github.event.issue.number || 'scheduled') || format('qwen-autofix-issue-run-{0}', github.run_id) }} cancel-in-progress: false permissions: contents: 'read' @@ -2331,6 +2355,7 @@ jobs: outputs: targets: '${{ steps.scan.outputs.targets }}' has_targets: '${{ steps.scan.outputs.has_targets }}' + enum_failed: '${{ steps.scan.outputs.enum_failed }}' env: REPO: '${{ github.repository }}' steps: @@ -2743,8 +2768,8 @@ jobs: # check-run in statusCheckRollup) does not exist yet, so the rollup # alone misses a just-triggered review; the runs API sees the run # by head SHA before its job starts. Empty on lookup failure — the - # gate then degrades to the rollup check only (fail-open, like - # BUSY_PRS). + # gate then degrades to the rollup check only (fail-open here, + # though the BUSY_PRS enumeration below is fail-closed). REVIEW_WF_ID="$(gh api "repos/${REPO}/actions/workflows/qwen-code-pr-review.yml" --jq '.id' 2> /dev/null || echo '')" REVIEW_RUNS_JSON='{"workflow_runs":[]}' if [[ -n "${REVIEW_WF_ID}" ]] \ @@ -2764,27 +2789,64 @@ jobs: # its queued PRs silently stop looking busy. Filtered this way the # limit applies to LIVE runs only (at most a handful), and one # jobs-view per live run stays cheap. + # + # FAIL-CLOSED: any enumeration failure (the run list, or one run's + # jobs view) empties THIS scan's candidate set. Measured 2026-08-16 + # (#9296): silently swallowing these errors re-dispatched PRs whose + # legs had been running or queued for 3-12 minutes; each duplicate + # burned one build-cli (~5 min) before cancelling a queued sibling + # leg through the per-PR group's latest-wins queue. A duplicate + # costs far more than one skipped scan — the next tick re-inspects + # with fresh reads. Only an EXPLICIT dispatch (workflow_dispatch + # with a PR number) keeps its override semantics and is NOT emptied + # by an enumeration failure — FORCED_PR is ALSO set for trusted + # pull_request_review scans, which are not explicit dispatches. The + # step also emits enum_failed: the scan exits 0, and an emptied set + # would otherwise read exactly like "no PR needs work" and flip the + # scheduled issue phase ON against its declared ordering. The + # dispatch-pending status check below additionally covers the + # window where the leg does not exist yet (behind build-cli). BUSY_PRS=' ' - while IFS= read -r LIVE_RUN; do - [[ -z "${LIVE_RUN}" ]] && continue - while IFS= read -r BUSY; do - [[ -n "${BUSY}" ]] && BUSY_PRS="${BUSY_PRS}${BUSY} " - done < <(gh run view "${LIVE_RUN}" --repo "${REPO}" --json jobs \ - --jq '.jobs[] | select(.status != "completed") | .name | capture("^review-address \\((?[0-9]+),") | .pr' 2> /dev/null) - done < <( - for LIVE_STATUS in in_progress queued; do - # || true: one status query failing must not hide the other. A - # DOUBLE failure yields an empty set — deliberately fail-open: - # this skip is an optimization, and the address-side live-marker - # revalidation is the correctness gate. If that revalidation is - # ever removed, this read must become fail-closed instead. - gh run list --repo "${REPO}" --workflow qwen-autofix.yml \ - --status "${LIVE_STATUS}" --limit 50 --json databaseId \ - --jq '.[].databaseId' 2> /dev/null || true - done | sort -u - ) + BUSY_ENUM_OK=1 + LIVE_RUNS='' + BUSY_ENUM_ERR="$(mktemp)" + for LIVE_STATUS in in_progress queued; do + if ! PART="$(gh run list --repo "${REPO}" --workflow qwen-autofix.yml \ + --status "${LIVE_STATUS}" --limit 50 --json databaseId \ + --jq '.[].databaseId' 2>> "${BUSY_ENUM_ERR}")"; then + BUSY_ENUM_OK=0 + break + fi + LIVE_RUNS="${LIVE_RUNS}${PART}"$'\n' + done + if [[ "${BUSY_ENUM_OK}" == '1' ]]; then + while IFS= read -r LIVE_RUN; do + [[ -z "${LIVE_RUN}" ]] && continue + if ! BUSY_OUT="$(gh run view "${LIVE_RUN}" --repo "${REPO}" --json jobs \ + --jq '.jobs[] | select(.status != "completed") | .name | capture("^review-address \\((?[0-9]+),") | .pr' 2>> "${BUSY_ENUM_ERR}")"; then + BUSY_ENUM_OK=0 + break + fi + while IFS= read -r BUSY; do + [[ -n "${BUSY}" ]] && BUSY_PRS="${BUSY_PRS}${BUSY} " + done <<< "${BUSY_OUT}" + done <<< "$(sort -u <<< "${LIVE_RUNS}")" + fi + if [[ "${BUSY_ENUM_OK}" != '1' ]]; then + BUSY_ENUM_ERR_TAIL="$(tail -c 200 "${BUSY_ENUM_ERR}" 2> /dev/null | tr '\r\n' ' ')" + echo "::warning::busy-PR enumeration failed (run list or jobs view unreadable) — failing closed: no scan targets dispatched this pass${BUSY_ENUM_ERR_TAIL:+ — last error: ${BUSY_ENUM_ERR_TAIL}}" + fleet_row '-' 'fail-closed' "busy enumeration unreadable; scan dispatch skipped this pass (next tick retries)${BUSY_ENUM_ERR_TAIL:+ — last error: ${BUSY_ENUM_ERR_TAIL}}" + echo "enum_failed=true" >> "${GITHUB_OUTPUT}" + if [[ -z "${FORCED_PR}" || "${EVENT_NAME}" != 'workflow_dispatch' ]]; then + CANDIDATES='' + fi + fi + rm -f "${BUSY_ENUM_ERR}" [[ "${BUSY_PRS}" != ' ' ]] && echo "🚧 address in flight/queued for PR(s):${BUSY_PRS}" + # Cutoff for the dispatch-pending marker check in the loop below. + DISPATCH_CUTOFF="$(date -u -d "${DISPATCH_STATUS_TTL_MINUTES} minutes ago" +%Y-%m-%dT%H:%M:%SZ)" + # Idle backoff, from the list's own updatedAt (no API call): a # candidate with no activity for >24h is inspected on about one # scan in four instead of every one. The pool doubled in two @@ -2876,6 +2938,24 @@ jobs: fleet_row "${PR}" 'skipped' "${SKIP_LABEL} label present" continue fi + # Dispatch-pending marker: a scan that dispatched this PR within + # DISPATCH_STATUS_TTL_MINUTES may still be building its CLI + # bundle — its matrix leg does not exist yet, so the live-run + # busy enumeration above cannot see it. The emitting scan stamped + # a PENDING status on the PR head at dispatch (same-repo heads + # only), and the leg re-stamps it SUCCESS on checkout; treat a + # fresh PENDING as busy. Costs no extra API call — the rollup is + # already in PR_META. Unlike the in-memory busy skip this runs + # after the metadata fetch, so it consumes inspection budget; + # acceptable because the case is rare (a PR dispatched <30m ago). + if jq -e --arg ctx "${DISPATCH_STATUS_CONTEXT}" --arg cut "${DISPATCH_CUTOFF}" ' + [.statusCheckRollup[]? | select(.__typename == "StatusContext") + | select(.context == $ctx) | select(.state == "PENDING") + | select((.startedAt // "") > $cut)] | length > 0' <<< "${PR_META}" > /dev/null; then + echo "⏳ #${PR}: dispatch pending (marker fresher than ${DISPATCH_STATUS_TTL_MINUTES}m, leg not materialized yet) — skipping" + fleet_row "${PR}" 'busy' "dispatch-pending marker live (<${DISPATCH_STATUS_TTL_MINUTES}m)" + continue + fi EFF_MAX_ROUNDS="${MAX_ROUNDS}" [[ "${HAS_TAKEOVER}" == "true" ]] && EFF_MAX_ROUNDS="${TAKEOVER_MAX_ROUNDS}" if [[ -z "${BRANCH}" ]]; then @@ -2994,12 +3074,19 @@ jobs: # startedAt is the only staleness clock: a check blocks only if it # started within the bound; one with no startedAt (queued, not yet # running) is not blocking (the next scan re-checks once it starts). + # The dispatch-pending marker is exempted by context: it is this + # loop's own StatusContext busy signal (no .workflowName/.name, so + # it passes the filters above) and its dedicated TTL check above is + # the authority on it — the 330-minute horizon here would keep a + # stranded marker blocking long past its TTL. HAS_PENDING_CHECKS="$(jq -r --arg cut "${PENDING_CUTOFF}" \ - --argjson nonblocking "${NON_BLOCKING_CHECKS}" ' + --argjson nonblocking "${NON_BLOCKING_CHECKS}" \ + --arg ctx "${DISPATCH_STATUS_CONTEXT}" ' [ .[] | select((.status // .state // "") | IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED")) | select(((.workflowName // "") != "Qwen Autofix") or (((.name // "") | startswith("review-address")))) | select((.name // "") as $n | ($nonblocking | index($n)) == null) + | select((.context // "") != $ctx) | select((.startedAt // $cut) > $cut) ] | length > 0 ' <<< "${CHECKS_JSON}")" @@ -3616,10 +3703,30 @@ jobs: --arg win "${REARM_KEY}" --arg hr "${HEAD_REPO_FULL}" \ '. + [{pr: $pr, branch: $branch, issue: $issue, round: $round, watermark: $wm, max_rounds: $mr, win: $win, head_repo: $hr}]' \ <<< "${TARGETS}")" + # Stamp the dispatch-pending marker now, while this scan still + # owns the decision: an overlapping scan inspecting this PR while + # build-cli runs (the leg has not materialized yet) sees the + # PENDING status in its rollup and skips. Same-repo heads only — + # a fork head sha cannot carry a status in this repo; fork + # duplicates stay covered by the address-time revalidation. A + # failed stamp degrades to a warning: the live-run enumeration + # and the address-time gates remain. Dry runs stamp nothing: a + # dry-run leg can die before any release runs (review-address is + # skipped when build-cli fails), and a stranded real PENDING + # would then block real scans — duplicate protection degrades to + # those same surviving layers. + if [[ "${DRY_RUN}" != "true" && "${HEAD_REPO_FULL}" == "${REPO}" ]]; then + gh api "repos/${REPO}/statuses/${LIVE_HEAD}" -X POST \ + -f state="pending" -f context="${DISPATCH_STATUS_CONTEXT}" \ + -f description="review-address dispatched (run ${GITHUB_RUN_ID}); the leg starts once the CLI bundle builds" \ + > /dev/null 2>&1 \ + || echo "⚠️ #${PR}: dispatch-pending status stamp failed (overlapping-scan skip degrades to live-run enumeration)" + fi # Fan out: emit EVERY eligible PR up to the per-scan budget. The # address matrix bounds simultaneity (max-parallel) and the per-PR - # concurrency groups plus the busy-PR skip above prevent duplicate - # same-PR runs, so one scan drains the whole backlog instead of + # concurrency groups plus the busy-PR skip and dispatch-pending + # marker above prevent duplicate same-PR runs, so one scan drains + # the whole backlog instead of # serving a single newest-first target per tick (which starved # older PRs for hours when cron ticks were sparse). The budget # break bounds this loop's RUNTIME and API usage too — each @@ -4380,6 +4487,7 @@ jobs: env: GITHUB_TOKEN: '${{ secrets.CI_DEV_BOT_PAT }}' TRUSTED_PATH: '${{ steps.stage.outputs.trusted_path }}' + DRY_RUN: '${{ needs.route.outputs.dry_run }}' run: |- mkdir -p "${WORKDIR}" # gh has its own $GITHUB_ENV-injectable channels: pin the host and @@ -4426,13 +4534,14 @@ jobs: # UNKNOWN and discards too (fail closed — the next scan re-emits a # still-valid target). PR_LIVE="$(gh pr view "${PR}" --repo "${REPO}" \ - --json state,author,isCrossRepository,baseRefName,headRefName,labels,maintainerCanModify,headRepositoryOwner,headRepository 2> /dev/null || echo '{}')" + --json state,author,isCrossRepository,baseRefName,headRefName,labels,maintainerCanModify,headRepositoryOwner,headRepository,headRefOid 2> /dev/null || echo '{}')" LIVE_STATE="$(jq -r '.state // ""' <<< "${PR_LIVE}")" LIVE_AUTHOR="$(jq -r '.author.login // ""' <<< "${PR_LIVE}")" LIVE_XREPO="$(jq -r 'if has("isCrossRepository") then .isCrossRepository else true end' <<< "${PR_LIVE}")" LIVE_CAN_MODIFY="$(jq -r '.maintainerCanModify // false' <<< "${PR_LIVE}")" LIVE_BASE="$(jq -r '.baseRefName // ""' <<< "${PR_LIVE}")" LIVE_BRANCH="$(jq -r '.headRefName // ""' <<< "${PR_LIVE}")" + LIVE_HEAD_OID="$(jq -r '.headRefOid // ""' <<< "${PR_LIVE}")" # Engagement labels are re-read LIVE too: a takeover label grants a # human-authored PR the author exemption, and a skip label applied # while this job sat queued withdraws consent — both must hold at @@ -4473,6 +4582,18 @@ jobs: fi if [[ -n "${INELIGIBLE}" ]]; then echo "🫥 target no longer eligible (${INELIGIBLE}) — discarding without action or marker" + # Release the dispatch-pending marker on the way out: this discard + # exits before checkout, so the checkout-path release never runs + # and a same-repo head would keep being skipped as dispatch- + # pending for the full TTL with no leg coming — against the + # re-emit promise above. Same-repo heads only (fork heads were + # never stamped) and never in a dry run (nothing was stamped). + if [[ "${DRY_RUN}" != "true" && "${LIVE_XREPO}" == "false" && -n "${LIVE_HEAD_OID}" ]]; then + gh api "repos/${REPO}/statuses/${LIVE_HEAD_OID}" -X POST \ + -f state="success" -f context="${DISPATCH_STATUS_CONTEXT}" \ + -f description="address leg discarded (${INELIGIBLE})" \ + > /dev/null 2>&1 || true + fi { echo "stale=true" echo "conflict=false" @@ -4543,6 +4664,23 @@ jobs: CHECKED_OUT_HEAD="$(git rev-parse HEAD)" echo "checked_out_head=${CHECKED_OUT_HEAD}" >> "${GITHUB_OUTPUT}" + # Release the dispatch-pending marker the emitting scan stamped on + # this head: the leg has materialized, so from here the live-run + # busy enumeration sees it and a PENDING status must not keep + # overlapping scans away. Best-effort — a miss only delays the next + # scan's view, and the marker expires by age anyway. A head that + # moved since the dispatch left the stamp on the old sha; the + # current head's rollup no longer shows it, so this re-stamp lands + # where the next scan actually looks. Same-repo heads only — a fork + # head sha is absent from this repo's object store and was never + # stamped; dry runs stamp nothing either. + if [[ "${DRY_RUN}" != "true" && "${HEAD_REPO:-${REPO}}" == "${REPO}" ]]; then + gh api "repos/${REPO}/statuses/${CHECKED_OUT_HEAD}" -X POST \ + -f state="success" -f context="${DISPATCH_STATUS_CONTEXT}" \ + -f description="address leg started (run ${GITHUB_RUN_ID})" \ + > /dev/null 2>&1 || true + fi + # Does the branch conflict with base? merge-tree computes the merge # without touching the tree; exit 1 means conflicts. UNKNOWN/errors are # treated as no-conflict so we never block on a transient state. diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index a7b150d6dfd..9204dd3f0a0 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -588,6 +588,7 @@ describe('qwen-autofix workflow', () => { ...process.env, PENDING_CUTOFF: '2026-07-21T00:00:00Z', NON_BLOCKING_CHECKS: JSON.stringify(nonBlocking), + DISPATCH_STATUS_CONTEXT: 'qwen-autofix/dispatch-pending', }, encoding: 'utf8', }, @@ -2113,6 +2114,119 @@ describe('qwen-autofix workflow', () => { expect(failed.log).toContain('metadata fetch failed (API error)'); }); + it('releases the dispatch-pending marker when the recheck discards a target', () => { + // The discard exits BEFORE checkout, where the leg-side release lives, so + // a discarded same-repo head would otherwise keep being skipped as + // dispatch-pending for the full TTL with no leg coming — against the + // re-emit promise its own comment makes. Replay the recheck block + // VERBATIM with a recording gh stub: the release must fire on same-repo + // discards and never where it cannot (fork heads were never stamped; + // dry runs stamp nothing; a missing head sha has nothing to stamp). + const recheck = prepareBranchAndFeedbackStep.match( + /(PR_LIVE="\$\(gh pr view[\s\S]*?exit 0\n {10}fi)/, + )?.[1]; + expect(recheck).toBeTruthy(); + // The live fetch carries the head sha the release stamps. + expect(recheck).toContain(',headRefOid'); + expect(recheck).toContain( + 'LIVE_HEAD_OID="$(jq -r \'.headRefOid // ""\' <<< "${PR_LIVE}")"', + ); + expect(recheck).toContain( + 'if [[ "${DRY_RUN}" != "true" && "${LIVE_XREPO}" == "false" && -n "${LIVE_HEAD_OID}" ]]; then', + ); + const runReleaseRecheck = (prJson, { dryRun = 'false' } = {}) => { + const dir = mkdtempSync(join(tmpdir(), 'autofix-release-')); + try { + writeFileSync( + join(dir, 'gh'), + [ + '#!/bin/bash', + `if [[ "$*" == *"/collaborators/"* ]]; then printf '%s' 'write';`, + `elif [[ "$1" == "api" && "$2" == repos/*/statuses/* ]]; then echo "API $*" >> '${join(dir, 'writes.log')}';`, + `else printf '%s' '${JSON.stringify(prJson)}'; fi`, + ].join('\n'), + ); + chmodSync(join(dir, 'gh'), 0o755); + writeFileSync(join(dir, 'writes.log'), ''); + const out = join(dir, 'out.txt'); + writeFileSync(out, ''); + const stdout = execFileSync( + 'bash', + ['-c', `${recheck.replace(/\n {10}/g, '\n')}\nprintf 'PASSED'`], + { + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + PR: '7163', + REPO: 'QwenLM/qwen-code', + BRANCH: 'ci/some-branch', + HEAD_REPO: 'QwenLM/qwen-code', + WATERMARK: '2026-07-18T08:00:00Z', + ROUND: '2', + AUTOFIX_BOT: 'qwen-code-dev-bot', + TAKEOVER_LABEL: 'autofix/takeover', + SKIP_LABEL: 'autofix/skip', + DISPATCH_STATUS_CONTEXT: 'qwen-autofix/dispatch-pending', + DRY_RUN: dryRun, + GITHUB_OUTPUT: out, + GITHUB_TOKEN: 'x', + }, + encoding: 'utf8', + }, + ); + return { + passed: stdout.endsWith('PASSED'), + out: readFileSync(out, 'utf8'), + writes: readFileSync(join(dir, 'writes.log'), 'utf8'), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; + const pr = (over = {}) => ({ + state: 'OPEN', + author: { login: 'qwen-code-dev-bot' }, + isCrossRepository: false, + baseRefName: 'main', + headRefName: 'ci/some-branch', + headRefOid: 'deadbeefcafe', + labels: [], + ...over, + }); + // Same-repo discard → the marker is released on the live head. + const closed = runReleaseRecheck(pr({ state: 'CLOSED' })); + expect(closed.passed).toBe(false); + expect(closed.out).toContain('stale=true'); + expect(closed.writes).toContain( + 'API api repos/QwenLM/qwen-code/statuses/deadbeefcafe -X POST', + ); + expect(closed.writes).toContain('state=success'); + expect(closed.writes).toContain('context=qwen-autofix/dispatch-pending'); + // A skip-label discard releases too (same-repo). + expect( + runReleaseRecheck(pr({ labels: [{ name: 'autofix/skip' }] })).writes, + ).toContain('statuses/deadbeefcafe'); + // Fork discard → no release: fork heads were never stamped. + expect( + runReleaseRecheck( + pr({ isCrossRepository: true, author: { login: 'human' } }), + ).writes, + ).toBe(''); + // Dry run → no release: no stamp was ever written. + expect( + runReleaseRecheck(pr({ state: 'CLOSED' }), { dryRun: 'true' }).writes, + ).toBe(''); + // Missing head sha → nothing to stamp. + expect( + runReleaseRecheck(pr({ state: 'CLOSED', headRefOid: '' })).writes, + ).toBe(''); + // Healthy target passes through WITHOUT a release here — the + // checkout-path release owns that stamp. + const ok = runReleaseRecheck(pr()); + expect(ok.passed).toBe(true); + expect(ok.writes).toBe(''); + }); + it('falls back to existing issue backlog only when review has no target', () => { expect(issueAutofixJob).toContain("needs: ['route', 'review-scan']"); // Anchor the job `if` opening: a bare toContain('always()') is also @@ -2122,7 +2236,7 @@ describe('qwen-autofix workflow', () => { ); expect(issueAutofixJob).toContain("needs.review-scan.result == 'success'"); expect(issueAutofixJob).toContain( - "github.event_name != 'schedule' || (needs.review-scan.result == 'success' && needs.review-scan.outputs.has_targets != 'true')", + "github.event_name != 'schedule' || (needs.review-scan.result == 'success' && needs.review-scan.outputs.has_targets != 'true' && needs.review-scan.outputs.enum_failed != 'true')", ); expect(findCandidateIssuesStep).toContain('OPEN_AUTOFIX_PR_COUNT'); expect(findCandidateIssuesStep).toContain('MAX_OPEN_AUTOFIX_PRS'); @@ -2281,7 +2395,7 @@ describe('qwen-autofix workflow', () => { // run-unique suffix appended as a continuation line would also become // part of the group value while a trailing-newline anchor stayed green. expect(issueAutofixJob).toContain( - "group: >-\n ${{ needs.route.outputs.do_issue == 'true' && needs.route.outputs.dry_run != 'true' && (github.event_name != 'schedule' || (needs.review-scan.result == 'success' && needs.review-scan.outputs.has_targets != 'true')) && format('qwen-autofix-issue-{0}', needs.route.outputs.issue_number || github.event.issue.number || 'scheduled') || format('qwen-autofix-issue-run-{0}', github.run_id) }}\n cancel-in-progress: false", + "group: >-\n ${{ needs.route.outputs.do_issue == 'true' && needs.route.outputs.dry_run != 'true' && (github.event_name != 'schedule' || (needs.review-scan.result == 'success' && needs.review-scan.outputs.has_targets != 'true' && needs.review-scan.outputs.enum_failed != 'true')) && format('qwen-autofix-issue-{0}', needs.route.outputs.issue_number || github.event.issue.number || 'scheduled') || format('qwen-autofix-issue-run-{0}', github.run_id) }}\n cancel-in-progress: false", ); expect(issueAutofixJob).not.toContain('|| github.run_id }}'); // The group identity and the scan step's FORCED_ISSUE env are @@ -3748,6 +3862,321 @@ describe('qwen-autofix workflow', () => { expect(reviewScanJob).toContain('still-red checks'); }); + it('fails closed on busy-enumeration failure, keeps explicit dispatches, and signals the issue phase', () => { + // Measured 2026-08-16 (#9296): a swallowed enumeration failure + // re-dispatched PRs with live legs. The fail-closed block must (a) empty + // the candidate set, (b) keep ONLY explicit workflow_dispatch dispatches + // (FORCED_PR is also set for trusted pull_request_review scans — those + // stay fail-closed), (c) emit enum_failed so the scheduled issue phase + // cannot flip ON over an emptied set (the scan exits 0, so has_targets + // alone would read exactly like "no PR needs work"), and (d) carry the + // underlying error tail — transient API instability and PAT decay need + // different oncall responses. Replay the block VERBATIM with a stubbed + // gh: the string pins alone would stay green on a flipped carve-out or a + // dropped enum_failed echo. + expect(reviewScanJob).toContain('BUSY_ENUM_OK=1'); + expect(reviewScanJob).toContain( + "enum_failed: '${{ steps.scan.outputs.enum_failed }}'", + ); + const enumBlock = reviewScanJob.match( + /BUSY_PRS=' '\n[\s\S]*?rm -f "\$\{BUSY_ENUM_ERR\}"/, + )?.[0]; + expect(enumBlock).toBeTruthy(); + // The carve-out is bounded to explicit dispatches (the cap-refused gate + // splits on EVENT_NAME the same way). + expect(enumBlock).toContain( + 'if [[ -z "${FORCED_PR}" || "${EVENT_NAME}" != \'workflow_dispatch\' ]]; then', + ); + // The warning and the fleet row carry the captured error tail. + expect(enumBlock).toContain( + 'failing closed: no scan targets dispatched this pass${BUSY_ENUM_ERR_TAIL:+ — last error: ${BUSY_ENUM_ERR_TAIL}}', + ); + const runEnum = ({ + listAnswer = '', + listError = '', + viewAnswer = '', + viewError = '', + forcedPr = '', + eventName = 'schedule', + }) => { + const dir = mkdtempSync(join(tmpdir(), 'autofix-enum-')); + try { + writeFileSync( + join(dir, 'gh'), + [ + '#!/bin/bash', + '# Emulate the gh CLI contract: answer with the fixture payload,', + '# applying the --jq filter when one is passed (like real gh).', + "jq_filter=''", + 'positional=()', + 'while [[ $# -gt 0 ]]; do', + ' case "$1" in', + ' --jq) jq_filter="$2"; shift 2 ;;', + ' *) positional+=("$1"); shift ;;', + ' esac', + 'done', + 'set -- "${positional[@]}"', + 'if [[ "$1" == "run" && "$2" == "list" ]]; then', + ' if [[ -n "${ENUM_LIST_ERROR}" ]]; then printf \'%s\' "${ENUM_LIST_ERROR}" >&2; exit 1; fi', + ' payload="${ENUM_LIST_ANSWER}"', + 'elif [[ "$1" == "run" && "$2" == "view" ]]; then', + ' if [[ -n "${ENUM_VIEW_ERROR}" ]]; then printf \'%s\' "${ENUM_VIEW_ERROR}" >&2; exit 1; fi', + ' payload="${ENUM_VIEW_ANSWER}"', + 'else', + ' exit 0', + 'fi', + 'if [[ -n "$jq_filter" ]]; then printf \'%s\' "$payload" | jq -r "$jq_filter"; else printf \'%s\' "$payload"; fi', + ].join('\n'), + ); + chmodSync(join(dir, 'gh'), 0o755); + const outFile = join(dir, 'out.txt'); + const fleetFile = join(dir, 'fleet.tsv'); + writeFileSync(outFile, ''); + writeFileSync(fleetFile, ''); + const stdout = execFileSync( + 'bash', + [ + '-c', + [ + 'set -uo pipefail', + `FLEET_FILE='${fleetFile}'`, + 'fleet_row() { printf \'%s\\t%s\\t%s\\n\' "$1" "$2" "$3" >> "${FLEET_FILE}"; }', + "CANDIDATES='101 102'", + enumBlock.replace(/\n {10}/g, '\n'), + 'printf \'CANDIDATES=[%s]|BUSY=[%s]\' "${CANDIDATES}" "${BUSY_PRS}"', + ].join('\n'), + ], + { + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + FORCED_PR: forcedPr, + EVENT_NAME: eventName, + REPO: 'QwenLM/qwen-code', + GITHUB_OUTPUT: outFile, + ENUM_LIST_ANSWER: listAnswer, + ENUM_LIST_ERROR: listError, + ENUM_VIEW_ANSWER: viewAnswer, + ENUM_VIEW_ERROR: viewError, + }, + encoding: 'utf8', + }, + ); + return { + candidates: stdout.match(/CANDIDATES=\[(.*)\]\|/)?.[1], + busy: stdout.match(/BUSY=\[(.*)\]/)?.[1], + out: readFileSync(outFile, 'utf8'), + fleet: readFileSync(fleetFile, 'utf8'), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; + // A run-list failure empties the candidates on a scheduled scan and + // emits the issue-phase signal with the error tail attached. + const listFailed = runEnum({ listError: 'HTTP 502: bad gateway' }); + expect(listFailed.candidates).toBe(''); + expect(listFailed.out).toContain('enum_failed=true'); + expect(listFailed.fleet).toContain('fail-closed'); + expect(listFailed.fleet).toContain('HTTP 502: bad gateway'); + // A jobs-view failure mid-enumeration fails closed the same way. + const viewFailed = runEnum({ + listAnswer: JSON.stringify([{ databaseId: 101 }]), + viewError: 'HTTP 500', + }); + expect(viewFailed.candidates).toBe(''); + expect(viewFailed.out).toContain('enum_failed=true'); + expect(viewFailed.fleet).toContain('HTTP 500'); + // A healthy enumeration accumulates the busy set and keeps candidates. + const ok = runEnum({ + listAnswer: JSON.stringify([{ databaseId: 101 }]), + viewAnswer: JSON.stringify({ + jobs: [ + { name: 'review-address (101, round 2)', status: 'in_progress' }, + ], + }), + }); + expect(ok.candidates).toBe('101 102'); + expect(ok.busy).toBe(' 101 '); + expect(ok.out).not.toContain('enum_failed'); + expect(ok.fleet).not.toContain('fail-closed'); + // ONLY an explicit workflow_dispatch dispatch keeps its candidates + // through an enumeration failure... + const explicit = runEnum({ + listError: 'x', + forcedPr: '77', + eventName: 'workflow_dispatch', + }); + expect(explicit.candidates).toBe('101 102'); + expect(explicit.out).toContain('enum_failed=true'); + // ...a trusted pull_request_review scan ALSO carries FORCED_PR but is + // not an explicit dispatch and stays fail-closed. + const reviewEvent = runEnum({ + listError: 'x', + forcedPr: '77', + eventName: 'pull_request_review', + }); + expect(reviewEvent.candidates).toBe(''); + }); + + it('pins the dispatch-pending marker lifecycle across its check, stamp, and release sites', () => { + // The marker closes the dispatch→leg visibility window (#9296). Its + // reader and writers live in three different steps; a context mismatch + // between them leaves the marker permanently unread, and a flipped TTL + // comparison re-opens the duplicate window — pin both behaviorally, and + // pin the HAS_PENDING_CHECKS exemption that keeps a stranded marker from + // blocking past its TTL. + expect(workflow).toContain( + "DISPATCH_STATUS_CONTEXT: 'qwen-autofix/dispatch-pending'", + ); + expect(workflow).toContain("DISPATCH_STATUS_TTL_MINUTES: '30'"); + expect(reviewScanJob).toContain( + 'DISPATCH_CUTOFF="$(date -u -d "${DISPATCH_STATUS_TTL_MINUTES} minutes ago"', + ); + // The freshness predicate's comparison direction: flipped, fresh markers + // would pass and stale markers block. + expect(reviewScanJob).toContain('select((.startedAt // "") > $cut)'); + // Behavioral replay of the scan-side skip predicate over fixture + // rollups. + const markerCheck = reviewScanJob.match( + /if jq -e --arg ctx "\$\{DISPATCH_STATUS_CONTEXT\}" --arg cut "\$\{DISPATCH_CUTOFF\}" '[\s\S]*?' <<< "\$\{PR_META\}" > \/dev\/null; then/, + )?.[0]; + expect(markerCheck).toBeTruthy(); + const runMarkerCheck = (rollup) => + execFileSync( + 'bash', + [ + '-c', + `${markerCheck.replace(/\n {12}/g, '\n')} printf skip; else printf pass; fi`, + ], + { + env: { + ...process.env, + DISPATCH_STATUS_CONTEXT: 'qwen-autofix/dispatch-pending', + DISPATCH_CUTOFF: '2026-08-17T06:00:00Z', + PR_META: JSON.stringify({ statusCheckRollup: rollup }), + }, + encoding: 'utf8', + }, + ); + const marker = ( + state, + startedAt, + context = 'qwen-autofix/dispatch-pending', + ) => ({ + __typename: 'StatusContext', + context, + state, + startedAt, + }); + // Fresh PENDING marker → busy. + expect(runMarkerCheck([marker('PENDING', '2026-08-17T06:30:00Z')])).toBe( + 'skip', + ); + // Stale PENDING marker (past the TTL) → dispatchable again. + expect(runMarkerCheck([marker('PENDING', '2026-08-17T05:00:00Z')])).toBe( + 'pass', + ); + // The leg materialized (SUCCESS re-stamp) → dispatchable. + expect(runMarkerCheck([marker('SUCCESS', '2026-08-17T06:30:00Z')])).toBe( + 'pass', + ); + // A foreign status context → dispatchable (the marker is keyed exactly). + expect( + runMarkerCheck([marker('PENDING', '2026-08-17T06:30:00Z', 'other-ci')]), + ).toBe('pass'); + // No rollup entries → dispatchable. + expect(runMarkerCheck([])).toBe('pass'); + + // A stranded marker must NOT keep blocking through the 330-minute + // HAS_PENDING_CHECKS gate after its TTL expired: replay the gate's jq + // over fixture rollups. + const pendingGate = reviewScanJob.match( + /HAS_PENDING_CHECKS="\$\(jq -r --arg cut "\$\{PENDING_CUTOFF\}" \\[\s\S]*?' <<< "\$\{CHECKS_JSON\}"\)"/, + )?.[0]; + expect(pendingGate).toBeTruthy(); + expect(pendingGate).toContain('--arg ctx "${DISPATCH_STATUS_CONTEXT}"'); + expect(pendingGate).toContain('select((.context // "") != $ctx)'); + const runPendingGate = (checks) => + execFileSync( + 'bash', + [ + '-c', + `${pendingGate.replace(/\n {12}/g, '\n')}\nprintf '%s' "$HAS_PENDING_CHECKS"`, + ], + { + env: { + ...process.env, + PENDING_CUTOFF: '2026-08-17T02:30:00Z', + NON_BLOCKING_CHECKS: '["review-pr"]', + DISPATCH_STATUS_CONTEXT: 'qwen-autofix/dispatch-pending', + CHECKS_JSON: JSON.stringify(checks), + }, + encoding: 'utf8', + }, + ); + const checkRun = (name, status, startedAt) => ({ + name, + workflowName: 'CI', + status, + startedAt, + }); + // A stranded marker ALONE blocks nobody... + expect(runPendingGate([marker('PENDING', '2026-08-17T07:00:00Z')])).toBe( + 'false', + ); + // ...a genuine in-flight check still blocks... + expect( + runPendingGate([ + checkRun('build', 'IN_PROGRESS', '2026-08-17T07:50:00Z'), + ]), + ).toBe('true'); + // ...including alongside the marker... + expect( + runPendingGate([ + marker('PENDING', '2026-08-17T07:00:00Z'), + checkRun('build', 'IN_PROGRESS', '2026-08-17T07:50:00Z'), + ]), + ).toBe('true'); + // ...a check stuck past the 330-minute horizon is aged out... + expect( + runPendingGate([ + checkRun('build', 'IN_PROGRESS', '2026-08-17T01:00:00Z'), + ]), + ).toBe('false'); + // ...and a foreign PENDING status context still blocks (the exemption is + // exactly the marker's own context). + expect( + runPendingGate([marker('PENDING', '2026-08-17T07:50:00Z', 'other-ci')]), + ).toBe('true'); + + // Writer/reader identity: all three status writes and both readers bind + // the SAME context variable — a mismatch on any site leaves the marker + // permanently unread (the fork-bridge tests pin their cross-site signal + // the same way). + const stampSites = + workflow.match( + /gh api "repos\/\$\{REPO\}\/statuses\/[^"]+" -X POST \\\n\s*-f state="\w+" -f context="\$\{DISPATCH_STATUS_CONTEXT\}"/g, + ) ?? []; + expect(stampSites).toHaveLength(3); + expect( + (reviewScanJob.match(/--arg ctx "\$\{DISPATCH_STATUS_CONTEXT\}"/g) ?? []) + .length, + ).toBe(2); + // Every write site is guarded same-repo and dry-run: fork head shas are + // absent from this repo's object store, and a dry run must leave no + // real, PR-visible status behind. + expect(reviewScanJob).toContain( + 'if [[ "${DRY_RUN}" != "true" && "${HEAD_REPO_FULL}" == "${REPO}" ]]; then', + ); + expect(prepareBranchAndFeedbackStep).toContain( + 'if [[ "${DRY_RUN}" != "true" && "${HEAD_REPO:-${REPO}}" == "${REPO}" ]]; then', + ); + expect(prepareBranchAndFeedbackStep).toContain( + "DRY_RUN: '${{ needs.route.outputs.dry_run }}'", + ); + }); + it('behaviorally replays the takeover-command toggle across all four paths', () => { // Extract the toggle VERBATIM (drift fails the test) and replay it with // a PATH-stubbed gh that records writes: add+absent applies the label, @@ -15527,6 +15956,9 @@ exit 1 '--argjson', 'nonblocking', '[]', + '--arg', + 'ctx', + 'qwen-autofix/dispatch-pending', jqFilter, ], { input: JSON.stringify(checks), encoding: 'utf8' },