From 213f3e987a47e890b3ace6d0a91e3f2e8c04b556 Mon Sep 17 00:00:00 2001 From: wenshao Date: Thu, 23 Jul 2026 12:07:17 +0800 Subject: [PATCH 1/5] feat(autofix): auto-rerun a check that died on infrastructure, once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed check can be red because the machine died, not the code — a self-hosted runner losing the server, the disk filling. #7490's E2E failed with "runner lost communication with the server" and went green on a rerun. The scan now reruns such a check's failed jobs automatically. Detection is a conservative annotation whitelist (INFRA_FAILURE_SIGNATURES) — only unambiguous machine failures, never a test-level timeout, which could be a real regression. The one-shot guard is run_attempt, not a marker: a run already retried to attempt 2 and still infra-failing is persistent, so it is left for a human; after a rerun the attempt increments, so the next scan will not rerun it. Every step is fail-safe (any API error → no rerun), it runs only when the PR actually has a failed check, and the gate carries the same review-address carve-out as the other check selectors so the loop never reruns its own runs. This is the transient-infra sibling of #7554 (stale-base): that merges current main when a check is base-inherited; this reruns when a check died on the runner. Neither touches a check that is a genuine failure. Note: rerun-failed-jobs needs the PAT to hold `actions: write`. --- .github/workflows/qwen-autofix.yml | 52 +++++++++ scripts/tests/qwen-autofix-workflow.test.js | 118 ++++++++++++++++++++ 2 files changed, 170 insertions(+) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 4c8d9ef1579..563f910a7e4 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -129,6 +129,15 @@ env: # invisible to the scan even when it already had unaddressed feedback. # Names must match job ids in qwen-code-pr-review.yml; a test pins that. NON_BLOCKING_CHECKS: '["review-pr"]' + # Failed-check annotation patterns that mean the INFRASTRUCTURE died, not the + # code — a self-hosted runner losing the server, the disk filling, a runner + # shutdown. Such a check is red for a reason unrelated to the PR and clears on + # a re-run (observed: #7490's E2E "runner lost communication" → green on the + # rerun). The scan auto-reruns those failed jobs ONCE, guarded by run_attempt + # so a persistent infra problem cannot loop. Deliberately conservative — only + # unambiguous machine failures, never a test-level timeout, which could be a + # real regression. Case-insensitive, matched against the check's annotations. + INFRA_FAILURE_SIGNATURES: 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (?:was|has been) (?:lost|terminated)' # Upper bound on review targets emitted per scan (fan-out defense-in-depth; # excess is logged and deferred to the next scan). MAX_TARGETS_PER_SCAN: '10' @@ -1809,6 +1818,49 @@ jobs: ISSUE="${PR}" fi CHECKS_JSON="$(jq -c '.statusCheckRollup // []' <<< "${PR_META}")" + + # Auto-rerun a check that died on INFRASTRUCTURE, not the code (see + # INFRA_FAILURE_SIGNATURES). Only reached when the PR has a FAILED + # check; then, for each, we read its annotations and — if they carry + # a machine-death signature — rerun that run's failed jobs ONCE. The + # once is enforced by run_attempt: a run already retried to attempt 2 + # and still infra-failing is persistent, so we stop and leave it. No + # marker needed; the attempt counter is the guard, and after a rerun + # the attempt increments so the next scan skips it. Any API failure + # here is fail-safe: it just means no rerun. + PR_HEAD_OID="$(jq -r '.headRefOid // ""' <<< "${PR_META}")" + if [[ -n "${PR_HEAD_OID}" ]] && jq -e 'any(.[]; ((.conclusion // .state // "") | IN("FAILURE","FAILED","ERROR","TIMED_OUT","ACTION_REQUIRED")) and (((.workflowName // "") != "Qwen Autofix") or ((.name // "") | startswith("review-address"))))' <<< "${CHECKS_JSON}" > /dev/null 2>&1; then + RERAN_INFRA=false + # Failed check-runs on this head, with their run id and annotation + # count — fetched once. External statuses (no check-run) are absent + # here, which is fine: only Actions runs can be reran. + FAILED_CRS="$(gh api --paginate "repos/${REPO}/commits/${PR_HEAD_OID}/check-runs" \ + --jq '.check_runs[] | select(.conclusion == "failure") + | select(((.output.annotations_count // 0) > 0)) + | [(.id | tostring), (.details_url // "")] | @tsv' 2> /dev/null | sort -u || true)" + while IFS=$'\t' read -r CR_ID DETAILS_URL; do + [[ -n "${CR_ID}" && "${DETAILS_URL}" == *"/actions/runs/"* ]] || continue + RUN_ID="${DETAILS_URL#*/actions/runs/}" + RUN_ID="${RUN_ID%%/*}" + [[ "${RUN_ID}" =~ ^[0-9]+$ ]] || continue + ANNS="$(gh api "repos/${REPO}/check-runs/${CR_ID}/annotations" \ + --jq '[.[].message] | join("\n")' 2> /dev/null || true)" + grep -qiE "${INFRA_FAILURE_SIGNATURES}" <<< "${ANNS}" || continue + ATTEMPT="$(gh api "repos/${REPO}/actions/runs/${RUN_ID}" --jq '.run_attempt // 0' 2> /dev/null || echo 0)" + if [[ "${ATTEMPT}" == '1' ]]; then + if gh api -X POST "repos/${REPO}/actions/runs/${RUN_ID}/rerun-failed-jobs" > /dev/null 2>&1; then + echo "♻️ #${PR}: a check died on infrastructure (run ${RUN_ID}, attempt 1) — reran its failed jobs; not the PR's code" + fleet_row "${PR}" 'infra-reran' "infra failure (run ${RUN_ID}) reran once" + RERAN_INFRA=true + break + fi + else + echo "⚠️ #${PR}: infra failure persisted after a rerun (run ${RUN_ID}, attempt ${ATTEMPT}) — leaving for a human" + fi + done <<< "${FAILED_CRS}" + [[ "${RERAN_INFRA}" == 'true' ]] && continue + fi + # startedAt is the only staleness clock: a check blocks only if it # started within the bound; one with no startedAt (queued, not yet # running) is not blocking (the next scan re-checks once it starts). diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index f344e75b413..1e20a41c2b3 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -401,6 +401,124 @@ describe('qwen-autofix workflow', () => { expect(run([{ ...llm, name: 'resolve-pr' }])).toBe('true'); }); + it('auto-reruns a check that died on infrastructure, once, guarded by run_attempt', () => { + // A self-hosted runner losing the server (or the disk filling) reds a check + // for a reason unrelated to the PR; it clears on a rerun (#7490's E2E: + // "runner lost communication" → green on the rerun). The scan reruns such a + // failed job ONCE, and run_attempt is the guard: a run already at attempt 2 + // and still infra-failing is persistent and left alone — no infinite loop. + const block = reviewScanJob.match( + /( {12}PR_HEAD_OID="\$\(jq -r '\.headRefOid[\s\S]*?\n {12}fi\n)\n {12}# startedAt is the only staleness/, + )?.[1]; + expect(block).toBeTruthy(); + const script = block.replace(/^ {12}/gm, ''); + + const run = ({ checks, annotations, attempt = 1, rerunOk = true }) => { + const dir = mkdtempSync(join(tmpdir(), 'infra-')); + const bin = join(dir, 'bin'); + mkdirSync(bin); + // Stubbed gh: check-runs → one failed run-1 check-run with annotations; + // annotations → the given message; runs/{id} → run_attempt; POST + // rerun-failed-jobs → success/fail. Records the rerun POST. + // The workflow calls check-runs with a --jq filter that yields, per + // failed check-run WITH annotations, a `\t` line; the + // stub emits what that filter would produce (a single line when there is + // an annotation, nothing otherwise) rather than raw JSON the stub can't + // filter. + const crTsv = annotations + ? '42\thttps://github.com/o/r/actions/runs/9001/job/5\n' + : ''; + writeFileSync( + join(bin, 'gh'), + [ + '#!/usr/bin/env bash', + `echo "$*" >> ${JSON.stringify(join(dir, 'calls.log'))}`, + 'args="$*"', + `case "$args" in`, + // %b so the \t/\n in the stubbed tsv become a real tab/newline (the + // filter's @tsv output), which `IFS=$'\\t' read` then splits. + ` *"/commits/"*"/check-runs"*) printf '%b' ${JSON.stringify(crTsv)}; exit 0;;`, + ` *"/check-runs/42/annotations"*) printf '%s' ${JSON.stringify(annotations || '')}; exit 0;;`, + ` *"/actions/runs/9001"*"rerun-failed-jobs"*) exit ${rerunOk ? 0 : 1};;`, + ` *"/actions/runs/9001"*) printf '${attempt}'; exit 0;;`, + 'esac', + 'exit 0', + ].join('\n'), + ); + chmodSync(join(bin, 'gh'), 0o755); + const out = execFileSync( + 'bash', + [ + '-c', + `set -uo pipefail\nfleet_row(){ :; }\nfor _ in x; do\n${script}\nprintf 'FELL_THROUGH'\ndone`, + ], + { + env: { + ...process.env, + REPO: 'o/r', + PR: '1', + PR_META: JSON.stringify({ headRefOid: 'headSHA' }), + CHECKS_JSON: JSON.stringify(checks), + INFRA_FAILURE_SIGNATURES: + 'lost communication with the server|No space left on device', + PATH: `${bin}:${process.env.PATH}`, + }, + encoding: 'utf8', + }, + ); + const calls = existsSync(join(dir, 'calls.log')) + ? readFileSync(join(dir, 'calls.log'), 'utf8') + : ''; + rmSync(dir, { recursive: true, force: true }); + return { + reran: /rerun-failed-jobs/.test(calls), + continued: !out.includes('FELL_THROUGH'), + }; + }; + const FAIL = { name: 'E2E', conclusion: 'FAILURE' }; + const OK = { name: 'E2E', conclusion: 'SUCCESS' }; + + // Infra death (runner lost the server) on attempt 1 → rerun & skip. + expect( + run({ + checks: [FAIL], + annotations: + 'The self-hosted runner lost communication with the server', + }), + ).toEqual({ reran: true, continued: true }); + // A REAL failure (no infra signature in the annotation) → never rerun; the + // agent/human handles it. This is the gate that stops masking real bugs. + expect( + run({ + checks: [FAIL], + annotations: 'Expected 1 argument but got 2 — src/foo.ts:10', + }), + ).toEqual({ reran: false, continued: false }); + // Already reran once (attempt 2) and still infra-failing → persistent, do + // not loop. + expect( + run({ + checks: [FAIL], + annotations: 'No space left on device', + attempt: 2, + }), + ).toEqual({ reran: false, continued: false }); + // No failed check at all → the block is skipped entirely. + expect(run({ checks: [OK], annotations: '' })).toEqual({ + reran: false, + continued: false, + }); + // Infra signature but the rerun POST fails (e.g. PAT lacks actions:write) → + // no crash, falls through to normal processing. + expect( + run({ + checks: [FAIL], + annotations: 'No space left on device', + rerunOk: false, + }), + ).toEqual({ reran: true, continued: false }); + }); + it('keeps a still-red check visible, but only once per head', () => { // A red check is a STATE, not the instant it turned red. Counting only // "failed since the watermark" made a still-failing PR invisible the From 3ffac12ac7e06a3335ca2da2c2daaaa5824434b9 Mon Sep 17 00:00:00 2001 From: Qwen Code Bot Date: Thu, 23 Jul 2026 04:52:16 +0000 Subject: [PATCH 2/5] fix(autofix): use POSIX ERE groups in infra-failure regex, cover all signatures in tests (#7562) --- .github/workflows/qwen-autofix.yml | 2 +- scripts/tests/qwen-autofix-workflow.test.js | 41 ++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 563f910a7e4..ce5b656fe8c 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -137,7 +137,7 @@ env: # so a persistent infra problem cannot loop. Deliberately conservative — only # unambiguous machine failures, never a test-level timeout, which could be a # real regression. Case-insensitive, matched against the check's annotations. - INFRA_FAILURE_SIGNATURES: 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (?:was|has been) (?:lost|terminated)' + INFRA_FAILURE_SIGNATURES: 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (was|has been) (lost|terminated)' # Upper bound on review targets emitted per scan (fan-out defense-in-depth; # excess is logged and deferred to the next scan). MAX_TARGETS_PER_SCAN: '10' diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 1e20a41c2b3..22513e6ddf0 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -460,7 +460,7 @@ describe('qwen-autofix workflow', () => { PR_META: JSON.stringify({ headRefOid: 'headSHA' }), CHECKS_JSON: JSON.stringify(checks), INFRA_FAILURE_SIGNATURES: - 'lost communication with the server|No space left on device', + 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (was|has been) (lost|terminated)', PATH: `${bin}:${process.env.PATH}`, }, encoding: 'utf8', @@ -517,6 +517,45 @@ describe('qwen-autofix workflow', () => { rerunOk: false, }), ).toEqual({ reran: true, continued: false }); + // Each remaining production signature also triggers a rerun. + for (const msg of [ + 'ENOSPC', + 'The runner has received a shutdown signal', + 'The runner has received an unexpected signal', + 'Failed to initialize container for job', + 'The runner was lost', + 'The runner was terminated', + 'The runner has been lost', + 'The runner has been terminated', + ]) { + expect(run({ checks: [FAIL], annotations: msg })).toEqual({ + reran: true, + continued: true, + }); + } + // Self-trigger guard: a "Qwen Autofix" workflow's own failed check must NOT + // be rerun (prevents the autofix from re-triggering itself), UNLESS the + // check is a review-address job (the exception carved out in the jq filter). + const AUTOFIX_CHECK = { + name: 'E2E', + conclusion: 'FAILURE', + workflowName: 'Qwen Autofix', + }; + expect( + run({ checks: [AUTOFIX_CHECK], annotations: 'No space left on device' }), + ).toEqual({ reran: false, continued: false }); + expect( + run({ + checks: [ + { + name: 'review-address issue-123', + conclusion: 'FAILURE', + workflowName: 'Qwen Autofix', + }, + ], + annotations: 'No space left on device', + }), + ).toEqual({ reran: true, continued: true }); }); it('keeps a still-red check visible, but only once per head', () => { From 3cd21953aa61d9166ba93aad5b2b541291b060a9 Mon Sep 17 00:00:00 2001 From: wenshao Date: Thu, 23 Jul 2026 13:11:36 +0800 Subject: [PATCH 3/5] fix(autofix): also treat a git fetch/clone transport death as infra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #6506's checkout died mid-transfer — "fetch-pack: invalid index-pack output" and "RPC failed; curl 92 ... CANCEL" — which then hung the job into the 20m limit. That is infra, not the PR (it only touches a doc), and a re-run made it green. But the infra-signature whitelist did not cover it, so the auto-rerun did not fire and it waited on a human. Add `invalid index-pack output` and `RPC failed` — the two canonical git-transport-death phrases — to INFRA_FAILURE_SIGNATURES. A co-present job-timeout line does not block the match (one matching line classifies the run), and a BARE timeout with no transport signature is still left alone, since it can be a real regression. Both new signatures are pinned in the test's per-signature loop, plus a case on #6506's real composite annotation and a bare-timeout-is-not-rerun guard. --- .github/workflows/qwen-autofix.yml | 17 +++++++----- scripts/tests/qwen-autofix-workflow.test.js | 30 +++++++++++++++++++-- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index ce5b656fe8c..d9b55c009f7 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -131,13 +131,16 @@ env: NON_BLOCKING_CHECKS: '["review-pr"]' # Failed-check annotation patterns that mean the INFRASTRUCTURE died, not the # code — a self-hosted runner losing the server, the disk filling, a runner - # shutdown. Such a check is red for a reason unrelated to the PR and clears on - # a re-run (observed: #7490's E2E "runner lost communication" → green on the - # rerun). The scan auto-reruns those failed jobs ONCE, guarded by run_attempt - # so a persistent infra problem cannot loop. Deliberately conservative — only - # unambiguous machine failures, never a test-level timeout, which could be a - # real regression. Case-insensitive, matched against the check's annotations. - INFRA_FAILURE_SIGNATURES: 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (was|has been) (lost|terminated)' + # shutdown, or a git fetch/clone dying mid-transfer. Such a check is red for a + # reason unrelated to the PR and clears on a re-run (observed: #7490's E2E + # "runner lost communication"; #6506's checkout "RPC failed; curl 92" / + # "fetch-pack: invalid index-pack output" — both green on the rerun). The scan + # auto-reruns those failed jobs ONCE, guarded by run_attempt so a persistent + # infra problem cannot loop. Deliberately conservative — only unambiguous + # machine/transport failures, never a bare test-level timeout, which could be + # a real regression (a co-present timeout does not block a match — one + # matching line classifies the run). Case-insensitive, vs the annotations. + INFRA_FAILURE_SIGNATURES: 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (was|has been) (lost|terminated)|invalid index-pack output|RPC failed' # Upper bound on review targets emitted per scan (fan-out defense-in-depth; # excess is logged and deferred to the next scan). MAX_TARGETS_PER_SCAN: '10' diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 22513e6ddf0..863369a1ea0 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -460,7 +460,7 @@ describe('qwen-autofix workflow', () => { PR_META: JSON.stringify({ headRefOid: 'headSHA' }), CHECKS_JSON: JSON.stringify(checks), INFRA_FAILURE_SIGNATURES: - 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (was|has been) (lost|terminated)', + 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (was|has been) (lost|terminated)|invalid index-pack output|RPC failed', PATH: `${bin}:${process.env.PATH}`, }, encoding: 'utf8', @@ -527,12 +527,36 @@ describe('qwen-autofix workflow', () => { 'The runner was terminated', 'The runner has been lost', 'The runner has been terminated', + 'fatal: fetch-pack: invalid index-pack output', + 'error: RPC failed; curl 92 HTTP/2 stream 5 was not closed cleanly: CANCEL (err 8)', ]) { expect(run({ checks: [FAIL], annotations: msg })).toEqual({ reran: true, continued: true, }); } + // #6506: a git fetch died mid-checkout, then hung the job into the 20m + // limit. The bare timeout line is deliberately NOT a signature (it can be a + // real regression), but the transport death IS — and one matching line + // classifies the whole run, so the co-present timeout does not block it. + expect( + run({ + checks: [FAIL], + annotations: [ + 'The job has exceeded the maximum execution time of 20m0s', + 'fatal: fetch-pack: invalid index-pack output', + 'error: RPC failed; curl 92 HTTP/2 stream 5 was not closed cleanly: CANCEL (err 8)', + ].join('\n'), + }), + ).toEqual({ reran: true, continued: true }); + // A BARE job timeout with no transport/infra signature is NOT rerun — it + // can be a real regression (a test hanging on the PR's own code). + expect( + run({ + checks: [FAIL], + annotations: 'The job has exceeded the maximum execution time of 20m0s', + }), + ).toEqual({ reran: false, continued: false }); // Self-trigger guard: a "Qwen Autofix" workflow's own failed check must NOT // be rerun (prevents the autofix from re-triggering itself), UNLESS the // check is a review-address job (the exception carved out in the jq filter). @@ -556,7 +580,9 @@ describe('qwen-autofix workflow', () => { annotations: 'No space left on device', }), ).toEqual({ reran: true, continued: true }); - }); + // Spawn-heavy: each run() forks bash + a stubbed gh. The default 5s per-test + // budget is tight for this many cases, so give it a comfortable margin. + }, 20000); it('keeps a still-red check visible, but only once per head', () => { // A red check is a STATE, not the instant it turned red. Counting only From 6fd7557de3e35d1d2c7e1c88504f0e78cd904741 Mon Sep 17 00:00:00 2001 From: wenshao Date: Thu, 23 Jul 2026 13:25:32 +0800 Subject: [PATCH 4/5] test(autofix): single-source the infra-signature list from the workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The infra-rerun test re-typed INFRA_FAILURE_SIGNATURES as an inline mirror of the workflow's env value. Two copies that must be hand-synced can drift — the test could keep passing against a stale list while production changed, or vice versa. That is exactly the copy the git-transport follow-up had to remember to update in two places. Extract the list from the workflow source instead, the same extract-from-source idiom the file already uses for NON_BLOCKING_CHECKS, so there is only one copy and drift is impossible. A toContain guard fails loudly if the env is renamed or the regex breaks, rather than letting an empty pattern match every line and silently pass. --- scripts/tests/qwen-autofix-workflow.test.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 863369a1ea0..b6d67fbceb6 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -413,6 +413,15 @@ describe('qwen-autofix workflow', () => { expect(block).toBeTruthy(); const script = block.replace(/^ {12}/gm, ''); + // Single-source the signature list from the workflow rather than re-typing + // it here, so the production value and this test can never drift out of + // sync (same extract-from-source idiom as NON_BLOCKING_CHECKS above). The + // toContain guard fails loudly if the env is renamed or the regex breaks — + // otherwise an empty pattern would match every line and silently pass. + const INFRA_SIGNATURES = + workflow.match(/INFRA_FAILURE_SIGNATURES: '([^']*)'/)?.[1] ?? ''; + expect(INFRA_SIGNATURES).toContain('lost communication with the server'); + const run = ({ checks, annotations, attempt = 1, rerunOk = true }) => { const dir = mkdtempSync(join(tmpdir(), 'infra-')); const bin = join(dir, 'bin'); @@ -459,8 +468,7 @@ describe('qwen-autofix workflow', () => { PR: '1', PR_META: JSON.stringify({ headRefOid: 'headSHA' }), CHECKS_JSON: JSON.stringify(checks), - INFRA_FAILURE_SIGNATURES: - 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (was|has been) (lost|terminated)|invalid index-pack output|RPC failed', + INFRA_FAILURE_SIGNATURES: INFRA_SIGNATURES, PATH: `${bin}:${process.env.PATH}`, }, encoding: 'utf8', From 59a079fc051a37bbae1d6f9a61c225b803116bd8 Mon Sep 17 00:00:00 2001 From: qwen-code-ci-bot Date: Thu, 23 Jul 2026 06:22:34 +0000 Subject: [PATCH 5/5] fix(autofix): paginate annotations and filter Autofix runs in infra-rerun loop (#7562) --- .github/workflows/qwen-autofix.yml | 15 +++++-- scripts/tests/qwen-autofix-workflow.test.js | 43 +++++++++++++++++---- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index d9b55c009f7..1357fb12749 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1840,16 +1840,23 @@ jobs: FAILED_CRS="$(gh api --paginate "repos/${REPO}/commits/${PR_HEAD_OID}/check-runs" \ --jq '.check_runs[] | select(.conclusion == "failure") | select(((.output.annotations_count // 0) > 0)) - | [(.id | tostring), (.details_url // "")] | @tsv' 2> /dev/null | sort -u || true)" - while IFS=$'\t' read -r CR_ID DETAILS_URL; do + | [(.id | tostring), (.details_url // ""), (.name // "")] | @tsv' 2> /dev/null | sort -u || true)" + while IFS=$'\t' read -r CR_ID DETAILS_URL CR_NAME; do [[ -n "${CR_ID}" && "${DETAILS_URL}" == *"/actions/runs/"* ]] || continue RUN_ID="${DETAILS_URL#*/actions/runs/}" RUN_ID="${RUN_ID%%/*}" [[ "${RUN_ID}" =~ ^[0-9]+$ ]] || continue - ANNS="$(gh api "repos/${REPO}/check-runs/${CR_ID}/annotations" \ + ANNS="$(gh api --paginate "repos/${REPO}/check-runs/${CR_ID}/annotations" \ --jq '[.[].message] | join("\n")' 2> /dev/null || true)" grep -qiE "${INFRA_FAILURE_SIGNATURES}" <<< "${ANNS}" || continue - ATTEMPT="$(gh api "repos/${REPO}/actions/runs/${RUN_ID}" --jq '.run_attempt // 0' 2> /dev/null || echo 0)" + RUN_META="$(gh api "repos/${REPO}/actions/runs/${RUN_ID}" --jq '[(.run_attempt // 0), (.name // "")] | @tsv' 2> /dev/null || printf '0\t')" + ATTEMPT="${RUN_META%%$'\t'*}" + WF_NAME="${RUN_META#*$'\t'}" + # Mirror the gate's self-trigger guard: skip Qwen Autofix runs + # unless the check is a review-address job. + if [[ "${WF_NAME}" == "Qwen Autofix" && "${CR_NAME}" != review-address* ]]; then + continue + fi if [[ "${ATTEMPT}" == '1' ]]; then if gh api -X POST "repos/${REPO}/actions/runs/${RUN_ID}/rerun-failed-jobs" > /dev/null 2>&1; then echo "♻️ #${PR}: a check died on infrastructure (run ${RUN_ID}, attempt 1) — reran its failed jobs; not the PR's code" diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 863369a1ea0..8c867d56ce0 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -413,7 +413,14 @@ describe('qwen-autofix workflow', () => { expect(block).toBeTruthy(); const script = block.replace(/^ {12}/gm, ''); - const run = ({ checks, annotations, attempt = 1, rerunOk = true }) => { + const run = ({ + checks, + annotations, + attempt = 1, + rerunOk = true, + crName = 'E2E', + wfName = 'CI', + }) => { const dir = mkdtempSync(join(tmpdir(), 'infra-')); const bin = join(dir, 'bin'); mkdirSync(bin); @@ -421,12 +428,12 @@ describe('qwen-autofix workflow', () => { // annotations → the given message; runs/{id} → run_attempt; POST // rerun-failed-jobs → success/fail. Records the rerun POST. // The workflow calls check-runs with a --jq filter that yields, per - // failed check-run WITH annotations, a `\t` line; the - // stub emits what that filter would produce (a single line when there is - // an annotation, nothing otherwise) rather than raw JSON the stub can't - // filter. + // failed check-run WITH annotations, a `\t\t` + // line; the stub emits what that filter would produce (a single line + // when there is an annotation, nothing otherwise) rather than raw JSON + // the stub can't filter. const crTsv = annotations - ? '42\thttps://github.com/o/r/actions/runs/9001/job/5\n' + ? `42\thttps://github.com/o/r/actions/runs/9001/job/5\t${crName}\n` : ''; writeFileSync( join(bin, 'gh'), @@ -440,7 +447,7 @@ describe('qwen-autofix workflow', () => { ` *"/commits/"*"/check-runs"*) printf '%b' ${JSON.stringify(crTsv)}; exit 0;;`, ` *"/check-runs/42/annotations"*) printf '%s' ${JSON.stringify(annotations || '')}; exit 0;;`, ` *"/actions/runs/9001"*"rerun-failed-jobs"*) exit ${rerunOk ? 0 : 1};;`, - ` *"/actions/runs/9001"*) printf '${attempt}'; exit 0;;`, + ` *"/actions/runs/9001"*) printf '${attempt}\\t${wfName}'; exit 0;;`, 'esac', 'exit 0', ].join('\n'), @@ -580,6 +587,28 @@ describe('qwen-autofix workflow', () => { annotations: 'No space left on device', }), ).toEqual({ reran: true, continued: true }); + // In-loop self-trigger guard: the gate above blocks a PR whose ONLY + // failed check is Qwen Autofix, but when a non-Autofix check ALSO failed + // the gate passes and FAILED_CRS returns ALL failed check-runs — the + // in-loop filter must skip the Autofix run so it cannot consume the + // single rerun slot. + expect( + run({ + checks: [FAIL], + annotations: 'No space left on device', + wfName: 'Qwen Autofix', + }), + ).toEqual({ reran: false, continued: false }); + // …but a review-address job from the Autofix workflow IS rerun (the + // exception carved out in both the gate and the in-loop filter). + expect( + run({ + checks: [FAIL], + annotations: 'No space left on device', + crName: 'review-address issue-123', + wfName: 'Qwen Autofix', + }), + ).toEqual({ reran: true, continued: true }); // Spawn-heavy: each run() forks bash + a stubbed gh. The default 5s per-test // budget is tight for this many cases, so give it a comfortable margin. }, 20000);