From c46fd920352f4b81f59e969c1e795f8b20f5ce57 Mon Sep 17 00:00:00 2001 From: verify Date: Wed, 29 Jul 2026 18:29:59 +0800 Subject: [PATCH 1/6] feat(autofix): back off scan inspection of idle candidates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scheduled scan inspects every candidate every tick. The takeover pool doubled in two days (28 open takeover PRs, 8 of them idle in 'nothing new' state for 10+ hours), and idle candidates crowd the two SHARED budgets: MAX_CANDIDATE_INSPECTIONS (60) and the 10-target cap. Observed on #8002: freshly engaged, admitted by the 09:03 scan, then deferred by the target budget while long-idle PRs re-confirmed their idleness yet again. Candidates whose list-provided updatedAt (no extra API call) is older than 24h are now inspected on roughly every 4th scan, on a deterministic slot keyed by PR number and UTC hour so no PR waits forever. The skip is free — it sits with the busy skip before the inspection-budget increment. Safe by construction: every real wake-up bumps updatedAt (reviews, comments, labels, pushes) or routes in real time anyway, so the only thing deferred is the scheduled re-confirmation of idleness plus worst-case a few hours of base-conflict-detection latency for a PR nobody touched in a day. The forced-dispatch path never builds the list files, so a forced PR is always inspected. --- .github/workflows/qwen-autofix.yml | 36 +++++++++++++++++-- scripts/tests/qwen-autofix-workflow.test.js | 39 ++++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index cc65d6dbb28..1ab2795f987 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1669,10 +1669,10 @@ jobs: else gh pr list --repo "${REPO}" --state open --author "${AUTOFIX_BOT}" \ --base main \ - --limit 100 --json number,headRefName,isCrossRepository,labels,author,maintainerCanModify > "${WORKDIR}/bot-prs.json" + --limit 100 --json number,headRefName,isCrossRepository,labels,author,maintainerCanModify,updatedAt > "${WORKDIR}/bot-prs.json" gh pr list --repo "${REPO}" --state open --label "${TAKEOVER_LABEL}" \ --base main \ - --limit 100 --json number,headRefName,isCrossRepository,labels,author,maintainerCanModify > "${WORKDIR}/takeover-prs.json" + --limit 100 --json number,headRefName,isCrossRepository,labels,author,maintainerCanModify,updatedAt > "${WORKDIR}/takeover-prs.json" # Skip-labeled PRs are excluded HERE, not only at the address # gate: that gate discards without writing a marker, so the # watermark never advances and an unfiltered scan would re-emit @@ -1815,6 +1815,30 @@ jobs: ) [[ "${BUSY_PRS}" != ' ' ]] && echo "🚧 address in flight/queued for PR(s):${BUSY_PRS}" + # Idle backoff, from the list's own updatedAt (no extra API call): + # a candidate with no activity for >24h gets inspected on roughly + # every 4th scan instead of every one. The pool doubled in two + # days (28 takeover PRs, 8 of them idle in "nothing new" state for + # 10+ hours), and idle candidates crowd the SHARED inspection and + # target budgets — #8002 was admitted and then deferred by the + # 10-target cap while long-idle PRs re-confirmed "nothing new". + # Safe because every real wake-up path bumps updatedAt (reviews, + # comments, labels, pushes) or routes in real time anyway; the only + # thing deferred is the scheduled re-confirmation of idleness, plus + # worst-case ~a few hours of latency on base-conflict detection for + # a PR nobody has touched in a day. Slotting is deterministic — + # one eligible hour in every four, keyed by PR number — so no PR + # is unlucky forever. The forced-dispatch path never builds the + # list files, so a forced PR is always inspected. + IDLE_UPDATED='{}' + if [[ -f "${WORKDIR}/bot-prs.json" && -f "${WORKDIR}/takeover-prs.json" ]]; then + IDLE_UPDATED="$(jq -cs 'add | unique_by(.number) + | map({(.number | tostring): (.updatedAt // "")}) | add // {}' \ + "${WORKDIR}/bot-prs.json" "${WORKDIR}/takeover-prs.json" 2> /dev/null || echo '{}')" + fi + IDLE_CUTOFF="$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ)" + IDLE_SLOT_NOW="$(( ($(date -u +%s) / 3600) % 4 ))" + TARGETS='[]' INSPECTED=0 for PR in ${CANDIDATES}; do @@ -1825,6 +1849,14 @@ jobs: fleet_row "${PR}" 'busy' 'address run in flight' continue fi + # The idle-backoff skip is free too (in-memory lookup) — it must + # not consume inspection budget either. + PR_UPDATED="$(jq -r --arg pr "${PR}" '.[$pr] // ""' <<< "${IDLE_UPDATED}")" + if [[ -n "${PR_UPDATED}" && "${PR_UPDATED}" < "${IDLE_CUTOFF}" && "$(( PR % 4 ))" != "${IDLE_SLOT_NOW}" ]]; then + echo "😴 #${PR}: idle since ${PR_UPDATED} (>24h) — deferring to its rotation slot (slot $(( PR % 4 )), current ${IDLE_SLOT_NOW})" + fleet_row "${PR}" 'idle-backoff' "idle since ${PR_UPDATED}; inspected roughly every 4th scan" + continue + fi INSPECTED=$(( INSPECTED + 1 )) if [[ "${INSPECTED}" -gt "${MAX_CANDIDATE_INSPECTIONS}" ]]; then echo "🧮 candidate-inspection budget (${MAX_CANDIDATE_INSPECTIONS}) reached — deferring the rest to the next scan" diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 094e4ebabdc..ec4abe4ea2f 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -2398,9 +2398,46 @@ describe('qwen-autofix workflow', () => { // starve the oldest tail forever once the pool exceeds the budget. expect(reviewScanJob).toContain('ROT_OFF='); expect(reviewScanJob).toContain('.[$o:] + .[:$o]'); + // The free skips (busy, idle-backoff) both come before the budget + // increment — neither may consume inspection budget. expect(reviewScanJob).toMatch( - /BUSY_PRS[\s\S]{0,240}INSPECTED=\$\(\( INSPECTED \+ 1 \)\)/, + /BUSY_PRS[\s\S]{0,1600}INSPECTED=\$\(\( INSPECTED \+ 1 \)\)/, ); + expect(reviewScanJob).toMatch( + /fleet_row "\$\{PR\}" 'idle-backoff'[\s\S]{0,120}INSPECTED=\$\(\( INSPECTED \+ 1 \)\)/, + ); + }); + + it('backs off idle candidates without spending inspection budget', () => { + // Measured 2026-07-29: 28 open takeover PRs, 8 of them idle in + // "nothing new" state for 10+ hours — every scan re-confirmed their + // idleness while #8002 (freshly engaged) was admitted and then + // deferred by the 10-target cap. Candidates idle >24h are inspected + // on roughly every 4th scan instead of every one. + // The staleness signal is the list's own updatedAt — no extra API + // call per candidate. + expect( + reviewScanJob.split('maintainerCanModify,updatedAt').length - 1, + ).toBe(2); + expect(reviewScanJob).toContain(`IDLE_CUTOFF="$(date -u -d '24 hours ago'`); + // Deterministic slotting by PR number and UTC hour — one eligible hour + // in every four, so no PR is unlucky forever. + expect(reviewScanJob).toContain( + 'IDLE_SLOT_NOW="$(( ($(date -u +%s) / 3600) % 4 ))"', + ); + expect(reviewScanJob).toContain('"$(( PR % 4 ))" != "${IDLE_SLOT_NOW}"'); + // Fail-open on the forced-dispatch path: it never builds the list + // files, so the lookup stays empty and a forced PR is always + // inspected. Same for a PR missing from the lookup. + expect(reviewScanJob).toContain("IDLE_UPDATED='{}'"); + expect(reviewScanJob).toMatch( + /if \[\[ -f "\$\{WORKDIR\}\/bot-prs\.json" && -f "\$\{WORKDIR\}\/takeover-prs\.json" \]\]; then\n\s+IDLE_UPDATED=/, + ); + expect(reviewScanJob).toContain( + '[[ -n "${PR_UPDATED}" && "${PR_UPDATED}" < "${IDLE_CUTOFF}"', + ); + // Visible in the fleet dashboard, like the busy skip. + expect(reviewScanJob).toContain("'idle-backoff'"); }); it('behaviorally replays the takeover-command toggle across all four paths', () => { From 29a96e391431a36e7d9d8ac49a451c36e0e59c78 Mon Sep 17 00:00:00 2001 From: verify Date: Wed, 29 Jul 2026 20:23:36 +0800 Subject: [PATCH 2/6] feat(autofix): idle-backoff review follow-ups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Corrected the comment's cost model: idle candidates hit 'continue' before the TARGETS append, so they never contend for the 10-target cap — the real win is the shared inspection budget plus the serial scan-walk latency (the walk is what delayed #8002's pickup by ~6 minutes), and the comment now says exactly that. - Slot quantum changed from the hour to the scan tick (600s, the same quantum as ROT_OFF): an hourly slot against the */10 cron meant 6 back-to-back inspections then a ~3h blind window per PR — same 25% average, terrible shape. The gap is now bounded at ~30 minutes, which is what the operator-facing strings promise ('gap ≤30m'). - The two scan-only signals updatedAt cannot see (a base conflict appearing when main moves; still-red checks awaiting the redcheck marker) are named in the comment instead of papered over. - The per-candidate jq fork became a single precomputed set + a bash substring test, matching the busy skip's idiom and the 'free' claim. - Tests: the skip predicate and set builder got a behavioral replay (idle+out-of-slot defers, idle+in-slot inspects, fresh inspects, missing-from-lookup inspects); the two byte-distance assertions became a loop-head slice (comment growth cannot red-light CI, and budget-consuming code between the skips and the increment fails); the --json field pin is order-independent; the 3600 quantum is pinned OUT. --- .github/workflows/qwen-autofix.yml | 60 +++++---- scripts/tests/qwen-autofix-workflow.test.js | 127 +++++++++++++++----- 2 files changed, 135 insertions(+), 52 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 1ab2795f987..be7604dc1df 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1815,29 +1815,39 @@ jobs: ) [[ "${BUSY_PRS}" != ' ' ]] && echo "🚧 address in flight/queued for PR(s):${BUSY_PRS}" - # Idle backoff, from the list's own updatedAt (no extra API call): - # a candidate with no activity for >24h gets inspected on roughly + # Idle backoff, from the list's own updatedAt (no API call): a + # candidate with no activity for >24h gets inspected on roughly # every 4th scan instead of every one. The pool doubled in two - # days (28 takeover PRs, 8 of them idle in "nothing new" state for - # 10+ hours), and idle candidates crowd the SHARED inspection and - # target budgets — #8002 was admitted and then deferred by the - # 10-target cap while long-idle PRs re-confirmed "nothing new". - # Safe because every real wake-up path bumps updatedAt (reviews, - # comments, labels, pushes) or routes in real time anyway; the only - # thing deferred is the scheduled re-confirmation of idleness, plus - # worst-case ~a few hours of latency on base-conflict detection for - # a PR nobody has touched in a day. Slotting is deterministic — - # one eligible hour in every four, keyed by PR number — so no PR - # is unlucky forever. The forced-dispatch path never builds the - # list files, so a forced PR is always inspected. - IDLE_UPDATED='{}' + # days (28 takeover PRs, 8 of them idle in "nothing new" state + # for 10+ hours), and every idle inspection costs twice: a unit + # of the SHARED MAX_CANDIDATE_INSPECTIONS budget, and a slice of + # the serial API walk over the candidate list — the walk is what + # delayed #8002's first pickup (admitted at 09:09, six minutes + # into the 09:03 scan's serial reads). Idle PRs never reach the + # 10-target budget (the "nothing new" branch continues before + # the TARGETS append), so that cap is NOT what this relieves — + # the win is inspection budget plus scan-walk latency for fresh + # engagements. Safe because comments, reviews, labels, and + # pushes all bump updatedAt or route in real time; the two + # scan-only signals that do NOT bump it — a base conflict + # appearing when main moves, and still-red checks awaiting the + # redcheck marker — wait at most the backoff gap (~30 min) on a + # PR nobody touched in a day, and self-correct (the eventual + # address run comments/pushes). Slotting is deterministic — one + # eligible 10-minute tick in every four, keyed by PR number, the + # same 600s quantum as ROT_OFF — so the gap is bounded at ~30 + # minutes and no PR is unlucky forever. The forced-dispatch path + # never builds the list files, so a forced PR is always + # inspected (fail-open, like a PR missing from the set). + IDLE_PRS=' ' if [[ -f "${WORKDIR}/bot-prs.json" && -f "${WORKDIR}/takeover-prs.json" ]]; then - IDLE_UPDATED="$(jq -cs 'add | unique_by(.number) - | map({(.number | tostring): (.updatedAt // "")}) | add // {}' \ - "${WORKDIR}/bot-prs.json" "${WORKDIR}/takeover-prs.json" 2> /dev/null || echo '{}')" + IDLE_CUTOFF="$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ)" + IDLE_PRS=" $(jq -rs --arg cut "${IDLE_CUTOFF}" 'add | unique_by(.number) + | map(select((.updatedAt // "") != "" and .updatedAt < $cut) | .number | tostring) + | join(" ")' \ + "${WORKDIR}/bot-prs.json" "${WORKDIR}/takeover-prs.json" 2> /dev/null || echo '') " fi - IDLE_CUTOFF="$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ)" - IDLE_SLOT_NOW="$(( ($(date -u +%s) / 3600) % 4 ))" + IDLE_SLOT_NOW="$(( ($(date -u +%s) / 600) % 4 ))" TARGETS='[]' INSPECTED=0 @@ -1849,12 +1859,12 @@ jobs: fleet_row "${PR}" 'busy' 'address run in flight' continue fi - # The idle-backoff skip is free too (in-memory lookup) — it must + # The idle-backoff skip is free too (a bash substring test on + # the precomputed set, same idiom as the busy skip) — it must # not consume inspection budget either. - PR_UPDATED="$(jq -r --arg pr "${PR}" '.[$pr] // ""' <<< "${IDLE_UPDATED}")" - if [[ -n "${PR_UPDATED}" && "${PR_UPDATED}" < "${IDLE_CUTOFF}" && "$(( PR % 4 ))" != "${IDLE_SLOT_NOW}" ]]; then - echo "😴 #${PR}: idle since ${PR_UPDATED} (>24h) — deferring to its rotation slot (slot $(( PR % 4 )), current ${IDLE_SLOT_NOW})" - fleet_row "${PR}" 'idle-backoff' "idle since ${PR_UPDATED}; inspected roughly every 4th scan" + if [[ "${IDLE_PRS}" == *" ${PR} "* && "$(( PR % 4 ))" != "${IDLE_SLOT_NOW}" ]]; then + echo "😴 #${PR}: idle >24h — deferring to its rotation slot (slot $(( PR % 4 )), current ${IDLE_SLOT_NOW}, next eligible tick ≤30m away)" + fleet_row "${PR}" 'idle-backoff' 'idle >24h; inspected roughly every 4th scan (gap ≤30m)' continue fi INSPECTED=$(( INSPECTED + 1 )) diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index ec4abe4ea2f..f7439505a3d 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -2398,46 +2398,119 @@ describe('qwen-autofix workflow', () => { // starve the oldest tail forever once the pool exceeds the budget. expect(reviewScanJob).toContain('ROT_OFF='); expect(reviewScanJob).toContain('.[$o:] + .[:$o]'); - // The free skips (busy, idle-backoff) both come before the budget - // increment — neither may consume inspection budget. - expect(reviewScanJob).toMatch( - /BUSY_PRS[\s\S]{0,1600}INSPECTED=\$\(\( INSPECTED \+ 1 \)\)/, - ); - expect(reviewScanJob).toMatch( - /fleet_row "\$\{PR\}" 'idle-backoff'[\s\S]{0,120}INSPECTED=\$\(\( INSPECTED \+ 1 \)\)/, - ); + // The free skips (busy, idle-backoff) both live in the loop head, + // BEFORE the budget increment — assert on the slice, not a byte + // distance (comment growth must not red-light CI, and arbitrary code + // between the skips and the increment must). + const loopHead = reviewScanJob.slice( + reviewScanJob.indexOf('for PR in ${CANDIDATES}'), + reviewScanJob.indexOf('INSPECTED=$(( INSPECTED + 1 ))'), + ); + expect(loopHead).toContain("'busy'"); + expect(loopHead).toContain("'idle-backoff'"); + expect(loopHead).not.toContain('INSPECTED='); }); it('backs off idle candidates without spending inspection budget', () => { - // Measured 2026-07-29: 28 open takeover PRs, 8 of them idle in - // "nothing new" state for 10+ hours — every scan re-confirmed their - // idleness while #8002 (freshly engaged) was admitted and then - // deferred by the 10-target cap. Candidates idle >24h are inspected - // on roughly every 4th scan instead of every one. - // The staleness signal is the list's own updatedAt — no extra API - // call per candidate. + // Measured 2026-07-29: 28 open takeover PRs, 8 idle in "nothing new" + // state for 10+ hours. Every idle inspection costs a unit of the + // shared MAX_CANDIDATE_INSPECTIONS budget and a slice of the serial + // API walk (the walk is what delayed #8002's first pickup by ~6 + // minutes). Candidates idle >24h are inspected on roughly every 4th + // scan. The staleness signal is the list's own updatedAt — no API + // call, and no per-candidate process fork either: one jq builds the + // idle SET, and the loop tests membership like the busy skip does. expect( - reviewScanJob.split('maintainerCanModify,updatedAt').length - 1, + (reviewScanJob.match(/--json [^\n]*\bupdatedAt\b/g) ?? []).length, ).toBe(2); expect(reviewScanJob).toContain(`IDLE_CUTOFF="$(date -u -d '24 hours ago'`); - // Deterministic slotting by PR number and UTC hour — one eligible hour - // in every four, so no PR is unlucky forever. + // Slotting quantum is the SCAN TICK (600s, same quantum as ROT_OFF), + // not the hour: an hourly slot against a */10 cron means 6 back-to-back + // inspections then a ~3h blind window — same 25% average, terrible + // shape. With 600s the gap is bounded at ~30 minutes, which is what + // the operator-facing strings promise. expect(reviewScanJob).toContain( - 'IDLE_SLOT_NOW="$(( ($(date -u +%s) / 3600) % 4 ))"', + 'IDLE_SLOT_NOW="$(( ($(date -u +%s) / 600) % 4 ))"', ); - expect(reviewScanJob).toContain('"$(( PR % 4 ))" != "${IDLE_SLOT_NOW}"'); + expect(reviewScanJob).not.toMatch(/IDLE_SLOT_NOW[^\n]*\/ 3600/); + expect(reviewScanJob).toContain('gap ≤30m'); // Fail-open on the forced-dispatch path: it never builds the list - // files, so the lookup stays empty and a forced PR is always - // inspected. Same for a PR missing from the lookup. - expect(reviewScanJob).toContain("IDLE_UPDATED='{}'"); + // files, so the set stays empty and a forced PR is always inspected. + expect(reviewScanJob).toContain("IDLE_PRS=' '"); expect(reviewScanJob).toMatch( - /if \[\[ -f "\$\{WORKDIR\}\/bot-prs\.json" && -f "\$\{WORKDIR\}\/takeover-prs\.json" \]\]; then\n\s+IDLE_UPDATED=/, - ); - expect(reviewScanJob).toContain( - '[[ -n "${PR_UPDATED}" && "${PR_UPDATED}" < "${IDLE_CUTOFF}"', + /if \[\[ -f "\$\{WORKDIR\}\/bot-prs\.json" && -f "\$\{WORKDIR\}\/takeover-prs\.json" \]\]; then\n\s+IDLE_CUTOFF=/, ); + + // Behavioral replay of the set builder + skip predicate (the string + // pins alone cannot catch a flipped comparison or slot equality): + // run the real jq and the real predicate over four candidate shapes. + const setSrc = reviewScanJob.match( + /IDLE_PRS=" \$\(jq -rs[\s\S]*?\) "/, + )?.[0]; + expect(setSrc).toBeTruthy(); + const runBackoff = ({ pr, updatedAt, slotNow, listed = true }) => { + const dir = mkdtempSync(join(tmpdir(), 'idle-backoff-')); + try { + const row = listed + ? [{ number: Number(pr), updatedAt }] + : [{ number: 99999, updatedAt: '2026-01-01T00:00:00Z' }]; + writeFileSync(join(dir, 'bot-prs.json'), JSON.stringify(row)); + writeFileSync(join(dir, 'takeover-prs.json'), '[]'); + return execFileSync( + 'bash', + [ + '-c', + [ + 'set -uo pipefail', + `WORKDIR='${dir}'`, + `IDLE_CUTOFF='2026-07-28T12:00:00Z'`, + setSrc, + `IDLE_SLOT_NOW='${slotNow}'`, + `PR='${pr}'`, + 'if [[ "${IDLE_PRS}" == *" ${PR} "* && "$(( PR % 4 ))" != "${IDLE_SLOT_NOW}" ]]; then printf skip; else printf inspect; fi', + ].join('\n'), + ], + { encoding: 'utf8' }, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; + // Idle and out of slot → deferred (8001 % 4 = 1). + expect( + runBackoff({ pr: '8001', updatedAt: '2026-07-27T00:00:00Z', slotNow: 0 }), + ).toBe('skip'); + // Idle but IN its slot → inspected (the gap is bounded). + expect( + runBackoff({ pr: '8001', updatedAt: '2026-07-27T00:00:00Z', slotNow: 1 }), + ).toBe('inspect'); + // Fresh activity → inspected regardless of slot. + expect( + runBackoff({ pr: '8001', updatedAt: '2026-07-28T13:00:00Z', slotNow: 0 }), + ).toBe('inspect'); + // Missing from the lookup (forced-dispatch shape) → inspected. + expect( + runBackoff({ + pr: '8001', + updatedAt: '2026-07-27T00:00:00Z', + slotNow: 0, + listed: false, + }), + ).toBe('inspect'); + // Visible in the fleet dashboard, like the busy skip. expect(reviewScanJob).toContain("'idle-backoff'"); + // The workflow comment must not claim the 10-target cap is relieved — + // idle candidates hit `continue` before the TARGETS append, so they + // never contend for it; the real win is inspection budget + walk + // latency, and the comment says so. + expect(reviewScanJob).toContain( + 'Idle PRs never reach the\n # 10-target budget', + ); + // The two scan-only signals updatedAt cannot see are named, not + // papered over by an absolute invariant. + expect(reviewScanJob).toContain('base conflict'); + expect(reviewScanJob).toContain('still-red checks'); }); it('behaviorally replays the takeover-command toggle across all four paths', () => { From 53d57df0814f743666f092d37e75764e738fa828 Mon Sep 17 00:00:00 2001 From: verify Date: Wed, 29 Jul 2026 15:20:43 +0000 Subject: [PATCH 3/6] test(autofix): pin the null-updatedAt defensive guard in idle-backoff replay (#8049) --- scripts/tests/qwen-autofix-workflow.test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index f7439505a3d..a1e61a33b01 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -2498,6 +2498,11 @@ describe('qwen-autofix workflow', () => { }), ).toBe('inspect'); + // Null/empty updatedAt (defensive guard) → inspected, not idle. + expect(runBackoff({ pr: '8001', updatedAt: null, slotNow: 0 })).toBe( + 'inspect', + ); + // Visible in the fleet dashboard, like the busy skip. expect(reviewScanJob).toContain("'idle-backoff'"); // The workflow comment must not claim the 10-target cap is relieved — From 0247b8933cc78b9789c120e01a9b6e911c2725f7 Mon Sep 17 00:00:00 2001 From: qwen-code-ci-bot Date: Wed, 29 Jul 2026 17:06:22 +0000 Subject: [PATCH 4/6] =?UTF-8?q?fix(autofix):=20extract=20idle-backoff=20pr?= =?UTF-8?q?edicate=20from=20workflow,=20fix=20gap=20bound=2030=E2=86=9240m?= =?UTF-8?q?=20(#8049)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/qwen-autofix.yml | 6 +++--- scripts/tests/qwen-autofix-workflow.test.js | 10 +++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index be7604dc1df..3ae731b7172 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1831,11 +1831,11 @@ jobs: # pushes all bump updatedAt or route in real time; the two # scan-only signals that do NOT bump it — a base conflict # appearing when main moves, and still-red checks awaiting the - # redcheck marker — wait at most the backoff gap (~30 min) on a + # redcheck marker — wait at most the backoff gap (~40 min) on a # PR nobody touched in a day, and self-correct (the eventual # address run comments/pushes). Slotting is deterministic — one # eligible 10-minute tick in every four, keyed by PR number, the - # same 600s quantum as ROT_OFF — so the gap is bounded at ~30 + # same 600s quantum as ROT_OFF — so the gap is bounded at ~40 # minutes and no PR is unlucky forever. The forced-dispatch path # never builds the list files, so a forced PR is always # inspected (fail-open, like a PR missing from the set). @@ -1864,7 +1864,7 @@ jobs: # not consume inspection budget either. if [[ "${IDLE_PRS}" == *" ${PR} "* && "$(( PR % 4 ))" != "${IDLE_SLOT_NOW}" ]]; then echo "😴 #${PR}: idle >24h — deferring to its rotation slot (slot $(( PR % 4 )), current ${IDLE_SLOT_NOW}, next eligible tick ≤30m away)" - fleet_row "${PR}" 'idle-backoff' 'idle >24h; inspected roughly every 4th scan (gap ≤30m)' + fleet_row "${PR}" 'idle-backoff' 'idle >24h; inspected roughly every 4th scan (gap ≤40m)' continue fi INSPECTED=$(( INSPECTED + 1 )) diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index a1e61a33b01..d4f49d48f0d 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -2427,13 +2427,13 @@ describe('qwen-autofix workflow', () => { // Slotting quantum is the SCAN TICK (600s, same quantum as ROT_OFF), // not the hour: an hourly slot against a */10 cron means 6 back-to-back // inspections then a ~3h blind window — same 25% average, terrible - // shape. With 600s the gap is bounded at ~30 minutes, which is what + // shape. With 600s the gap is bounded at ~40 minutes, which is what // the operator-facing strings promise. expect(reviewScanJob).toContain( 'IDLE_SLOT_NOW="$(( ($(date -u +%s) / 600) % 4 ))"', ); expect(reviewScanJob).not.toMatch(/IDLE_SLOT_NOW[^\n]*\/ 3600/); - expect(reviewScanJob).toContain('gap ≤30m'); + expect(reviewScanJob).toContain('gap ≤40m'); // Fail-open on the forced-dispatch path: it never builds the list // files, so the set stays empty and a forced PR is always inspected. expect(reviewScanJob).toContain("IDLE_PRS=' '"); @@ -2448,6 +2448,10 @@ describe('qwen-autofix workflow', () => { /IDLE_PRS=" \$\(jq -rs[\s\S]*?\) "/, )?.[0]; expect(setSrc).toBeTruthy(); + const predSrc = reviewScanJob.match( + /if \[\[ "\$\{IDLE_PRS\}" == \*" \$\{PR\} "\* &&[^\n]*\]\]; then/, + )?.[0]; + expect(predSrc).toBeTruthy(); const runBackoff = ({ pr, updatedAt, slotNow, listed = true }) => { const dir = mkdtempSync(join(tmpdir(), 'idle-backoff-')); try { @@ -2467,7 +2471,7 @@ describe('qwen-autofix workflow', () => { setSrc, `IDLE_SLOT_NOW='${slotNow}'`, `PR='${pr}'`, - 'if [[ "${IDLE_PRS}" == *" ${PR} "* && "$(( PR % 4 ))" != "${IDLE_SLOT_NOW}" ]]; then printf skip; else printf inspect; fi', + `${predSrc} printf skip; else printf inspect; fi`, ].join('\n'), ], { encoding: 'utf8' }, From 0fe54dc447713178bf3b61424dac0604ff3945bc Mon Sep 17 00:00:00 2001 From: qwen-code-ci-bot Date: Wed, 29 Jul 2026 18:20:31 +0000 Subject: [PATCH 5/6] test(autofix): pin takeover-prs.json in idle-backoff replay (#8049) --- scripts/tests/qwen-autofix-workflow.test.js | 28 ++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index d4f49d48f0d..efd50cd97b4 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -2448,18 +2448,31 @@ describe('qwen-autofix workflow', () => { /IDLE_PRS=" \$\(jq -rs[\s\S]*?\) "/, )?.[0]; expect(setSrc).toBeTruthy(); + expect(setSrc).toContain('takeover-prs.json'); const predSrc = reviewScanJob.match( /if \[\[ "\$\{IDLE_PRS\}" == \*" \$\{PR\} "\* &&[^\n]*\]\]; then/, )?.[0]; expect(predSrc).toBeTruthy(); - const runBackoff = ({ pr, updatedAt, slotNow, listed = true }) => { + const runBackoff = ({ + pr, + updatedAt, + slotNow, + listed = true, + inTakeover = false, + }) => { const dir = mkdtempSync(join(tmpdir(), 'idle-backoff-')); try { const row = listed ? [{ number: Number(pr), updatedAt }] : [{ number: 99999, updatedAt: '2026-01-01T00:00:00Z' }]; - writeFileSync(join(dir, 'bot-prs.json'), JSON.stringify(row)); - writeFileSync(join(dir, 'takeover-prs.json'), '[]'); + writeFileSync( + join(dir, 'bot-prs.json'), + inTakeover ? '[]' : JSON.stringify(row), + ); + writeFileSync( + join(dir, 'takeover-prs.json'), + inTakeover ? JSON.stringify(row) : '[]', + ); return execFileSync( 'bash', [ @@ -2506,6 +2519,15 @@ describe('qwen-autofix workflow', () => { expect(runBackoff({ pr: '8001', updatedAt: null, slotNow: 0 })).toBe( 'inspect', ); + // Idle candidate in takeover-prs.json (the cohort this targets) → deferred. + expect( + runBackoff({ + pr: '8001', + updatedAt: '2026-07-27T00:00:00Z', + slotNow: 0, + inTakeover: true, + }), + ).toBe('skip'); // Visible in the fleet dashboard, like the busy skip. expect(reviewScanJob).toContain("'idle-backoff'"); From 7c4eaf027a05cc653d3847a75fc50aa4d255e56a Mon Sep 17 00:00:00 2001 From: qwen-code-ci-bot Date: Thu, 30 Jul 2026 00:34:30 +0000 Subject: [PATCH 6/6] fix(autofix): state idle-backoff gap probabilistically, unpin quantum (#8049) Round-2 verification showed the scheduled scan lands every ~40-70 min on this repo, not every 10 min, so the (epoch/600)%4 slot is an independent ~25% draw per scan, not a deterministic 1-in-4 rotation. The gap is geometric (measured median ~2h, p90 ~6h), not bounded at ~40m. Reword the operator-facing strings (skip echo, fleet row) and the workflow comment to state the behaviour probabilistically and drop the false "bounded" / "no PR is unlucky forever" claims; correct the misattributed #8002 latency (queue/startup, not the serial walk). Relax the tests so a truthful number is not a CI failure: pin the mod-4 time-quantum shape instead of the exact 600s constant, and drop the /3600 exclusion that forbade the better-tailed quantum. Mechanism logic is unchanged. --- .github/workflows/qwen-autofix.yml | 50 +++++++++++---------- scripts/tests/qwen-autofix-workflow.test.js | 33 ++++++++------ 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 3ae731b7172..11a2e0b60c6 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1816,29 +1816,33 @@ jobs: [[ "${BUSY_PRS}" != ' ' ]] && echo "🚧 address in flight/queued for PR(s):${BUSY_PRS}" # Idle backoff, from the list's own updatedAt (no API call): a - # candidate with no activity for >24h gets inspected on roughly - # every 4th scan instead of every one. The pool doubled in two + # candidate with no activity for >24h is inspected on about one + # scan in four instead of every one. The pool doubled in two # days (28 takeover PRs, 8 of them idle in "nothing new" state - # for 10+ hours), and every idle inspection costs twice: a unit - # of the SHARED MAX_CANDIDATE_INSPECTIONS budget, and a slice of - # the serial API walk over the candidate list — the walk is what - # delayed #8002's first pickup (admitted at 09:09, six minutes - # into the 09:03 scan's serial reads). Idle PRs never reach the + # for 10+ hours), and every idle inspection costs a unit of the + # SHARED MAX_CANDIDATE_INSPECTIONS budget plus a slice of the + # serial API walk over the candidate list. The win is small: a + # few fewer gh round-trips per scan (~2-3 of the pool) and less + # rate-limit pressure. It does NOT recover the job's queue or + # startup latency, which dwarfed the walk in the #8002 + # measurement that motivated this. Idle PRs never reach the # 10-target budget (the "nothing new" branch continues before - # the TARGETS append), so that cap is NOT what this relieves — - # the win is inspection budget plus scan-walk latency for fresh - # engagements. Safe because comments, reviews, labels, and - # pushes all bump updatedAt or route in real time; the two - # scan-only signals that do NOT bump it — a base conflict - # appearing when main moves, and still-red checks awaiting the - # redcheck marker — wait at most the backoff gap (~40 min) on a - # PR nobody touched in a day, and self-correct (the eventual - # address run comments/pushes). Slotting is deterministic — one - # eligible 10-minute tick in every four, keyed by PR number, the - # same 600s quantum as ROT_OFF — so the gap is bounded at ~40 - # minutes and no PR is unlucky forever. The forced-dispatch path - # never builds the list files, so a forced PR is always - # inspected (fail-open, like a PR missing from the set). + # the TARGETS append), so that cap is NOT what this relieves. + # Safe because comments, reviews, labels, and pushes all bump + # updatedAt or route in real time; the two scan-only signals + # that do NOT bump it — a base conflict appearing when main + # moves, and still-red checks awaiting the redcheck marker — + # wait out the backoff on a PR nobody touched in a day, then + # self-correct (the eventual address run comments/pushes). The + # slot is keyed by PR number mod 4 against a 600s time quantum + # (same quantum as ROT_OFF), so each scan is an independent + # ~25% draw per idle PR — about one scan in four. This is NOT a + # bounded gap: the scheduled scan lands every ~40-70 min on + # this repo (not the */10 the cron implies), so the wait is + # geometric — measured median ~2h, p90 ~6h across 100 real + # scans. The forced-dispatch path never builds the list files, + # so a forced PR is always inspected (fail-open, like a PR + # missing from the set). IDLE_PRS=' ' if [[ -f "${WORKDIR}/bot-prs.json" && -f "${WORKDIR}/takeover-prs.json" ]]; then IDLE_CUTOFF="$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ)" @@ -1863,8 +1867,8 @@ jobs: # the precomputed set, same idiom as the busy skip) — it must # not consume inspection budget either. if [[ "${IDLE_PRS}" == *" ${PR} "* && "$(( PR % 4 ))" != "${IDLE_SLOT_NOW}" ]]; then - echo "😴 #${PR}: idle >24h — deferring to its rotation slot (slot $(( PR % 4 )), current ${IDLE_SLOT_NOW}, next eligible tick ≤30m away)" - fleet_row "${PR}" 'idle-backoff' 'idle >24h; inspected roughly every 4th scan (gap ≤40m)' + echo "😴 #${PR}: idle >24h — deferring to its rotation slot (slot $(( PR % 4 )), current ${IDLE_SLOT_NOW}; inspected ~1 scan in 4)" + fleet_row "${PR}" 'idle-backoff' 'idle >24h; inspected ~1 scan in 4 (median ~2h, p90 ~6h)' continue fi INSPECTED=$(( INSPECTED + 1 )) diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index efd50cd97b4..d1eb8e6ba82 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -2415,25 +2415,30 @@ describe('qwen-autofix workflow', () => { // Measured 2026-07-29: 28 open takeover PRs, 8 idle in "nothing new" // state for 10+ hours. Every idle inspection costs a unit of the // shared MAX_CANDIDATE_INSPECTIONS budget and a slice of the serial - // API walk (the walk is what delayed #8002's first pickup by ~6 - // minutes). Candidates idle >24h are inspected on roughly every 4th - // scan. The staleness signal is the list's own updatedAt — no API - // call, and no per-candidate process fork either: one jq builds the - // idle SET, and the loop tests membership like the busy skip does. + // API walk (a few fewer gh round-trips per scan; the #8002 delay was + // queue/startup, which the candidate loop cannot recover). Candidates + // idle >24h are inspected on about one scan in four. The staleness + // signal is the list's own updatedAt — no API call, and no + // per-candidate process fork either: one jq builds the idle SET, and + // the loop tests membership like the busy skip does. expect( (reviewScanJob.match(/--json [^\n]*\bupdatedAt\b/g) ?? []).length, ).toBe(2); expect(reviewScanJob).toContain(`IDLE_CUTOFF="$(date -u -d '24 hours ago'`); - // Slotting quantum is the SCAN TICK (600s, same quantum as ROT_OFF), - // not the hour: an hourly slot against a */10 cron means 6 back-to-back - // inspections then a ~3h blind window — same 25% average, terrible - // shape. With 600s the gap is bounded at ~40 minutes, which is what - // the operator-facing strings promise. + // The slot is a time quantum mod 4 keyed against PR % 4 (same quantum + // family as ROT_OFF). The exact quantum is deliberately NOT pinned: + // against the real irregular scan cadence (~40-70 min gaps, not */10) + // no constant quantum bounds the gap — the wait is geometric (measured + // median ~2h, p90 ~6h) — and 3600 s actually measures better on p90/max + // than 600 s, so CI must not forbid a future quantum change. The + // behavioral replay below (which passes slotNow explicitly) protects + // the slot logic; this pin only asserts the mod-4 time-quantum shape. + expect(reviewScanJob).toMatch( + /IDLE_SLOT_NOW="\$\(\(.*\$\(date -u \+%s\) \/ \d+\) % 4 \)\)"/, + ); expect(reviewScanJob).toContain( - 'IDLE_SLOT_NOW="$(( ($(date -u +%s) / 600) % 4 ))"', + 'inspected ~1 scan in 4 (median ~2h, p90 ~6h)', ); - expect(reviewScanJob).not.toMatch(/IDLE_SLOT_NOW[^\n]*\/ 3600/); - expect(reviewScanJob).toContain('gap ≤40m'); // Fail-open on the forced-dispatch path: it never builds the list // files, so the set stays empty and a forced PR is always inspected. expect(reviewScanJob).toContain("IDLE_PRS=' '"); @@ -2497,7 +2502,7 @@ describe('qwen-autofix workflow', () => { expect( runBackoff({ pr: '8001', updatedAt: '2026-07-27T00:00:00Z', slotNow: 0 }), ).toBe('skip'); - // Idle but IN its slot → inspected (the gap is bounded). + // Idle but IN its slot → inspected (its slot matched this scan). expect( runBackoff({ pr: '8001', updatedAt: '2026-07-27T00:00:00Z', slotNow: 1 }), ).toBe('inspect');