Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 48 additions & 2 deletions .github/workflows/qwen-autofix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1669,10 +1669,10 @@ jobs:
else
gh pr list --repo "${REPO}" --state open --author "${AUTOFIX_BOT}" \
--base main \
--limit 100 --json number,headRefName,isCrossRepository,labels,author,maintainerCanModify > "${WORKDIR}/bot-prs.json"
--limit 100 --json number,headRefName,isCrossRepository,labels,author,maintainerCanModify,updatedAt > "${WORKDIR}/bot-prs.json"
gh pr list --repo "${REPO}" --state open --label "${TAKEOVER_LABEL}" \
--base main \
--limit 100 --json number,headRefName,isCrossRepository,labels,author,maintainerCanModify > "${WORKDIR}/takeover-prs.json"
--limit 100 --json number,headRefName,isCrossRepository,labels,author,maintainerCanModify,updatedAt > "${WORKDIR}/takeover-prs.json"
# Skip-labeled PRs are excluded HERE, not only at the address
# gate: that gate discards without writing a marker, so the
# watermark never advances and an unfiltered scan would re-emit
Expand Down Expand Up @@ -1815,6 +1815,44 @@ jobs:
)
[[ "${BUSY_PRS}" != ' ' ]] && echo "🚧 address in flight/queued for PR(s):${BUSY_PRS}"

# Idle backoff, from the list's own updatedAt (no API call): a
# candidate with no activity for >24h is inspected on about one
# scan in four instead of every one. The pool doubled in two
# days (28 takeover PRs, 8 of them idle in "nothing new" state
# for 10+ hours), and every idle inspection costs a unit of the
# SHARED MAX_CANDIDATE_INSPECTIONS budget plus a slice of the
# serial API walk over the candidate list. The win is small: a
# few fewer gh round-trips per scan (~2-3 of the pool) and less
# rate-limit pressure. It does NOT recover the job's queue or
# startup latency, which dwarfed the walk in the #8002
# measurement that motivated this. Idle PRs never reach the
# 10-target budget (the "nothing new" branch continues before
# the TARGETS append), so that cap is NOT what this relieves.
# Safe because comments, reviews, labels, and pushes all bump
# updatedAt or route in real time; the two scan-only signals
# that do NOT bump it — a base conflict appearing when main
# moves, and still-red checks awaiting the redcheck marker —
# wait out the backoff on a PR nobody touched in a day, then
# self-correct (the eventual address run comments/pushes). The
# slot is keyed by PR number mod 4 against a 600s time quantum
# (same quantum as ROT_OFF), so each scan is an independent
# ~25% draw per idle PR — about one scan in four. This is NOT a
# bounded gap: the scheduled scan lands every ~40-70 min on
# this repo (not the */10 the cron implies), so the wait is
# geometric — measured median ~2h, p90 ~6h across 100 real
# scans. The forced-dispatch path never builds the list files,
# so a forced PR is always inspected (fail-open, like a PR
# missing from the set).
IDLE_PRS=' '
if [[ -f "${WORKDIR}/bot-prs.json" && -f "${WORKDIR}/takeover-prs.json" ]]; then
IDLE_CUTOFF="$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ)"
IDLE_PRS=" $(jq -rs --arg cut "${IDLE_CUTOFF}" 'add | unique_by(.number)
| map(select((.updatedAt // "") != "" and .updatedAt < $cut) | .number | tostring)
| join(" ")' \
"${WORKDIR}/bot-prs.json" "${WORKDIR}/takeover-prs.json" 2> /dev/null || echo '') "
fi
IDLE_SLOT_NOW="$(( ($(date -u +%s) / 600) % 4 ))"

TARGETS='[]'
INSPECTED=0
for PR in ${CANDIDATES}; do
Expand All @@ -1825,6 +1863,14 @@ jobs:
fleet_row "${PR}" 'busy' 'address run in flight'
continue
fi
# The idle-backoff skip is free too (a bash substring test on
# the precomputed set, same idiom as the busy skip) — it must
# not consume inspection budget either.
if [[ "${IDLE_PRS}" == *" ${PR} "* && "$(( PR % 4 ))" != "${IDLE_SLOT_NOW}" ]]; then
echo "😴 #${PR}: idle >24h — deferring to its rotation slot (slot $(( PR % 4 )), current ${IDLE_SLOT_NOW}; inspected ~1 scan in 4)"
fleet_row "${PR}" 'idle-backoff' 'idle >24h; inspected ~1 scan in 4 (median ~2h, p90 ~6h)'
continue
fi
INSPECTED=$(( INSPECTED + 1 ))
if [[ "${INSPECTED}" -gt "${MAX_CANDIDATE_INSPECTIONS}" ]]; then
echo "🧮 candidate-inspection budget (${MAX_CANDIDATE_INSPECTIONS}) reached — deferring the rest to the next scan"
Expand Down
148 changes: 147 additions & 1 deletion scripts/tests/qwen-autofix-workflow.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -2398,9 +2398,155 @@ describe('qwen-autofix workflow', () => {
// starve the oldest tail forever once the pool exceeds the budget.
expect(reviewScanJob).toContain('ROT_OFF=');
expect(reviewScanJob).toContain('.[$o:] + .[:$o]');
// The free skips (busy, idle-backoff) both live in the loop head,
// BEFORE the budget increment — assert on the slice, not a byte
// distance (comment growth must not red-light CI, and arbitrary code
// between the skips and the increment must).
const loopHead = reviewScanJob.slice(
reviewScanJob.indexOf('for PR in ${CANDIDATES}'),
reviewScanJob.indexOf('INSPECTED=$(( INSPECTED + 1 ))'),
);
expect(loopHead).toContain("'busy'");
expect(loopHead).toContain("'idle-backoff'");
expect(loopHead).not.toContain('INSPECTED=');
});

it('backs off idle candidates without spending inspection budget', () => {
// Measured 2026-07-29: 28 open takeover PRs, 8 idle in "nothing new"
// state for 10+ hours. Every idle inspection costs a unit of the
// shared MAX_CANDIDATE_INSPECTIONS budget and a slice of the serial
// API walk (a few fewer gh round-trips per scan; the #8002 delay was
// queue/startup, which the candidate loop cannot recover). Candidates
// idle >24h are inspected on about one scan in four. The staleness
// signal is the list's own updatedAt — no API call, and no
// per-candidate process fork either: one jq builds the idle SET, and
// the loop tests membership like the busy skip does.
expect(
(reviewScanJob.match(/--json [^\n]*\bupdatedAt\b/g) ?? []).length,
).toBe(2);
expect(reviewScanJob).toContain(`IDLE_CUTOFF="$(date -u -d '24 hours ago'`);
// The slot is a time quantum mod 4 keyed against PR % 4 (same quantum
// family as ROT_OFF). The exact quantum is deliberately NOT pinned:
// against the real irregular scan cadence (~40-70 min gaps, not */10)
// no constant quantum bounds the gap — the wait is geometric (measured
// median ~2h, p90 ~6h) — and 3600 s actually measures better on p90/max
// than 600 s, so CI must not forbid a future quantum change. The
// behavioral replay below (which passes slotNow explicitly) protects
// the slot logic; this pin only asserts the mod-4 time-quantum shape.
expect(reviewScanJob).toMatch(
/IDLE_SLOT_NOW="\$\(\(.*\$\(date -u \+%s\) \/ \d+\) % 4 \)\)"/,
);
expect(reviewScanJob).toContain(
'inspected ~1 scan in 4 (median ~2h, p90 ~6h)',
);
// Fail-open on the forced-dispatch path: it never builds the list
// files, so the set stays empty and a forced PR is always inspected.
expect(reviewScanJob).toContain("IDLE_PRS=' '");
expect(reviewScanJob).toMatch(
/BUSY_PRS[\s\S]{0,240}INSPECTED=\$\(\( INSPECTED \+ 1 \)\)/,
/if \[\[ -f "\$\{WORKDIR\}\/bot-prs\.json" && -f "\$\{WORKDIR\}\/takeover-prs\.json" \]\]; then\n\s+IDLE_CUTOFF=/,
);

// Behavioral replay of the set builder + skip predicate (the string
// pins alone cannot catch a flipped comparison or slot equality):
// run the real jq and the real predicate over four candidate shapes.
const setSrc = reviewScanJob.match(
/IDLE_PRS=" \$\(jq -rs[\s\S]*?\) "/,
)?.[0];
expect(setSrc).toBeTruthy();
expect(setSrc).toContain('takeover-prs.json');
const predSrc = reviewScanJob.match(
/if \[\[ "\$\{IDLE_PRS\}" == \*" \$\{PR\} "\* &&[^\n]*\]\]; then/,
)?.[0];
expect(predSrc).toBeTruthy();
const runBackoff = ({
pr,
updatedAt,
slotNow,
listed = true,
inTakeover = false,
}) => {
const dir = mkdtempSync(join(tmpdir(), 'idle-backoff-'));
try {
const row = listed
? [{ number: Number(pr), updatedAt }]
: [{ number: 99999, updatedAt: '2026-01-01T00:00:00Z' }];
writeFileSync(
join(dir, 'bot-prs.json'),
inTakeover ? '[]' : JSON.stringify(row),
);
writeFileSync(
join(dir, 'takeover-prs.json'),
inTakeover ? JSON.stringify(row) : '[]',
);
return execFileSync(
'bash',
[
'-c',
[
'set -uo pipefail',
`WORKDIR='${dir}'`,
`IDLE_CUTOFF='2026-07-28T12:00:00Z'`,
setSrc,
`IDLE_SLOT_NOW='${slotNow}'`,
`PR='${pr}'`,
`${predSrc} printf skip; else printf inspect; fi`,
].join('\n'),
],
{ encoding: 'utf8' },
);
} finally {
rmSync(dir, { recursive: true, force: true });
}
};
// Idle and out of slot → deferred (8001 % 4 = 1).
expect(
runBackoff({ pr: '8001', updatedAt: '2026-07-27T00:00:00Z', slotNow: 0 }),
).toBe('skip');
// Idle but IN its slot → inspected (its slot matched this scan).
expect(
runBackoff({ pr: '8001', updatedAt: '2026-07-27T00:00:00Z', slotNow: 1 }),
).toBe('inspect');
// Fresh activity → inspected regardless of slot.
expect(
runBackoff({ pr: '8001', updatedAt: '2026-07-28T13:00:00Z', slotNow: 0 }),
).toBe('inspect');
// Missing from the lookup (forced-dispatch shape) → inspected.
expect(
runBackoff({
pr: '8001',
updatedAt: '2026-07-27T00:00:00Z',
slotNow: 0,
listed: false,
}),
).toBe('inspect');

// Null/empty updatedAt (defensive guard) → inspected, not idle.
expect(runBackoff({ pr: '8001', updatedAt: null, slotNow: 0 })).toBe(
'inspect',
);
// Idle candidate in takeover-prs.json (the cohort this targets) → deferred.
expect(
runBackoff({
pr: '8001',
updatedAt: '2026-07-27T00:00:00Z',
slotNow: 0,
inTakeover: true,
}),
).toBe('skip');

// Visible in the fleet dashboard, like the busy skip.
expect(reviewScanJob).toContain("'idle-backoff'");
// The workflow comment must not claim the 10-target cap is relieved —
// idle candidates hit `continue` before the TARGETS append, so they
// never contend for it; the real win is inspection budget + walk
// latency, and the comment says so.
expect(reviewScanJob).toContain(
'Idle PRs never reach the\n # 10-target budget',
);
// The two scan-only signals updatedAt cannot see are named, not
// papered over by an absolute invariant.
expect(reviewScanJob).toContain('base conflict');
expect(reviewScanJob).toContain('still-red checks');
});

it('behaviorally replays the takeover-command toggle across all four paths', () => {
Expand Down
Loading