From 22abbf1cb21956841bd9ee16abbdf1ac24d93c2a Mon Sep 17 00:00:00 2001 From: wenshao Date: Tue, 21 Jul 2026 00:37:08 +0800 Subject: [PATCH 1/2] fix(autofix): retry a verification-gate crash instead of burying the agent's fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A gate failure had two very different meanings collapsed into one outcome. When the gate DECLARES a verdict (outcome=failed) it evaluated the agent's attempt and rejected it, so advancing the watermark is right — the same feedback would reproduce the same rejection, and MAX_ROUNDS bounds it. But when the gate dies WITHOUT a verdict it never judged the work at all, and advancing buries a fix the agent had already written: the next scan sees "nothing new" and the PR sits until a human deletes the marker by hand. That is exactly how the nested-package ENOENT stranded #7329 and #7336. Both agents had implemented the review feedback — the handoff even quoted the implemented changes — but the gate crashed on its own bug while resolving packages/channels/*, the commit was discarded, and the PRs read as "Could not address the latest feedback automatically". Two halves: - The review-address gate now declares every rejection it can legitimately reach: build, typecheck, lint and the per-package tests each call a `reject_fix` helper that writes outcome=failed before exiting. (The resolver call is deliberately left undeclared — a resolver error IS a gate bug.) - The handoff treats an EMPTY outcome on a non-success job as the gate's own crash and routes it to the existing sentinel/retry path, so the feedback stays live and the next scan retries. The round still increments, so a persistently crashing gate is bounded exactly as before, and the headline names the real cause ("hit a verification-gate error before reaching a verdict") and, on the final attempt, points at the gate logs. Unchanged: a declared rejection still advances and reads as before, a no-output crash keeps its own wording and retry, and a crash before the feedback was read stays terminal. Tests: the real extracted decision block is replayed under bash across declared rejection (advances to NEWEST), gate crash (sentinel + retry + round+1), no output (sentinel, original wording), the round cap (operator fix), and a successful job (never a crash); plus the reject_fix helper is driven for real to prove a rejection writes outcome=failed. Both mutation-verified — dropping the crash arm, or unwiring one known rejection, turns them red. --- .github/workflows/qwen-autofix.yml | 49 ++++++++-- scripts/tests/qwen-autofix-workflow.test.js | 100 ++++++++++++++++++++ 2 files changed, 142 insertions(+), 7 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 177cbb9a231..655735afe28 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -2498,10 +2498,22 @@ jobs: exit 1 fi + # Every check below can legitimately REJECT the agent's attempt, so + # each declares that verdict explicitly. That is what lets the handoff + # tell a rejection apart from the gate's OWN death: an empty outcome + # on a failed job means the gate never reached a verdict (its own bug, + # an infra blip), and the agent's work must then be retried rather + # than buried by a watermark advance. + reject_fix() { + echo "❌ ${1}" + echo "outcome=failed" >> "${GITHUB_OUTPUT}" + exit 1 + } + echo '🔬 Re-running deterministic checks (independent of the agent)...' - npm run build - npm run typecheck - npm run lint + npm run build || reject_fix 'build failed on the agent-committed fix' + npm run typecheck || reject_fix 'typecheck failed on the agent-committed fix' + npm run lint || reject_fix 'lint failed on the agent-committed fix' # Test changed/related files for the packages this PR touches. # --changed follows the import graph so transitive breakage is caught. @@ -2532,7 +2544,8 @@ jobs: continue fi echo "🧪 Testing ${p} (changed files only)..." - npm run test --workspace "${p}" --if-present -- --changed origin/main --passWithNoTests + npm run test --workspace "${p}" --if-present -- --changed origin/main --passWithNoTests \ + || reject_fix "tests failed in ${p}" done fi echo "outcome=fixed" >> "${GITHUB_OUTPUT}" @@ -2769,10 +2782,22 @@ jobs: # ISO-8601 date is used (not a bare word) so it is both non-empty AND # sorts above any real timestamp in EVAL_WM's max, belt-and-suspenders # with the terminal round. + # The gate declares its verdict explicitly (failed / noop / fixed). + # An EMPTY outcome on a non-success job means it died BEFORE reaching + # one - its own crash (a gate bug, an infra blip, a resolver error), + # not a judgement on the agent's work. That must retry like any other + # pre-verdict crash instead of advancing the watermark: the + # nested-package ENOENT that stranded #7329/#7336 looked exactly like + # a rejection, so a fix the agent had already written was discarded + # and the PR sat idle until a human deleted the marker by hand. + GATE_CRASHED=false + if [[ -z "${OUTCOME}" && "${JOB_STATUS:-}" != 'success' ]]; then + GATE_CRASHED=true + fi MARK_TS="${NEWEST:-${WATERMARK:-9999-12-31T23:59:59Z}}" if [[ -n "${NEWEST:-}" ]]; then MARK_ROUND="$(( ROUND + 1 ))" - if [[ -z "${DETAIL_FILE}" ]]; then + if [[ -z "${DETAIL_FILE}" || "${GATE_CRASHED}" == 'true' ]]; then # Prepare ran (NEWEST is set) but the agent produced NO output # at all — it crashed before writing any verdict (e.g. a staged # runner that fails to boot). It evaluated NOTHING, so the @@ -2790,10 +2815,20 @@ jobs: # final attempt must say so itself, or the maintainer waits for # a retry that never comes. No Run log here: the report block # below appends it to every handoff (avoid a duplicate URL). + # Name the real cause: a gate crash points the maintainer at + # the gate logs (the agent's work may be fine and is preserved + # for the retry), while a no-output crash points at the run. + if [[ -z "${DETAIL_FILE}" ]]; then + CAUSE='crashed before it could evaluate the feedback' + LAST_FIX='a human should take over this PR' + else + CAUSE='hit a verification-gate error before reaching a verdict' + LAST_FIX='a maintainer should check the gate logs, then re-arm' + fi if [[ "${MARK_ROUND}" -lt "${MAX_ROUNDS}" ]]; then - HEADLINE="🤖 AutoFix crashed before it could evaluate the feedback (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — it will retry on the next scan." + HEADLINE="🤖 AutoFix ${CAUSE} (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — it will retry on the next scan." else - HEADLINE="🤖 AutoFix crashed before it could evaluate the feedback (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — this was the last automatic attempt; a human should take over this PR." + HEADLINE="🤖 AutoFix ${CAUSE} (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — this was the last automatic attempt; ${LAST_FIX}." fi else HEADLINE="🤖 Could not address the latest feedback automatically (round ${MARK_ROUND}/${MAX_ROUNDS}). A human should take over this PR." diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 64292cc0e70..12ed05350a8 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -3114,6 +3114,106 @@ describe('qwen-autofix workflow', () => { ).toBe(0); }); + it('retries a verification-gate crash instead of burying the fix', () => { + // A gate that DECLARES a verdict (outcome=failed) evaluated the agent's + // attempt and rejected it - the watermark advances, bounded by MAX_ROUNDS. + // A gate that dies WITHOUT a verdict (empty outcome on a failed job) never + // judged the work at all; advancing there strands a fix the agent had + // already written, which is exactly how the nested-package ENOENT stranded + // #7329/#7336 until a human deleted the marker. + const decision = reviewAddressReportStep.match( + /(GATE_CRASHED=false\n[\s\S]*?\n {12}fi)\n {12}\{/, + )?.[1]; + expect(decision).toBeTruthy(); + const SENTINEL = '9999-12-31T23:59:59Z'; + const NEWEST = '2026-07-20T10:00:00Z'; + const run = (env) => + execFileSync( + 'bash', + [ + '-c', + `${decision}\nprintf '%s|%s|%s' "$MARK_TS" "$MARK_ROUND" "$HEADLINE"`, + ], + { + env: { + ...process.env, + NEWEST, + WATERMARK: '2026-07-20T09:00:00Z', + ROUND: '1', + MAX_ROUNDS: '5', + DETAIL_FILE: '/w/address-summary.md', + OUTCOME: '', + JOB_STATUS: 'failure', + ...env, + }, + encoding: 'utf8', + }, + ); + + // Declared rejection: the agent was judged -> advance the watermark. + const rejected = run({ OUTCOME: 'failed' }); + expect(rejected.split('|')[0]).toBe(NEWEST); + expect(rejected).toContain('Could not address the latest feedback'); + + // Gate crash (no verdict): keep the feedback live and retry. + const crashed = run({ OUTCOME: '' }); + expect(crashed.split('|')[0]).toBe(SENTINEL); + expect(crashed).toContain( + 'verification-gate error before reaching a verdict', + ); + expect(crashed).toContain('it will retry on the next scan'); + expect(crashed.split('|')[1]).toBe('2'); + + // A no-output crash keeps its own (pre-existing) wording, still a retry. + const noOutput = run({ OUTCOME: '', DETAIL_FILE: '' }); + expect(noOutput.split('|')[0]).toBe(SENTINEL); + expect(noOutput).toContain('crashed before it could evaluate the feedback'); + + // At the cap the gate crash names the operator fix rather than promising a + // retry the scan's round gate would refuse. + const capped = run({ OUTCOME: '', ROUND: '4' }); + expect(capped).toContain('this was the last automatic attempt'); + expect(capped).toContain('check the gate logs, then re-arm'); + + // A successful job never counts as a crash (dry-run reporting path). + expect(run({ OUTCOME: '', JOB_STATUS: 'success' }).split('|')[0]).toBe( + NEWEST, + ); + }); + + it('makes every known gate rejection declare its verdict', () => { + // The retry/advance split above is only sound while each real rejection + // writes outcome=failed; an unwired check would read as a gate crash and be + // retried instead of reported. Drive the extracted helper for real. + const gate = verificationGateSteps[1]; + for (const check of [ + "npm run build || reject_fix 'build failed on the agent-committed fix'", + "npm run typecheck || reject_fix 'typecheck failed on the agent-committed fix'", + "npm run lint || reject_fix 'lint failed on the agent-committed fix'", + 'reject_fix "tests failed in ${p}"', + ]) { + expect(gate).toContain(check); + } + const helper = gate.match(/reject_fix\(\) \{\n[\s\S]*?\n {10}\}/)?.[0]; + expect(helper).toBeTruthy(); + const dir = mkdtempSync(join(tmpdir(), 'reject-')); + const out = join(dir, 'gh_output'); + writeFileSync(out, ''); + let status = 0; + try { + execFileSync( + 'bash', + ['-c', `set -eo pipefail\n${helper}\nfalse || reject_fix 'boom'`], + { env: { ...process.env, GITHUB_OUTPUT: out }, encoding: 'utf8' }, + ); + } catch (e) { + status = e.status; + } + expect(status).not.toBe(0); + expect(readFileSync(out, 'utf8')).toContain('outcome=failed'); + rmSync(dir, { recursive: true, force: true }); + }); + it('replays the handoff decision and terminal-round transitions under bash', () => { // The agent step is bounded below the 120-minute job timeout so a runaway // agent fails the STEP, not the job, leaving the always() report step time to From 9f00d1008033630cf1f3d2578e910d4d34ddcdea Mon Sep 17 00:00:00 2001 From: Qwen Code Bot Date: Tue, 21 Jul 2026 01:45:37 +0000 Subject: [PATCH 2/2] fix(autofix): clarify retry-branch comments per review nits (#7351) --- .github/workflows/qwen-autofix.yml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 26a3ff90cf7..b026deb50fa 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -2885,16 +2885,18 @@ jobs: if [[ -n "${NEWEST:-}" ]]; then MARK_ROUND="$(( ROUND + 1 ))" if [[ -z "${DETAIL_FILE}" || "${GATE_CRASHED}" == 'true' ]]; then - # Prepare ran (NEWEST is set) but the agent produced NO output - # at all — it crashed before writing any verdict (e.g. a staged - # runner that fails to boot). It evaluated NOTHING, so the - # watermark must NOT advance past this feedback: an advance makes - # the next scan see "nothing new" and never retry, stranding the - # PR on a transient crash (a deploy, an infra blip, a base-image - # bug fixed minutes later). Stamp the sentinel ts (excluded from - # EVAL_WM) so the feedback stays live and the next scan retries; - # the incremented round still bounds retries to MAX_ROUNDS before - # a terminal handoff, so a PERSISTENT crash cannot loop forever. + # Prepare ran (NEWEST is set) but no verdict was reached — either + # the agent produced NO output at all (crashed before writing any + # verdict, e.g. a staged runner that fails to boot), or the gate + # crashed after the agent wrote its summary. It evaluated NOTHING, + # so the watermark must NOT advance past this feedback: an advance + # makes the next scan see "nothing new" and never retry, + # stranding the PR on a transient crash (a deploy, an infra blip, + # a base-image bug fixed minutes later). Stamp the sentinel ts + # (excluded from EVAL_WM) so the feedback stays live and the next + # scan retries; the incremented round still bounds retries to + # MAX_ROUNDS before a terminal handoff, so a PERSISTENT crash + # cannot loop forever. MARK_TS='9999-12-31T23:59:59Z' # Only promise a retry when one will actually happen: at # MARK_ROUND == MAX_ROUNDS the next scan's round-cap gate skips @@ -2903,8 +2905,9 @@ jobs: # a retry that never comes. No Run log here: the report block # below appends it to every handoff (avoid a duplicate URL). # Name the real cause: a gate crash points the maintainer at - # the gate logs (the agent's work may be fine and is preserved - # for the retry), while a no-output crash points at the run. + # the gate logs (the agent's commit is discarded with the runner, + # but the feedback watermark is preserved so the retry re-attempts + # the same feedback), while a no-output crash points at the run. if [[ -z "${DETAIL_FILE}" ]]; then CAUSE='crashed before it could evaluate the feedback' LAST_FIX='a human should take over this PR'