Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 55 additions & 17 deletions .github/workflows/qwen-autofix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2585,10 +2585,22 @@ jobs:
exit 1
fi

# Every check below can legitimately REJECT the agent's attempt, so
# each declares that verdict explicitly. That is what lets the handoff
# tell a rejection apart from the gate's OWN death: an empty outcome
# on a failed job means the gate never reached a verdict (its own bug,
# an infra blip), and the agent's work must then be retried rather
# than buried by a watermark advance.
reject_fix() {
echo "❌ ${1}"
echo "outcome=failed" >> "${GITHUB_OUTPUT}"
exit 1
}

echo '🔬 Re-running deterministic checks (independent of the agent)...'
npm run build
npm run typecheck
npm run lint
npm run build || reject_fix 'build failed on the agent-committed fix'
npm run typecheck || reject_fix 'typecheck failed on the agent-committed fix'
npm run lint || reject_fix 'lint failed on the agent-committed fix'

# Test changed/related files for the packages this PR touches.
# --changed follows the import graph so transitive breakage is caught.
Expand Down Expand Up @@ -2619,7 +2631,8 @@ jobs:
continue
fi
echo "🧪 Testing ${p} (changed files only)..."
npm run test --workspace "${p}" --if-present -- --changed origin/main --passWithNoTests
npm run test --workspace "${p}" --if-present -- --changed origin/main --passWithNoTests \
|| reject_fix "tests failed in ${p}"
done
fi
echo "outcome=fixed" >> "${GITHUB_OUTPUT}"
Expand Down Expand Up @@ -2856,31 +2869,56 @@ jobs:
# ISO-8601 date is used (not a bare word) so it is both non-empty AND
# sorts above any real timestamp in EVAL_WM's max, belt-and-suspenders
# with the terminal round.
# The gate declares its verdict explicitly (failed / noop / fixed).
# An EMPTY outcome on a non-success job means it died BEFORE reaching
# one - its own crash (a gate bug, an infra blip, a resolver error),
# not a judgement on the agent's work. That must retry like any other
# pre-verdict crash instead of advancing the watermark: the
# nested-package ENOENT that stranded #7329/#7336 looked exactly like
# a rejection, so a fix the agent had already written was discarded
# and the PR sat idle until a human deleted the marker by hand.
GATE_CRASHED=false
if [[ -z "${OUTCOME}" && "${JOB_STATUS:-}" != 'success' ]]; then
GATE_CRASHED=true
fi
MARK_TS="${NEWEST:-${WATERMARK:-9999-12-31T23:59:59Z}}"
if [[ -n "${NEWEST:-}" ]]; then
MARK_ROUND="$(( ROUND + 1 ))"
if [[ -z "${DETAIL_FILE}" ]]; then
# Prepare ran (NEWEST is set) but the agent produced NO output
# at all — it crashed before writing any verdict (e.g. a staged
# runner that fails to boot). It evaluated NOTHING, so the
# watermark must NOT advance past this feedback: an advance makes
# the next scan see "nothing new" and never retry, stranding the
# PR on a transient crash (a deploy, an infra blip, a base-image
# bug fixed minutes later). Stamp the sentinel ts (excluded from
# EVAL_WM) so the feedback stays live and the next scan retries;
# the incremented round still bounds retries to MAX_ROUNDS before
# a terminal handoff, so a PERSISTENT crash cannot loop forever.
if [[ -z "${DETAIL_FILE}" || "${GATE_CRASHED}" == 'true' ]]; then
# Prepare ran (NEWEST is set) but no verdict was reached — either
# the agent produced NO output at all (crashed before writing any
# verdict, e.g. a staged runner that fails to boot), or the gate
# crashed after the agent wrote its summary. It evaluated NOTHING,
# so the watermark must NOT advance past this feedback: an advance
# makes the next scan see "nothing new" and never retry,
# stranding the PR on a transient crash (a deploy, an infra blip,
# a base-image bug fixed minutes later). Stamp the sentinel ts
# (excluded from EVAL_WM) so the feedback stays live and the next
# scan retries; the incremented round still bounds retries to
# MAX_ROUNDS before a terminal handoff, so a PERSISTENT crash
# cannot loop forever.
MARK_TS='9999-12-31T23:59:59Z'
# Only promise a retry when one will actually happen: at
# MARK_ROUND == MAX_ROUNDS the next scan's round-cap gate skips
# the PR, and the cap-reached notice is takeover-only — so the
# final attempt must say so itself, or the maintainer waits for
# a retry that never comes. No Run log here: the report block
# below appends it to every handoff (avoid a duplicate URL).
# Name the real cause: a gate crash points the maintainer at
# the gate logs (the agent's commit is discarded with the runner,
# but the feedback watermark is preserved so the retry re-attempts
# the same feedback), while a no-output crash points at the run.
if [[ -z "${DETAIL_FILE}" ]]; then
CAUSE='crashed before it could evaluate the feedback'
LAST_FIX='a human should take over this PR'
else
CAUSE='hit a verification-gate error before reaching a verdict'
LAST_FIX='a maintainer should check the gate logs, then re-arm'
fi
if [[ "${MARK_ROUND}" -lt "${MAX_ROUNDS}" ]]; then
HEADLINE="🤖 AutoFix crashed before it could evaluate the feedback (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — it will retry on the next scan."
HEADLINE="🤖 AutoFix ${CAUSE} (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — it will retry on the next scan."
else
HEADLINE="🤖 AutoFix crashed before it could evaluate the feedback (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — this was the last automatic attempt; a human should take over this PR."
HEADLINE="🤖 AutoFix ${CAUSE} (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — this was the last automatic attempt; ${LAST_FIX}."
fi
else
HEADLINE="🤖 Could not address the latest feedback automatically (round ${MARK_ROUND}/${MAX_ROUNDS}). A human should take over this PR."
Expand Down
100 changes: 100 additions & 0 deletions scripts/tests/qwen-autofix-workflow.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -3114,6 +3114,106 @@ describe('qwen-autofix workflow', () => {
).toBe(0);
});

it('retries a verification-gate crash instead of burying the fix', () => {
// A gate that DECLARES a verdict (outcome=failed) evaluated the agent's
// attempt and rejected it - the watermark advances, bounded by MAX_ROUNDS.
// A gate that dies WITHOUT a verdict (empty outcome on a failed job) never
// judged the work at all; advancing there strands a fix the agent had
// already written, which is exactly how the nested-package ENOENT stranded
// #7329/#7336 until a human deleted the marker.
const decision = reviewAddressReportStep.match(
/(GATE_CRASHED=false\n[\s\S]*?\n {12}fi)\n {12}\{/,
)?.[1];
expect(decision).toBeTruthy();
const SENTINEL = '9999-12-31T23:59:59Z';
const NEWEST = '2026-07-20T10:00:00Z';
const run = (env) =>
execFileSync(
'bash',
[
'-c',
`${decision}\nprintf '%s|%s|%s' "$MARK_TS" "$MARK_ROUND" "$HEADLINE"`,
],
{
env: {
...process.env,
NEWEST,
WATERMARK: '2026-07-20T09:00:00Z',
ROUND: '1',
MAX_ROUNDS: '5',
DETAIL_FILE: '/w/address-summary.md',
OUTCOME: '',
JOB_STATUS: 'failure',
...env,
},
encoding: 'utf8',
},
);

// Declared rejection: the agent was judged -> advance the watermark.
const rejected = run({ OUTCOME: 'failed' });
expect(rejected.split('|')[0]).toBe(NEWEST);
expect(rejected).toContain('Could not address the latest feedback');

// Gate crash (no verdict): keep the feedback live and retry.
const crashed = run({ OUTCOME: '' });
expect(crashed.split('|')[0]).toBe(SENTINEL);
expect(crashed).toContain(
'verification-gate error before reaching a verdict',
);
expect(crashed).toContain('it will retry on the next scan');
expect(crashed.split('|')[1]).toBe('2');

// A no-output crash keeps its own (pre-existing) wording, still a retry.
const noOutput = run({ OUTCOME: '', DETAIL_FILE: '' });
expect(noOutput.split('|')[0]).toBe(SENTINEL);
expect(noOutput).toContain('crashed before it could evaluate the feedback');

// At the cap the gate crash names the operator fix rather than promising a
// retry the scan's round gate would refuse.
const capped = run({ OUTCOME: '', ROUND: '4' });
expect(capped).toContain('this was the last automatic attempt');
expect(capped).toContain('check the gate logs, then re-arm');

// A successful job never counts as a crash (dry-run reporting path).
expect(run({ OUTCOME: '', JOB_STATUS: 'success' }).split('|')[0]).toBe(
NEWEST,
);
});

it('makes every known gate rejection declare its verdict', () => {
// The retry/advance split above is only sound while each real rejection
// writes outcome=failed; an unwired check would read as a gate crash and be
// retried instead of reported. Drive the extracted helper for real.
const gate = verificationGateSteps[1];
for (const check of [
"npm run build || reject_fix 'build failed on the agent-committed fix'",
"npm run typecheck || reject_fix 'typecheck failed on the agent-committed fix'",
"npm run lint || reject_fix 'lint failed on the agent-committed fix'",
'reject_fix "tests failed in ${p}"',
]) {
expect(gate).toContain(check);
}
const helper = gate.match(/reject_fix\(\) \{\n[\s\S]*?\n {10}\}/)?.[0];
expect(helper).toBeTruthy();
const dir = mkdtempSync(join(tmpdir(), 'reject-'));
const out = join(dir, 'gh_output');
writeFileSync(out, '');
let status = 0;
try {
execFileSync(
'bash',
['-c', `set -eo pipefail\n${helper}\nfalse || reject_fix 'boom'`],
{ env: { ...process.env, GITHUB_OUTPUT: out }, encoding: 'utf8' },
);
} catch (e) {
status = e.status;
}
expect(status).not.toBe(0);
expect(readFileSync(out, 'utf8')).toContain('outcome=failed');
rmSync(dir, { recursive: true, force: true });
});

it('re-arms a stranded PR from a marker instead of a deleted comment', () => {
// Recovery used to mean `gh api -X DELETE` on the bot's own eval marker:
// raw API access, an erased audit trail, undiscoverable. `@qwen-code
Expand Down
Loading