Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 103 additions & 8 deletions .github/workflows/qwen-autofix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2425,6 +2425,27 @@ jobs:
| select((.completedAt // .updatedAt // "") > $wm)
| "- \(((.name // .workflowName) // "external check") | gsub("[^A-Za-z0-9 _./()-]"; "") | .[0:80]): \(.conclusion // .state // "?")"' \
"${WORKDIR}/checks.json"
# If the LAST round ended in a gate rejection, show the agent WHY.
# It is otherwise invisible on the retry: the reason lives in the
# bot's own handoff comment, which the feedback filter above
# (correctly) excludes, so the agent would re-read only the original
# review points and re-make the same mistake - #7208 was handed to a
# human over a two-character TS4111 fix its own compiler output had
# already spelled out.
LAST_REJECTION="$(jq -r --arg ab "${AUTOFIX_BOT}" '
[ .[] | select((.user.login // "") == $ab)
| select((.body // "") | contains("<!-- autofix-eval ts=")) ]
| sort_by(.created_at) | last | .body // ""' "${WORKDIR}/ic.json" \
| sed -n '/<!-- autofix-gate-rejection-start -->/,/<!-- autofix-gate-rejection-end -->/p' \
| sed '1d;$d')"
if [[ -n "${LAST_REJECTION}" ]]; then
echo
echo '## Your previous attempt was REJECTED by the verification gate'
echo
echo 'Fix this first — the same change will be rejected again otherwise.'
echo
printf '%s\n' "${LAST_REJECTION}"
fi
} > "${WORKDIR}/feedback.md"
echo '--- feedback.md ---'
cat "${WORKDIR}/feedback.md"
Expand Down Expand Up @@ -2585,10 +2606,47 @@ jobs:
exit 1
fi

# Every check below can legitimately REJECT the agent's attempt, so
# each declares that verdict explicitly. That is what lets the handoff
# tell a rejection apart from the gate's OWN death: an empty outcome
# on a failed job means the gate never reached a verdict (its own bug,
# an infra blip), and the agent's work must then be retried rather
# than buried by a watermark advance.
# Capture each check's output. A rejection has to tell the agent WHY
# its change was refused: without that, the next round re-reads only
# the original review feedback and re-makes the same mistake - #7208
# was handed to a human over a two-character TS4111 fix its own
# compiler output already spelled out.
GATE_LOG="${WORKDIR}/gate-output.log"
: > "${GATE_LOG}"
reject_fix() {
echo "❌ ${1}"
{
echo "**${1}**"
echo
# A four-backtick fence cannot be closed by a ``` line, so
# captured output containing its own fences stays inside the
# block when this is posted verbatim as a PR comment.
echo '````'
tail -c 3000 "${GATE_LOG}" 2> /dev/null
echo '````'
} > "${WORKDIR}/gate-rejection.md"
echo "outcome=failed" >> "${GITHUB_OUTPUT}"
exit 1
}
run_check() {
# pipefail makes the pipeline carry the command's status, not tee's.
local label="${1}"
shift
if ! "$@" 2>&1 | tee -a "${GATE_LOG}"; then
reject_fix "${label}"
fi
}

echo '🔬 Re-running deterministic checks (independent of the agent)...'
npm run build
npm run typecheck
npm run lint
run_check 'build failed on the agent-committed fix' npm run build
run_check 'typecheck failed on the agent-committed fix' npm run typecheck
run_check 'lint failed on the agent-committed fix' npm run lint

# Test changed/related files for the packages this PR touches.
# --changed follows the import graph so transitive breakage is caught.
Expand Down Expand Up @@ -2619,7 +2677,8 @@ jobs:
continue
fi
echo "🧪 Testing ${p} (changed files only)..."
npm run test --workspace "${p}" --if-present -- --changed origin/main --passWithNoTests
run_check "tests failed in ${p}" \
npm run test --workspace "${p}" --if-present -- --changed origin/main --passWithNoTests
done
fi
echo "outcome=fixed" >> "${GITHUB_OUTPUT}"
Expand All @@ -2631,7 +2690,7 @@ jobs:
if git rev-parse --verify "${BRANCH}" > /dev/null 2>&1; then
git diff "origin/main...${BRANCH}" > "${WORKDIR}/pr.diff" || true
fi
for f in feedback.md address-summary.md no-action.md failure.md handoff.md pr.diff; do
for f in feedback.md address-summary.md no-action.md failure.md handoff.md gate-rejection.md pr.diff; do
if [[ -f "${WORKDIR}/${f}" ]]; then
echo "=============== ${f} ==============="
cat "${WORKDIR}/${f}"
Expand Down Expand Up @@ -2856,10 +2915,22 @@ jobs:
# ISO-8601 date is used (not a bare word) so it is both non-empty AND
# sorts above any real timestamp in EVAL_WM's max, belt-and-suspenders
# with the terminal round.
# The gate declares its verdict explicitly (failed / noop / fixed).
# An EMPTY outcome on a non-success job means it died BEFORE reaching
# one - its own crash (a gate bug, an infra blip, a resolver error),
# not a judgement on the agent's work. That must retry like any other
# pre-verdict crash instead of advancing the watermark: the
# nested-package ENOENT that stranded #7329/#7336 looked exactly like
# a rejection, so a fix the agent had already written was discarded
# and the PR sat idle until a human deleted the marker by hand.
GATE_CRASHED=false
if [[ -z "${OUTCOME}" && "${JOB_STATUS:-}" != 'success' ]]; then
GATE_CRASHED=true
fi
MARK_TS="${NEWEST:-${WATERMARK:-9999-12-31T23:59:59Z}}"
if [[ -n "${NEWEST:-}" ]]; then
MARK_ROUND="$(( ROUND + 1 ))"
if [[ -z "${DETAIL_FILE}" ]]; then
if [[ -z "${DETAIL_FILE}" || "${GATE_CRASHED}" == 'true' ]]; then
# Prepare ran (NEWEST is set) but the agent produced NO output
# at all — it crashed before writing any verdict (e.g. a staged
# runner that fails to boot). It evaluated NOTHING, so the
Expand All @@ -2877,10 +2948,20 @@ jobs:
# final attempt must say so itself, or the maintainer waits for
# a retry that never comes. No Run log here: the report block
# below appends it to every handoff (avoid a duplicate URL).
# Name the real cause: a gate crash points the maintainer at
# the gate logs (the agent's work may be fine and is preserved
# for the retry), while a no-output crash points at the run.
if [[ -z "${DETAIL_FILE}" ]]; then
CAUSE='crashed before it could evaluate the feedback'
LAST_FIX='a human should take over this PR'
else
CAUSE='hit a verification-gate error before reaching a verdict'
LAST_FIX='a maintainer should check the gate logs, then re-arm'
fi
if [[ "${MARK_ROUND}" -lt "${MAX_ROUNDS}" ]]; then
HEADLINE="🤖 AutoFix crashed before it could evaluate the feedback (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — it will retry on the next scan."
HEADLINE="🤖 AutoFix ${CAUSE} (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — it will retry on the next scan."
else
HEADLINE="🤖 AutoFix crashed before it could evaluate the feedback (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — this was the last automatic attempt; a human should take over this PR."
HEADLINE="🤖 AutoFix ${CAUSE} (attempt ${MARK_ROUND}/${MAX_ROUNDS}) — this was the last automatic attempt; ${LAST_FIX}."
fi
else
HEADLINE="🤖 Could not address the latest feedback automatically (round ${MARK_ROUND}/${MAX_ROUNDS}). A human should take over this PR."
Expand Down Expand Up @@ -2923,6 +3004,20 @@ jobs:
else
echo "AutoFix failed before producing a verified commit (the run crashed or timed out before it could explain why)."
fi
# Carry the gate's own rejection reason. Without it the comment
# shows only the agent's optimistic summary, so neither a human
# nor the NEXT round can see why the change was refused - the
# agent then re-reads the original review feedback and repeats
# the same mistake. Delimited so `Prepare branch and feedback`
# can lift it back out on the retry.
if [[ -s "${WORKDIR}/gate-rejection.md" ]]; then
echo
echo '<!-- autofix-gate-rejection-start -->'
echo "**Why it was not pushed:**"
echo
head -c 3500 "${WORKDIR}/gate-rejection.md" | iconv -f utf-8 -t utf-8 -c | sed 's/<!--/<!\-\-/g' || true
echo '<!-- autofix-gate-rejection-end -->'
fi
echo
echo
echo "Run log: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
Expand Down
217 changes: 217 additions & 0 deletions scripts/tests/qwen-autofix-workflow.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -3114,6 +3114,109 @@ describe('qwen-autofix workflow', () => {
).toBe(0);
});

it('retries a verification-gate crash instead of burying the fix', () => {
// A gate that DECLARES a verdict (outcome=failed) evaluated the agent's
// attempt and rejected it - the watermark advances, bounded by MAX_ROUNDS.
// A gate that dies WITHOUT a verdict (empty outcome on a failed job) never
// judged the work at all; advancing there strands a fix the agent had
// already written, which is exactly how the nested-package ENOENT stranded
// #7329/#7336 until a human deleted the marker.
const decision = reviewAddressReportStep.match(
/(GATE_CRASHED=false\n[\s\S]*?\n {12}fi)\n {12}\{/,
)?.[1];
expect(decision).toBeTruthy();
const SENTINEL = '9999-12-31T23:59:59Z';
const NEWEST = '2026-07-20T10:00:00Z';
const run = (env) =>
execFileSync(
'bash',
[
'-c',
`${decision}\nprintf '%s|%s|%s' "$MARK_TS" "$MARK_ROUND" "$HEADLINE"`,
],
{
env: {
...process.env,
NEWEST,
WATERMARK: '2026-07-20T09:00:00Z',
ROUND: '1',
MAX_ROUNDS: '5',
DETAIL_FILE: '/w/address-summary.md',
OUTCOME: '',
JOB_STATUS: 'failure',
...env,
},
encoding: 'utf8',
},
);

// Declared rejection: the agent was judged -> advance the watermark.
const rejected = run({ OUTCOME: 'failed' });
expect(rejected.split('|')[0]).toBe(NEWEST);
expect(rejected).toContain('Could not address the latest feedback');

// Gate crash (no verdict): keep the feedback live and retry.
const crashed = run({ OUTCOME: '' });
expect(crashed.split('|')[0]).toBe(SENTINEL);
expect(crashed).toContain(
'verification-gate error before reaching a verdict',
);
expect(crashed).toContain('it will retry on the next scan');
expect(crashed.split('|')[1]).toBe('2');

// A no-output crash keeps its own (pre-existing) wording, still a retry.
const noOutput = run({ OUTCOME: '', DETAIL_FILE: '' });
expect(noOutput.split('|')[0]).toBe(SENTINEL);
expect(noOutput).toContain('crashed before it could evaluate the feedback');

// At the cap the gate crash names the operator fix rather than promising a
// retry the scan's round gate would refuse.
const capped = run({ OUTCOME: '', ROUND: '4' });
expect(capped).toContain('this was the last automatic attempt');
expect(capped).toContain('check the gate logs, then re-arm');

// A successful job never counts as a crash (dry-run reporting path).
expect(run({ OUTCOME: '', JOB_STATUS: 'success' }).split('|')[0]).toBe(
NEWEST,
);
});

it('makes every known gate rejection declare its verdict', () => {
// The retry/advance split above is only sound while each real rejection
// writes outcome=failed; an unwired check would read as a gate crash and be
// retried instead of reported. Drive the extracted helper for real.
const gate = verificationGateSteps[1];
// Each check runs through run_check, which tees its output to GATE_LOG and
// calls reject_fix on failure - so the verdict is declared AND the reason
// is captured for the retry.
for (const check of [
"run_check 'build failed on the agent-committed fix' npm run build",
"run_check 'typecheck failed on the agent-committed fix' npm run typecheck",
"run_check 'lint failed on the agent-committed fix' npm run lint",
'run_check "tests failed in ${p}"',
]) {
expect(gate).toContain(check);
}
const helper = gate.match(/reject_fix\(\) \{\n[\s\S]*?\n {10}\}/)?.[0];
expect(helper).toBeTruthy();
const dir = mkdtempSync(join(tmpdir(), 'reject-'));
const out = join(dir, 'gh_output');
writeFileSync(out, '');
let status = 0;
try {
execFileSync(
'bash',
['-c', `set -eo pipefail\n${helper}\nfalse || reject_fix 'boom'`],
{ env: { ...process.env, GITHUB_OUTPUT: out }, encoding: 'utf8' },
);
} catch (e) {
status = e.status;
}
expect(status).not.toBe(0);
expect(readFileSync(out, 'utf8')).toContain('outcome=failed');
rmSync(dir, { recursive: true, force: true });
});

it('re-arms a stranded PR from a marker instead of a deleted comment', () => {
// Recovery used to mean `gh api -X DELETE` on the bot's own eval marker:
// raw API access, an erased audit trail, undiscoverable. `@qwen-code
Expand Down Expand Up @@ -3406,6 +3509,120 @@ describe('qwen-autofix workflow', () => {
expect(failed.stdout).toContain('Bad credentials');
});

it('feeds the gate rejection back so the retry can fix what it broke', () => {
// #7208 was handed to a human over a two-character TS4111 error its own
// compiler output already spelled out: the gate rejected the commit, the
// handoff showed only the agent's optimistic summary, and the next round
// re-read the original review points with no idea why it had been refused.
const gate = verificationGateSteps[1];
const prep =
workflow.match(
/- name: 'Prepare branch and feedback'[\s\S]*?(?=\n {6}- name: )/,
)?.[0] ?? '';

// 1. A failing check records WHY, not just THAT, it failed.
const capture = gate.match(
/GATE_LOG="\$\{WORKDIR\}\/gate-output\.log"[\s\S]*?\n {10}\}\n {10}run_check\(\) \{[\s\S]*?\n {10}\}/,
)?.[0];
expect(capture).toBeTruthy();
const dir = mkdtempSync(join(tmpdir(), 'gate-'));
const out = join(dir, 'gh_output');
writeFileSync(out, '');
let status = 0;
try {
execFileSync(
'bash',
[
'-c',
[
'set -eo pipefail',
`WORKDIR=${JSON.stringify(dir)}`,
capture,
"run_check 'build failed on the agent-committed fix' bash -c \"echo 'src/goals/goalJudge.ts(364,13): error TS4111'; exit 1\"",
].join('\n'),
],
{ env: { ...process.env, GITHUB_OUTPUT: out }, encoding: 'utf8' },
);
} catch (e) {
status = e.status;
}
expect(status).not.toBe(0);
expect(readFileSync(out, 'utf8')).toContain('outcome=failed');
const rejection = readFileSync(join(dir, 'gate-rejection.md'), 'utf8');
expect(rejection).toContain('build failed on the agent-committed fix');
// The compiler's own words must survive - that is the whole point.
expect(rejection).toContain('error TS4111');
// A four-backtick fence cannot be closed by captured ``` output.
expect(rejection).toContain('````');

// 2. The handoff delimits it so the retry can lift it back out.
expect(reviewAddressReportStep).toContain(
'<!-- autofix-gate-rejection-start -->',
);
expect(reviewAddressReportStep).toContain(
'<!-- autofix-gate-rejection-end -->',
);

// 3. Round-trip: the prepare step recovers it from the bot's newest comment.
const extract = prep.match(
/LAST_REJECTION="\$\(jq[\s\S]*?\n {14}\| sed '1d;\$d'\)"/,
)?.[0];
expect(extract).toBeTruthy();
const runExtract = (comments) => {
const d = mkdtempSync(join(tmpdir(), 'fb-'));
writeFileSync(join(d, 'ic.json'), JSON.stringify(comments));
const res = execFileSync(
'bash',
[
'-c',
[
'set -uo pipefail',
`WORKDIR=${JSON.stringify(d)}`,
'AUTOFIX_BOT=qwen-code-dev-bot',
extract,
'printf "%s" "${LAST_REJECTION}"',
].join('\n'),
],
{ encoding: 'utf8' },
);
rmSync(d, { recursive: true, force: true });
return res;
};
const withRejection = [
{
user: { login: 'qwen-code-dev-bot' },
created_at: '2026-07-20T10:00:00Z',
body: 'old <!-- autofix-eval ts=1 acted=true round=1 -->',
},
{
user: { login: 'qwen-code-dev-bot' },
created_at: '2026-07-20T19:32:00Z',
body: [
'handoff',
'<!-- autofix-gate-rejection-start -->',
'**build failed on the agent-committed fix**',
"error TS4111: Property must be accessed with ['truncated']",
'<!-- autofix-gate-rejection-end -->',
'<!-- autofix-eval ts=2 acted=false round=5 -->',
].join('\n'),
},
];
const recovered = runExtract(withRejection);
expect(recovered).toContain('error TS4111');
expect(recovered).not.toContain('autofix-gate-rejection'); // markers stripped
// A round that pushed carries no rejection - nothing to replay.
expect(
runExtract([
{
user: { login: 'qwen-code-dev-bot' },
created_at: '2026-07-20T19:32:00Z',
body: 'pushed <!-- autofix-eval ts=2 acted=true round=5 -->',
},
]).trim(),
).toBe('');
rmSync(dir, { recursive: true, force: true });
});

it('replays the handoff decision and terminal-round transitions under bash', () => {
// The agent step is bounded below the 120-minute job timeout so a runaway
// agent fails the STEP, not the job, leaving the always() report step time to
Expand Down
Loading