Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion .github/workflows/qwen-autofix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5356,7 +5356,7 @@ jobs:
repository(owner:$owner,name:$name){
pullRequest(number:$pr){
reviewThreads(first:100, after:$endCursor){
nodes{id isResolved comments(first:100){nodes{databaseId} pageInfo{hasNextPage}}}
nodes{id isResolved comments(first:100){nodes{databaseId author{login} body} pageInfo{hasNextPage}}}
pageInfo{hasNextPage endCursor}
}
}
Expand Down Expand Up @@ -5471,6 +5471,27 @@ jobs:
root_id="$(jq -r --argjson id "${rc_id}" \
'map(select(any(.comments.nodes[]; .databaseId == $id)))
| .[0].comments.nodes[0].databaseId // $id' <<< "${THREADS_JSON}")"
# Idempotence gate: a crash-and-rerun of this round, a
# same-run repair that regenerates the dispositions, or a
# later round whose agent rewrites an unchanged declination
# must not post the same bot reply twice on one thread
# (observed 2026-08-16: an identical reply posted three
# times, #9296). Skip when the thread already carries a
# comment by the bot whose body EQUALS the neutralised body
# about to be posted; a changed body — a new reason in a
# later round — still posts. Best-effort like the rest: with
# a stale or empty threads view this degrades to the old
# post-always behavior.
if jq -e --argjson id "${root_id}" --arg bot "${AUTOFIX_BOT}" \
--arg body "${REPLY_BODY}" '
map(select(any(.comments.nodes[]; .databaseId == $id)))
| .[0].comments.nodes // []
| any(.[]; (.author.login // "") == $bot
and (.body // "") == $body)' \
<<< "${THREADS_JSON}" > /dev/null 2>&1; then
echo "⏭️ reply to review comment ${rc_id} skipped — identical bot reply already on the thread"
continue
fi
if gh api "repos/${REPO}/pulls/${PR}/comments/${root_id}/replies" \
-f body="${REPLY_BODY}" > /dev/null 2>&1; then
REPLIED_N=$(( REPLIED_N + 1 ))
Expand Down
168 changes: 159 additions & 9 deletions scripts/tests/qwen-autofix-workflow.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -16288,6 +16288,16 @@ exit 1
// exit 0 and no warning, restoring the oldest-hundred bug silently. Do not
// "fix" this pin by reordering it.
expect(block).toContain('pageInfo{hasNextPage endCursor}');
// Supply side of the reply gate: author{login} and body feed the dedup
// check, and pageInfo belongs INSIDE comments(...) —
// PullRequestReviewThread has no pageInfo field, so hoisting it to the
// thread level makes GitHub reject the whole query, THREADS_JSON becomes
// [], and both this block and the reply gate silently degrade. Pin the
// exact shape so the field list and pageInfo's inner position regress
// loudly.
expect(block).toContain(
'comments(first:100){nodes{databaseId author{login} body} pageInfo{hasNextPage}}',
);

const matching = runResolve();
expect(matching.status).toBe(0);
Expand Down Expand Up @@ -16642,7 +16652,17 @@ exit 1
].join('\n'),
);
chmodSync(join(bin, 'gh'), 0o755);
const runBlock = () =>
// The threads fetch is hoisted above the reply block (shared with the
// resolve block). One thread holds root comment 100 with a reply 222, so
// a reply aimed at the REPLY id 222 must be remapped to root 100 —
// GitHub rejects a reply whose target is itself a reply. 444 is in no
// thread, which exercises the fall-back to the id as given. The nodes
// omit author/body on purpose: the idempotence gate must tolerate a
// threads view without them (pre-existing shape) and post as before.
const DEFAULT_THREADS = [
{ comments: { nodes: [{ databaseId: 100 }, { databaseId: 222 }] } },
];
const runBlock = (threads = DEFAULT_THREADS) =>
execFileSync('bash', ['-c', `set -uo pipefail\n${block}`], {
env: {
...process.env,
Expand All @@ -16651,14 +16671,8 @@ exit 1
REPO: 'QwenLM/qwen-code',
PR: '7731',
REPLIED_LOG: repliedLog,
// The threads fetch is hoisted above the reply block (shared with the
// resolve block). One thread holds root comment 100 with a reply 222,
// so a reply aimed at the REPLY id 222 must be remapped to root 100 —
// GitHub rejects a reply whose target is itself a reply. 444 is in no
// thread, which exercises the fall-back to the id as given.
THREADS_JSON: JSON.stringify([
{ comments: { nodes: [{ databaseId: 100 }, { databaseId: 222 }] } },
]),
AUTOFIX_BOT: 'qwen-code-dev-bot',
THREADS_JSON: JSON.stringify(threads),
},
encoding: 'utf8',
});
Expand Down Expand Up @@ -16720,6 +16734,142 @@ exit 1
out = runBlock();
expect(readFileSync(repliedLog, 'utf8').trim()).toBe('');
expect(out).toContain('replied on 0 thread');

// Idempotence: a crash-and-rerun of the round, a same-run repair, or a
// later round re-declining the same finding regenerates the same
// disposition — the reply must not land twice on one thread (#9296,
// where one identical reply was posted three times). The match is on
// the bot login plus the exact NEUTRALISED body, so a CHANGED body —
// new information from a later round — still posts.
writeFileSync(repliedLog, '');
rmSync(join(dir, 'resolved-comments.txt'), { force: true });
writeFileSync(
join(dir, 'comment-replies.json'),
JSON.stringify([
{ id: 222, body: 'Deferred — follow-up.\n\n中文:已延后。' },
{ id: 222, body: 'Changed reason — new round.' },
]),
);
out = runBlock([
{
// isResolved is set on purpose: the gate must not filter resolved
// threads — a re-armed round can re-decline a finding whose thread a
// reviewer resolved after the bot's first reply, and a duplicate is
// still a duplicate. The trailing human comment is the common live
// shape (root, bot declination, human answer): the matching comment
// is not the newest node, so the gate must scan every comment.
isResolved: true,
comments: {
nodes: [
{ databaseId: 100 },
{ databaseId: 222 },
{
databaseId: 300,
author: { login: 'qwen-code-dev-bot' },
Comment on lines +16767 to +16768

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Suggestion] In both new fixtures the compared comment (bot or human) is the LAST node of its thread, so a mutant narrowing the gate's scan to the newest comment — (.[0].comments.nodes // [])[-1:] — passes the whole suite (measured: 180/180 green with the mutant). The common real thread shape is root comment, bot declination X, then a human reply answering it; on that shape a crash-and-rerun regenerating the identical disposition X meets a newest-comment-only gate that sees only the human's comment, finds no match, and re-posts X — the #9296 duplicate returns on the most common thread shape while every test stays green (probe-verified: original gate MATCH/skip, mutant NO MATCH/repost). Fix: add one fixture where a human (or authorless) comment FOLLOWS the bot's matching comment in the same thread, and assert the reply is still skipped.

中文说明

两个新 fixture 里被比对的评论(bot 或人类)都是各自线程的最后一个节点,因此把门的扫描收窄到最新一条评论的变异体——(.[0].comments.nodes // [])[-1:]——能通过整个套件(实测:变异体下 180/180 全绿)。真实线程的常见形态是:根评论、bot 的 decline X、随后人类回复作答;在这种形态下,crash 重跑重新生成相同 disposition X 时,只看最新评论的门只会看到人类那条,匹配失败,X 被再次发布——#9296 的重复回复在最常见线程形态上复活,而所有测试保持绿(探针验证:原门 MATCH/跳过,变异体 NO MATCH/重发)。修复:新增一个 fixture,让一条人类(或无作者)评论跟在该线程中 bot 的匹配评论之后,断言回复仍被跳过。

— qwen3.8-max via Qwen Code /review (v0.21.14)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Suggestion] R1-5: Still standing at this commit (probe re-measured): in both new fixtures the compared comment (bot or human) is the LAST node of its thread, so a mutant narrowing the gate's scan to the newest comment survives the whole suite. A/B at this commit: with the matching bot comment followed by a later unrelated human comment in the same thread, the shipped gate exits 0 (skip — correct) while the newest-only mutant exits 1 (post — duplicate). Fix: add a fixture where the matching bot comment is followed by another comment in the same thread.

中文说明

[Suggestion] R1-5:在本 commit 仍然存在(探针重新实测):两个新 fixture 中被比较的评论(bot 或人类)都是其线程的最后一个节点,因此把门的扫描收窄到最新评论的突变能通过整个套件。本 commit 上的 A/B:匹配的 bot 评论之后同一线程还有一条无关的人类评论时,现有门 exit=0(跳过——正确),只看最新评论的突变 exit=1(发布——重复)。修复:新增一个 fixture,让匹配的 bot 评论之后同线程还有另一条评论。

— qwen3.8-max via Qwen Code /review (v0.21.14)

body: 'Deferred — follow-up.\n\n中文:已延后。',
Comment on lines +16768 to +16769

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Suggestion] No reply-gate fixture sets isResolved on a thread, so a mutant adding select(.isResolved | not) to the gate's selector — a plausible harmonisation with the resolve block's selector ~15 lines above in the workflow, which carries exactly that filter — survives the entire suite (measured: 180/180 green with the mutant; probe on a resolved thread flips MATCH→NO MATCH). The harm path is real: after a re-arm marker resets the watermark the next round re-reads all feedback, and the REST pulls/{pr}/comments enumeration carries no resolved state — so the agent can legitimately re-decline an old finding whose thread was resolved after the bot's first reply, and an isResolved-filtered gate would repost the duplicate onto the resolved thread. The current code is correct (the gate ignores isResolved); only the pin is missing. Fix: add isResolved: true to the dedupe scenario's thread object (or add a scenario) and keep the skip assertions.

中文说明

没有任何回复门的 fixture 给线程设置 isResolved,因此在门的选择器上加 select(.isResolved | not) 的变异体——与工作流上方约 15 行处 resolve 块选择器(恰好带这个过滤条件)的“合理对齐”——能在整个套件下存活(实测:变异体下 180/180 全绿;在已解决线程上的探针显示 MATCH→NO MATCH 翻转)。危害路径真实存在:re-arm 标记重置水位线后,下一轮会重新读取全部反馈,而 REST pulls/{pr}/comments 枚举不携带线程解决状态——因此 agent 完全可能对一条“bot 首次回复后被 reviewer 解决”的旧 finding 重新 decline,而带 isResolved 过滤的门会把重复回复重发到已解决的线程上。当前代码是正确的(门忽略 isResolved);缺的只是钉住这一点的测试。修复:在去重场景的线程对象上加 isResolved: true(或新增场景),并保留跳过断言。

— qwen3.8-max via Qwen Code /review (v0.21.14)

Comment on lines +16768 to +16769

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Suggestion] R1-6: Still standing at this commit (probe re-measured): no reply-gate fixture sets isResolved on a thread, so a mutant adding select(.isResolved | not) to the gate's selector — a plausible harmonisation with the resolve block's selector ~15 lines above in the workflow — survives the suite (null | not is true when the field is absent). With that mutant in place, a reply to a comment in a resolved thread is silently skipped (no post, no distinct log line) and the suite stays green — the intended semantics for resolved threads are unpinned. Fix: add a fixture with isResolved: true on the thread asserting the reply still posts (or is deliberately skipped with a distinct log line, per the intended semantics).

中文说明

[Suggestion] R1-6:在本 commit 仍然存在(探针重新实测):没有任何回复门 fixture 给线程设置 isResolved,因此在门的选择器上加 select(.isResolved | not) 的突变能通过套件——这是一种与工作流上方约 15 行 resolve 块选择器的「合理对齐」,且字段缺失时 null | not 为 true。该突变生效后,对已 resolve 线程中评论的回复会被静默跳过(不发布、也没有独立日志),套件仍全绿——resolved 线程下的预期语义没有被 pin。修复:新增一个线程带 isResolved: true 的 fixture,断言回复仍会发布(或按预期语义被有意跳过并输出独立日志)。

— qwen3.8-max via Qwen Code /review (v0.21.14)

},
{ databaseId: 350, author: { login: 'wenshao' }, body: 'ack' },
],
},
},
]);
const deduped = readFileSync(repliedLog, 'utf8').trim().split('\n');
expect(deduped).toHaveLength(1);
expect(deduped[0]).toContain('pulls/7731/comments/100/replies');
expect(deduped[0]).toContain('body=Changed reason');

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Suggestion] None of the new fixture bodies contains <!--, so a one-line mutant — the gate comparing the raw decoded body instead of the neutralised REPLY_BODY — survives the suite (measured: mutant applied, 1 passed | 179 skipped). In production, reply bodies containing <!-- are neutralised to <!\-\- before posting (the block's own comment anticipates model output smuggling control markers); a later round regenerating the same raw body would then compare raw against the stored neutralised form, miss the match, and repost the duplicate. The code as written compares the neutralised body correctly — only the pin is missing. Fix: add one scenario where the pending reply body contains <!-- and the existing bot comment's body holds the neutralised <!\-\- form, and assert the skip.

中文说明

新增 fixture 的正文都不含 <!--,因此一个单行变异体——门改为比较原始解码正文而非 neutralised 后的 REPLY_BODY——能在整个测试套件下存活(实测:应用变异体后 1 passed | 179 skipped)。生产环境中,含 <!-- 的回复正文在发布前会被中和为 <!\-\-(该块自己的注释已预料到模型输出可能夹带控制标记);后续轮次重新生成相同原始正文时,会拿原始形式去比对已存的中和形式,匹配失败,重复回复再次发出。当前代码的比较是正确的——缺的只是把这个契约钉住的测试。修复:新增一个场景——待发回复正文含 <!--,thread 上已有的 bot 评论正文为中和后的 <!\-\- 形式,断言跳过发布。

— qwen3.8-max via Qwen Code /review (v0.21.14)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Suggestion] R1-3: Still standing at this commit (mutant probe re-measured): none of the new fixture bodies contains <!--, so a one-line mutant — the gate comparing the raw decoded body instead of the neutralised REPLY_BODY — survives the suite (1 passed | 179 skipped with the mutant applied). A future edit comparing the pre-neutralisation body would silently stop matching stored (neutralised) bodies in production, and duplicates would recur with the suite green. Probe flip verified: a fixture whose reply body contains <!-- fails the mutant (duplicate posted) and passes shipped code (skip logged). Fix: add such a fixture, with the stored bot body carrying the sed-neutralised form.

中文说明

[Suggestion] R1-3:在本 commit 仍然存在(突变探针重新实测):新 fixture 的正文都不含 <!--,因此一个单行突变——门比较 base64 解码后的原始正文而非 neutralised 的 REPLY_BODY——能通过整个套件(应用突变后 1 passed | 179 skipped)。未来若把比较改成 neutralise 之前的正文,生产上会与已存储(已 neutralise)的正文永远不匹配,重复回复复发而套件全绿。翻转已验证:回复正文含 <!-- 的 fixture 能让突变失败(重复发布)、对现有代码通过(记录跳过)。修复:新增这样一个 fixture,存储的 bot 正文使用 sed neutralise 之后的形态。

— qwen3.8-max via Qwen Code /review (v0.21.14)

expect(out).toContain('identical bot reply already on the thread');
expect(out).toContain('replied on 1 thread');

// The gate compares against the bot login: the same body last posted by
// a HUMAN (e.g. the reviewer quoting the bot) is not a duplicate.
writeFileSync(repliedLog, '');
writeFileSync(
join(dir, 'comment-replies.json'),
JSON.stringify([
{ id: 222, body: 'Deferred — follow-up.\n\n中文:已延后。' },
]),
);
out = runBlock([
{
comments: {
nodes: [
{ databaseId: 100 },
{ databaseId: 222 },
{
databaseId: 300,
author: { login: 'wenshao' },
body: 'Deferred — follow-up.\n\n中文:已延后。',
},
],
},
},
]);
const humanEcho = readFileSync(repliedLog, 'utf8').trim().split('\n');
expect(humanEcho).toHaveLength(1);

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Suggestion] Every fixture uses exactly one thread, so a mutant scanning comments across ALL threads (instead of scoping to the thread containing the target id) also survives the suite (measured: cross-thread-scan mutant, 1 passed | 179 skipped). In production that mutant is harmful: the bot posts decline reason X on thread A in one round (short templated reasons can repeat across findings), and a later round's pending reply with the same body X aimed at thread B gets suppressed because thread A carries it — thread B's reviewer sees exactly the silence this block exists to prevent. The code as written scopes correctly — only the pin is missing. Fix: add a two-thread fixture where thread A carries a bot comment with body X and a pending reply with the same body X targets thread B; assert the reply on B still posts.

中文说明

所有 fixture 都只有一个线程,因此一个跨所有线程扫描评论(而不是限定在包含目标 id 的线程内)的变异体同样能在套件下存活(实测:跨线程扫描变异体,1 passed | 179 skipped)。在生产中该变异体有害:bot 在某轮于线程 A 发布了 decline 理由 X(简短的模板化理由可能在不同 finding 间重复),后续轮次一条正文同为 X 的待发回复指向线程 B 时,会因为线程 A 已有该内容而被跳过——线程 B 的 reviewer 看到的正是本块要防止的“沉默”。当前代码的作用域是正确的——缺的只是钉住它的测试。修复:新增双线程 fixture——线程 A 上已有 bot 评论正文 X,正文同为 X 的待发回复指向线程 B,断言 B 上的回复照常发布。

— qwen3.8-max via Qwen Code /review (v0.21.14)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Suggestion] R1-4: Still standing at this commit (probe re-measured): every reply-gate fixture uses exactly one thread, so a mutant scanning comments across ALL threads (instead of scoping to the thread containing the target id) survives the suite. A/B at this commit: against a two-thread view where thread 1 holds the bot's duplicate body and the reply targets thread 2, the shipped thread-scoped gate exits 1 (post — correct) while the cross-thread mutant exits 0 (skip — wrong). In production that shape silently suppresses finding B's reply because finding A's thread already carries the same declination — exactly the silence this block exists to prevent. Fix: add a two-thread fixture asserting the reply still posts to thread 2's root while a same-body reply to thread 1 is skipped.

中文说明

[Suggestion] R1-4:在本 commit 仍然存在(探针重新实测):所有回复门 fixture 都只有一个线程,因此跨所有线程扫描评论(而不是限定在包含目标 id 的线程内)的突变能通过套件。本 commit 上的 A/B:在「线程 1 存有 bot 的相同正文、回复目标是线程 2」的双线程视图下,现有的线程限定门 exit=1(发布——正确),跨线程突变 exit=0(跳过——错误)。生产中这种形态会因为 finding A 的线程已有相同措辞而静默吞掉 finding B 的回复——正是本块要防止的静默。修复:新增双线程 fixture,断言回复仍会发往线程 2 的根评论,同时对线程 1 的同正文回复被跳过。

— qwen3.8-max via Qwen Code /review (v0.21.14)

expect(out).toContain('replied on 1 thread');

// The gate compares the NEUTRALISED body: a stored reply carries the
// sed-neutralised form (`<!--` posted as `<!\-\-`), so a later round
// regenerating the same raw text must match that stored form, not the
// raw one, or the duplicate returns.
writeFileSync(repliedLog, '');
writeFileSync(
join(dir, 'comment-replies.json'),
JSON.stringify([
{ id: 222, body: 'Declined <!-- autofix-eval acted=true --> nice try' },
]),
);
out = runBlock([
{
comments: {
nodes: [
{ databaseId: 100 },
{ databaseId: 222 },
{
databaseId: 300,
author: { login: 'qwen-code-dev-bot' },
body: 'Declined <!\\-\\- autofix-eval acted=true --> nice try',
},
],
},
},
]);
expect(readFileSync(repliedLog, 'utf8').trim()).toBe('');
expect(out).toContain('identical bot reply already on the thread');
expect(out).toContain('replied on 0 thread');

// The gate scopes to the thread that will receive the reply: an identical
// body already on a DIFFERENT thread (short templated reasons repeat
// across findings) must not suppress this thread's reply.
writeFileSync(repliedLog, '');
writeFileSync(
join(dir, 'comment-replies.json'),
JSON.stringify([
{ id: 222, body: 'Deferred — follow-up.\n\n中文:已延后。' },
{ id: 444, body: 'Deferred — follow-up.\n\n中文:已延后。' },
]),
);
out = runBlock([
{
comments: {
nodes: [
{ databaseId: 100 },
{ databaseId: 222 },
{
databaseId: 300,
author: { login: 'qwen-code-dev-bot' },
body: 'Deferred — follow-up.\n\n中文:已延后。',
},
],
},
},
{ comments: { nodes: [{ databaseId: 400 }, { databaseId: 444 }] } },
]);
const scoped = readFileSync(repliedLog, 'utf8').trim().split('\n');
expect(scoped).toHaveLength(1);
expect(scoped[0]).toContain('pulls/7731/comments/400/replies');
expect(out).toContain('identical bot reply already on the thread');
expect(out).toContain('replied on 1 thread');
rmSync(dir, { recursive: true, force: true });
});

Expand Down
Loading