From 9ddfc9327f9a84ccac0ef753a8792291428a5a4e Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Wed, 12 Aug 2026 03:44:58 +0800 Subject: [PATCH 1/3] fix(ci): seed the dist-rebuild warning on every retryable A/B exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The baseline leg rebuilds dist/ from baseline sources, and every retryable exit of the verify gate hands that tree to the repair agent — but the "run npm run build first" steering note only reached the green-baseline rejection. The comm -23 comparison failure and different-signature exits sent the repair agent in blind, free to trust or test against stale baseline artifacts (the different-signature exit carried the note on #8765's branch; the #8878 port kept it on the green exit only). #8765's post-close round-3 review flagged the comm path as Critical. Append the note on both missing exits and pin all three paths: the DIFFERENT-reason test now asserts the note, a new test stubs comm to fail and asserts the same, and the pre-existing test pins the note OUT of its document — no repair runs for that verdict. Mutation-tested, 3 of 3 caught: comm-path note dropped, different-reason note dropped, note leaked into the pre-existing document. --- .../run-autofix-review-verification.sh | 17 ++++++---- scripts/tests/qwen-autofix-workflow.test.js | 32 +++++++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/.github/scripts/run-autofix-review-verification.sh b/.github/scripts/run-autofix-review-verification.sh index 4a46a182fa3..c09cbd97162 100755 --- a/.github/scripts/run-autofix-review-verification.sh +++ b/.github/scripts/run-autofix-review-verification.sh @@ -153,12 +153,14 @@ baseline_also_fails() { } > "${WORKDIR}/gate-rejection.md" || true exit 1 fi + # Every retryable exit below hands the tree to the repair agent with + # dist/ REBUILT FROM BASELINE SOURCES (the restore checkout brings back + # tracked files only) — the mirror of the dist confound that exempted + # typecheck from the A/B. The note seeds the repair feedback so the + # agent rebuilds before it trusts any dist-consuming check. The + # pre-existing exit is the exception: no repair runs for it, so the + # note stays out of its document. if [[ "${rc}" -ne 1 ]]; then - # Both retryable exits below hand the tree to the repair agent with - # dist/ REBUILT FROM BASELINE SOURCES (the restore checkout brings back - # tracked files only) — the mirror of the dist confound that exempted - # typecheck from the A/B. The note seeds the repair feedback so the - # agent rebuilds before it trusts any dist-consuming check. echo "⚠️ the baseline leg rebuilt dist/ from baseline sources — run npm run build before typecheck/tests" >> "${GATE_LOG}" echo "🔁 baseline is green — the failure belongs to this round" \ | tee -a "${GATE_LOG}" @@ -185,9 +187,12 @@ baseline_also_fails() { # verdict-less gate crash. # (sig_head was extracted before the detach.) sig_base="$(fail_signature "${ab_log}")" || true - new_in_round="$(comm -23 <(printf '%s\n' "${sig_head}") <(printf '%s\n' "${sig_base}"))" || + new_in_round="$(comm -23 <(printf '%s\n' "${sig_head}") <(printf '%s\n' "${sig_base}"))" || { + echo "⚠️ the baseline leg rebuilt dist/ from baseline sources — run npm run build before typecheck/tests" >> "${GATE_LOG}" return 1 + } if [[ -z "${sig_head}" || -z "${sig_base}" ]] || [[ -n "${new_in_round}" ]]; then + echo "⚠️ the baseline leg rebuilt dist/ from baseline sources — run npm run build before typecheck/tests" >> "${GATE_LOG}" echo "🔁 baseline fails for a DIFFERENT reason — charged to the round" \ | tee -a "${GATE_LOG}" return 1 diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 1ea08ed3189..31341a14d65 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -11582,6 +11582,7 @@ describe('review verification gate: baseline A/B on deterministic rejection', () hugeFail = false, noIdentity = false, trackedDirt = false, + commFail = false, }) => { const dir = mkdtempSync(join(tmpdir(), 'gate-ab-')); try { @@ -11698,6 +11699,13 @@ describe('review verification gate: baseline A/B on deterministic rejection', () ].join('\n'), ); chmodSync(join(bin, 'npm'), 0o755); + if (commFail) { + // Shadows the system comm via PATH precedence: the signature + // comparison itself fails (the SIGPIPE-under-pipefail class), so + // the gate takes its fail-closed retryable exit. + writeFileSync(join(bin, 'comm'), '#!/bin/bash\nexit 1\n'); + chmodSync(join(bin, 'comm'), 0o755); + } const rt = join(dir, 'rt'); mkdirSync(rt); writeFileSync( @@ -11795,6 +11803,11 @@ describe('review verification gate: baseline A/B on deterministic rejection', () // The baseline leg's own transcript is the ONLY proof behind the // verdict — it must reach the rejection document. expect(r.rejection).toContain(`stub build FAILED at ${r.baselineSha}`); + // No repair runs for a pre-existing failure — the dist/ steering note + // is for the repair agent and stays out of this document. + expect(r.rejection).not.toContain( + 'run npm run build before typecheck/tests', + ); expect(r.headAfter).toBe('feature'); }); @@ -11934,6 +11947,25 @@ describe('review verification gate: baseline A/B on deterministic rejection', () expect(r.outputs).toContain('retryable=true'); expect(r.outputs).not.toContain('preexisting=true'); expect(r.stdout).toContain('DIFFERENT reason'); + // The same repair handoff as the green path — the dist/ warning must + // seed this rejection too. + expect(r.rejection).toContain('run npm run build before typecheck/tests'); + }); + + it('seeds the dist-rebuild warning when the signature comparison itself fails', () => { + // comm failing (SIGPIPE under pipefail, an infrastructure hiccup) + // takes the same retryable handoff as the green/different-signature + // exits — without the note the repair agent trusts baseline-built + // dist/ and chases phantom dist-consuming failures. + const r = runGate({ + failAt: ['feature', 'origin/feature'], + commFail: true, + }); + expect(r.status).toBe(1); + expect(r.outputs).toContain('retryable=true'); + expect(r.outputs).not.toContain('preexisting=true'); + expect(r.rejection).toContain('run npm run build before typecheck/tests'); + expect(r.headAfter).toBe('feature'); }); it('keeps the green path intact', () => { From 807cdfda2d3529f4d2f5044dc35189483ee7703f Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Wed, 12 Aug 2026 07:59:35 +0800 Subject: [PATCH 2/3] fix(ci): single emit point for the dist note, name the comm-failure exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the two review suggestions on #8958: - The steering note existed as three byte-identical copies, and the "every retryable exit seeds the note" invariant depended on copy-paste — the exact drift this PR was patching (one exit seeded on #8765's branch, one lost in the #8878 port). Both reviewers flagged it. The string now lives in seed_dist_note(), called from all three exits. - The comm-failure exit seeded the note but, unlike its sibling retryable exits, emitted no verdict-rationale line — an oncall could not distinguish "the comparison itself failed" from "baseline is green" without re-running the A/B. It now says so. Mutation-tested: mutating the string inside the helper fails all three path assertions at once; mutating the rationale line fails the comm-exit test. --- .../scripts/run-autofix-review-verification.sh | 18 +++++++++++++----- scripts/tests/qwen-autofix-workflow.test.js | 4 ++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/scripts/run-autofix-review-verification.sh b/.github/scripts/run-autofix-review-verification.sh index c09cbd97162..1e334ea923a 100755 --- a/.github/scripts/run-autofix-review-verification.sh +++ b/.github/scripts/run-autofix-review-verification.sh @@ -156,12 +156,12 @@ baseline_also_fails() { # Every retryable exit below hands the tree to the repair agent with # dist/ REBUILT FROM BASELINE SOURCES (the restore checkout brings back # tracked files only) — the mirror of the dist confound that exempted - # typecheck from the A/B. The note seeds the repair feedback so the - # agent rebuilds before it trusts any dist-consuming check. The + # typecheck from the A/B. seed_dist_note seeds the repair feedback so + # the agent rebuilds before it trusts any dist-consuming check. The # pre-existing exit is the exception: no repair runs for it, so the # note stays out of its document. if [[ "${rc}" -ne 1 ]]; then - echo "⚠️ the baseline leg rebuilt dist/ from baseline sources — run npm run build before typecheck/tests" >> "${GATE_LOG}" + seed_dist_note echo "🔁 baseline is green — the failure belongs to this round" \ | tee -a "${GATE_LOG}" return 1 @@ -188,11 +188,13 @@ baseline_also_fails() { # (sig_head was extracted before the detach.) sig_base="$(fail_signature "${ab_log}")" || true new_in_round="$(comm -23 <(printf '%s\n' "${sig_head}") <(printf '%s\n' "${sig_base}"))" || { - echo "⚠️ the baseline leg rebuilt dist/ from baseline sources — run npm run build before typecheck/tests" >> "${GATE_LOG}" + seed_dist_note + echo "🔁 signature comparison failed — fail-closed, charged to the round" \ + | tee -a "${GATE_LOG}" return 1 } if [[ -z "${sig_head}" || -z "${sig_base}" ]] || [[ -n "${new_in_round}" ]]; then - echo "⚠️ the baseline leg rebuilt dist/ from baseline sources — run npm run build before typecheck/tests" >> "${GATE_LOG}" + seed_dist_note echo "🔁 baseline fails for a DIFFERENT reason — charged to the round" \ | tee -a "${GATE_LOG}" return 1 @@ -217,6 +219,12 @@ fail_signature() { grep -oE "[^ '\"]+\([0-9]+,[0-9]+\): error TS[0-9]+.*" "${1}" 2> /dev/null \ | sed -E 's/\([0-9]+,[0-9]+\)//' | sort -u } +# The one emit point for the dist-rebuild steering note — every retryable +# exit of baseline_also_fails after the baseline leg calls this, so the +# guidance cannot drift across exits. +seed_dist_note() { + echo "⚠️ the baseline leg rebuilt dist/ from baseline sources — run npm run build before typecheck/tests" >> "${GATE_LOG}" +} run_check() { # pipefail makes the pipeline carry the command's status, not tee's. The # side copy holds THIS check's transcript alone — the identity comparison diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 31341a14d65..325aa17747e 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -11965,6 +11965,10 @@ describe('review verification gate: baseline A/B on deterministic rejection', () expect(r.outputs).toContain('retryable=true'); expect(r.outputs).not.toContain('preexisting=true'); expect(r.rejection).toContain('run npm run build before typecheck/tests'); + // Like its sibling exits, this one names its verdict — an oncall must + // distinguish "the comparison itself failed" from "baseline is green" + // without re-running the A/B. + expect(r.rejection).toContain('signature comparison failed'); expect(r.headAfter).toBe('feature'); }); From 5240948e731536fc60f8f5f852a7900c94fc3261 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Wed, 12 Aug 2026 00:21:37 +0000 Subject: [PATCH 3/3] test(ci): pin the no-identity baseline arm of the A/B gate (#8958) Co-authored-by: Qwen-Coder --- scripts/tests/qwen-autofix-workflow.test.js | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 325aa17747e..02b428e3d45 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -11581,6 +11581,7 @@ describe('review verification gate: baseline A/B on deterministic rejection', () restoreClash = false, hugeFail = false, noIdentity = false, + baselineNoIdentity = false, trackedDirt = false, commFail = false, }) => { @@ -11664,6 +11665,11 @@ describe('review verification gate: baseline A/B on deterministic rejection', () // vite/esbuild shape: a red build with no tsc diagnostic at all. ' echo "error during build: something exploded"; exit 1', ' fi', + ' if [[ "$head" == "${BASELINE_SHA:-}" && "${BASELINE_NO_IDENTITY:-}" == "1" ]]; then', + // Same shape restricted to the baseline leg — the head keeps its + // tsc identity while the baseline loses its. + ' echo "error during build: something exploded"; exit 1', + ' fi', ' if [[ "$head" == "${BASELINE_SHA:-}" && "${TRACKED_DIRT:-}" == "1" ]]; then', // The build rewrites a TRACKED file (the settings-schema shape): // f.txt differs across refs, so an undiscarded rewrite makes the @@ -11749,6 +11755,7 @@ describe('review verification gate: baseline A/B on deterministic rejection', () EXTRA_BASELINE_DIAG: extraBaselineDiag ? '1' : '', RESTORE_CLASH: restoreClash ? '1' : '', NO_IDENTITY: noIdentity ? '1' : '', + BASELINE_NO_IDENTITY: baselineNoIdentity ? '1' : '', TRACKED_DIRT: trackedDirt ? '1' : '', SCHEMA_FAIL: schemaFail ? '1' : '', TYPECHECK_FAIL: typecheckFail ? '1' : '', @@ -11952,6 +11959,23 @@ describe('review verification gate: baseline A/B on deterministic rejection', () expect(r.rejection).toContain('run npm run build before typecheck/tests'); }); + it('charges the round when the baseline fails without a failure identity', () => { + // Mirror of the head-side noIdentity shape on the other leg: the + // baseline crashes vite/esbuild-style with no tsc diagnostic, so its + // signature is empty and identity cannot be established — fail + // closed and charge the round, with the same repair handoff (and + // dist/ note) as the sibling retryable exits. + const r = runGate({ + failAt: ['feature', 'origin/feature'], + baselineNoIdentity: true, + }); + expect(r.status).toBe(1); + expect(r.outputs).toContain('retryable=true'); + expect(r.outputs).not.toContain('preexisting=true'); + expect(r.stdout).toContain('DIFFERENT reason'); + expect(r.rejection).toContain('run npm run build before typecheck/tests'); + }); + it('seeds the dist-rebuild warning when the signature comparison itself fails', () => { // comm failing (SIGPIPE under pipefail, an infrastructure hiccup) // takes the same retryable handoff as the green/different-signature