From 4cb72de00f599ea3e4d7c38507241a63d091c1d6 Mon Sep 17 00:00:00 2001 From: wenshao Date: Fri, 14 Aug 2026 12:43:32 +0800 Subject: [PATCH 01/18] feat(triage): add a deterministic flakiness gate to sandboxed verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #9125. PR #9086's ~50% mtime-assertion flake passed every automated layer because each executed the changed tests exactly once — a coin flip a single green run cannot distinguish from health. The gate re-runs the PR's added/modified unit-test files N times (default 5, vars.QWEN_VERIFY_FLAKE_ROUNDS to override, clamped to 2..10) through the same entry points CI uses and compares outcomes per group across rounds. Design constraints, each pinned by a workflow test: - One-way authority: 'flaky' demotes the published headline (even a trusted agent merge-ready); no gate value can raise or soften one. The gate runs the PR's own test code, so it can always be neutered — but a gate that can only demote is not worth forging. - Divergence-only signal: a group failing identically every round is deterministic (CI owns it) and an environment-sensitive suite must not false-positive here; both report informationally, never demote. - Fail open: the gate is not under -e and every terminal path exits 0 — a gate bug reports verdict 'error' instead of taking down the verify lane. - Honest file list: recorded from HEAD^1..HEAD before install/build hands the workspace (and .git) to PR lifecycle code; the gate consumes the root-owned recorded list and never re-derives the diff. - Untrusted text stays out of outputs: summaries are fixed text plus counters; PR-controlled paths live in flake-gate.log, embedded through the publisher's escaping emit_block. Job timeout raised 150 -> 175 for the gate's ~25m worst case (15m round budget checked before each invocation + one 10m-capped in-flight run). --- .github/scripts/qwen-triage-workflow.test.mjs | 174 ++++++++++- .github/workflows/qwen-triage.yml | 282 +++++++++++++++++- 2 files changed, 446 insertions(+), 10 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index 6c4f66e50f8..657ff744bae 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -219,7 +219,10 @@ describe('qwen-triage: fork-PR runner routing', () => { it('keeps the authorize gate itself on the same-repo guard', () => { // authorize IS the permission check (and loads CI_BOT_PAT); it cannot // route on its own output and must not widen to association-based trust. - assert.match(authorizeRunsOn, /head\.repo\.full_name == github\.repository/); + assert.match( + authorizeRunsOn, + /head\.repo\.full_name == github\.repository/, + ); assert.doesNotMatch(authorizeRunsOn, /author_association/); assert.doesNotMatch(authorizeRunsOn, /needs\./); }); @@ -651,8 +654,14 @@ describe('qwen-triage: npm cache restore-only invariant', () => { const restoreIdx = jobDef.steps.findIndex( (s) => s.name === 'Restore npm cache', ); - assert.ok(clearIdx !== -1, `'Clear stale npm cache' step must exist in ${jobName}`); - assert.ok(restoreIdx !== -1, `'Restore npm cache' step must exist in ${jobName}`); + assert.ok( + clearIdx !== -1, + `'Clear stale npm cache' step must exist in ${jobName}`, + ); + assert.ok( + restoreIdx !== -1, + `'Restore npm cache' step must exist in ${jobName}`, + ); assert.ok( clearIdx < restoreIdx, 'clear step must come before restore step', @@ -708,8 +717,8 @@ describe('qwen-triage: npm cache producer workflow', () => { }); it('saves with the same key and path the triage lanes restore', () => { - const saveStep = saveJob.steps.find( - (s) => s.uses?.startsWith('actions/cache/save@'), + const saveStep = saveJob.steps.find((s) => + s.uses?.startsWith('actions/cache/save@'), ); assert.ok(saveStep, 'must have an actions/cache/save step'); for (const [jobName, jobDef] of [ @@ -733,8 +742,8 @@ describe('qwen-triage: npm cache producer workflow', () => { }); it('populates the cache directory it saves', () => { - const saveStep = saveJob.steps.find( - (s) => s.uses?.startsWith('actions/cache/save@'), + const saveStep = saveJob.steps.find((s) => + s.uses?.startsWith('actions/cache/save@'), ); assert.ok(saveStep, 'must have an actions/cache/save step'); const dir = saveStep.with.path.replace( @@ -780,3 +789,154 @@ describe('qwen-triage: npm cache producer workflow', () => { ); }); }); + +describe('qwen-triage: flakiness gate (#9125)', () => { + const recordStep = verifyJob.steps.find( + (s) => s.name === 'Record changed test files for the flakiness gate', + ); + const flakeStep = verifyJob.steps.find((s) => s.id === 'flake'); + const prepareStep = verifyJob.steps.find( + (s) => s.name === 'Install and build PR app', + ); + const agentStep = verifyJob.steps.find( + (s) => s.name === 'Run verification agent', + ); + const publishStep = doc.jobs['publish-verify'].steps.find( + (s) => s.name === 'Post verification report comment', + ); + + it('records the changed-test list BEFORE the workspace is handed to the build user', () => { + assert.ok(recordStep, 'record step must exist'); + assert.ok(flakeStep, 'flake gate step must exist'); + const recordIdx = verifyJob.steps.indexOf(recordStep); + const prepareIdx = verifyJob.steps.indexOf(prepareStep); + const flakeIdx = verifyJob.steps.indexOf(flakeStep); + // After npm ci, PR lifecycle code owns .git and could rewrite the diff + // to hide a test file — the list must be pinned while .git is still + // root-owned, and the gate must consume that pinned list after the build. + assert.ok( + recordIdx < prepareIdx, + 'the list must be recorded before install/build runs PR lifecycle code', + ); + assert.ok( + prepareIdx < flakeIdx, + 'the gate needs node_modules, so it must run after install/build', + ); + assert.match( + recordStep.run, + /HEAD\^1/, + 'diff must be against the merge base', + ); + assert.match( + flakeStep.run, + /flake-gate-files/, + 'the gate must read the recorded list, not re-derive the diff', + ); + assert.doesNotMatch( + flakeStep.run, + /git diff/, + 'the gate must not re-derive the diff from post-build git metadata', + ); + }); + + it('runs PR test code as the build user with no tokens, and fails open', () => { + assert.equal(flakeStep.env.GITHUB_TOKEN, '', 'no GitHub token in the gate'); + assert.equal(flakeStep.env.GH_TOKEN, '', 'no gh token in the gate'); + assert.match( + flakeStep.run, + /runuser -u node --/, + 'PR test code must run as the unprivileged build user', + ); + assert.match( + flakeStep.run, + /^\s*set -uo pipefail/m, + 'the gate must not run under -e', + ); + assert.doesNotMatch( + flakeStep.run, + /set -euo/, + 'a gate bug must fail OPEN (verdict error), never abort the verify job', + ); + assert.equal( + flakeStep.if, + "steps.pr.outputs.decision == 'run' && steps.prepare.outputs.verdict == ''", + 'the gate runs exactly when the agent would (after a clean build)', + ); + }); + + it('exposes the gate outcome to the publisher and preserves its log', () => { + assert.equal( + verifyJob.outputs.flake_verdict, + '${{ steps.flake.outputs.flake_verdict }}', + ); + assert.equal( + verifyJob.outputs.flake_summary, + '${{ steps.flake.outputs.flake_summary }}', + ); + assert.equal( + publishStep.env.FLAKE_VERDICT, + '${{ needs.verify.outputs.flake_verdict }}', + ); + assert.equal( + publishStep.env.FLAKE_SUMMARY, + '${{ needs.verify.outputs.flake_summary }}', + ); + // The agent step recreates verify-results from scratch; the gate log is + // root-owned in RUNNER_TEMP and must be copied back in afterwards, or + // the artifact and the comment lose the per-round matrix. + const wipeIdx = agentStep.run.indexOf( + 'rm -rf "$RUNNER_TEMP/verify-results"', + ); + const copyIdx = agentStep.run.indexOf( + 'cp "$RUNNER_TEMP/flake-gate.log" "$RUNNER_TEMP/verify-results/flake-gate.log"', + ); + assert.ok(wipeIdx !== -1, 'agent step must still recreate verify-results'); + assert.ok(copyIdx !== -1, 'agent step must copy the gate log back'); + assert.ok(copyIdx > wipeIdx, 'the copy must happen AFTER the recreation'); + assert.match( + publishStep.run, + /emit_block 'Flakiness gate log' "\$FLAKE_LOG"/, + 'the publisher must embed the gate log through the escaping emit_block', + ); + }); + + it('gate authority is one-way: only `flaky` may touch the headline, and only to demote', () => { + const block = publishStep.run.match( + /case "\$\{FLAKE_VERDICT:-\}" in[\s\S]*?esac/, + ); + assert.ok( + block, + 'the publisher must map FLAKE_VERDICT through one case block', + ); + const arms = block[0].split(/;;/); + for (const arm of arms) { + const touchesHeadline = /(QUAL|HEADLINE)(_ZH)?=/.test(arm); + if (!touchesHeadline) continue; + assert.match( + arm, + /^\s*flaky\)/m, + `only the flaky arm may reassign the headline, found: ${arm.trim().slice(0, 60)}`, + ); + assert.match( + arm, + /QUAL='❌ not passed'/, + 'flaky must demote to not-passed', + ); + assert.doesNotMatch( + arm, + /QUAL='✅/, + 'no gate value may ever set a passing headline', + ); + } + }); + + it('the verify job timeout still covers agent + prepare + gate', () => { + // agent 120m + install/build 15m + gate ~25m + misc ~5m — the job limit + // must stay comfortably above the sum or the container is killed mid-run + // and the ship-what-ran path is bypassed (see the budget comment). + assert.ok( + verifyJob['timeout-minutes'] >= 170, + `timeout-minutes must cover the gate budget (got ${verifyJob['timeout-minutes']})`, + ); + }); +}); diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index b867587bcba..34a1149c5a3 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2137,14 +2137,17 @@ jobs: # install + build ~6m measured (run 30284341325: npm ci # 3m00 + build 2m40), budget 15m for a # cold cache or a heavier dependency tree + # flakiness gate ~25m worst case (15m round budget, + # checked before each invocation, plus one + # in-flight invocation capped at 10m) # resolver/tools, checkout, # pin, upload, cleanup ~5m # ------------------------------------ - # worst case ~140m ⇒ 150 leaves 10m of headroom. + # worst case ~165m ⇒ 175 leaves 10m of headroom. # # Cost of the raise, stated so it is a decision and not a surprise: a - # verify run now occupies one ECS slot for up to 2.5h instead of 1h. - timeout-minutes: 150 + # verify run now occupies one ECS slot for up to ~3h instead of 1h. + timeout-minutes: 175 runs-on: ['self-hosted', 'linux', 'x64', 'ecs-qwen'] # The job checks out and executes PR code. Run the steps in a container so # package scripts/builds cannot persist changes in the self-hosted runner's @@ -2165,6 +2168,11 @@ jobs: verdict: '${{ steps.run.outputs.verdict || steps.prepare.outputs.verdict || steps.pr.outputs.verdict }}' failure_phase: '${{ steps.prepare.outputs.failure_phase }}' agent_verdict: '${{ steps.run.outputs.agent_verdict }}' + # Deterministic flakiness gate (#9125): `flaky` demotes the published + # headline; every other value is informational. The summary is + # gate-authored fixed text plus counters — never PR-controlled strings. + flake_verdict: '${{ steps.flake.outputs.flake_verdict }}' + flake_summary: '${{ steps.flake.outputs.flake_summary }}' skip_reason: '${{ steps.pr.outputs.skip_reason }}' steps: - name: 'Install PR resolver tools' @@ -2729,6 +2737,31 @@ jobs: fi echo "agent inputs pinned from base $(git rev-parse --short 'HEAD^1'); head ${ACTUAL_HEAD} matches the authorized head" + # The flakiness gate (#9125) re-runs the PR's changed test files and + # compares outcomes across identical rounds. The file list is recorded + # HERE — before the workspace is handed to the build user — because + # after `npm ci` a PR lifecycle script owns .git and could rewrite the + # diff to hide a test file from the gate. The list lives root-owned in + # RUNNER_TEMP, so the gate later runs exactly what was recorded. This + # pin is honesty, not a security boundary: the gate's verdict can only + # ever DEMOTE the published outcome (see the publish job), so hiding a + # file from it merely returns the PR to today's baseline of a single + # execution. + - name: 'Record changed test files for the flakiness gate' + if: "steps.pr.outputs.decision == 'run'" + run: |- + set -euo pipefail + # Two statements, not a pipeline: `git diff | grep || true` would + # swallow a git failure as "no changed test files", silently + # narrowing the gate to n/a. A git failure here is pre-build + # infrastructure and must fail the step loudly; only a no-match + # grep may produce an empty list. + files="$(git diff --name-only --diff-filter=ACMR 'HEAD^1' HEAD)" + printf '%s\n' "$files" \ + | grep -E '\.(test|spec)\.(ts|tsx|js|jsx|mjs|cjs)$' \ + > "${RUNNER_TEMP:?}/flake-gate-files" || true + echo "Recorded $(grep -c . "$RUNNER_TEMP/flake-gate-files" || true) changed test file(s) for the flakiness gate." + - name: 'Clear stale npm cache' if: "steps.pr.outputs.decision == 'run'" run: 'rm -rf "$RUNNER_TEMP/npm-cache"' @@ -2860,6 +2893,193 @@ jobs: fi echo "Install/build completed before verification." >> "$GITHUB_STEP_SUMMARY" + # Deterministic flakiness gate (#9125): re-run the PR's changed test + # files N times through the same entry points CI uses and compare the + # outcomes. A single execution has no power against non-deterministic + # failures — a ~50% flake passes half of all CI runs (PR #9086's mtime + # assertion was certified exactly that way) — while N=5 identical + # re-runs catch it with ~97% probability. Only run-to-run DIVERGENCE is + # a gate signal: a test that fails identically every round is + # deterministic, CI already owns that, and an environment-sensitive + # suite must not false-positive here. + # + # Authority is one-way by construction: `flaky` demotes the published + # headline, and no gate value can raise or soften one. The test code + # under execution is the PR's own, so a PR can always neuter its gate — + # but a gate that can only demote is not worth forging, which is what + # keeps its evidence meaningful. + - name: 'Flakiness gate: re-run changed test files' + id: 'flake' + if: "steps.pr.outputs.decision == 'run' && steps.prepare.outputs.verdict == ''" + env: + GITHUB_TOKEN: '' + GH_TOKEN: '' + FLAKE_ROUNDS: '${{ vars.QWEN_VERIFY_FLAKE_ROUNDS }}' + run: |- + # Deliberately NOT -e: the gate is advisory-and-demoting only, so a + # bug in it must fail OPEN (verdict `error`, agent still runs) — the + # alternative is a gate outage taking down the whole verify lane. + set -uo pipefail + unset ACTIONS_RUNTIME_TOKEN ACTIONS_RUNTIME_URL ACTIONS_CACHE_URL + LOG="${RUNNER_TEMP:?}/flake-gate.log" + LIST="$RUNNER_TEMP/flake-gate-files" + : > "$LOG" + finish() { + # Outputs carry FIXED strings and counters only — file paths are + # PR-controlled text and belong in the log, which the publisher + # HTML-escapes before embedding. + echo "flake_verdict=$1" >> "$GITHUB_OUTPUT" + echo "flake_summary=$2" >> "$GITHUB_OUTPUT" + printf '\nverdict: %s\nsummary: %s\n' "$1" "$2" >> "$LOG" + echo "Flakiness gate: $1 — $2" >> "$GITHUB_STEP_SUMMARY" + exit 0 + } + + ROUNDS="${FLAKE_ROUNDS:-}" + case "$ROUNDS" in ''|*[!0-9]*) ROUNDS=5 ;; esac + [ "$ROUNDS" -ge 2 ] || ROUNDS=2 + [ "$ROUNDS" -le 10 ] || ROUNDS=10 + + [ -f "$LIST" ] || finish error 'the recorded changed-test list is missing' + + # Partition the recorded files into runnable groups. Every skipped + # file is logged with its reason — a silently narrowed gate would + # read as "covered" when it was not. + scripts_files=() + helper_files=() + skipped=0 + declare -A pkg_files=() + while IFS= read -r f; do + [ -n "$f" ] || continue + if [ ! -f "$f" ]; then + printf 'not present in the merge tree, skipped: %s\n' "$f" >> "$LOG" + skipped=$((skipped + 1)) + continue + fi + case "$f" in + scripts/tests/*) scripts_files+=("$f") ;; + .github/scripts/*.test.mjs) helper_files+=("$f") ;; + integration-tests/*) + # E2E suites need sandbox/model plumbing this gate does not have. + printf 'integration test, out of gate scope: %s\n' "$f" >> "$LOG" + skipped=$((skipped + 1)) + ;; + packages/*/*) + pkg="${f#packages/}" + pkg="packages/${pkg%%/*}" + pkg_files["$pkg"]+="${f#"$pkg"/}"$'\n' + ;; + *) + printf 'no runner mapped for this path, skipped: %s\n' "$f" >> "$LOG" + skipped=$((skipped + 1)) + ;; + esac + done < "$LIST" + + group_labels=() + group_dirs=() + group_cmds=() + if [ "${#scripts_files[@]}" -gt 0 ]; then + group_labels+=('scripts/tests') + group_dirs+=('.') + group_cmds+=("npx --no-install vitest run --config ./scripts/tests/vitest.config.ts $(printf '%q ' "${scripts_files[@]}")") + fi + if [ "${#helper_files[@]}" -gt 0 ]; then + group_labels+=('.github/scripts') + group_dirs+=('.') + group_cmds+=("node --test $(printf '%q ' "${helper_files[@]}")") + fi + for pkg in "${!pkg_files[@]}"; do + mapfile -t rel <<< "${pkg_files[$pkg]%$'\n'}" + group_labels+=("$pkg") + group_dirs+=("$pkg") + # From the package CWD (vitest configs assume it); npx resolves + # the workspace-root binary by walking up node_modules. + group_cmds+=("npx --no-install vitest run $(printf '%q ' "${rel[@]}")") + done + + total="${#group_labels[@]}" + if [ "$total" -eq 0 ]; then + finish n/a "no changed unit-test files to re-run (${skipped} out-of-scope file(s) noted in the log)" + fi + { + printf 'rounds=%s groups=%s skipped=%s\n' "$ROUNDS" "$total" "$skipped" + for i in "${!group_labels[@]}"; do + printf 'group %s: (cd %s) %s\n' "${group_labels[$i]}" "${group_dirs[$i]}" "${group_cmds[$i]}" + done + printf '\n' + } >> "$LOG" + + # 15-minute wall budget, checked before every invocation, plus a + # 10-minute cap per invocation — worst case ~25m, which the job + # timeout budget above accounts for. Divergence needs no uniform + # round count: a P and an F for the same group is non-determinism + # no matter how many rounds fit the budget. + declare -a results=() + deadline=$(( $(date +%s) + 900 )) + rounds_done=0 + timed_out=false + round=1 + while [ "$round" -le "$ROUNDS" ]; do + for i in "${!group_labels[@]}"; do + if [ "$(date +%s)" -ge "$deadline" ]; then + timed_out=true + break + fi + out="$RUNNER_TEMP/flake-gate-round-out" + ( + cd "${group_dirs[$i]}" && + timeout -k 30 600 runuser -u node -- \ + env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ + NODE_OPTIONS='--max-old-space-size=3072' CI=true \ + bash -c "${group_cmds[$i]}" + ) > "$out" 2>&1 + status=$? + mark='P' + [ "$status" -eq 0 ] || mark='F' + results[$i]="${results[$i]:-}${mark}" + printf 'round %s · %s: %s (exit %s)\n' "$round" "${group_labels[$i]}" "$mark" "$status" >> "$LOG" + if [ "$status" -ne 0 ]; then + { + printf -- '--- output tail · round %s · %s ---\n' "$round" "${group_labels[$i]}" + tail -c 8000 "$out" + printf '\n' + } >> "$LOG" + fi + done + [ "$timed_out" = true ] && break + rounds_done="$round" + round=$((round + 1)) + done + + flaky=0 + failing=0 + for i in "${!group_labels[@]}"; do + case "${results[$i]:-}" in + *P*F*|*F*P*) flaky=$((flaky + 1)) ;; + *P*) : ;; + *F*) failing=$((failing + 1)) ;; + esac + done + printf '\nper-group results (P=pass F=fail, one letter per run):\n' >> "$LOG" + for i in "${!group_labels[@]}"; do + printf ' %s: %s\n' "${group_labels[$i]}" "${results[$i]:-}" >> "$LOG" + done + + if [ "$flaky" -gt 0 ]; then + finish flaky "${flaky} of ${total} changed-test group(s) returned different results across identical re-runs (${rounds_done} full round(s))" + fi + if [ "$timed_out" = true ] && [ "$rounds_done" -lt 2 ]; then + finish timeout "the 15-minute budget elapsed before two full rounds completed (${rounds_done} done) — no flakiness signal either way" + fi + if [ "$failing" -gt 0 ]; then + finish consistent-fail "${failing} of ${total} changed-test group(s) failed identically in every round — deterministic, so CI owns that signal" + fi + if [ "$timed_out" = true ]; then + finish timeout "only ${rounds_done} of ${ROUNDS} rounds fit the 15-minute budget; the completed rounds agreed" + fi + finish pass "${total} changed-test group(s) x ${rounds_done} identical rounds, no divergence" + - name: 'Install evidence browser' if: "steps.pr.outputs.decision == 'run' && steps.prepare.outputs.verdict == ''" env: @@ -3008,6 +3228,12 @@ jobs: if [ -f "$RUNNER_TEMP/prepare.log.keep" ]; then mv "$RUNNER_TEMP/prepare.log.keep" "$RUNNER_TEMP/verify-results/prepare.log" fi + # The flakiness gate's log is root-owned in RUNNER_TEMP (the build + # user cannot rewrite it); copy it into the freshly recreated + # results dir so the artifact and the publish job can see it. + if [ -f "$RUNNER_TEMP/flake-gate.log" ]; then + cp "$RUNNER_TEMP/flake-gate.log" "$RUNNER_TEMP/verify-results/flake-gate.log" + fi # Bypass the runner proxy before launching qwen: the proxy cuts the # SSE stream to the model host, and qwen reads HTTP(S)_PROXY directly @@ -3573,6 +3799,8 @@ jobs: PR_NUMBER: '${{ needs.verify.outputs.pr_number || github.event.issue.number }}' VERDICT: '${{ needs.verify.outputs.verdict }}' AGENT_VERDICT: '${{ needs.verify.outputs.agent_verdict }}' + FLAKE_VERDICT: '${{ needs.verify.outputs.flake_verdict }}' + FLAKE_SUMMARY: '${{ needs.verify.outputs.flake_summary }}' SKIP_REASON: '${{ needs.verify.outputs.skip_reason }}' PREPARE_FAILURE_PHASE: '${{ needs.verify.outputs.failure_phase }}' VERIFY_RESULT: '${{ needs.verify.result }}' @@ -4117,6 +4345,7 @@ jobs: # whatever directory find visits first, which is unordered. REPORT="$(find verify-results -mindepth 2 -type f -path '*-verify-*/report.md' 2>/dev/null | sort | head -1 || true)" ASSERTIONS_FILE="$(find verify-results -mindepth 2 -type f -path '*-verify-*/assertions.json' 2>/dev/null | sort | head -1 || true)" + FLAKE_LOG="$(find verify-results -type f -name 'flake-gate.log' 2>/dev/null | sort | head -1 || true)" ASSERT_LINE='' ASSERT_LINE_ZH='' if [ -n "$ASSERTIONS_FILE" ]; then @@ -4198,6 +4427,40 @@ jobs: fi fi fi + # The flakiness gate (#9125) is deterministic evidence with + # ONE-WAY authority: `flaky` demotes any headline — including a + # trusted agent `merge-ready` — because non-deterministic tests + # are a PR defect the agent's single execution cannot see. No + # other gate value may raise or soften an outcome: the gate ran + # the PR's own test code, and a gate that can only demote is not + # worth forging, which is what keeps its evidence trustworthy. + # FLAKE_SUMMARY is gate-authored fixed text plus counters (the + # gate keeps PR-controlled paths in its log, which is embedded + # through the escaping emit_block below). + FLAKE_LINE='' + FLAKE_LINE_ZH='' + case "${FLAKE_VERDICT:-}" in + flaky) + QUAL='❌ not passed' + QUAL_ZH='❌ 不通过' + HEADLINE='non-deterministic tests (flakiness gate)' + HEADLINE_ZH='测试结果不确定（抖动门）' + FLAKE_LINE="Flakiness gate: ❌ ${FLAKE_SUMMARY:-changed test files returned different results across identical re-runs}" + FLAKE_LINE_ZH="抖动门：❌ ${FLAKE_SUMMARY:-changed test files returned different results across identical re-runs}" + ;; + pass) + FLAKE_LINE="Flakiness gate: ✅ ${FLAKE_SUMMARY:-changed test files re-ran identically}" + FLAKE_LINE_ZH="抖动门：✅ ${FLAKE_SUMMARY:-changed test files re-ran identically}" + ;; + n/a) + FLAKE_LINE="Flakiness gate: not applicable — ${FLAKE_SUMMARY:-no changed unit-test files}" + FLAKE_LINE_ZH="抖动门：不适用 — ${FLAKE_SUMMARY:-no changed unit-test files}" + ;; + consistent-fail|timeout|error) + FLAKE_LINE="Flakiness gate: ⚠️ ${FLAKE_VERDICT} — ${FLAKE_SUMMARY:-see flake-gate.log in the run artifacts}" + FLAKE_LINE_ZH="抖动门：⚠️ ${FLAKE_VERDICT} — ${FLAKE_SUMMARY:-see flake-gate.log in the run artifacts}" + ;; + esac if [ -z "$REPORT" ]; then MISSING_REPORT_NOTE='No report.md was found in the run artifacts, so the report section is omitted — see the workflow run output.' echo "::warning::${MISSING_REPORT_NOTE}" @@ -4232,9 +4495,15 @@ jobs: if [ -n "$ASSERT_LINE" ]; then printf '%s\n\n' "$ASSERT_LINE" fi + if [ -n "$FLAKE_LINE" ]; then + printf '%s\n\n' "$FLAKE_LINE" + fi if [ "${AGENT_VERDICT:-}" = 'merge-ready' ] && [ "$TRUST_AGENT_VERDICT" != true ] && [ "${A_FAIL:-0}" != '0' ]; then printf 'The agent reported `merge-ready`, but `assertions.json` recorded %s failures, so that claim was not trusted — see the report for whether these are expected A/B control cells.\n\n' "$A_FAIL" fi + if [ "${FLAKE_VERDICT:-}" = 'flaky' ]; then + printf 'The deterministic flakiness gate re-ran the test files this PR changes and got different outcomes from identical runs%s. A test that can fail with no code changing lands as intermittent red on unrelated PRs, so this run is reported as not passed regardless of the agent verdict — the per-round matrix is in the flakiness gate log below.\n\n' "${AGENT_VERDICT:+ (agent verdict: \`${AGENT_VERDICT}\`)}" + fi printf '
\n中文 — 判定：%s · %s\n\n' "$QUAL_ZH" "${HEADLINE_ZH:-$HEADLINE}" if [ "${VERDICT:-}" = 'pass' ]; then printf '%s\n\n' "$SCOPE_ZH" @@ -4244,14 +4513,21 @@ jobs: if [ -n "$ASSERT_LINE_ZH" ]; then printf '%s\n\n' "$ASSERT_LINE_ZH" fi + if [ -n "$FLAKE_LINE_ZH" ]; then + printf '%s\n\n' "$FLAKE_LINE_ZH" + fi if [ "${AGENT_VERDICT:-}" = 'merge-ready' ] && [ "$TRUST_AGENT_VERDICT" != true ] && [ "${A_FAIL:-0}" != '0' ]; then printf 'agent 报告了 `merge-ready`，但 `assertions.json` 记录了 %s 个失败，因此该判定未被采信——请参阅报告确认这些是否为预期的 A/B 对照单元。\n\n' "$A_FAIL" fi + if [ "${FLAKE_VERDICT:-}" = 'flaky' ]; then + printf '确定性抖动门将本 PR 改动的测试文件原样重跑了多轮，得到了不一致的结果%s。一个在代码不变时也会失败的测试会以间歇性红灯落在无关的 PR 上，因此无论 agent 判定如何，本次运行按不通过报告——各轮结果矩阵见下方抖动门日志。\n\n' "${AGENT_VERDICT:+（agent 判定：\`${AGENT_VERDICT}\`）}" + fi printf '
\n\n' if [ -n "${MISSING_REPORT_NOTE:-}" ]; then printf '%s\n\n' "$MISSING_REPORT_NOTE" fi emit_report "$REPORT" 45000 + emit_block 'Flakiness gate log' "$FLAKE_LOG" 20000 if [ -n "$EVIDENCE_SECTION" ]; then printf '%s' "$EVIDENCE_SECTION" fi From b03407bc9d312be8f9ab44d321d90dc11b909fee Mon Sep 17 00:00:00 2001 From: wenshao Date: Fri, 14 Aug 2026 14:42:56 +0800 Subject: [PATCH 02/18] fix(triage): survive the runner wrapper's -e, per-file gate granularity, hardened log staging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-1 review + sandboxed-verify feedback, all seven findings: - set +e after set -uo pipefail: the runner wraps every run: block in 'bash -e -o pipefail' and set -uo does NOT clear that inherited -e, so the first failing test invocation killed the step — fail-open inverted to fail-closed for exactly the flaky/consistent-fail populations the gate classifies (verify cells C/D). An EXIT trap additionally converts any abnormal ending (set -u death) into the fixed 'error' verdict. - Per-FILE groups: one runner invocation per changed test file, so a consistently failing file can no longer mask another file's run-to-run divergence behind a shared exit bit. - Owning-package resolution: nearest ancestor package.json (nested workspaces like packages/channels/base are entered themselves) plus a vitest-config probe; unsupported runner families (packages/desktop's bun test) and */e2e/* specs are logged out-of-scope instead of being mis-run as permanent consistent-fail noise. - Operands are ./-prefixed before %q, so a checked-in filename beginning with '-' (e.g. --config=x) can never be parsed as a runner option. - Log staging moved to a dedicated always() root step after the agent exits — the last write to verify-results/flake-gate.log — and the publisher pins that exact path instead of find|sort|head, so an early agent abort cannot lose the matrix and agent-era PR code (which owns a chowned verify-results) cannot control or shadow what is embedded. - Detection math corrected: N=5 catches a 50/50 flake with ~94% (1 - 2*(1/2)^5), not ~97% — all-pass and all-fail rounds both miss. - New behavioral suite executes the extracted gate and publisher fragments under the production wrapper itself (bash --noprofile --norc -e -o pipefail) with scripted per-file P/F sequences: pass, flaky-next-to-consistent-fail, consistent-fail, missing-list error, out-of-scope n/a, nested-package + leading-dash operand, and the seven-value one-way demotion — closing the structural blindness where YAML-string tests stayed green while the shipped behavior regressed. --- .github/scripts/qwen-triage-workflow.test.mjs | 328 +++++++++++++++++- .github/workflows/qwen-triage.yml | 204 +++++++---- 2 files changed, 457 insertions(+), 75 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index 657ff744bae..468768387a4 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -7,7 +7,14 @@ // catch it — this file is that test. import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { after, before, describe, it } from 'node:test'; @@ -850,7 +857,22 @@ describe('qwen-triage: flakiness gate (#9125)', () => { assert.match( flakeStep.run, /^\s*set -uo pipefail/m, - 'the gate must not run under -e', + 'the gate must not opt into -e', + ); + // The runner wraps run: blocks in `bash -e -o pipefail`, and `set -uo` + // does NOT clear that inherited -e — only an explicit `set +e` does. + // Without it the first failing test invocation kills the step (round-1 + // sandboxed verify blocker), and the behavioral suite below proves the + // same end to end under the wrapper. + assert.match( + flakeStep.run, + /^\s*set \+e$/m, + 'the gate must explicitly clear the runner wrapper -e', + ); + assert.match( + flakeStep.run, + /trap on_gate_exit EXIT/, + 'abnormal exits (set -u deaths) must be converted to the error verdict', ); assert.doesNotMatch( flakeStep.run, @@ -881,18 +903,45 @@ describe('qwen-triage: flakiness gate (#9125)', () => { publishStep.env.FLAKE_SUMMARY, '${{ needs.verify.outputs.flake_summary }}', ); - // The agent step recreates verify-results from scratch; the gate log is - // root-owned in RUNNER_TEMP and must be copied back in afterwards, or - // the artifact and the comment lose the per-round matrix. - const wipeIdx = agentStep.run.indexOf( - 'rm -rf "$RUNNER_TEMP/verify-results"', + // The authoritative log stays root-owned in RUNNER_TEMP and is staged + // into verify-results by a dedicated always() root step AFTER the agent + // exits — the last write to that filename. Staging it earlier loses it + // on an early agent abort, and verify-results is chowned to the build + // user while PR-controlled agent code runs, so an earlier copy could be + // rewritten before upload (round-1 review). + const stageStep = verifyJob.steps.find( + (s) => s.name === 'Stage flakiness gate log for upload', + ); + assert.ok(stageStep, 'the staging step must exist'); + assert.equal( + stageStep.if, + "always() && steps.pr.outputs.decision == 'run'", + 'staging must survive a failed agent step', + ); + const agentIdx = verifyJob.steps.indexOf(agentStep); + const stageIdx = verifyJob.steps.indexOf(stageStep); + const uploadIdx = verifyJob.steps.findIndex( + (s) => s.name === 'Upload verify results', ); - const copyIdx = agentStep.run.indexOf( - 'cp "$RUNNER_TEMP/flake-gate.log" "$RUNNER_TEMP/verify-results/flake-gate.log"', + assert.ok( + agentIdx < stageIdx && stageIdx < uploadIdx, + 'staging must run after the agent and before the upload', + ); + assert.match( + stageStep.run, + /cp -f "\$RUNNER_TEMP\/flake-gate\.log" "\$RUNNER_TEMP\/verify-results\/flake-gate\.log"/, + 'staging must overwrite whatever the agent era left under that name', + ); + assert.doesNotMatch( + agentStep.run, + /flake-gate\.log/, + 'the agent step must not stage the log — that is the wrong trust boundary', + ); + assert.match( + publishStep.run, + /FLAKE_LOG='verify-results\/flake-gate\.log'/, + 'the publisher must pin the exact root-level path, never find/sort', ); - assert.ok(wipeIdx !== -1, 'agent step must still recreate verify-results'); - assert.ok(copyIdx !== -1, 'agent step must copy the gate log back'); - assert.ok(copyIdx > wipeIdx, 'the copy must happen AFTER the recreation'); assert.match( publishStep.run, /emit_block 'Flakiness gate log' "\$FLAKE_LOG"/, @@ -940,3 +989,258 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); }); }); + +describe('qwen-triage: flakiness gate — behavioral, under the production wrapper', () => { + // The structural tests above pin the YAML text; these execute the + // extracted gate and publisher fragments, because YAML inspection cannot + // observe the runner's own shell contract: every run: block executes + // under `bash --noprofile --norc -e -o pipefail`, and a `set -uo` script + // does NOT clear that inherited -e. That exact blind spot shipped the + // round-1 blocker — the first failing test invocation killed the step — + // so every scenario here runs under the wrapper, not under a bare bash. + const flakeRun = verifyJob.steps.find((s) => s.id === 'flake').run; + const publishRun = doc.jobs['publish-verify'].steps.find( + (s) => s.name === 'Post verification report comment', + ).run; + + const STUB_RUNUSER = [ + '#!/bin/bash', + 'while [ "$1" != "--" ]; do shift; done', + 'shift', + 'exec "$@"', + '', + ].join('\n'); + // npx/node stub: the last argument is the ./file operand; its scripted + // P/F sequence lives at $FLAKE_SEQ_DIR/, consumed one letter + // per invocation and cycled (missing sequence file = always pass). + const STUB_TESTRUNNER = [ + '#!/bin/bash', + 'f="${@: -1}"', + 'key="$(basename "$f")"', + 'n_file="$FLAKE_SEQ_DIR/.count-$key"', + 'n=$(cat "$n_file" 2>/dev/null || echo 0)', + 'echo $((n+1)) > "$n_file"', + 'seq="$(cat "$FLAKE_SEQ_DIR/$key" 2>/dev/null || echo P)"', + 'i=$((n % ${#seq}))', + '[ "${seq:$i:1}" = P ] && exit 0', + 'echo "stub failure for $key run $((n+1))"', + 'exit 1', + '', + ].join('\n'); + + const scenarioRoot = mkdtempSync(join(tmpdir(), 'flake-behavioral-')); + after(() => rmSync(scenarioRoot, { recursive: true, force: true })); + + const runGate = ({ layout = {}, list, sequences = {} }) => { + const root = mkdtempSync(join(scenarioRoot, 'case-')); + const ws = join(root, 'ws'); + const rt = join(root, 'rt'); + const bin = join(root, 'bin'); + const seqDir = join(root, 'seq'); + for (const d of [ws, rt, bin, seqDir]) mkdirSync(d, { recursive: true }); + for (const [p, content] of Object.entries(layout)) { + mkdirSync(dirname(join(ws, p)), { recursive: true }); + writeFileSync(join(ws, p), content); + } + if (list !== null) writeFileSync(join(rt, 'flake-gate-files'), list); + for (const [k, v] of Object.entries(sequences)) { + writeFileSync(join(seqDir, k), v); + } + for (const [name, content] of [ + ['runuser', STUB_RUNUSER], + ['npx', STUB_TESTRUNNER], + ['node', STUB_TESTRUNNER], + ]) { + writeFileSync(join(bin, name), content); + chmodSync(join(bin, name), 0o755); + } + const gateFile = join(root, 'gate.sh'); + writeFileSync(gateFile, flakeRun); + const out = join(rt, 'github-output'); + writeFileSync(out, ''); + writeFileSync(join(rt, 'github-summary'), ''); + const res = spawnSync( + 'bash', + ['--noprofile', '--norc', '-e', '-o', 'pipefail', gateFile], + { + cwd: ws, + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + RUNNER_TEMP: rt, + GITHUB_OUTPUT: out, + GITHUB_STEP_SUMMARY: join(rt, 'github-summary'), + FLAKE_ROUNDS: '5', + FLAKE_SEQ_DIR: seqDir, + }, + encoding: 'utf8', + timeout: 60_000, + }, + ); + const outputs = Object.fromEntries( + readFileSync(out, 'utf8') + .split('\n') + .filter((l) => l.includes('=')) + .map((l) => [l.slice(0, l.indexOf('=')), l.slice(l.indexOf('=') + 1)]), + ); + let log = ''; + try { + log = readFileSync(join(rt, 'flake-gate.log'), 'utf8'); + } catch { + // a scenario may legitimately abort before creating the log + } + return { res, outputs, log }; + }; + + const UNIT = { + 'scripts/tests/a.test.js': '', + 'scripts/tests/b.test.js': '', + }; + + it('all-pass rounds land as `pass` with exit 0', () => { + const { res, outputs } = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\nscripts/tests/b.test.js\n', + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'pass'); + }); + + it('per-file P/F alternation is `flaky` even next to a consistently failing file, and the wrapper -e does not kill the step', () => { + // One shared exit bit would classify this pair consistent-fail (the + // FFFFF file masks the PFPFP one); per-file groups must still see the + // divergence. Every F also exercises the errexit hazard: without + // `set +e` the first one kills the script under the wrapper. + const { res, outputs, log } = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\nscripts/tests/b.test.js\n', + sequences: { 'a.test.js': 'F', 'b.test.js': 'PF' }, + }); + assert.equal(res.status, 0, `gate died under the wrapper: ${res.stderr}`); + assert.equal(outputs.flake_verdict, 'flaky'); + assert.match(log, /a\.test\.js: FFFFF/); + assert.match(log, /b\.test\.js: PFPFP/); + }); + + it('identical failure every round stays informational `consistent-fail`, exit 0', () => { + const { res, outputs } = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\n', + sequences: { 'a.test.js': 'F' }, + }); + assert.equal(res.status, 0, `gate died under the wrapper: ${res.stderr}`); + assert.equal(outputs.flake_verdict, 'consistent-fail'); + }); + + it('a missing recorded list degrades to the `error` verdict, exit 0', () => { + const { res, outputs } = runGate({ layout: UNIT, list: null }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'error'); + }); + + it('out-of-scope families are skipped with logged reasons and land `n/a`', () => { + const { res, outputs, log } = runGate({ + layout: { + 'integration-tests/x.test.ts': '', + 'packages/web/client/e2e/y.spec.ts': '', + 'packages/bunpkg/package.json': '{}', + 'packages/bunpkg/z.test.ts': '', + }, + list: [ + 'integration-tests/x.test.ts', + 'packages/web/client/e2e/y.spec.ts', + 'packages/bunpkg/z.test.ts', + '', + ].join('\n'), + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'n/a'); + assert.match(log, /integration test, out of gate scope/); + assert.match(log, /e2e suite, out of gate scope/); + assert.match(log, /no vitest config \(unsupported runner family\)/); + }); + + it('a nested-workspace file runs from its OWN package, and a leading-dash filename stays an operand', () => { + const { res, outputs, log } = runGate({ + layout: { + 'packages/channels/base/package.json': '{}', + 'packages/channels/base/vitest.config.ts': '', + 'packages/channels/base/src/p.test.ts': '', + 'scripts/tests/--config=evil.test.js': '', + }, + list: 'packages/channels/base/src/p.test.ts\nscripts/tests/--config=evil.test.js\n', + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match( + log, + /\(cd packages\/channels\/base\) npx --no-install vitest run \.\/src\/p\.test\.ts/, + 'the nested package must be entered itself, not its parent', + ); + assert.match( + log, + /\.\/scripts\/tests\/--config=evil\.test\.js/, + 'operands must be ./-prefixed so vitest cannot parse them as options', + ); + }); + + it('publisher demotion executes one-way: only `flaky` demotes, and it MUST demote', () => { + const block = publishRun.match( + /case "\$\{FLAKE_VERDICT:-\}" in[\s\S]*?esac/, + ); + assert.ok(block, 'the publisher must map FLAKE_VERDICT in a case block'); + const drive = (verdict) => { + const res = spawnSync( + 'bash', + [ + '--noprofile', + '--norc', + '-e', + '-o', + 'pipefail', + '-c', + [ + "QUAL='✅ passed'", + "QUAL_ZH='✅ 通过'", + "HEADLINE='merge-ready (agent verdict)'", + "HEADLINE_ZH='可合入'", + "FLAKE_LINE=''", + "FLAKE_LINE_ZH=''", + block[0], + 'printf \'%s|%s\' "$QUAL" "$HEADLINE"', + ].join('\n'), + ], + { + env: { + ...process.env, + FLAKE_VERDICT: verdict, + FLAKE_SUMMARY: '1 of 2 changed test file(s) diverged', + }, + encoding: 'utf8', + timeout: 15_000, + }, + ); + assert.equal(res.status, 0, res.stderr); + return res.stdout; + }; + for (const v of [ + '', + 'pass', + 'n/a', + 'consistent-fail', + 'timeout', + 'error', + ]) { + assert.equal( + drive(v), + '✅ passed|merge-ready (agent verdict)', + `'${v}' must not touch the headline`, + ); + } + assert.equal( + drive('flaky'), + '❌ not passed|non-deterministic tests (flakiness gate)', + 'flaky must demote — deleting the flaky arm has to fail this test', + ); + }); +}); diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index 34a1149c5a3..dd2f638a59d 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2895,13 +2895,16 @@ jobs: # Deterministic flakiness gate (#9125): re-run the PR's changed test # files N times through the same entry points CI uses and compare the - # outcomes. A single execution has no power against non-deterministic - # failures — a ~50% flake passes half of all CI runs (PR #9086's mtime - # assertion was certified exactly that way) — while N=5 identical - # re-runs catch it with ~97% probability. Only run-to-run DIVERGENCE is - # a gate signal: a test that fails identically every round is + # outcomes per FILE. A single execution has no power against + # non-deterministic failures — a ~50% flake passes half of all CI runs + # (PR #9086's mtime assertion was certified exactly that way) — while + # N=5 identical re-runs catch it with ~94% probability (all-pass and + # all-fail rounds both miss: 1 - 2*(1/2)^5). Only run-to-run DIVERGENCE + # is a gate signal: a test that fails identically every round is # deterministic, CI already owns that, and an environment-sensitive - # suite must not false-positive here. + # suite must not false-positive here. Granularity is one runner + # invocation per changed file, so a consistently failing file cannot + # mask another file's divergence behind a shared exit bit. # # Authority is one-way by construction: `flaky` demotes the published # headline, and no gate value can raise or soften one. The test code @@ -2916,10 +2919,32 @@ jobs: GH_TOKEN: '' FLAKE_ROUNDS: '${{ vars.QWEN_VERIFY_FLAKE_ROUNDS }}' run: |- - # Deliberately NOT -e: the gate is advisory-and-demoting only, so a - # bug in it must fail OPEN (verdict `error`, agent still runs) — the - # alternative is a gate outage taking down the whole verify lane. + # Fail OPEN by contract: the gate is advisory-and-demoting only, so + # a bug in it must degrade to the fixed `error` verdict — never + # take down the verify lane. + # + # `set +e` is load-bearing, not style: the runner wraps every run: + # block in `bash -e -o pipefail`, and `set -uo pipefail` alone does + # NOT clear that inherited -e. Without the explicit +e the first + # failing test invocation kills the step, inverting fail-open into + # fail-closed for exactly the flaky/consistent-fail populations the + # gate exists to classify (round-1 sandboxed verify, cells C/D). set -uo pipefail + set +e + # Abnormal-exit net: `set -u` (or any unforeseen fatal) still + # aborts non-zero. Convert that ending to the fixed `error` verdict + # and a zero exit, so even a gate implementation bug is + # information, not an outage. finish() marks completion — the trap + # only rewrites endings that never reached a verdict. + GATE_DONE='' + on_gate_exit() { + if [ -z "$GATE_DONE" ] && [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "flake_verdict=error" >> "$GITHUB_OUTPUT" + echo "flake_summary=the gate aborted before reaching a verdict — see the step log" >> "$GITHUB_OUTPUT" + fi + exit 0 + } + trap on_gate_exit EXIT unset ACTIONS_RUNTIME_TOKEN ACTIONS_RUNTIME_URL ACTIONS_CACHE_URL LOG="${RUNNER_TEMP:?}/flake-gate.log" LIST="$RUNNER_TEMP/flake-gate-files" @@ -2932,6 +2957,7 @@ jobs: echo "flake_summary=$2" >> "$GITHUB_OUTPUT" printf '\nverdict: %s\nsummary: %s\n' "$1" "$2" >> "$LOG" echo "Flakiness gate: $1 — $2" >> "$GITHUB_STEP_SUMMARY" + GATE_DONE=1 exit 0 } @@ -2942,70 +2968,107 @@ jobs: [ -f "$LIST" ] || finish error 'the recorded changed-test list is missing' - # Partition the recorded files into runnable groups. Every skipped - # file is logged with its reason — a silently narrowed gate would - # read as "covered" when it was not. - scripts_files=() - helper_files=() + # Partition into per-FILE groups. Every skipped file is logged + # with its reason — a silently narrowed gate would read as + # "covered" when it was not. Operands are `./`-prefixed BEFORE %q, + # so a checked-in filename beginning with `-` (e.g. `--config=x`) + # can never be parsed as a runner option. + group_labels=() + group_dirs=() + group_cmds=() skipped=0 - declare -A pkg_files=() + add_skip() { + printf '%s: %s\n' "$2" "$1" >> "$LOG" + skipped=$((skipped + 1)) + } + owning_pkg_dir() { + # Nearest ancestor directory carrying package.json — nested + # workspaces (packages/channels/base) own their runner and must + # be entered themselves, never their parent. + local d="$1" + d="${d%/*}" + while [ -n "$d" ] && [ "$d" != '.' ]; do + if [ -f "$d/package.json" ]; then + printf '%s' "$d" + return 0 + fi + case "$d" in + */*) d="${d%/*}" ;; + *) d='' ;; + esac + done + return 1 + } + has_vitest_config() { + local c + for c in vitest.config.ts vitest.config.mts vitest.config.js vitest.config.mjs vitest.workspace.ts; do + [ -f "$1/$c" ] && return 0 + done + return 1 + } while IFS= read -r f; do [ -n "$f" ] || continue if [ ! -f "$f" ]; then - printf 'not present in the merge tree, skipped: %s\n' "$f" >> "$LOG" - skipped=$((skipped + 1)) + add_skip "$f" 'not present in the merge tree, skipped' continue fi case "$f" in - scripts/tests/*) scripts_files+=("$f") ;; - .github/scripts/*.test.mjs) helper_files+=("$f") ;; integration-tests/*) # E2E suites need sandbox/model plumbing this gate does not have. - printf 'integration test, out of gate scope: %s\n' "$f" >> "$LOG" - skipped=$((skipped + 1)) + add_skip "$f" 'integration test, out of gate scope' + continue + ;; + */e2e/*|e2e/*) + # Browser/E2E specs (e.g. web-shell client/e2e) are excluded + # by their vitest configs — running them here would be + # permanent "No test files found" noise, not coverage. + add_skip "$f" 'e2e suite, out of gate scope' + continue ;; - packages/*/*) - pkg="${f#packages/}" - pkg="packages/${pkg%%/*}" - pkg_files["$pkg"]+="${f#"$pkg"/}"$'\n' + esac + case "$f" in + scripts/tests/*) + group_labels+=("$f") + group_dirs+=('.') + group_cmds+=("npx --no-install vitest run --config ./scripts/tests/vitest.config.ts $(printf '%q' "./$f")") + ;; + .github/scripts/*.test.mjs) + group_labels+=("$f") + group_dirs+=('.') + group_cmds+=("node --test $(printf '%q' "./$f")") + ;; + packages/*) + if ! pkg="$(owning_pkg_dir "$f")"; then + add_skip "$f" 'no owning package.json, skipped' + continue + fi + if ! has_vitest_config "$pkg"; then + # e.g. packages/desktop runs `bun test` — an unsupported + # runner family stays explicitly out of scope rather than + # being mis-run through vitest. + add_skip "$f" "owning package ${pkg} has no vitest config (unsupported runner family), skipped" + continue + fi + # From the owning package CWD (vitest configs assume it); + # npx resolves the binary by walking up node_modules. + group_labels+=("$f") + group_dirs+=("$pkg") + group_cmds+=("npx --no-install vitest run $(printf '%q' "./${f#"$pkg"/}")") ;; *) - printf 'no runner mapped for this path, skipped: %s\n' "$f" >> "$LOG" - skipped=$((skipped + 1)) + add_skip "$f" 'no runner mapped for this path, skipped' ;; esac done < "$LIST" - group_labels=() - group_dirs=() - group_cmds=() - if [ "${#scripts_files[@]}" -gt 0 ]; then - group_labels+=('scripts/tests') - group_dirs+=('.') - group_cmds+=("npx --no-install vitest run --config ./scripts/tests/vitest.config.ts $(printf '%q ' "${scripts_files[@]}")") - fi - if [ "${#helper_files[@]}" -gt 0 ]; then - group_labels+=('.github/scripts') - group_dirs+=('.') - group_cmds+=("node --test $(printf '%q ' "${helper_files[@]}")") - fi - for pkg in "${!pkg_files[@]}"; do - mapfile -t rel <<< "${pkg_files[$pkg]%$'\n'}" - group_labels+=("$pkg") - group_dirs+=("$pkg") - # From the package CWD (vitest configs assume it); npx resolves - # the workspace-root binary by walking up node_modules. - group_cmds+=("npx --no-install vitest run $(printf '%q ' "${rel[@]}")") - done - total="${#group_labels[@]}" if [ "$total" -eq 0 ]; then - finish n/a "no changed unit-test files to re-run (${skipped} out-of-scope file(s) noted in the log)" + finish n/a "no runnable changed test files (${skipped} out-of-scope file(s) noted in the log)" fi { - printf 'rounds=%s groups=%s skipped=%s\n' "$ROUNDS" "$total" "$skipped" + printf 'rounds=%s files=%s skipped=%s\n' "$ROUNDS" "$total" "$skipped" for i in "${!group_labels[@]}"; do - printf 'group %s: (cd %s) %s\n' "${group_labels[$i]}" "${group_dirs[$i]}" "${group_cmds[$i]}" + printf 'file %s: (cd %s) %s\n' "${group_labels[$i]}" "${group_dirs[$i]}" "${group_cmds[$i]}" done printf '\n' } >> "$LOG" @@ -3061,24 +3124,24 @@ jobs: *F*) failing=$((failing + 1)) ;; esac done - printf '\nper-group results (P=pass F=fail, one letter per run):\n' >> "$LOG" + printf '\nper-file results (P=pass F=fail, one letter per run):\n' >> "$LOG" for i in "${!group_labels[@]}"; do printf ' %s: %s\n' "${group_labels[$i]}" "${results[$i]:-}" >> "$LOG" done if [ "$flaky" -gt 0 ]; then - finish flaky "${flaky} of ${total} changed-test group(s) returned different results across identical re-runs (${rounds_done} full round(s))" + finish flaky "${flaky} of ${total} changed test file(s) returned different results across identical re-runs (${rounds_done} full round(s))" fi if [ "$timed_out" = true ] && [ "$rounds_done" -lt 2 ]; then finish timeout "the 15-minute budget elapsed before two full rounds completed (${rounds_done} done) — no flakiness signal either way" fi if [ "$failing" -gt 0 ]; then - finish consistent-fail "${failing} of ${total} changed-test group(s) failed identically in every round — deterministic, so CI owns that signal" + finish consistent-fail "${failing} of ${total} changed test file(s) failed identically in every round — deterministic, so CI owns that signal" fi if [ "$timed_out" = true ]; then finish timeout "only ${rounds_done} of ${ROUNDS} rounds fit the 15-minute budget; the completed rounds agreed" fi - finish pass "${total} changed-test group(s) x ${rounds_done} identical rounds, no divergence" + finish pass "${total} changed test file(s) x ${rounds_done} identical rounds, no divergence" - name: 'Install evidence browser' if: "steps.pr.outputs.decision == 'run' && steps.prepare.outputs.verdict == ''" @@ -3228,12 +3291,6 @@ jobs: if [ -f "$RUNNER_TEMP/prepare.log.keep" ]; then mv "$RUNNER_TEMP/prepare.log.keep" "$RUNNER_TEMP/verify-results/prepare.log" fi - # The flakiness gate's log is root-owned in RUNNER_TEMP (the build - # user cannot rewrite it); copy it into the freshly recreated - # results dir so the artifact and the publish job can see it. - if [ -f "$RUNNER_TEMP/flake-gate.log" ]; then - cp "$RUNNER_TEMP/flake-gate.log" "$RUNNER_TEMP/verify-results/flake-gate.log" - fi # Bypass the runner proxy before launching qwen: the proxy cuts the # SSE stream to the model host, and qwen reads HTTP(S)_PROXY directly @@ -3661,6 +3718,24 @@ jobs: echo "agent_verdict=$AGENT_VERDICT" >> "$GITHUB_OUTPUT" echo "verify verdict: $VERDICT agent: ${AGENT_VERDICT:-none} (exit $EXIT_CODE)" >> "$GITHUB_STEP_SUMMARY" + # The gate log's authoritative copy lives root-owned at + # $RUNNER_TEMP/flake-gate.log, outside every agent-writable directory. + # It is staged into verify-results HERE — after the agent exits, from + # an always() root step — as the LAST write to that filename: an early + # agent-step abort cannot lose it, and agent-era PR code (which owns a + # chowned verify-results while it runs) cannot control what is + # uploaded under this name. The publisher pins this exact root-level + # path, so a nested same-named file planted in a collected artifact + # dir cannot shadow it either. + - name: 'Stage flakiness gate log for upload' + if: "always() && steps.pr.outputs.decision == 'run'" + run: |- + set -euo pipefail + if [ -f "${RUNNER_TEMP:?}/flake-gate.log" ]; then + mkdir -p "$RUNNER_TEMP/verify-results" + cp -f "$RUNNER_TEMP/flake-gate.log" "$RUNNER_TEMP/verify-results/flake-gate.log" + fi + - name: 'Upload verify results' if: "always() && steps.pr.outputs.decision == 'run'" # Don't let a missing/empty results dir (qwen crashed before writing @@ -4345,7 +4420,10 @@ jobs: # whatever directory find visits first, which is unordered. REPORT="$(find verify-results -mindepth 2 -type f -path '*-verify-*/report.md' 2>/dev/null | sort | head -1 || true)" ASSERTIONS_FILE="$(find verify-results -mindepth 2 -type f -path '*-verify-*/assertions.json' 2>/dev/null | sort | head -1 || true)" - FLAKE_LOG="$(find verify-results -type f -name 'flake-gate.log' 2>/dev/null | sort | head -1 || true)" + # Exact root-level path, never a find: the staging step writes + # this name last as root, and a same-named file nested inside a + # collected agent artifact dir must not shadow it. + FLAKE_LOG='verify-results/flake-gate.log' ASSERT_LINE='' ASSERT_LINE_ZH='' if [ -n "$ASSERTIONS_FILE" ]; then From 13708068bd9406d4392dc73ca440359e062ec003 Mon Sep 17 00:00:00 2001 From: wenshao Date: Fri, 14 Aug 2026 10:51:00 +0000 Subject: [PATCH 03/18] fix(triage): isolate flake-gate rounds, classify infra exits, widen runner resolution (#9130) Review-round fixes for the deterministic flakiness gate: - Reset shared state between rounds (restore tracked files, tear down test-user processes, fresh per-invocation TMPDIR) so a deterministic test cannot fail on its own residue and fake a divergence (R1-8). - Classify timeout/signal exits (124, 128+N) as infrastructure, not F marks, and report the informational timeout verdict instead of a fake flaky (R2-3/R3-2). - Resolve the vitest runner by owning package + vitest's real config list (vite.config.* included), keyed on the package lookup instead of a packages/* prefix, so webui and integrations workspaces are re-run instead of skipped (R2-2/R3-3). - Narrow the scripts/tests arm to the pinned config's *.test.{js,ts} include set so admitted-but-rejected files are skipped, not mis-run into a bogus consistent-fail (R3-11). - Harden the gate-log staging: kill leftover build-user processes, and remove a planted destination entry before copying so a FIFO/symlink can neither hang the copy nor redirect it (R1-5). - Cap the embedded gate log at 10000 chars to keep the assembled comment under GitHub's 65,536-char limit (R3-4). - Record changed files with core.quotePath=false so non-ASCII test filenames are not silently dropped (R3-5). - Behavioral tests: hermetic timeout/pkill stubs (the suite no longer depends on GNU coreutils, fixing the macOS red), infra-exit and round-reset scenarios, trap-abort fail-open, node --test arm, FLAKE_ROUNDS clamping, fixed-shape summary, record-step shape pins. --- .github/scripts/qwen-triage-workflow.test.mjs | 336 ++++++++++++++++-- .github/workflows/qwen-triage.yml | 90 ++++- 2 files changed, 391 insertions(+), 35 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index 468768387a4..be6afbb5a51 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -834,6 +834,29 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /HEAD\^1/, 'diff must be against the merge base', ); + assert.match( + recordStep.run, + /git -c core\.quotePath=false diff --name-only/, + 'non-ASCII filenames must not be C-quoted out of the gate list', + ); + // The "two statements, not a pipeline" property is load-bearing: a git + // failure must fail the step loudly; only a no-match grep may yield an + // empty list. A `git diff ... | grep ... || true` pipeline would + // swallow the failure as "no changed test files" and certify the PR + // with an n/a gate. + assert.match( + recordStep.run, + /^\s*files="\$\(git [^|)]*\)"\s*$/m, + 'git diff must run as a standalone assignment, not feed a pipeline', + ); + assert.doesNotMatch( + recordStep.run + .split('\n') + .filter((l) => !l.trim().startsWith('#')) + .join('\n'), + /git diff[^\n]*\|/, + 'a git failure must not be swallowable by a pipeline || true', + ); assert.match( flakeStep.run, /flake-gate-files/, @@ -879,6 +902,19 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /set -euo/, 'a gate bug must fail OPEN (verdict error), never abort the verify job', ); + // Repeated samples share one tree and one process table: without a + // per-round reset, a deterministic test fails on its own residue + // (mutated fixture, leftover daemon/port) and fakes a divergence. + assert.match( + flakeStep.run, + /^\s*git checkout -- \. \|\| true$/m, + 'each round must restore tracked files before the next sample', + ); + assert.match( + flakeStep.run, + /^\s*pkill -u node \|\| true$/m, + 'each round must tear down leftover test-user processes', + ); assert.equal( flakeStep.if, "steps.pr.outputs.decision == 'run' && steps.prepare.outputs.verdict == ''", @@ -932,6 +968,16 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /cp -f "\$RUNNER_TEMP\/flake-gate\.log" "\$RUNNER_TEMP\/verify-results\/flake-gate\.log"/, 'staging must overwrite whatever the agent era left under that name', ); + assert.match( + stageStep.run, + /rm -rf -- "\$RUNNER_TEMP\/verify-results\/flake-gate\.log"/, + 'staging must remove a planted destination BEFORE copying — cp would open a FIFO and block, or follow a symlink', + ); + assert.match( + stageStep.run, + /pkill -KILL -u node/, + 'staging must kill leftover build-user processes so nothing races the copy', + ); assert.doesNotMatch( agentStep.run, /flake-gate\.log/, @@ -944,8 +990,8 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( publishStep.run, - /emit_block 'Flakiness gate log' "\$FLAKE_LOG"/, - 'the publisher must embed the gate log through the escaping emit_block', + /emit_block 'Flakiness gate log' "\$FLAKE_LOG" 10000/, + 'the gate-log cap must leave headroom under GitHub 65,536-char comment limit next to the 45000 report block', ); }); @@ -1011,8 +1057,10 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp '', ].join('\n'); // npx/node stub: the last argument is the ./file operand; its scripted - // P/F sequence lives at $FLAKE_SEQ_DIR/, consumed one letter - // per invocation and cycled (missing sequence file = always pass). + // outcome sequence lives at $FLAKE_SEQ_DIR/, consumed one + // letter per invocation and cycled (missing sequence file = always + // pass). Letters: P=exit 0, F=exit 1, T=exit 124 (timeout), K=exit 137 + // (signal kill) — T/K model infrastructure exits, not test failures. const STUB_TESTRUNNER = [ '#!/bin/bash', 'f="${@: -1}"', @@ -1022,16 +1070,45 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp 'echo $((n+1)) > "$n_file"', 'seq="$(cat "$FLAKE_SEQ_DIR/$key" 2>/dev/null || echo P)"', 'i=$((n % ${#seq}))', - '[ "${seq:$i:1}" = P ] && exit 0', + 'm="${seq:$i:1}"', + '[ "$m" = P ] && exit 0', + '[ "$m" = T ] && exit 124', + '[ "$m" = K ] && exit 137', 'echo "stub failure for $key run $((n+1))"', 'exit 1', '', ].join('\n'); + // GNU coreutils `timeout` exists on the Linux runner but not on stock + // macOS: without a stub every invocation exits 127 off-Linux and every + // scenario reads consistent-fail. Consume the gate's `-k 30 600` shape + // and exec the wrapped command. + const STUB_TIMEOUT = [ + '#!/bin/bash', + 'while [ $# -gt 0 ]; do', + ' case "$1" in', + ' -k|--kill-after) shift 2 ;;', + ' *) break ;;', + ' esac', + 'done', + 'shift', + 'exec "$@"', + '', + ].join('\n'); + // pkill is stubbed for the harness's sake, not the gate's: the real + // binary would kill processes owned by whoever runs these tests. + const STUB_PKILL = ['#!/bin/bash', 'exit 0', ''].join('\n'); const scenarioRoot = mkdtempSync(join(tmpdir(), 'flake-behavioral-')); after(() => rmSync(scenarioRoot, { recursive: true, force: true })); - const runGate = ({ layout = {}, list, sequences = {} }) => { + const runGate = ({ + layout = {}, + list, + sequences = {}, + stubs = {}, + env: envOverrides = {}, + git = false, + }) => { const root = mkdtempSync(join(scenarioRoot, 'case-')); const ws = join(root, 'ws'); const rt = join(root, 'rt'); @@ -1046,11 +1123,35 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp for (const [k, v] of Object.entries(sequences)) { writeFileSync(join(seqDir, k), v); } - for (const [name, content] of [ - ['runuser', STUB_RUNUSER], - ['npx', STUB_TESTRUNNER], - ['node', STUB_TESTRUNNER], - ]) { + if (git) { + // A committed tree, so the gate's between-round `git checkout -- .` + // reset has something to restore (round-state isolation scenarios). + for (const args of [ + ['init', '-q'], + ['add', '-A'], + [ + '-c', + 'user.name=flake-gate', + '-c', + 'user.email=flake@gate', + 'commit', + '-qm', + 'fixture', + ], + ]) { + const g = spawnSync('git', args, { cwd: ws, encoding: 'utf8' }); + assert.equal(g.status, 0, `git ${args.join(' ')}: ${g.stderr}`); + } + } + const stubSet = { + runuser: STUB_RUNUSER, + npx: STUB_TESTRUNNER, + node: STUB_TESTRUNNER, + timeout: STUB_TIMEOUT, + pkill: STUB_PKILL, + ...stubs, + }; + for (const [name, content] of Object.entries(stubSet)) { writeFileSync(join(bin, name), content); chmodSync(join(bin, name), 0o755); } @@ -1059,20 +1160,26 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp const out = join(rt, 'github-output'); writeFileSync(out, ''); writeFileSync(join(rt, 'github-summary'), ''); + const env = { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + RUNNER_TEMP: rt, + GITHUB_OUTPUT: out, + GITHUB_STEP_SUMMARY: join(rt, 'github-summary'), + FLAKE_ROUNDS: '5', + FLAKE_SEQ_DIR: seqDir, + ...envOverrides, + }; + for (const [k, v] of Object.entries(env)) { + // An override of undefined deletes the variable (unset scenarios). + if (v === undefined) delete env[k]; + } const res = spawnSync( 'bash', ['--noprofile', '--norc', '-e', '-o', 'pipefail', gateFile], { cwd: ws, - env: { - ...process.env, - PATH: `${bin}:${process.env.PATH}`, - RUNNER_TEMP: rt, - GITHUB_OUTPUT: out, - GITHUB_STEP_SUMMARY: join(rt, 'github-summary'), - FLAKE_ROUNDS: '5', - FLAKE_SEQ_DIR: seqDir, - }, + env, encoding: 'utf8', timeout: 60_000, }, @@ -1120,6 +1227,18 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.equal(outputs.flake_verdict, 'flaky'); assert.match(log, /a\.test\.js: FFFFF/); assert.match(log, /b\.test\.js: PFPFP/); + // Gate outputs are embedded UNESCAPED into the published comment: they + // must stay fixed text plus counters, never PR-controlled strings. + assert.match( + outputs.flake_summary, + /^\d+ of \d+ changed test file\(s\) returned different results across identical re-runs \(\d+ full round\(s\)\)$/, + 'the summary must be fixed text plus counters', + ); + assert.doesNotMatch( + outputs.flake_summary, + /a\.test\.js|b\.test\.js/, + 'PR-controlled filenames must stay out of the outputs', + ); }); it('identical failure every round stays informational `consistent-fail`, exit 0', () => { @@ -1132,6 +1251,62 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.equal(outputs.flake_verdict, 'consistent-fail'); }); + it('timeout/signal exits are infrastructure, never F marks or fake flakiness', () => { + // A pass next to an exit-124 round used to publish `flaky`; an OOM + // kill (137) is the same class. Infra exits must stay out of P/F + // divergence and land the informational `timeout` verdict instead. + const { res, outputs, log } = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\nscripts/tests/b.test.js\n', + sequences: { 'a.test.js': 'PT', 'b.test.js': 'K' }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'timeout'); + assert.match(log, /a\.test\.js: PIPIP/); + assert.match(log, /b\.test\.js: IIIII/); + assert.match(log, /exit 124/); + assert.doesNotMatch(outputs.flake_summary, /a\.test\.js|b\.test\.js/); + }); + + it('real divergence still outranks an infra exit in another file', () => { + const { res, outputs } = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\nscripts/tests/b.test.js\n', + sequences: { 'a.test.js': 'PF', 'b.test.js': 'PT' }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'flaky'); + }); + + it('round-state reset keeps a tree-mutating deterministic test from faking flakiness', () => { + // The stub passes only while fixture.txt is pristine, then mutates it. + // Without the gate's between-round `git checkout -- .` reset, rounds + // 2-5 fail on round 1's residue (PFFFF -> false flaky); with it every + // round starts from the committed state again (PPPPP -> pass). + const STUB_STATEFUL = [ + '#!/bin/bash', + 'if ! grep -q pristine fixture.txt; then', + ' echo "fixture mutated by an earlier round"', + ' exit 1', + 'fi', + 'echo mutated > fixture.txt', + 'exit 0', + '', + ].join('\n'); + const { res, outputs, log } = runGate({ + layout: { + 'scripts/tests/stateful.test.js': '', + 'fixture.txt': 'pristine\n', + }, + list: 'scripts/tests/stateful.test.js\n', + stubs: { npx: STUB_STATEFUL }, + git: true, + }); + assert.equal(res.status, 0, `gate died under the wrapper: ${res.stderr}`); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match(log, /stateful\.test\.js: PPPPP/); + }); + it('a missing recorded list degrades to the `error` verdict, exit 0', () => { const { res, outputs } = runGate({ layout: UNIT, list: null }); assert.equal(res.status, 0, res.stderr); @@ -1160,6 +1335,66 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.match(log, /no vitest config \(unsupported runner family\)/); }); + it('vite.config-only packages and root workspaces outside packages/** are RUN, not skipped', () => { + // packages/webui's only config is vite.config.ts (vitest resolves it), + // and integrations/external-context is a root npm workspace no path + // prefix covers — both are CI-tested, so the gate must re-run them + // through their owning package instead of logging them out of scope. + const { res, outputs, log } = runGate({ + layout: { + 'packages/webui/package.json': '{}', + 'packages/webui/vite.config.ts': '', + 'packages/webui/src/x.test.ts': '', + 'integrations/external-context/package.json': '{}', + 'integrations/external-context/vitest.config.ts': '', + 'integrations/external-context/src/y.test.ts': '', + }, + list: 'packages/webui/src/x.test.ts\nintegrations/external-context/src/y.test.ts\n', + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match( + log, + /\(cd packages\/webui\) npx --no-install vitest run \.\/src\/x\.test\.ts/, + 'a vite.config-only package must be entered and run', + ); + assert.match( + log, + /\(cd integrations\/external-context\) npx --no-install vitest run \.\/src\/y\.test\.ts/, + 'a root workspace outside packages/** must be entered and run', + ); + assert.doesNotMatch(log, /, skipped:/); + }); + + it('scripts/tests files outside the pinned vitest include set are skipped, not mis-run', () => { + // The pinned config only includes *.test.{js,ts}: an admitted .spec.js + // or .test.mjs would fail collection EVERY round otherwise and publish + // a bogus consistent-fail. + const { res, outputs, log } = runGate({ + layout: { + 'scripts/tests/probe.spec.js': '', + 'scripts/tests/probe.test.mjs': '', + 'scripts/tests/probe.test.js': '', + }, + list: 'scripts/tests/probe.spec.js\nscripts/tests/probe.test.mjs\nscripts/tests/probe.test.js\n', + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match( + log, + /not in the scripts\/tests vitest include set \(\*\.test\.\{js,ts\}\), skipped: scripts\/tests\/probe\.spec\.js/, + ); + assert.match( + log, + /not in the scripts\/tests vitest include set \(\*\.test\.\{js,ts\}\), skipped: scripts\/tests\/probe\.test\.mjs/, + ); + assert.match( + log, + /\(cd \.\) npx --no-install vitest run --config \.\/scripts\/tests\/vitest\.config\.ts \.\/scripts\/tests\/probe\.test\.js/, + 'the admitted .test.js file must still run', + ); + }); + it('a nested-workspace file runs from its OWN package, and a leading-dash filename stays an operand', () => { const { res, outputs, log } = runGate({ layout: { @@ -1184,6 +1419,67 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp ); }); + it('an abort before any verdict still fails open via the EXIT trap', () => { + // RUNNER_TEMP unset kills the script at ${RUNNER_TEMP:?} before a + // verdict exists; the trap must rewrite that ending into the fixed + // error outputs and a zero exit (fail-open, never a red step). + const { res, outputs } = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\n', + env: { RUNNER_TEMP: undefined }, + }); + assert.equal(res.status, 0, `the trap must convert the abort to exit 0: ${res.stderr}`); + assert.equal(outputs.flake_verdict, 'error'); + assert.match(outputs.flake_summary, /aborted before reaching a verdict/); + }); + + it('the .github/scripts node --test arm runs and is logged', () => { + const { res, outputs, log } = runGate({ + layout: { '.github/scripts/foo.test.mjs': '' }, + list: '.github/scripts/foo.test.mjs\n', + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match(log, /node --test \.\/\.github\/scripts\/foo\.test\.mjs/); + }); + + it('FLAKE_ROUNDS is parsed and clamped (default 5, floor 2, cap 10)', () => { + const roundsInLog = ({ log }) => { + const m = log.match(/^rounds=(\d+)/m); + assert.ok(m, 'the gate log header must carry the rounds count'); + return m[1]; + }; + for (const [env, expected] of [ + [{ FLAKE_ROUNDS: undefined }, '5'], + [{ FLAKE_ROUNDS: 'abc' }, '5'], + [{ FLAKE_ROUNDS: '99' }, '10'], + ]) { + const r = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\n', + env, + }); + assert.equal(r.res.status, 0, r.res.stderr); + assert.equal( + roundsInLog(r), + expected, + `FLAKE_ROUNDS=${JSON.stringify(env)} must clamp to ${expected}`, + ); + } + // The floor is load-bearing: classification needs at least two marks, + // so at rounds=1 divergence is impossible by construction and every + // flaky PR would read as pass. + const floored = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\n', + sequences: { 'a.test.js': 'PF' }, + env: { FLAKE_ROUNDS: '1' }, + }); + assert.equal(floored.res.status, 0, floored.res.stderr); + assert.equal(roundsInLog(floored), '2'); + assert.equal(floored.outputs.flake_verdict, 'flaky'); + }); + it('publisher demotion executes one-way: only `flaky` demotes, and it MUST demote', () => { const block = publishRun.match( /case "\$\{FLAKE_VERDICT:-\}" in[\s\S]*?esac/, diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index dd2f638a59d..63527704e53 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2756,7 +2756,10 @@ jobs: # narrowing the gate to n/a. A git failure here is pre-build # infrastructure and must fail the step loudly; only a no-match # grep may produce an empty list. - files="$(git diff --name-only --diff-filter=ACMR 'HEAD^1' HEAD)" + # core.quotePath=false: with the default, a non-ASCII filename is + # emitted C-quoted and silently fails the extension grep below, + # narrowing the gate without a skip-log entry. + files="$(git -c core.quotePath=false diff --name-only --diff-filter=ACMR 'HEAD^1' HEAD)" printf '%s\n' "$files" \ | grep -E '\.(test|spec)\.(ts|tsx|js|jsx|mjs|cjs)$' \ > "${RUNNER_TEMP:?}/flake-gate-files" || true @@ -3000,10 +3003,16 @@ jobs: return 1 } has_vitest_config() { - local c - for c in vitest.config.ts vitest.config.mts vitest.config.js vitest.config.mjs vitest.workspace.ts; do - [ -f "$1/$c" ] && return 0 + # Mirror vitest's own resolution: it accepts vitest.config AND + # vite.config in six extensions. A narrower probe skips runnable + # packages (packages/webui's only config is vite.config.ts). + local n e + for n in vitest.config vite.config; do + for e in ts mts cts js mjs cjs; do + [ -f "$1/$n.$e" ] && return 0 + done done + [ -f "$1/vitest.workspace.ts" ] && return 0 return 1 } while IFS= read -r f; do @@ -3027,17 +3036,27 @@ jobs: ;; esac case "$f" in - scripts/tests/*) + scripts/tests/*.test.js|scripts/tests/*.test.ts) group_labels+=("$f") group_dirs+=('.') group_cmds+=("npx --no-install vitest run --config ./scripts/tests/vitest.config.ts $(printf '%q' "./$f")") ;; + scripts/tests/*) + # The pinned config's include set is narrower than the gate's + # intake regex; a file it rejects would fail collection every + # round and masquerade as a deterministic failure. + add_skip "$f" 'not in the scripts/tests vitest include set (*.test.{js,ts}), skipped' + ;; .github/scripts/*.test.mjs) group_labels+=("$f") group_dirs+=('.') group_cmds+=("node --test $(printf '%q' "./$f")") ;; - packages/*) + *) + # Generic vitest resolution keyed on the OWNING PACKAGE, not + # a path prefix: root npm workspaces outside packages/** + # (integrations/*) are CI-tested too and must be re-run + # through their own entry point like any nested workspace. if ! pkg="$(owning_pkg_dir "$f")"; then add_skip "$f" 'no owning package.json, skipped' continue @@ -3055,9 +3074,6 @@ jobs: group_dirs+=("$pkg") group_cmds+=("npx --no-install vitest run $(printf '%q' "./${f#"$pkg"/}")") ;; - *) - add_skip "$f" 'no runner mapped for this path, skipped' - ;; esac done < "$LIST" @@ -3082,6 +3098,7 @@ jobs: deadline=$(( $(date +%s) + 900 )) rounds_done=0 timed_out=false + infra_exits=0 round=1 while [ "$round" -le "$ROUNDS" ]; do for i in "${!group_labels[@]}"; do @@ -3089,17 +3106,31 @@ jobs: timed_out=true break fi + # Fresh per-invocation temp/cache dir: samples must not share + # caches any more than they share the tree or processes. + inv_tmp="$RUNNER_TEMP/flake-inv-tmp" + rm -rf "$inv_tmp" + mkdir -p "$inv_tmp" + chown node:node "$inv_tmp" out="$RUNNER_TEMP/flake-gate-round-out" ( cd "${group_dirs[$i]}" && timeout -k 30 600 runuser -u node -- \ env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ - NODE_OPTIONS='--max-old-space-size=3072' CI=true \ + NODE_OPTIONS='--max-old-space-size=3072' CI=true TMPDIR="$inv_tmp" \ bash -c "${group_cmds[$i]}" ) > "$out" 2>&1 status=$? + # Timeout/signal exits (124, 128+N) are infrastructure events — + # a runner cap or an OOM kill — not test outcomes: recorded as + # F they would publish a fake `flaky` next to any pass. mark='P' - [ "$status" -eq 0 ] || mark='F' + if [ "$status" -eq 124 ] || [ "$status" -gt 128 ]; then + mark='I' + infra_exits=$((infra_exits + 1)) + elif [ "$status" -ne 0 ]; then + mark='F' + fi results[$i]="${results[$i]:-}${mark}" printf 'round %s · %s: %s (exit %s)\n' "$round" "${group_labels[$i]}" "$mark" "$status" >> "$LOG" if [ "$status" -ne 0 ]; then @@ -3110,6 +3141,13 @@ jobs: } >> "$LOG" fi done + # Round-state reset: repeated samples share this tree and its + # process table, so restore the equivalent of a clean checkout + # before the next sample — a deterministic test that mutates a + # fixture or leaves a daemon/port behind must not fail later + # rounds on its own residue and fake a divergence (PFF). + git checkout -- . || true + pkill -u node || true [ "$timed_out" = true ] && break rounds_done="$round" round=$((round + 1)) @@ -3124,7 +3162,7 @@ jobs: *F*) failing=$((failing + 1)) ;; esac done - printf '\nper-file results (P=pass F=fail, one letter per run):\n' >> "$LOG" + printf '\nper-file results (P=pass F=fail I=infra-exit, one letter per run):\n' >> "$LOG" for i in "${!group_labels[@]}"; do printf ' %s: %s\n' "${group_labels[$i]}" "${results[$i]:-}" >> "$LOG" done @@ -3135,6 +3173,9 @@ jobs: if [ "$timed_out" = true ] && [ "$rounds_done" -lt 2 ]; then finish timeout "the 15-minute budget elapsed before two full rounds completed (${rounds_done} done) — no flakiness signal either way" fi + if [ "$infra_exits" -gt 0 ]; then + finish timeout "${infra_exits} invocation(s) ended in a timeout/signal exit — infrastructure, not test nondeterminism, so these rounds carry no flakiness signal" + fi if [ "$failing" -gt 0 ]; then finish consistent-fail "${failing} of ${total} changed test file(s) failed identically in every round — deterministic, so CI owns that signal" fi @@ -3665,8 +3706,10 @@ jobs: find tmp -maxdepth 2 -type d -name '*-verify-*' -exec cp -r {} "$RUNNER_TEMP/verify-results/" \; 2>/dev/null || true # cp -r copies symlinks as symlinks (no deref), but # actions/upload-artifact FOLLOWS them — a node-planted link would - # exfiltrate whatever it points at into the artifact. Drop links. - find "$RUNNER_TEMP/verify-results" -type l -delete 2>/dev/null || true + # exfiltrate whatever it points at into the artifact; a planted + # FIFO would hang whoever opens it next. Drop every non-regular + # entry except directories (the collected artifact dirs). + find "$RUNNER_TEMP/verify-results" \( -type l -o -type p -o -type s -o -type b -o -type c \) -delete 2>/dev/null || true # 137 is ambiguous: the watchdog escalating past --kill-after looks # identical to an OOM kill. Use the elapsed budget to tell them @@ -3732,7 +3775,21 @@ jobs: run: |- set -euo pipefail if [ -f "${RUNNER_TEMP:?}/flake-gate.log" ]; then + # verify-results was chowned to the build user while PR code ran, + # and node daemons can outlive the agent step: kill leftover + # test-user processes first so nothing races this copy. Never + # OPEN a planted destination either — rm unlinks the entry + # itself, while `cp -f` would open a planted FIFO O_WRONLY and + # block until the job timeout, follow a symlink to rewrite a + # victim, or copy INTO a planted directory so the log vanishes + # from the pinned publisher path. + pkill -KILL -u node 2>/dev/null || true + if [ -L "$RUNNER_TEMP/verify-results" ] || + { [ -e "$RUNNER_TEMP/verify-results" ] && [ ! -d "$RUNNER_TEMP/verify-results" ]; }; then + rm -rf -- "$RUNNER_TEMP/verify-results" + fi mkdir -p "$RUNNER_TEMP/verify-results" + rm -rf -- "$RUNNER_TEMP/verify-results/flake-gate.log" cp -f "$RUNNER_TEMP/flake-gate.log" "$RUNNER_TEMP/verify-results/flake-gate.log" fi @@ -4605,7 +4662,10 @@ jobs: printf '%s\n\n' "$MISSING_REPORT_NOTE" fi emit_report "$REPORT" 45000 - emit_block 'Flakiness gate log' "$FLAKE_LOG" 20000 + # Cap 10000: the per-block caps must leave headroom under + # GitHub's 65,536-char comment limit once the 45000 report + # block and the mandatory prose are added (20000 crossed it). + emit_block 'Flakiness gate log' "$FLAKE_LOG" 10000 if [ -n "$EVIDENCE_SECTION" ]; then printf '%s' "$EVIDENCE_SECTION" fi From e76221cf9d15517e83afc9f0b5d1700892680874 Mon Sep 17 00:00:00 2001 From: wenshao Date: Fri, 14 Aug 2026 19:55:57 +0800 Subject: [PATCH 04/18] test(triage): follow the widened special-file strip into the vitest twin pins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 13708068bd widened the verify-lane artifact strip from symlinks only to \( -type l -o -type p -o -type s -o -type b -o -type c \) so a planted FIFO/socket/device cannot hang or redirect the collection — but two pins in scripts/tests/qwen-triage-workflow.test.js still asserted the old '-type l -delete' literal and went red (the Test job's only failures). Update both pins to the full new expression; the intent they guard (strip present, and AFTER the artifact copy) is unchanged, and the tmux-side pin keeps the old literal because the tmux lane still uses it. --- scripts/tests/qwen-triage-workflow.test.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/tests/qwen-triage-workflow.test.js b/scripts/tests/qwen-triage-workflow.test.js index 9f121ad0191..fae9ab5e14f 100644 --- a/scripts/tests/qwen-triage-workflow.test.js +++ b/scripts/tests/qwen-triage-workflow.test.js @@ -1480,9 +1480,12 @@ describe('qwen-triage verify workflow', () => { expect(runStep.indexOf(sweep)).toBeLessThan( runStep.indexOf('start_openai_proxy'), ); - // Uploaded artifacts must not carry node-planted symlinks: - // actions/upload-artifact dereferences them. - expect(runStep).toContain('-type l -delete'); + // Uploaded artifacts must not carry node-planted symlinks (or FIFOs/ + // sockets/devices, which can hang or redirect the collection): + // actions/upload-artifact dereferences symlinks. + expect(runStep).toContain( + 'find "$RUNNER_TEMP/verify-results" \\( -type l -o -type p -o -type s -o -type b -o -type c \\) -delete', + ); }); // RUNNER_TEMP hygiene between jobs is runner-managed; this pool is @@ -2903,7 +2906,9 @@ describe('qwen-triage verify hardening round 2', () => { const copy = runStep.indexOf( '-exec cp -r {} "$RUNNER_TEMP/verify-results/"', ); - const strip = runStep.indexOf('-type l -delete'); + const strip = runStep.indexOf( + 'find "$RUNNER_TEMP/verify-results" \\( -type l -o -type p -o -type s -o -type b -o -type c \\) -delete', + ); expect(copy).toBeGreaterThan(-1); expect(strip).toBeGreaterThan(copy); }); From 9f3b548e05dadc441db42d4ad9a6ca4509b178f2 Mon Sep 17 00:00:00 2001 From: wenshao Date: Fri, 14 Aug 2026 22:41:54 +0800 Subject: [PATCH 05/18] fix(triage): build-user round resets incl. pre-round-1, artifact-loss-proof flaky demotion, mechanism-anchored pins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-4 review, all 22 findings (2 Critical dual-anchored + 20): - Round-state reset (Critical): run the reset AS THE BUILD USER — a root checkout restores node-mutated tracked files as root-owned inodes that later node rounds cannot write (EACCES divergence) — add 'git clean -fd' (no -x) so untracked round residue is dropped, and run one reset BEFORE round 1 so lifecycle-script mutations cannot make the first sample differ from the rest. Behavioral scenarios: untracked residue, pre-gate tree mutation (both PPPPP/pass), each red without its fix. - Staging step: continue-on-error (evidence-copying must not let the publisher discard a recorded verdict as 'infrastructure failure'), and the order chain (pkill -> dir guard -> mkdir -> unlink dst -> cp) is now pinned by index comparison, not presence-only regexes. - Publisher: the flaky demotion now fires in the artifact-download- failure branch too — FLAKE_VERDICT travels via job outputs and does not need the artifact — instead of a neutral 'results unavailable'. - Pins anchored to mechanisms, not adjacency (R2-P1): exact-line record assignment (kills ;/& status swallowing and covers the -c form), word- based no-re-derivation, line-anchored 'timeout -k 30 600 runuser' invocation (comment-proof, also pins the per-invocation cap), build- user reset lines, agent/gate if-equivalence. - Unpinned guards now pinned (R2-P2/P3): ACTIONS_* credential strip, the env -u runner-file isolation, whole-env key set (a future secret in the gate env must be an explicit test decision), intake extension set, record->gate handoff filename, child-env line (CI/heap/TMPDIR). - New behavioral scenarios: wall-budget expiry via a scripted date stub (both timeout branches, pinning rounds_done placement), space-bearing filename through %q as one operand, bilingual one-way demotion (the collapsed Chinese summary is the one verdict a zh reader sees). --- .github/scripts/qwen-triage-workflow.test.mjs | 308 +++++++++++++++--- .github/workflows/qwen-triage.yml | 63 +++- 2 files changed, 321 insertions(+), 50 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index be6afbb5a51..04a7eb698b0 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -844,38 +844,78 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // empty list. A `git diff ... | grep ... || true` pipeline would // swallow the failure as "no changed test files" and certify the PR // with an n/a gate. + // Exact-line pin (round-4 R2-P1): the earlier `[^|)]` separator class + // admitted `;`/`&` status swallowing inside the substitution, and a + // `git diff` adjacency check can never match the mandated `-c` form. + // Pinning the whole assignment leaves no room for any separator. assert.match( recordStep.run, - /^\s*files="\$\(git [^|)]*\)"\s*$/m, - 'git diff must run as a standalone assignment, not feed a pipeline', + /^\s*files="\$\(git -c core\.quotePath=false diff --name-only --diff-filter=ACMR 'HEAD\^1' HEAD\)"$/m, + 'git diff must run as a standalone, exactly-shaped assignment — no pipeline, no `;`/`&` status swallowing', ); - assert.doesNotMatch( - recordStep.run - .split('\n') - .filter((l) => !l.trim().startsWith('#')) - .join('\n'), - /git diff[^\n]*\|/, - 'a git failure must not be swallowable by a pipeline || true', + // The intake extension set is load-bearing: dropping `mjs` would + // silently drop .github/scripts/*.test.mjs files from the gate list in + // production — behavioral scenarios plant the list by hand and cannot + // see the record step narrow. + assert.ok( + recordStep.run.includes( + "grep -E '\\.(test|spec)\\.(ts|tsx|js|jsx|mjs|cjs)$'", + ), + 'the intake extension set must stay pinned', + ); + assert.match( + recordStep.run, + /flake-gate-files/, + 'the record step must write the list where the gate reads it', ); assert.match( flakeStep.run, /flake-gate-files/, 'the gate must read the recorded list, not re-derive the diff', ); + // Word-based, not adjacency: `git -c … diff` is still a re-derivation. + // The gate's own `git checkout`/`git clean` reset lines stay legal. assert.doesNotMatch( flakeStep.run, - /git diff/, - 'the gate must not re-derive the diff from post-build git metadata', + /\bgit\b[^\n]*\bdiff\b/, + 'the gate must not re-derive the diff from post-build git metadata in any spelling', ); }); it('runs PR test code as the build user with no tokens, and fails open', () => { assert.equal(flakeStep.env.GITHUB_TOKEN, '', 'no GitHub token in the gate'); assert.equal(flakeStep.env.GH_TOKEN, '', 'no gh token in the gate'); + // Line-anchored (round-4 R2-P1): an unanchored substring is satisfied + // by a comment while the invocation itself runs as root — and this one + // line also pins the per-invocation `timeout` cap, without which a + // single hung test holds the invocation to the job timeout's SIGKILL, + // which the EXIT-trap fail-open cannot survive. + assert.match( + flakeStep.run, + /^\s*timeout -k 30 600 runuser -u node -- \\$/m, + 'PR test code must run as the build user under the per-invocation timeout cap', + ); assert.match( flakeStep.run, - /runuser -u node --/, - 'PR test code must run as the unprivileged build user', + /unset ACTIONS_RUNTIME_TOKEN ACTIONS_RUNTIME_URL ACTIONS_CACHE_URL/, + 'cache-service credentials must be stripped before PR test code runs', + ); + assert.match( + flakeStep.run, + /env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY/, + 'runner-injection files must be invisible to PR test code', + ); + // Whole-env pin: any future secret added to this step env reaches + // process.env of PR test code, so it must be an explicit test decision. + assert.deepEqual( + Object.keys(flakeStep.env).sort(), + ['FLAKE_ROUNDS', 'GH_TOKEN', 'GITHUB_TOKEN'], + 'the gate env must stay tokens-blanked and secret-free', + ); + assert.match( + flakeStep.run, + /NODE_OPTIONS='--max-old-space-size=3072' CI=true TMPDIR="\$inv_tmp"/, + 'child env must pin CI parity, the heap limit, and the per-invocation TMPDIR', ); assert.match( flakeStep.run, @@ -902,24 +942,51 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /set -euo/, 'a gate bug must fail OPEN (verdict error), never abort the verify job', ); - // Repeated samples share one tree and one process table: without a - // per-round reset, a deterministic test fails on its own residue - // (mutated fixture, leftover daemon/port) and fakes a divergence. + // Round-4 Critical: the reset must run AS THE BUILD USER — a root + // checkout restores node-mutated tracked files as new root-owned + // inodes that later node rounds cannot write (EACCES divergence) — + // and must also drop untracked residue (`git clean -fd`, no -x so + // gitignored node_modules/dist survive). Line-anchored: a comment + // cannot satisfy these. assert.match( flakeStep.run, - /^\s*git checkout -- \. \|\| true$/m, - 'each round must restore tracked files before the next sample', + /^\s*runuser -u node -- git checkout -- \. 2>\/dev\/null \|\| true$/m, + 'the tracked-file restore must run as the build user', + ); + assert.match( + flakeStep.run, + /^\s*runuser -u node -- git clean -fd 2>\/dev\/null \|\| true$/m, + 'untracked round residue must be cleaned without touching gitignored build outputs', ); assert.match( flakeStep.run, /^\s*pkill -u node \|\| true$/m, 'each round must tear down leftover test-user processes', ); + // And the reset must also run BEFORE round 1: lifecycle scripts (npm + // ci/build, run as node) mutate the tree between list-record and the + // first sample, so round 1 must sample the same restored tree as + // rounds 2..N. + const firstReset = flakeStep.run.search(/^\s*reset_round_state$/m); + const loopStart = flakeStep.run.indexOf('while [ "$round" -le "$ROUNDS" ]'); + assert.ok( + firstReset !== -1 && loopStart !== -1 && firstReset < loopStart, + 'one reset must precede round 1, not only the between-round resets', + ); assert.equal( flakeStep.if, "steps.pr.outputs.decision == 'run' && steps.prepare.outputs.verdict == ''", 'the gate runs exactly when the agent would (after a clean build)', ); + // The message above is only true while both conditions stay identical: + // if the agent's `if:` drifts wider, the agent publishes a verdict on + // runs the gate never sampled, and the empty FLAKE_VERDICT leaves the + // headline untouched (the behavioral drive test proves '' is a no-op). + assert.equal( + agentStep.if, + flakeStep.if, + 'gate and agent must run under identical conditions', + ); }); it('exposes the gate outcome to the publisher and preserves its log', () => { @@ -963,20 +1030,43 @@ describe('qwen-triage: flakiness gate (#9125)', () => { agentIdx < stageIdx && stageIdx < uploadIdx, 'staging must run after the agent and before the upload', ); - assert.match( - stageStep.run, - /cp -f "\$RUNNER_TEMP\/flake-gate\.log" "\$RUNNER_TEMP\/verify-results\/flake-gate\.log"/, - 'staging must overwrite whatever the agent era left under that name', - ); - assert.match( - stageStep.run, - /rm -rf -- "\$RUNNER_TEMP\/verify-results\/flake-gate\.log"/, - 'staging must remove a planted destination BEFORE copying — cp would open a FIFO and block, or follow a symlink', - ); - assert.match( - stageStep.run, - /pkill -KILL -u node/, - 'staging must kill leftover build-user processes so nothing races the copy', + // Evidence-copying only, after the verdict outputs are written: a + // staging failure (ENOSPC, hostile mount) must not flip the job red or + // the publisher discards the recorded verdict as "infrastructure". + assert.equal( + stageStep['continue-on-error'], + true, + 'staging must not be able to fail the job', + ); + // The ORDER is the guard (round-4 R2-P1): presence-only pins stayed + // green with `cp` reordered before the unlinks, reopening the planted + // FIFO/symlink hazard. Kill racers → drop a planted dir/symlink at the + // directory level → recreate → unlink the destination entry → copy. + const sr = stageStep.run; + const iPkill = sr.indexOf('pkill -KILL -u node'); + const iDirGuard = sr.search(/\[ -L "\$RUNNER_TEMP\/verify-results" \]/); + const iMkdir = sr.indexOf('mkdir -p "$RUNNER_TEMP/verify-results"'); + const iRmDst = sr.indexOf( + 'rm -rf -- "$RUNNER_TEMP/verify-results/flake-gate.log"', + ); + const iCp = sr.indexOf( + 'cp -f "$RUNNER_TEMP/flake-gate.log" "$RUNNER_TEMP/verify-results/flake-gate.log"', + ); + for (const [label, idx] of [ + ['pkill', iPkill], + ['directory-level symlink/non-dir guard', iDirGuard], + ['mkdir -p', iMkdir], + ['destination unlink', iRmDst], + ['copy', iCp], + ]) { + assert.ok(idx !== -1, `staging must contain the ${label}`); + } + assert.ok( + iPkill < iDirGuard && + iDirGuard < iMkdir && + iMkdir < iRmDst && + iRmDst < iCp, + 'staging order must be: kill racers, dir guard, recreate, unlink destination, copy', ); assert.doesNotMatch( agentStep.run, @@ -1025,6 +1115,27 @@ describe('qwen-triage: flakiness gate (#9125)', () => { } }); + it('the flaky demotion also fires when the result artifact is unavailable', () => { + // FLAKE_VERDICT travels via job outputs, independent of the artifact. + // Without this branch handling, a flaky PR whose artifact download + // failed got a neutral "results unavailable" notice — the demotion + // contract silently not firing. + const branch = publishStep.run.match( + /elif \[ "\$\{DOWNLOAD_OUTCOME:-success\}" != "success" \];[\s\S]*?\nelif /, + ); + assert.ok(branch, 'the download-failure branch must exist'); + assert.match( + branch[0], + /"\$\{FLAKE_VERDICT:-\}" = 'flaky'/, + 'the branch must consult the gate verdict', + ); + assert.match( + branch[0], + /❌ not passed — non-deterministic tests \(flakiness gate\)/, + 'flaky must still demote the headline without the artifact', + ); + }); + it('the verify job timeout still covers agent + prepare + gate', () => { // agent 120m + install/build 15m + gate ~25m + misc ~5m — the job limit // must stay comfortably above the sum or the container is killed mid-run @@ -1108,6 +1219,7 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp stubs = {}, env: envOverrides = {}, git = false, + mutate, }) => { const root = mkdtempSync(join(scenarioRoot, 'case-')); const ws = join(root, 'ws'); @@ -1143,6 +1255,9 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.equal(g.status, 0, `git ${args.join(' ')}: ${g.stderr}`); } } + // Applied AFTER the commit: models PR lifecycle scripts (npm ci/build) + // mutating the tree between list-record and round 1. + if (mutate) mutate(ws); const stubSet = { runuser: STUB_RUNUSER, npx: STUB_TESTRUNNER, @@ -1307,6 +1422,115 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.match(log, /stateful\.test\.js: PPPPP/); }); + it('untracked residue from round 1 is cleaned, not left to fail later rounds', () => { + // `git checkout -- .` never removes untracked files (round-4 Critical + // mechanism a): a test whose first invocation leaves an untracked + // lock/output dir would fail rounds 2-5 on that residue (PFFFF -> + // false flaky) unless the reset also runs `git clean -fd`. + const STUB_UNTRACKED = [ + '#!/bin/bash', + 'if [ -e out-residue ]; then', + ' echo "untracked residue from an earlier round"', + ' exit 1', + 'fi', + 'mkdir out-residue', + 'exit 0', + '', + ].join('\n'); + const { res, outputs, log } = runGate({ + layout: { 'scripts/tests/untracked.test.js': '' }, + list: 'scripts/tests/untracked.test.js\n', + stubs: { npx: STUB_UNTRACKED }, + git: true, + }); + assert.equal(res.status, 0, `gate died under the wrapper: ${res.stderr}`); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match(log, /untracked\.test\.js: PPPPP/); + }); + + it('a pre-gate tree mutation cannot make round 1 sample different content than rounds 2..N', () => { + // Round-4 Critical mechanism c: PR lifecycle scripts run as node + // between list-record and round 1. Without a reset BEFORE round 1 the + // first sample sees the mutated tree and later samples see the + // restored one — the difference reads as divergence (F then PPPP). + const STUB_MARKER = [ + '#!/bin/bash', + 'if grep -q clean marker.txt; then exit 0; fi', + 'echo "sampled the lifecycle-mutated tree"', + 'exit 1', + '', + ].join('\n'); + const { res, outputs, log } = runGate({ + layout: { + 'scripts/tests/marker.test.js': '', + 'marker.txt': 'clean\n', + }, + list: 'scripts/tests/marker.test.js\n', + stubs: { npx: STUB_MARKER }, + git: true, + mutate: (ws) => writeFileSync(join(ws, 'marker.txt'), 'dirty\n'), + }); + assert.equal(res.status, 0, `gate died under the wrapper: ${res.stderr}`); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match(log, /marker\.test\.js: PPPPP/); + }); + + it('wall-budget expiry before two full rounds is the informational timeout verdict', () => { + // A scripted `date` stub drives the clock: deadline init at 0 (so the + // budget ends at 900), round 1 checked at 100 and run, round 2 checked + // at 1000 — expired with only one full round done. + const STUB_DATE = [ + '#!/bin/bash', + 'n_file="$FLAKE_SEQ_DIR/.count-date"', + 'n=$(cat "$n_file" 2>/dev/null || echo 0)', + 'echo $((n+1)) > "$n_file"', + 'mapfile -t vals < "$FLAKE_SEQ_DIR/dates"', + 'i=$n', + '[ "$i" -ge "${#vals[@]}" ] && i=$((${#vals[@]} - 1))', + 'echo "${vals[$i]}"', + '', + ].join('\n'); + const one = runGate({ + layout: { 'scripts/tests/a.test.js': '' }, + list: 'scripts/tests/a.test.js\n', + stubs: { date: STUB_DATE }, + sequences: { dates: '0\n100\n1000\n' }, + }); + assert.equal(one.res.status, 0, one.res.stderr); + assert.equal(one.outputs.flake_verdict, 'timeout'); + assert.match(one.outputs.flake_summary, /before two full rounds/); + // With two agreeing rounds completed before expiry, the summary must + // say the completed rounds agreed — rounds_done must track completed + // rounds, not the loop counter. + const two = runGate({ + layout: { 'scripts/tests/a.test.js': '' }, + list: 'scripts/tests/a.test.js\n', + stubs: { date: STUB_DATE }, + sequences: { dates: '0\n100\n200\n1000\n' }, + }); + assert.equal(two.res.status, 0, two.res.stderr); + assert.equal(two.outputs.flake_verdict, 'timeout'); + assert.match(two.outputs.flake_summary, /the completed rounds agreed/); + }); + + it('a space-bearing filename survives the %q quoting as one operand', () => { + // The operands are re-parsed by `bash -c`: without `printf %q` a space + // splits the path into two operands, vitest finds no file, and every + // round fails identically — a bogus consistent-fail without one + // sample. (Round-4 R2-P3.) + const { res, outputs, log } = runGate({ + layout: { 'scripts/tests/has space.test.js': '' }, + list: 'scripts/tests/has space.test.js\n', + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match( + log, + /has\\ space\.test\.js/, + 'the logged command must carry the escaped, single operand', + ); + }); + it('a missing recorded list degrades to the `error` verdict, exit 0', () => { const { res, outputs } = runGate({ layout: UNIT, list: null }); assert.equal(res.status, 0, res.stderr); @@ -1428,7 +1652,11 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp list: 'scripts/tests/a.test.js\n', env: { RUNNER_TEMP: undefined }, }); - assert.equal(res.status, 0, `the trap must convert the abort to exit 0: ${res.stderr}`); + assert.equal( + res.status, + 0, + `the trap must convert the abort to exit 0: ${res.stderr}`, + ); assert.equal(outputs.flake_verdict, 'error'); assert.match(outputs.flake_summary, /aborted before reaching a verdict/); }); @@ -1503,7 +1731,7 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp "FLAKE_LINE=''", "FLAKE_LINE_ZH=''", block[0], - 'printf \'%s|%s\' "$QUAL" "$HEADLINE"', + 'printf \'%s|%s|%s|%s\' "$QUAL" "$HEADLINE" "$QUAL_ZH" "$HEADLINE_ZH"', ].join('\n'), ], { @@ -1519,6 +1747,10 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.equal(res.status, 0, res.stderr); return res.stdout; }; + // Both language pairs are asserted (round 4): the Chinese summary line + // is the ONE verdict a collapsed-details reader sees — if the _ZH + // assignments drop, a demoted PR renders `判定：✅ 通过` in Chinese + // while the English headline says ❌. for (const v of [ '', 'pass', @@ -1529,14 +1761,14 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp ]) { assert.equal( drive(v), - '✅ passed|merge-ready (agent verdict)', - `'${v}' must not touch the headline`, + '✅ passed|merge-ready (agent verdict)|✅ 通过|可合入', + `'${v}' must not touch the headline in either language`, ); } assert.equal( drive('flaky'), - '❌ not passed|non-deterministic tests (flakiness gate)', - 'flaky must demote — deleting the flaky arm has to fail this test', + '❌ not passed|non-deterministic tests (flakiness gate)|❌ 不通过|测试结果不确定（抖动门）', + 'flaky must demote in BOTH languages — deleting either pair has to fail this test', ); }); }); diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index 63527704e53..34bccf1bdf5 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -3094,12 +3094,31 @@ jobs: # timeout budget above accounts for. Divergence needs no uniform # round count: a P and an F for the same group is non-determinism # no matter how many rounds fit the budget. + # Round-state reset — run AS THE BUILD USER, not root: a root + # checkout restores node-mutated tracked files as new root-owned + # inodes that later node rounds cannot write (EACCES), + # manufacturing the very divergence the reset exists to prevent. + # `git clean -fd` (no -x: gitignored node_modules/dist must + # survive) drops the untracked residue `checkout -- .` never + # touches — lock files, output dirs a test creates in round 1. + reset_round_state() { + runuser -u node -- git checkout -- . 2>/dev/null || true + runuser -u node -- git clean -fd 2>/dev/null || true + pkill -u node || true + } + declare -a results=() deadline=$(( $(date +%s) + 900 )) rounds_done=0 timed_out=false infra_exits=0 round=1 + # Reset BEFORE round 1 too: PR lifecycle scripts (npm ci/build, + # run as node) may have mutated the tree since the list was + # recorded — without this, round 1 samples mutated content while + # rounds 2..N sample restored content, and the difference reads + # as divergence. + reset_round_state while [ "$round" -le "$ROUNDS" ]; do for i in "${!group_labels[@]}"; do if [ "$(date +%s)" -ge "$deadline" ]; then @@ -3141,13 +3160,11 @@ jobs: } >> "$LOG" fi done - # Round-state reset: repeated samples share this tree and its - # process table, so restore the equivalent of a clean checkout - # before the next sample — a deterministic test that mutates a - # fixture or leaves a daemon/port behind must not fail later - # rounds on its own residue and fake a divergence (PFF). - git checkout -- . || true - pkill -u node || true + # Restore the equivalent of a clean checkout before the next + # sample — a deterministic test that mutates a fixture or leaves + # a daemon/port/untracked file behind must not fail later rounds + # on its own residue and fake a divergence (PFF). + reset_round_state [ "$timed_out" = true ] && break rounds_done="$round" round=$((round + 1)) @@ -3772,6 +3789,13 @@ jobs: # dir cannot shadow it either. - name: 'Stage flakiness gate log for upload' if: "always() && steps.pr.outputs.decision == 'run'" + # Evidence-copying only, and the verdict outputs are already + # written: a failure here (ENOSPC from disk-filling PR tests, a + # hostile mount) must not flip the job red — the publisher's + # VERIFY_RESULT != success branch would then discard the recorded + # verdict and report "infrastructure failure" instead. Same rule + # the Upload step below documents. + continue-on-error: true run: |- set -euo pipefail if [ -f "${RUNNER_TEMP:?}/flake-gate.log" ]; then @@ -4404,13 +4428,28 @@ jobs: # claims the A/B, the harnesses and the gates were delivered — # when nothing was. WEAK_BODY=true + # The flake verdict travels via job outputs, independent of the + # artifact — the gate's demotion contract ("reported as not + # passed regardless of the agent verdict") must fire even when + # the download did not, or a flaky PR gets a neutral + # "results unavailable" notice instead of its ❌. { printf '%s\n\n' '' - printf '**Sandboxed verification: ⚠️ incomplete — results unavailable** - [workflow run](%s)\n\n' "$RUN_URL" - printf 'The verification ran, but its result artifact could not be retrieved for publishing, so there is nothing to report here. The run log still has the agent output; re-run `@qwen-code /verify` for a fresh report.\n\n' - printf '
\n中文 — 判定：⚠️ 未完成 · 结果不可用\n\n' - printf '验证已执行，但结果产物未能取回用于发布，因此此处没有可报告的内容。运行日志中仍有 agent 输出；如需完整报告请重新运行 `@qwen-code /verify`。\n\n' - printf '
\n\n' + if [ "${FLAKE_VERDICT:-}" = 'flaky' ]; then + printf '**Sandboxed verification: ❌ not passed — non-deterministic tests (flakiness gate); result artifact unavailable** - [workflow run](%s)\n\n' "$RUN_URL" + printf 'Flakiness gate: ❌ %s\n\n' "${FLAKE_SUMMARY:-changed test files returned different results across identical re-runs}" + printf 'The verification ran and its deterministic flakiness gate observed run-to-run divergence, but the result artifact (report and per-round matrix) could not be retrieved for publishing. The run log still has the full output; re-run `@qwen-code /verify` for a fresh report.\n\n' + printf '
\n中文 — 判定：❌ 不通过 · 测试结果不确定（抖动门）；结果产物不可用\n\n' + printf '抖动门：❌ %s\n\n' "${FLAKE_SUMMARY:-changed test files returned different results across identical re-runs}" + printf '验证已执行，其确定性抖动门观测到轮间分歧，但结果产物（报告与各轮矩阵）未能取回用于发布。运行日志中仍有完整输出；如需完整报告请重新运行 `@qwen-code /verify`。\n\n' + printf '
\n\n' + else + printf '**Sandboxed verification: ⚠️ incomplete — results unavailable** - [workflow run](%s)\n\n' "$RUN_URL" + printf 'The verification ran, but its result artifact could not be retrieved for publishing, so there is nothing to report here. The run log still has the agent output; re-run `@qwen-code /verify` for a fresh report.\n\n' + printf '
\n中文 — 判定：⚠️ 未完成 · 结果不可用\n\n' + printf '验证已执行，但结果产物未能取回用于发布，因此此处没有可报告的内容。运行日志中仍有 agent 输出；如需完整报告请重新运行 `@qwen-code /verify`。\n\n' + printf '
\n\n' + fi printf '%s\n' '— _Qwen Code · sandboxed verification_' } > "$BODY_FILE" elif [ -n "${PREPARE_FAILURE_PHASE:-}" ]; then From 3d91a9c34fe007e5119165906444f96e80b4e1ee Mon Sep 17 00:00:00 2001 From: wenshao Date: Sat, 15 Aug 2026 09:00:59 +0800 Subject: [PATCH 06/18] fix(triage): NUL-delimited gate intake, zero-collection class, front-loaded matrix, hardened resets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-5 review, all 28 findings: - NUL-delimited record end to end (git diff -z, grep -z, gate read -d ''): quotePath=false only stops quoting of bytes >= 0x80 — ASCII specials (backslash, tab, quote, control chars) stayed C-quoted and silently failed the $-anchored line grep with no skip-log entry. The raw diff never passes through $( ) (command substitution strips NUL). Intake extension set gains .mts/.cts (vitest's default include collects them). - Zero-collection class ('N' mark): a file the runner's include set rejects exits 1 every round with 'No test files found' — publishing that as consistent-fail claimed 'deterministic, CI owns it' with both clauses false. All-uncollected lands n/a; mixed runs pass with the not-collected count in the summary. Faking the marker can only SUPPRESS a demotion the PR could already dodge — one-way authority. - Per-invocation detail moved behind the matrix/verdict: the publisher embeds the FIRST 10,000 chars, and failure tails (8 KB each) pushed the promised per-round matrix past the cap in exactly the flaky runs the demotion points at. Plus a bilingual fallback note when the gate log could not be staged into the artifact. - reset_round_state kills FIRST (a live daemon re-dirties the tree after checkout), with SIGKILL + a bounded wait replacing the one-shot TERM. - Behavioral hardening: runner-injection-env and operand-resolution guards baked into the default stub (pins the cd and %q for every arm), hostile filenames through the generic and node --test arms, mid-round budget expiry, flaky-outranks-timeout, infra-exit amid divergence, vitest.workspace.ts entry, exact summary counters, step-summary read-back, gate status line in the demotion drive. - Structural pins anchored to mechanisms: exact NUL-record statements, continuation-proof no-re-derivation (plus log/show/whatchanged), unset-before-invocation ordering, FLAKE_ROUNDS wired to the repo var, inv_tmp lifecycle, record/gate if-pins, flake-before-agent order, staging line-anchored order chain incl. both guard halves and the guard's rm reaction, DOWNLOAD_OUTCOME wiring. --- .github/scripts/qwen-triage-workflow.test.mjs | 440 +++++++++++++++--- .github/workflows/qwen-triage.yml | 89 +++- 2 files changed, 443 insertions(+), 86 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index 04a7eb698b0..6e36d7edaae 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -829,57 +829,75 @@ describe('qwen-triage: flakiness gate (#9125)', () => { prepareIdx < flakeIdx, 'the gate needs node_modules, so it must run after install/build', ); - assert.match( - recordStep.run, - /HEAD\^1/, - 'diff must be against the merge base', + // And before the agent: the sampled tree must be the built tree the + // agent verifies, not one the agent era has already mutated. + assert.ok( + flakeIdx < verifyJob.steps.indexOf(agentStep), + 'the gate must sample before the agent runs', ); + // The record step must fire on every run the gate can fire on — a + // narrower `if:` here silently starves the gate into `error`. + assert.equal( + recordStep.if, + "steps.pr.outputs.decision == 'run'", + 'the record step must run whenever the lane runs', + ); + // NUL-delimited end to end (round 5): quotePath=false only stops + // quoting of bytes >= 0x80; ASCII specials (backslash, tab, quote, + // control chars) stay C-quoted and silently failed a $-anchored line + // grep. Exact-line pins: the git statement must stand alone (a + // pipeline or a command substitution would swallow the exit status or + // the NUL bytes respectively), and the grep must read the file and + // carry the full extension set (.mts/.cts included — vitest's default + // include collects them). assert.match( recordStep.run, - /git -c core\.quotePath=false diff --name-only/, - 'non-ASCII filenames must not be C-quoted out of the gate list', - ); - // The "two statements, not a pipeline" property is load-bearing: a git - // failure must fail the step loudly; only a no-match grep may yield an - // empty list. A `git diff ... | grep ... || true` pipeline would - // swallow the failure as "no changed test files" and certify the PR - // with an n/a gate. - // Exact-line pin (round-4 R2-P1): the earlier `[^|)]` separator class - // admitted `;`/`&` status swallowing inside the substitution, and a - // `git diff` adjacency check can never match the mandated `-c` form. - // Pinning the whole assignment leaves no room for any separator. + /^\s*git -c core\.quotePath=false diff -z --name-only --diff-filter=ACMR 'HEAD\^1' HEAD \\$/m, + 'the diff must be NUL-delimited, against the merge base, standalone', + ); assert.match( recordStep.run, - /^\s*files="\$\(git -c core\.quotePath=false diff --name-only --diff-filter=ACMR 'HEAD\^1' HEAD\)"$/m, - 'git diff must run as a standalone, exactly-shaped assignment — no pipeline, no `;`/`&` status swallowing', + /^\s*> "\$\{RUNNER_TEMP:\?\}\/flake-gate-files-all"$/m, + 'the raw diff must land in a file — $( ) strips NUL bytes', ); - // The intake extension set is load-bearing: dropping `mjs` would - // silently drop .github/scripts/*.test.mjs files from the gate list in - // production — behavioral scenarios plant the list by hand and cannot - // see the record step narrow. assert.ok( recordStep.run.includes( - "grep -E '\\.(test|spec)\\.(ts|tsx|js|jsx|mjs|cjs)$'", + "grep -zE '\\.(test|spec)\\.(ts|tsx|js|jsx|mjs|cjs|mts|cts)$'", ), - 'the intake extension set must stay pinned', + 'the NUL-record grep must carry the full extension set', ); assert.match( recordStep.run, - /flake-gate-files/, - 'the record step must write the list where the gate reads it', + /^\s*> "\$RUNNER_TEMP\/flake-gate-files" \|\| true$/m, + 'only a no-match grep may yield an empty list, into the exact handoff path', + ); + assert.doesNotMatch( + recordStep.run + .split('\n') + .filter((l) => !l.trim().startsWith('#')) + .join('\n'), + /git[^\n]*diff[^\n]*\|/, + 'a git failure must never be swallowable by a pipeline', ); assert.match( flakeStep.run, - /flake-gate-files/, - 'the gate must read the recorded list, not re-derive the diff', + /read -r -d '' f/, + 'the gate must consume the list NUL-delimited — the one framing a filename cannot break', ); - // Word-based, not adjacency: `git -c … diff` is still a re-derivation. - // The gate's own `git checkout`/`git clean` reset lines stay legal. + // Word-based and continuation-proof (round 5): `git -c … diff`, a + // backslash-continued `git \⏎ diff`, and log/show/whatchanged + // --name-only are all re-derivations. The gate's own `git checkout`/ + // `git clean` reset lines stay legal. assert.doesNotMatch( - flakeStep.run, + flakeStep.run.replace(/\\\n/g, ' '), /\bgit\b[^\n]*\bdiff\b/, 'the gate must not re-derive the diff from post-build git metadata in any spelling', ); + assert.doesNotMatch( + flakeStep.run.replace(/\\\n/g, ' '), + /\bgit\b[^\n]*\b(log|show|whatchanged)\b[^\n]*--name-only/, + 'nor via history-walking verbs', + ); }); it('runs PR test code as the build user with no tokens, and fails open', () => { @@ -895,9 +913,16 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /^\s*timeout -k 30 600 runuser -u node -- \\$/m, 'PR test code must run as the build user under the per-invocation timeout cap', ); - assert.match( - flakeStep.run, - /unset ACTIONS_RUNTIME_TOKEN ACTIONS_RUNTIME_URL ACTIONS_CACHE_URL/, + // Presence AND position: the strip must precede the first invocation, + // or the credentials are already in the child env when PR code runs. + const unsetIdx = flakeStep.run.indexOf( + 'unset ACTIONS_RUNTIME_TOKEN ACTIONS_RUNTIME_URL ACTIONS_CACHE_URL', + ); + const firstInvocation = flakeStep.run.search( + /^\s*timeout -k 30 600 runuser -u node -- \\$/m, + ); + assert.ok( + unsetIdx !== -1 && firstInvocation !== -1 && unsetIdx < firstInvocation, 'cache-service credentials must be stripped before PR test code runs', ); assert.match( @@ -912,11 +937,36 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ['FLAKE_ROUNDS', 'GH_TOKEN', 'GITHUB_TOKEN'], 'the gate env must stay tokens-blanked and secret-free', ); + // Keys AND the one non-blank value: FLAKE_ROUNDS must come from the + // repo variable, not a hardcoded count or a PR-influenced expression. + assert.equal( + flakeStep.env.FLAKE_ROUNDS, + '${{ vars.QWEN_VERIFY_FLAKE_ROUNDS }}', + 'round count must be operator-controlled', + ); assert.match( flakeStep.run, /NODE_OPTIONS='--max-old-space-size=3072' CI=true TMPDIR="\$inv_tmp"/, 'child env must pin CI parity, the heap limit, and the per-invocation TMPDIR', ); + // The per-invocation temp dir must be recreated fresh and handed to + // the build user — a shared or stale TMPDIR is exactly the cross-round + // cache leakage the reset exists to prevent. + assert.match( + flakeStep.run, + /^\s*rm -rf "\$inv_tmp"$/m, + 'inv_tmp must be flushed per invocation', + ); + assert.match( + flakeStep.run, + /^\s*mkdir -p "\$inv_tmp"$/m, + 'inv_tmp must be recreated per invocation', + ); + assert.match( + flakeStep.run, + /^\s*chown node:node "\$inv_tmp"$/m, + 'inv_tmp must be writable by the build user', + ); assert.match( flakeStep.run, /^\s*set -uo pipefail/m, @@ -958,10 +1008,23 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /^\s*runuser -u node -- git clean -fd 2>\/dev\/null \|\| true$/m, 'untracked round residue must be cleaned without touching gitignored build outputs', ); + // Kill FIRST (a live daemon can re-dirty the tree after the checkout), + // with SIGKILL + a bounded wait — one-shot SIGTERM races slow-draining + // daemons (round 5), the same reasoning as the agent step's guard. + const resetKill = flakeStep.run.search( + /^\s*pkill -KILL -u node 2>\/dev\/null \|\| true$/m, + ); + const resetCheckout = flakeStep.run.search( + /^\s*runuser -u node -- git checkout/m, + ); + assert.ok( + resetKill !== -1 && resetCheckout !== -1 && resetKill < resetCheckout, + 'the reset must SIGKILL leftover test-user processes BEFORE restoring the tree', + ); assert.match( flakeStep.run, - /^\s*pkill -u node \|\| true$/m, - 'each round must tear down leftover test-user processes', + /ps -o pid=,stat= -u node[^\n]*awk '\$2 !~ \/\^Z\/'/, + 'the kill must wait out survivors, zombies disregarded', ); // And the reset must also run BEFORE round 1: lifecycle scripts (npm // ci/build, run as node) mutate the tree between list-record and the @@ -1042,15 +1105,33 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // green with `cp` reordered before the unlinks, reopening the planted // FIFO/symlink hazard. Kill racers → drop a planted dir/symlink at the // directory level → recreate → unlink the destination entry → copy. + // Line-anchored (round 5): unanchored substrings are satisfied by + // comments, and the guard's REACTION (the directory-level rm) must be + // part of the chain — an inert guard body lets mkdir/cp write through + // a planted symlink. const sr = stageStep.run; - const iPkill = sr.indexOf('pkill -KILL -u node'); - const iDirGuard = sr.search(/\[ -L "\$RUNNER_TEMP\/verify-results" \]/); - const iMkdir = sr.indexOf('mkdir -p "$RUNNER_TEMP/verify-results"'); - const iRmDst = sr.indexOf( - 'rm -rf -- "$RUNNER_TEMP/verify-results/flake-gate.log"', + const iPkill = sr.search(/^\s*pkill -KILL -u node/m); + const iDirGuard = sr.search( + /^\s*if \[ -L "\$RUNNER_TEMP\/verify-results" \] \|\|$/m, ); - const iCp = sr.indexOf( - 'cp -f "$RUNNER_TEMP/flake-gate.log" "$RUNNER_TEMP/verify-results/flake-gate.log"', + const iMkdir = sr.search(/^\s*mkdir -p "\$RUNNER_TEMP\/verify-results"$/m); + const iRmDst = sr.search( + /^\s*rm -rf -- "\$RUNNER_TEMP\/verify-results\/flake-gate\.log"$/m, + ); + const iCp = sr.search( + /^\s*cp -f "\$RUNNER_TEMP\/flake-gate\.log" "\$RUNNER_TEMP\/verify-results\/flake-gate\.log"$/m, + ); + // The guard must cover BOTH halves (symlink and non-directory) and + // must actually remove the entry it detects. + assert.match( + sr, + /\[ -L "\$RUNNER_TEMP\/verify-results" \] \|\|\s*\n\s*\{ \[ -e "\$RUNNER_TEMP\/verify-results" \] && \[ ! -d "\$RUNNER_TEMP\/verify-results" \]; \}/, + 'the guard must catch symlinks AND non-directory plants', + ); + assert.match( + sr, + /\[ -L "\$RUNNER_TEMP\/verify-results" \][\s\S]{0,200}?rm -rf -- "\$RUNNER_TEMP\/verify-results"$/m, + 'the guard must REMOVE the planted entry, not merely detect it', ); for (const [label, idx] of [ ['pkill', iPkill], @@ -1134,6 +1215,14 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /❌ not passed — non-deterministic tests \(flakiness gate\)/, 'flaky must still demote the headline without the artifact', ); + // The branch's only input: DOWNLOAD_OUTCOME must stay wired to the + // download step's outcome, or the branch is unreachable and the + // full-report path lies about artifacts that never arrived. + assert.equal( + publishStep.env.DOWNLOAD_OUTCOME, + '${{ steps.download.outcome }}', + 'the download-failure branch input must stay wired', + ); }); it('the verify job timeout still covers agent + prepare + gate', () => { @@ -1174,7 +1263,15 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp // (signal kill) — T/K model infrastructure exits, not test failures. const STUB_TESTRUNNER = [ '#!/bin/bash', + // Round-5 guards baked into EVERY default-runner scenario: PR test code + // must never see the runner-injection files (env -u strip), and the + // operand must resolve from the invocation cwd (pins the `cd` into the + // owning package and the %q quoting for every arm). + 'for v in GITHUB_OUTPUT GITHUB_STATE GITHUB_ENV GITHUB_PATH GITHUB_STEP_SUMMARY; do', + ' [ -z "${!v:-}" ] || { echo "runner-injection env leaked: $v"; exit 97; }', + 'done', 'f="${@: -1}"', + '[ -f "$f" ] || { echo "operand not resolvable from $PWD: $f"; exit 96; }', 'key="$(basename "$f")"', 'n_file="$FLAKE_SEQ_DIR/.count-$key"', 'n=$(cat "$n_file" 2>/dev/null || echo 0)', @@ -1205,9 +1302,12 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp 'exec "$@"', '', ].join('\n'); - // pkill is stubbed for the harness's sake, not the gate's: the real - // binary would kill processes owned by whoever runs these tests. + // pkill/ps are stubbed for the harness's sake, not the gate's: the real + // pkill would kill processes owned by whoever runs these tests, and a + // real `ps -u node` on a box with a live node user would spin the + // reset's wait loop. const STUB_PKILL = ['#!/bin/bash', 'exit 0', ''].join('\n'); + const STUB_PS = ['#!/bin/bash', 'exit 0', ''].join('\n'); const scenarioRoot = mkdtempSync(join(tmpdir(), 'flake-behavioral-')); after(() => rmSync(scenarioRoot, { recursive: true, force: true })); @@ -1231,7 +1331,17 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp mkdirSync(dirname(join(ws, p)), { recursive: true }); writeFileSync(join(ws, p), content); } - if (list !== null) writeFileSync(join(rt, 'flake-gate-files'), list); + if (list !== null) { + // Scenarios describe lists as newline text; the wire format is + // NUL-delimited (the record step emits `git diff -z` through + // `grep -z`), so convert here. + const framed = list + .split('\n') + .filter(Boolean) + .map((f) => `${f}\u0000`) + .join(''); + writeFileSync(join(rt, 'flake-gate-files'), framed); + } for (const [k, v] of Object.entries(sequences)) { writeFileSync(join(seqDir, k), v); } @@ -1264,6 +1374,7 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp node: STUB_TESTRUNNER, timeout: STUB_TIMEOUT, pkill: STUB_PKILL, + ps: STUB_PS, ...stubs, }; for (const [name, content] of Object.entries(stubSet)) { @@ -1311,7 +1422,13 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp } catch { // a scenario may legitimately abort before creating the log } - return { res, outputs, log }; + let summary = ''; + try { + summary = readFileSync(join(rt, 'github-summary'), 'utf8'); + } catch { + // ditto + } + return { res, outputs, log, summary }; }; const UNIT = { @@ -1320,12 +1437,19 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp }; it('all-pass rounds land as `pass` with exit 0', () => { - const { res, outputs } = runGate({ + const { res, outputs, summary } = runGate({ layout: UNIT, list: 'scripts/tests/a.test.js\nscripts/tests/b.test.js\n', }); assert.equal(res.status, 0, res.stderr); assert.equal(outputs.flake_verdict, 'pass'); + // Exact counters (round 5), not just the verdict word — and the step + // summary line finish() owes the run must actually be written. + assert.equal( + outputs.flake_summary, + '2 changed test file(s) x 5 identical rounds, no divergence', + ); + assert.match(summary, /Flakiness gate: pass — 2 changed test file\(s\)/); }); it('per-file P/F alternation is `flaky` even next to a consistently failing file, and the wrapper -e does not kill the step', () => { @@ -1342,6 +1466,27 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.equal(outputs.flake_verdict, 'flaky'); assert.match(log, /a\.test\.js: FFFFF/); assert.match(log, /b\.test\.js: PFPFP/); + // The publisher embeds only the FIRST 10,000 chars: the matrix and + // verdict must precede the failure tails or the demotion's promised + // evidence is truncated away in exactly the flaky runs it points at. + const matrixAt = log.indexOf('per-file results'); + const verdictAt = log.indexOf('\nverdict: flaky'); + const detailAt = log.indexOf('--- per-invocation detail'); + assert.ok( + matrixAt !== -1 && verdictAt !== -1 && detailAt !== -1, + 'log must carry matrix, verdict, and detail sections', + ); + assert.ok( + matrixAt < verdictAt && verdictAt < detailAt, + 'matrix and verdict must precede the failure detail', + ); + // And the failure tails themselves — the content that can outgrow the + // embed cap — must all sit behind the verdict, not just the marker. + const firstTail = log.indexOf('--- output tail'); + assert.ok( + firstTail !== -1 && firstTail > verdictAt, + 'failure tails must never precede the matrix/verdict', + ); // Gate outputs are embedded UNESCAPED into the published comment: they // must stay fixed text plus counters, never PR-controlled strings. assert.match( @@ -1364,6 +1509,10 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp }); assert.equal(res.status, 0, `gate died under the wrapper: ${res.stderr}`); assert.equal(outputs.flake_verdict, 'consistent-fail'); + assert.equal( + outputs.flake_summary, + '1 of 1 changed test file(s) failed identically in every round — deterministic, so CI owns that signal', + ); }); it('timeout/signal exits are infrastructure, never F marks or fake flakiness', () => { @@ -1475,21 +1624,23 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.match(log, /marker\.test\.js: PPPPP/); }); + // Scripted clock shared by the wall-budget scenarios: one value per + // `date +%s` call, last value repeating. + const STUB_DATE = [ + '#!/bin/bash', + 'n_file="$FLAKE_SEQ_DIR/.count-date"', + 'n=$(cat "$n_file" 2>/dev/null || echo 0)', + 'echo $((n+1)) > "$n_file"', + 'mapfile -t vals < "$FLAKE_SEQ_DIR/dates"', + 'i=$n', + '[ "$i" -ge "${#vals[@]}" ] && i=$((${#vals[@]} - 1))', + 'echo "${vals[$i]}"', + '', + ].join('\n'); + it('wall-budget expiry before two full rounds is the informational timeout verdict', () => { - // A scripted `date` stub drives the clock: deadline init at 0 (so the - // budget ends at 900), round 1 checked at 100 and run, round 2 checked - // at 1000 — expired with only one full round done. - const STUB_DATE = [ - '#!/bin/bash', - 'n_file="$FLAKE_SEQ_DIR/.count-date"', - 'n=$(cat "$n_file" 2>/dev/null || echo 0)', - 'echo $((n+1)) > "$n_file"', - 'mapfile -t vals < "$FLAKE_SEQ_DIR/dates"', - 'i=$n', - '[ "$i" -ge "${#vals[@]}" ] && i=$((${#vals[@]} - 1))', - 'echo "${vals[$i]}"', - '', - ].join('\n'); + // Deadline init at 0 (so the budget ends at 900), round 1 checked at + // 100 and run, round 2 checked at 1000 — expired, one full round done. const one = runGate({ layout: { 'scripts/tests/a.test.js': '' }, list: 'scripts/tests/a.test.js\n', @@ -1531,6 +1682,121 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp ); }); + it('hostile filenames survive the generic-package and node --test arms too', () => { + // Round 5: tame names quote to byte-identical output, so a per-arm + // weakening of %q was invisible while only scripts/tests carried the + // hostile fixtures. The stub's operand-resolution guard makes a + // word-split fail loudly in any arm. + const { res, outputs, log } = runGate({ + layout: { + 'packages/pkga/package.json': '{}', + 'packages/pkga/vitest.config.ts': '', + 'packages/pkga/src/has space.test.ts': '', + '.github/scripts/al so.test.mjs': '', + }, + list: 'packages/pkga/src/has space.test.ts\n.github/scripts/al so.test.mjs\n', + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match( + log, + /\(cd packages\/pkga\) npx --no-install vitest run \.\/src\/has\\ space\.test\.ts/, + 'the generic arm must quote its operand', + ); + assert.match( + log, + /node --test \.\/\.github\/scripts\/al\\ so\.test\.mjs/, + 'the node --test arm must quote its operand', + ); + }); + + it('a mid-round wall-budget expiry stops the remaining files of that round', () => { + // Two files, clock 0/100/1000: file a is checked at 100 and runs, file + // b is checked at 1000 — the deadline gates every INVOCATION, not just + // round boundaries. Hoisting the check to the round loop would run + // every remaining file after expiry (up to N×10 min via the caps), + // blowing the ~25-minute budget the job timeout accounting relies on. + const { res, outputs, log } = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\nscripts/tests/b.test.js\n', + stubs: { date: STUB_DATE }, + sequences: { dates: '0\n100\n1000\n' }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'timeout'); + assert.match(outputs.flake_summary, /before two full rounds/); + assert.ok( + !log.includes('round 1 · scripts/tests/b.test.js'), + 'the second file must never run after the budget expired', + ); + }); + + it('an observed divergence outranks a later wall-budget expiry', () => { + // Divergence needs no full round count: once PF exists the verdict is + // flaky even when the clock then expires — the flaky check must stay + // ahead of the timeout branches. + const { res, outputs } = runGate({ + layout: { 'scripts/tests/a.test.js': '' }, + list: 'scripts/tests/a.test.js\n', + stubs: { date: STUB_DATE }, + sequences: { 'a.test.js': 'PF', dates: '0\n100\n200\n1000\n' }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'flaky'); + }); + + it('an infra exit between a P and an F does not mask the divergence', () => { + // Marks P,I,F,P,I from a cycled PTF sequence: the I letters are + // neutral — the P..F subsequence is still non-determinism. + const { res, outputs, log } = runGate({ + layout: { 'scripts/tests/a.test.js': '' }, + list: 'scripts/tests/a.test.js\n', + sequences: { 'a.test.js': 'PTF' }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'flaky'); + assert.match(log, /a\.test\.js: PIFPI/); + }); + + it('zero-collection is reported as not-collected, never as consistent-fail', () => { + // A file the runner's include set rejects exits 1 every round with + // "No test files found" — publishing that as "deterministic, CI owns + // it" would be false on both clauses (round 5). All-uncollected lands + // n/a; a mixed run passes with the not-collected count in the summary. + const STUB_UNCOLLECTED = [ + '#!/bin/bash', + 'echo "No test files found, exiting with code 1"', + 'exit 1', + '', + ].join('\n'); + const alone = runGate({ + layout: { 'scripts/tests/a.test.js': '' }, + list: 'scripts/tests/a.test.js\n', + stubs: { npx: STUB_UNCOLLECTED }, + }); + assert.equal(alone.res.status, 0, alone.res.stderr); + assert.equal(alone.outputs.flake_verdict, 'n/a'); + assert.match( + alone.outputs.flake_summary, + /none of the 1 changed test file\(s\) were collected/, + ); + assert.match(alone.log, /a\.test\.js: NNNNN/); + const mixed = runGate({ + layout: { + 'scripts/tests/a.test.js': '', + '.github/scripts/ok.test.mjs': '', + }, + list: 'scripts/tests/a.test.js\n.github/scripts/ok.test.mjs\n', + stubs: { npx: STUB_UNCOLLECTED }, + }); + assert.equal(mixed.res.status, 0, mixed.res.stderr); + assert.equal(mixed.outputs.flake_verdict, 'pass'); + assert.match( + mixed.outputs.flake_summary, + /\(1 not collected by the runner — see the log\)$/, + ); + }); + it('a missing recorded list degrades to the `error` verdict, exit 0', () => { const { res, outputs } = runGate({ layout: UNIT, list: null }); assert.equal(res.status, 0, res.stderr); @@ -1572,11 +1838,24 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp 'integrations/external-context/package.json': '{}', 'integrations/external-context/vitest.config.ts': '', 'integrations/external-context/src/y.test.ts': '', + 'packages/wspkg/package.json': '{}', + 'packages/wspkg/vitest.workspace.ts': '', + 'packages/wspkg/src/z.test.ts': '', }, - list: 'packages/webui/src/x.test.ts\nintegrations/external-context/src/y.test.ts\n', + list: [ + 'packages/webui/src/x.test.ts', + 'integrations/external-context/src/y.test.ts', + 'packages/wspkg/src/z.test.ts', + '', + ].join('\n'), }); assert.equal(res.status, 0, res.stderr); assert.equal(outputs.flake_verdict, 'pass'); + assert.match( + log, + /\(cd packages\/wspkg\) npx --no-install vitest run \.\/src\/z\.test\.ts/, + 'a vitest.workspace.ts-only package must be entered and run', + ); assert.match( log, /\(cd packages\/webui\) npx --no-install vitest run \.\/src\/x\.test\.ts/, @@ -1731,7 +2010,7 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp "FLAKE_LINE=''", "FLAKE_LINE_ZH=''", block[0], - 'printf \'%s|%s|%s|%s\' "$QUAL" "$HEADLINE" "$QUAL_ZH" "$HEADLINE_ZH"', + 'printf \'%s|%s|%s|%s|%s\' "$QUAL" "$HEADLINE" "$QUAL_ZH" "$HEADLINE_ZH" "$FLAKE_LINE"', ].join('\n'), ], { @@ -1751,23 +2030,38 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp // is the ONE verdict a collapsed-details reader sees — if the _ZH // assignments drop, a demoted PR renders `判定：✅ 通过` in Chinese // while the English headline says ❌. - for (const v of [ - '', - 'pass', - 'n/a', - 'consistent-fail', - 'timeout', - 'error', + // The fifth field pins the gate STATUS LINE (round 5): informational + // verdicts must render their line without touching the headline, and + // the flaky line must carry the ❌. + for (const [v, line] of [ + ['', ''], + ['pass', 'Flakiness gate: ✅ 1 of 2 changed test file(s) diverged'], + [ + 'n/a', + 'Flakiness gate: not applicable — 1 of 2 changed test file(s) diverged', + ], + [ + 'consistent-fail', + 'Flakiness gate: ⚠️ consistent-fail — 1 of 2 changed test file(s) diverged', + ], + [ + 'timeout', + 'Flakiness gate: ⚠️ timeout — 1 of 2 changed test file(s) diverged', + ], + [ + 'error', + 'Flakiness gate: ⚠️ error — 1 of 2 changed test file(s) diverged', + ], ]) { assert.equal( drive(v), - '✅ passed|merge-ready (agent verdict)|✅ 通过|可合入', + `✅ passed|merge-ready (agent verdict)|✅ 通过|可合入|${line}`, `'${v}' must not touch the headline in either language`, ); } assert.equal( drive('flaky'), - '❌ not passed|non-deterministic tests (flakiness gate)|❌ 不通过|测试结果不确定（抖动门）', + '❌ not passed|non-deterministic tests (flakiness gate)|❌ 不通过|测试结果不确定（抖动门）|Flakiness gate: ❌ 1 of 2 changed test file(s) diverged', 'flaky must demote in BOTH languages — deleting either pair has to fail this test', ); }); diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index 34bccf1bdf5..63d1e37b40c 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2756,14 +2756,22 @@ jobs: # narrowing the gate to n/a. A git failure here is pre-build # infrastructure and must fail the step loudly; only a no-match # grep may produce an empty list. - # core.quotePath=false: with the default, a non-ASCII filename is - # emitted C-quoted and silently fails the extension grep below, - # narrowing the gate without a skip-log entry. - files="$(git -c core.quotePath=false diff --name-only --diff-filter=ACMR 'HEAD^1' HEAD)" - printf '%s\n' "$files" \ - | grep -E '\.(test|spec)\.(ts|tsx|js|jsx|mjs|cjs)$' \ - > "${RUNNER_TEMP:?}/flake-gate-files" || true - echo "Recorded $(grep -c . "$RUNNER_TEMP/flake-gate-files" || true) changed test file(s) for the flakiness gate." + # NUL-delimited END TO END (-z / grep -z / gate `read -d ''`): + # line-based intake silently dropped every filename git C-quotes — + # quotePath=false only stops quoting of bytes >= 0x80, while ASCII + # specials (backslash, tab, quote, control chars, and the + # line-breaking newline itself) stay quoted and fail a `$`-anchored + # line grep with no skip-log entry. NUL is the one byte a path + # cannot contain. NOTE: the git output must never pass through a + # command substitution — `$( )` strips NUL bytes. + git -c core.quotePath=false diff -z --name-only --diff-filter=ACMR 'HEAD^1' HEAD \ + > "${RUNNER_TEMP:?}/flake-gate-files-all" + # .mts/.cts included: vitest's default include set collects them. + grep -zE '\.(test|spec)\.(ts|tsx|js|jsx|mjs|cjs|mts|cts)$' \ + "$RUNNER_TEMP/flake-gate-files-all" \ + > "$RUNNER_TEMP/flake-gate-files" || true + rm -f "$RUNNER_TEMP/flake-gate-files-all" + echo "Recorded $(tr -cd '\0' < "$RUNNER_TEMP/flake-gate-files" | wc -c) changed test file(s) for the flakiness gate." - name: 'Clear stale npm cache' if: "steps.pr.outputs.decision == 'run'" @@ -2951,7 +2959,13 @@ jobs: unset ACTIONS_RUNTIME_TOKEN ACTIONS_RUNTIME_URL ACTIONS_CACHE_URL LOG="${RUNNER_TEMP:?}/flake-gate.log" LIST="$RUNNER_TEMP/flake-gate-files" + # Per-invocation detail (round lines, failure tails) goes to a + # SEPARATE file and is appended after the verdict: the publisher + # embeds the FIRST 10,000 chars of the log, so the per-file matrix + # and verdict must sit ahead of detail that can outgrow the cap. + DETAIL="$RUNNER_TEMP/flake-gate-detail" : > "$LOG" + : > "$DETAIL" finish() { # Outputs carry FIXED strings and counters only — file paths are # PR-controlled text and belong in the log, which the publisher @@ -2959,6 +2973,13 @@ jobs: echo "flake_verdict=$1" >> "$GITHUB_OUTPUT" echo "flake_summary=$2" >> "$GITHUB_OUTPUT" printf '\nverdict: %s\nsummary: %s\n' "$1" "$2" >> "$LOG" + # Detail LAST: the publisher embeds the first 10,000 chars, and + # the matrix/verdict must never be truncated away behind + # failure tails (full copy stays in the artifact). + if [ -s "$DETAIL" ]; then + printf -- '\n--- per-invocation detail (full copy in the artifact) ---\n' >> "$LOG" + cat "$DETAIL" >> "$LOG" + fi echo "Flakiness gate: $1 — $2" >> "$GITHUB_STEP_SUMMARY" GATE_DONE=1 exit 0 @@ -3015,7 +3036,9 @@ jobs: [ -f "$1/vitest.workspace.ts" ] && return 0 return 1 } - while IFS= read -r f; do + # NUL-delimited to match the record step: `read -d ''` is the one + # framing a filename cannot break out of. + while IFS= read -r -d '' f; do [ -n "$f" ] || continue if [ ! -f "$f" ]; then add_skip "$f" 'not present in the merge tree, skipped' @@ -3102,9 +3125,18 @@ jobs: # survive) drops the untracked residue `checkout -- .` never # touches — lock files, output dirs a test creates in round 1. reset_round_state() { + # Kill FIRST, then restore — a live daemon can re-dirty the tree + # after the checkout. One-shot SIGTERM races slow-draining + # daemons, so SIGKILL with a bounded wait, zombies disregarded + # (same reasoning as the agent step's process guard). + pkill -KILL -u node 2>/dev/null || true + for _ in 1 2 3; do + [ -n "$(ps -o pid=,stat= -u node 2>/dev/null | awk '$2 !~ /^Z/')" ] || break + sleep 1 + pkill -KILL -u node 2>/dev/null || true + done runuser -u node -- git checkout -- . 2>/dev/null || true runuser -u node -- git clean -fd 2>/dev/null || true - pkill -u node || true } declare -a results=() @@ -3112,6 +3144,7 @@ jobs: rounds_done=0 timed_out=false infra_exits=0 + uncollected=0 round=1 # Reset BEFORE round 1 too: PR lifecycle scripts (npm ci/build, # run as node) may have mutated the tree since the list was @@ -3143,21 +3176,31 @@ jobs: # Timeout/signal exits (124, 128+N) are infrastructure events — # a runner cap or an OOM kill — not test outcomes: recorded as # F they would publish a fake `flaky` next to any pass. + # Zero-collection is its own class (`N`): a file the runner's + # include set rejects fails every round without one test + # executing — reporting that as consistent-fail would publish + # "deterministic, CI owns it" with both clauses false. The + # marker is matched from PR-controlled output, but faking it + # can only SUPPRESS a demotion the PR could already dodge by + # deleting its tests — one-way authority holds. mark='P' if [ "$status" -eq 124 ] || [ "$status" -gt 128 ]; then mark='I' infra_exits=$((infra_exits + 1)) + elif [ "$status" -ne 0 ] && grep -q 'No test files found' "$out"; then + mark='N' + uncollected=$((uncollected + 1)) elif [ "$status" -ne 0 ]; then mark='F' fi results[$i]="${results[$i]:-}${mark}" - printf 'round %s · %s: %s (exit %s)\n' "$round" "${group_labels[$i]}" "$mark" "$status" >> "$LOG" + printf 'round %s · %s: %s (exit %s)\n' "$round" "${group_labels[$i]}" "$mark" "$status" >> "$DETAIL" if [ "$status" -ne 0 ]; then { printf -- '--- output tail · round %s · %s ---\n' "$round" "${group_labels[$i]}" tail -c 8000 "$out" printf '\n' - } >> "$LOG" + } >> "$DETAIL" fi done # Restore the equivalent of a clean checkout before the next @@ -3172,11 +3215,13 @@ jobs: flaky=0 failing=0 + ncoll=0 for i in "${!group_labels[@]}"; do case "${results[$i]:-}" in *P*F*|*F*P*) flaky=$((flaky + 1)) ;; *P*) : ;; *F*) failing=$((failing + 1)) ;; + *N*) ncoll=$((ncoll + 1)) ;; esac done printf '\nper-file results (P=pass F=fail I=infra-exit, one letter per run):\n' >> "$LOG" @@ -3196,10 +3241,18 @@ jobs: if [ "$failing" -gt 0 ]; then finish consistent-fail "${failing} of ${total} changed test file(s) failed identically in every round — deterministic, so CI owns that signal" fi + if [ "$ncoll" -gt 0 ] && [ "$ncoll" -eq "$total" ]; then + # Every file hit a runner include-set mismatch: claiming these + # rounds as sampling would publish evidence about tests that + # never executed. + finish n/a "none of the ${total} changed test file(s) were collected by their runner (include-set mismatch — reasons in the log)" + fi if [ "$timed_out" = true ]; then finish timeout "only ${rounds_done} of ${ROUNDS} rounds fit the 15-minute budget; the completed rounds agreed" fi - finish pass "${total} changed test file(s) x ${rounds_done} identical rounds, no divergence" + pass_note='' + [ "$ncoll" -gt 0 ] && pass_note=" (${ncoll} not collected by the runner — see the log)" + finish pass "${total} changed test file(s) x ${rounds_done} identical rounds, no divergence${pass_note}" - name: 'Install evidence browser' if: "steps.pr.outputs.decision == 'run' && steps.prepare.outputs.verdict == ''" @@ -4677,6 +4730,13 @@ jobs: fi if [ "${FLAKE_VERDICT:-}" = 'flaky' ]; then printf 'The deterministic flakiness gate re-ran the test files this PR changes and got different outcomes from identical runs%s. A test that can fail with no code changing lands as intermittent red on unrelated PRs, so this run is reported as not passed regardless of the agent verdict — the per-round matrix is in the flakiness gate log below.\n\n' "${AGENT_VERDICT:+ (agent verdict: \`${AGENT_VERDICT}\`)}" + if [ ! -f "$FLAKE_LOG" ]; then + # emit_block no-ops on a missing file: without this note + # the demotion's only evidence pointer dangles when the + # staging step failed (its continue-on-error keeps the + # job green precisely so the verdict survives). + printf 'The flakiness gate log could not be staged into the result artifact — the per-round matrix is in the gate step output of the workflow run log.\n\n' + fi fi printf '
\n中文 — 判定：%s · %s\n\n' "$QUAL_ZH" "${HEADLINE_ZH:-$HEADLINE}" if [ "${VERDICT:-}" = 'pass' ]; then @@ -4695,6 +4755,9 @@ jobs: fi if [ "${FLAKE_VERDICT:-}" = 'flaky' ]; then printf '确定性抖动门将本 PR 改动的测试文件原样重跑了多轮，得到了不一致的结果%s。一个在代码不变时也会失败的测试会以间歇性红灯落在无关的 PR 上，因此无论 agent 判定如何，本次运行按不通过报告——各轮结果矩阵见下方抖动门日志。\n\n' "${AGENT_VERDICT:+（agent 判定：\`${AGENT_VERDICT}\`）}" + if [ ! -f "$FLAKE_LOG" ]; then + printf '抖动门日志未能暂存进结果产物——各轮结果矩阵请查看工作流运行日志中 gate step 的输出。\n\n' + fi fi printf '
\n\n' if [ -n "${MISSING_REPORT_NOTE:-}" ]; then From 35cd43ef6c1e9386cea34e15b63996097bf9e914 Mon Sep 17 00:00:00 2001 From: wenshao Date: Sat, 15 Aug 2026 12:18:42 +0800 Subject: [PATCH 07/18] fix(triage): close the desktop-app runnability hole and the staging replant race; demote in every terminal branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-6 review (2 Critical + 19 Suggestion; 17 applied, 2 declined with rationale on-thread): - Desktop/docs-site exclusion (Critical): packages/desktop/apps/* each carry a package.json plus a BUILD vite.config.ts, so the generic resolver treated bun-family tests as runnable — 102/319 real desktop test files misclassified, published under a false include-set-mismatch diagnosis while draining the shared wall budget. Explicit skip arm ahead of the generic arm; behavioral scenario pins both trees. - Staging replant race (Critical): the one-shot pkill lost to setsid daemons/continuous forkers, and verify-results stayed node-owned — a survivor could swap the staged log for a symlink between cp and upload-artifact's link-following enumeration: root-readable-file exfiltration into the public comment. The kill now uses the bounded survivor wait, and the directory is chown -R root:root before the copy, revoking the replant capability regardless of the race. - A recorded flaky now demotes in EVERY terminal publisher branch: cancelled and job-failure used to post the neutral notice while needs.verify.outputs still carried the verdict (the download-failure branch already honored it). Bilingual, with run-log pointers. - Dead 'uncollected' counter removed (ncoll already counts N marks). - Tests: ghost-file and desktop-skip scenarios; behavioral child-env guards in the default stub (CI/heap/TMPDIR-under-RUNNER_TEMP); invocation-count ground truth for the mid-round budget stop; sixth drive field pins the Chinese gate status line; adjacency-pinned record statements; anchored credential-strip/wait-loop/publisher pins; cancelled/failure/download branch pins; upload transport pins; workflow/job-level env emptiness pins; guard then-body pin. Declined (reasoning on the threads): whitespace-spelling re-derivation evasions (bounded by one-way authority — re-derivation can only narrow the gate), and an inv_tmp position pin (superseded by the stub's behavioral TMPDIR guard, which every scenario now enforces). --- .github/scripts/qwen-triage-workflow.test.mjs | 258 ++++++++++++++---- .github/workflows/qwen-triage.yml | 69 ++++- 2 files changed, 260 insertions(+), 67 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index 6e36d7edaae..5b4afca0ee2 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -850,15 +850,13 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // the NUL bytes respectively), and the grep must read the file and // carry the full extension set (.mts/.cts included — vitest's default // include collects them). + // Adjacency-pinned as WHOLE statements (round 6): pinning the git line + // and its redirect as independent shapes let an interposed pipeline + // stage satisfy both. assert.match( recordStep.run, - /^\s*git -c core\.quotePath=false diff -z --name-only --diff-filter=ACMR 'HEAD\^1' HEAD \\$/m, - 'the diff must be NUL-delimited, against the merge base, standalone', - ); - assert.match( - recordStep.run, - /^\s*> "\$\{RUNNER_TEMP:\?\}\/flake-gate-files-all"$/m, - 'the raw diff must land in a file — $( ) strips NUL bytes', + /^\s*git -c core\.quotePath=false diff -z --name-only --diff-filter=ACMR 'HEAD\^1' HEAD \\\n\s*> "\$\{RUNNER_TEMP:\?\}\/flake-gate-files-all"$/m, + 'the NUL diff must flow straight into its file — $( ) strips NUL bytes, a pipeline swallows the exit status', ); assert.ok( recordStep.run.includes( @@ -868,11 +866,14 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( recordStep.run, - /^\s*> "\$RUNNER_TEMP\/flake-gate-files" \|\| true$/m, - 'only a no-match grep may yield an empty list, into the exact handoff path', + /^\s*grep -zE '[^']+' \\\n\s*"\$RUNNER_TEMP\/flake-gate-files-all" \\\n\s*> "\$RUNNER_TEMP\/flake-gate-files" \|\| true$/m, + 'the grep must read the raw file and only a no-match may yield an empty list', ); + // Continuation-collapsed (round 6): a `\⏎|` split pipeline is still a + // pipeline. assert.doesNotMatch( recordStep.run + .replace(/\\\n/g, ' ') .split('\n') .filter((l) => !l.trim().startsWith('#')) .join('\n'), @@ -915,8 +916,8 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); // Presence AND position: the strip must precede the first invocation, // or the credentials are already in the child env when PR code runs. - const unsetIdx = flakeStep.run.indexOf( - 'unset ACTIONS_RUNTIME_TOKEN ACTIONS_RUNTIME_URL ACTIONS_CACHE_URL', + const unsetIdx = flakeStep.run.search( + /^\s*unset ACTIONS_RUNTIME_TOKEN ACTIONS_RUNTIME_URL ACTIONS_CACHE_URL$/m, ); const firstInvocation = flakeStep.run.search( /^\s*timeout -k 30 600 runuser -u node -- \\$/m, @@ -937,6 +938,14 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ['FLAKE_ROUNDS', 'GH_TOKEN', 'GITHUB_TOKEN'], 'the gate env must stay tokens-blanked and secret-free', ); + // Actions merges workflow- and job-level env into every step: the + // step-key pin above is only exhaustive while those levels stay empty. + assert.equal(doc.env, undefined, 'no workflow-level env may appear'); + assert.equal( + verifyJob.env, + undefined, + 'no verify-job-level env may appear — it would flow into the gate', + ); // Keys AND the one non-blank value: FLAKE_ROUNDS must come from the // repo variable, not a hardcoded count or a PR-influenced expression. assert.equal( @@ -1021,10 +1030,14 @@ describe('qwen-triage: flakiness gate (#9125)', () => { resetKill !== -1 && resetCheckout !== -1 && resetKill < resetCheckout, 'the reset must SIGKILL leftover test-user processes BEFORE restoring the tree', ); - assert.match( - flakeStep.run, - /ps -o pid=,stat= -u node[^\n]*awk '\$2 !~ \/\^Z\/'/, - 'the kill must wait out survivors, zombies disregarded', + // Line-anchored and position-pinned (round 6): the survivor wait must + // sit between the kill and the restore, not merely exist somewhere. + const resetWait = flakeStep.run.search( + /^\s*\[ -n "\$\(ps -o pid=,stat= -u node 2>\/dev\/null \| awk '\$2 !~ \/\^Z\/'\)" \] \|\| break$/m, + ); + assert.ok( + resetWait !== -1 && resetKill < resetWait && resetWait < resetCheckout, + 'the kill must wait out survivors (zombies disregarded) before the restore', ); // And the reset must also run BEFORE round 1: lifecycle scripts (npm // ci/build, run as node) mutate the tree between list-record and the @@ -1093,6 +1106,25 @@ describe('qwen-triage: flakiness gate (#9125)', () => { agentIdx < stageIdx && stageIdx < uploadIdx, 'staging must run after the agent and before the upload', ); + // The transport itself (round 6): the chain is only closed if the + // upload actually ships the staged directory and cannot fail the job + // out from under the recorded verdict. + const uploadStep = verifyJob.steps[uploadIdx]; + assert.equal( + uploadStep.with.path, + '${{ runner.temp }}/verify-results/', + 'the artifact must ship the staged directory', + ); + assert.match( + uploadStep.with.name, + /^verify-results-/, + 'the artifact name must stay in the family the publisher downloads', + ); + assert.equal( + uploadStep['continue-on-error'], + true, + 'a missing/empty results dir must not fail the job and mask the original error', + ); // Evidence-copying only, after the verdict outputs are written: a // staging failure (ENOSPC, hostile mount) must not flip the job red or // the publisher discards the recorded verdict as "infrastructure". @@ -1115,6 +1147,18 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /^\s*if \[ -L "\$RUNNER_TEMP\/verify-results" \] \|\|$/m, ); const iMkdir = sr.search(/^\s*mkdir -p "\$RUNNER_TEMP\/verify-results"$/m); + // Round 6: a one-shot pkill loses the race against setsid daemons and + // continuous forkers; the kill needs the bounded survivor wait, and the + // directory must be re-owned by root before the copy — a survivor that + // cannot create or unlink names cannot replant a symlink for + // upload-artifact to follow (root-readable exfiltration into the + // public comment). + const iWait = sr.search( + /^\s*\[ -n "\$\(ps -o pid=,stat= -u node 2>\/dev\/null \| awk '\$2 !~ \/\^Z\/'\)" \] \|\| break$/m, + ); + const iChown = sr.search( + /^\s*chown -R root:root "\$RUNNER_TEMP\/verify-results"$/m, + ); const iRmDst = sr.search( /^\s*rm -rf -- "\$RUNNER_TEMP\/verify-results\/flake-gate\.log"$/m, ); @@ -1128,26 +1172,32 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /\[ -L "\$RUNNER_TEMP\/verify-results" \] \|\|\s*\n\s*\{ \[ -e "\$RUNNER_TEMP\/verify-results" \] && \[ ! -d "\$RUNNER_TEMP\/verify-results" \]; \}/, 'the guard must catch symlinks AND non-directory plants', ); + // The reaction is pinned as the guard's own then-body (round 6), not + // by loose proximity to any -L mention. assert.match( sr, - /\[ -L "\$RUNNER_TEMP\/verify-results" \][\s\S]{0,200}?rm -rf -- "\$RUNNER_TEMP\/verify-results"$/m, - 'the guard must REMOVE the planted entry, not merely detect it', + /^\s*if \[ -L "\$RUNNER_TEMP\/verify-results" \] \|\|\n\s*\{ \[ -e "\$RUNNER_TEMP\/verify-results" \] && \[ ! -d "\$RUNNER_TEMP\/verify-results" \]; \}; then\n\s*rm -rf -- "\$RUNNER_TEMP\/verify-results"\n\s*fi$/m, + 'the guard must REMOVE the planted entry in its own then-body', ); for (const [label, idx] of [ ['pkill', iPkill], + ['bounded survivor wait', iWait], ['directory-level symlink/non-dir guard', iDirGuard], ['mkdir -p', iMkdir], + ['root re-own', iChown], ['destination unlink', iRmDst], ['copy', iCp], ]) { assert.ok(idx !== -1, `staging must contain the ${label}`); } assert.ok( - iPkill < iDirGuard && + iPkill < iWait && + iWait < iDirGuard && iDirGuard < iMkdir && - iMkdir < iRmDst && + iMkdir < iChown && + iChown < iRmDst && iRmDst < iCp, - 'staging order must be: kill racers, dir guard, recreate, unlink destination, copy', + 'staging order must be: kill+wait, dir guard, recreate, root re-own, unlink destination, copy', ); assert.doesNotMatch( agentStep.run, @@ -1156,19 +1206,19 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( publishStep.run, - /FLAKE_LOG='verify-results\/flake-gate\.log'/, + /^\s*FLAKE_LOG='verify-results\/flake-gate\.log'$/m, 'the publisher must pin the exact root-level path, never find/sort', ); assert.match( publishStep.run, - /emit_block 'Flakiness gate log' "\$FLAKE_LOG" 10000/, + /^\s*emit_block 'Flakiness gate log' "\$FLAKE_LOG" 10000$/m, 'the gate-log cap must leave headroom under GitHub 65,536-char comment limit next to the 45000 report block', ); }); it('gate authority is one-way: only `flaky` may touch the headline, and only to demote', () => { const block = publishStep.run.match( - /case "\$\{FLAKE_VERDICT:-\}" in[\s\S]*?esac/, + /^\s*case "\$\{FLAKE_VERDICT:-\}" in[\s\S]*?^\s*esac$/m, ); assert.ok( block, @@ -1197,24 +1247,38 @@ describe('qwen-triage: flakiness gate (#9125)', () => { }); it('the flaky demotion also fires when the result artifact is unavailable', () => { - // FLAKE_VERDICT travels via job outputs, independent of the artifact. - // Without this branch handling, a flaky PR whose artifact download - // failed got a neutral "results unavailable" notice — the demotion - // contract silently not firing. - const branch = publishStep.run.match( - /elif \[ "\$\{DOWNLOAD_OUTCOME:-success\}" != "success" \];[\s\S]*?\nelif /, - ); - assert.ok(branch, 'the download-failure branch must exist'); - assert.match( - branch[0], - /"\$\{FLAKE_VERDICT:-\}" = 'flaky'/, - 'the branch must consult the gate verdict', - ); - assert.match( - branch[0], - /❌ not passed — non-deterministic tests \(flakiness gate\)/, - 'flaky must still demote the headline without the artifact', - ); + // FLAKE_VERDICT travels via job outputs, independent of the artifact + // AND of job completion: cancelled, job-failure, and download-failure + // branches must each consult it (round 6) — a recorded flaky must + // never collapse into a neutral ⚠️ notice. + const branches = [ + [ + 'cancelled', + /if \[ "\$\{VERIFY_RESULT:-\}" = "cancelled" \];[\s\S]*?\n\s*elif /, + ], + [ + 'job-failure', + /elif \[ "\$\{VERIFY_RESULT:-\}" != "success" \] \|\| \[ -z "\$\{VERDICT:-\}" \];[\s\S]*?\n\s*elif /, + ], + [ + 'download-failure', + /elif \[ "\$\{DOWNLOAD_OUTCOME:-success\}" != "success" \];[\s\S]*?\n\s*elif /, + ], + ]; + for (const [label, re] of branches) { + const branch = publishStep.run.match(re); + assert.ok(branch, `the ${label} branch must exist`); + assert.match( + branch[0], + /"\$\{FLAKE_VERDICT:-\}" = 'flaky'/, + `the ${label} branch must consult the gate verdict`, + ); + assert.match( + branch[0], + /^\s*printf '\*\*Sandboxed verification: ❌ not passed — non-deterministic tests \(flakiness gate\)/m, + `flaky must still demote the ${label} headline`, + ); + } // The branch's only input: DOWNLOAD_OUTCOME must stay wired to the // download step's outcome, or the branch is unreachable and the // full-report path lies about artifacts that never arrived. @@ -1272,6 +1336,12 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp 'done', 'f="${@: -1}"', '[ -f "$f" ] || { echo "operand not resolvable from $PWD: $f"; exit 96; }', + // Round 6: the child-env handoff (CI parity, heap cap, per-invocation + // TMPDIR under RUNNER_TEMP) is enforced behaviorally by every + // default-runner scenario, not just by a textual pin. + '[ "${CI:-}" = true ] || { echo "CI parity lost"; exit 95; }', + 'case "${NODE_OPTIONS:-}" in *max-old-space-size*) ;; *) echo "heap cap lost"; exit 95 ;; esac', + 'case "${TMPDIR:-}" in "$RUNNER_TEMP"/*) ;; *) echo "shared TMPDIR"; exit 95 ;; esac', 'key="$(basename "$f")"', 'n_file="$FLAKE_SEQ_DIR/.count-$key"', 'n=$(cat "$n_file" 2>/dev/null || echo 0)', @@ -1428,7 +1498,19 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp } catch { // ditto } - return { res, outputs, log, summary }; + // Ground-truth invocation count per operand basename, read from the + // stub's own counter files — log-line absence alone cannot prove an + // invocation never ran. + const counts = (key) => { + try { + return Number( + readFileSync(join(seqDir, `.count-${key}`), 'utf8').trim(), + ); + } catch { + return 0; + } + }; + return { res, outputs, log, summary, counts }; }; const UNIT = { @@ -1716,7 +1798,7 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp // round boundaries. Hoisting the check to the round loop would run // every remaining file after expiry (up to N×10 min via the caps), // blowing the ~25-minute budget the job timeout accounting relies on. - const { res, outputs, log } = runGate({ + const { res, outputs, log, counts } = runGate({ layout: UNIT, list: 'scripts/tests/a.test.js\nscripts/tests/b.test.js\n', stubs: { date: STUB_DATE }, @@ -1727,8 +1809,12 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.match(outputs.flake_summary, /before two full rounds/); assert.ok( !log.includes('round 1 · scripts/tests/b.test.js'), - 'the second file must never run after the budget expired', + 'the second file must never be logged after the budget expired', ); + // Ground truth, not log absence (round 6): the stub's own counter + // proves the invocation was never made. + assert.equal(counts('a.test.js'), 1, 'file a ran exactly once'); + assert.equal(counts('b.test.js'), 0, 'file b never ran'); }); it('an observed divergence outranks a later wall-budget expiry', () => { @@ -1797,6 +1883,58 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp ); }); + it('desktop-app and docs-site trees are skipped despite their build vite.config', () => { + // Round-6 Critical: each packages/desktop/apps/* carries its own + // package.json plus a BUILD vite.config.ts, which fooled the generic + // resolver into treating bun-family tests as runnable — 102 of 319 + // real desktop test files misclassified, published under a false + // "include-set mismatch" diagnosis while draining the wall budget. + const { res, outputs, log, counts } = runGate({ + layout: { + 'packages/desktop/apps/electron/package.json': '{}', + 'packages/desktop/apps/electron/vite.config.ts': '', + 'packages/desktop/apps/electron/src/a.test.ts': '', + 'docs-site/package.json': '{}', + 'docs-site/vitest.config.js': '', + 'docs-site/b.test.ts': '', + }, + list: 'packages/desktop/apps/electron/src/a.test.ts\ndocs-site/b.test.ts\n', + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'n/a'); + assert.match( + log, + /outside the npm-workspace install set \(unsupported runner family\), skipped: packages\/desktop\/apps\/electron\/src\/a\.test\.ts/, + ); + assert.match( + log, + /outside the npm-workspace install set \(unsupported runner family\), skipped: docs-site\/b\.test\.ts/, + ); + assert.equal(counts('a.test.ts'), 0, 'no invocation may be attempted'); + }); + + it('a recorded file missing at gate time is skip-logged, never marked F', () => { + // Round 6: the record list is pinned pre-build, but PR lifecycle + // scripts can delete/rename a recorded file before the gate runs. + // Without the absent-file guard the ghost would reach a runnable arm + // and publish consistent-fail about a test that never executed. + const { res, outputs, log } = runGate({ + layout: { 'scripts/tests/a.test.js': '' }, + list: 'scripts/tests/a.test.js\nscripts/tests/ghost.test.js\n', + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match( + log, + /not present in the merge tree, skipped: scripts\/tests\/ghost\.test\.js/, + ); + assert.match( + outputs.flake_summary, + /^1 changed test file\(s\)/, + 'the summary must count only the runnable file', + ); + }); + it('a missing recorded list degrades to the `error` verdict, exit 0', () => { const { res, outputs } = runGate({ layout: UNIT, list: null }); assert.equal(res.status, 0, res.stderr); @@ -1989,7 +2127,7 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp it('publisher demotion executes one-way: only `flaky` demotes, and it MUST demote', () => { const block = publishRun.match( - /case "\$\{FLAKE_VERDICT:-\}" in[\s\S]*?esac/, + /^\s*case "\$\{FLAKE_VERDICT:-\}" in[\s\S]*?^\s*esac$/m, ); assert.ok(block, 'the publisher must map FLAKE_VERDICT in a case block'); const drive = (verdict) => { @@ -2010,7 +2148,7 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp "FLAKE_LINE=''", "FLAKE_LINE_ZH=''", block[0], - 'printf \'%s|%s|%s|%s|%s\' "$QUAL" "$HEADLINE" "$QUAL_ZH" "$HEADLINE_ZH" "$FLAKE_LINE"', + 'printf \'%s|%s|%s|%s|%s|%s\' "$QUAL" "$HEADLINE" "$QUAL_ZH" "$HEADLINE_ZH" "$FLAKE_LINE" "$FLAKE_LINE_ZH"', ].join('\n'), ], { @@ -2030,39 +2168,49 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp // is the ONE verdict a collapsed-details reader sees — if the _ZH // assignments drop, a demoted PR renders `判定：✅ 通过` in Chinese // while the English headline says ❌. - // The fifth field pins the gate STATUS LINE (round 5): informational - // verdicts must render their line without touching the headline, and - // the flaky line must carry the ❌. - for (const [v, line] of [ - ['', ''], - ['pass', 'Flakiness gate: ✅ 1 of 2 changed test file(s) diverged'], + // The fifth and sixth fields pin BOTH gate status lines (rounds 5-6): + // informational verdicts must render their lines without touching the + // headline, the flaky lines must carry the ❌, and the Chinese line — + // the one verdict a collapsed-details reader sees — must never drop + // out while the English one keeps the suite green. + for (const [v, line, zh] of [ + ['', '', ''], + [ + 'pass', + 'Flakiness gate: ✅ 1 of 2 changed test file(s) diverged', + '抖动门：✅ 1 of 2 changed test file(s) diverged', + ], [ 'n/a', 'Flakiness gate: not applicable — 1 of 2 changed test file(s) diverged', + '抖动门：不适用 — 1 of 2 changed test file(s) diverged', ], [ 'consistent-fail', 'Flakiness gate: ⚠️ consistent-fail — 1 of 2 changed test file(s) diverged', + '抖动门：⚠️ consistent-fail — 1 of 2 changed test file(s) diverged', ], [ 'timeout', 'Flakiness gate: ⚠️ timeout — 1 of 2 changed test file(s) diverged', + '抖动门：⚠️ timeout — 1 of 2 changed test file(s) diverged', ], [ 'error', 'Flakiness gate: ⚠️ error — 1 of 2 changed test file(s) diverged', + '抖动门：⚠️ error — 1 of 2 changed test file(s) diverged', ], ]) { assert.equal( drive(v), - `✅ passed|merge-ready (agent verdict)|✅ 通过|可合入|${line}`, + `✅ passed|merge-ready (agent verdict)|✅ 通过|可合入|${line}|${zh}`, `'${v}' must not touch the headline in either language`, ); } assert.equal( drive('flaky'), - '❌ not passed|non-deterministic tests (flakiness gate)|❌ 不通过|测试结果不确定（抖动门）|Flakiness gate: ❌ 1 of 2 changed test file(s) diverged', - 'flaky must demote in BOTH languages — deleting either pair has to fail this test', + '❌ not passed|non-deterministic tests (flakiness gate)|❌ 不通过|测试结果不确定（抖动门）|Flakiness gate: ❌ 1 of 2 changed test file(s) diverged|抖动门：❌ 1 of 2 changed test file(s) diverged', + 'flaky must demote in BOTH languages — deleting either pair or status line has to fail this test', ); }); }); diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index 63d1e37b40c..ce5d8ab72a9 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -3075,6 +3075,15 @@ jobs: group_dirs+=('.') group_cmds+=("node --test $(printf '%q' "./$f")") ;; + packages/desktop/*|docs-site/*) + # Outside the npm-workspace install set (root workspaces + # exclude packages/desktop; docs-site is standalone), so the + # gate's vitest can never collect them — and desktop's + # nested apps each carry a package.json plus a BUILD + # vite.config.ts that would otherwise fool the generic + # resolver into treating bun-family tests as runnable. + add_skip "$f" 'outside the npm-workspace install set (unsupported runner family), skipped' + ;; *) # Generic vitest resolution keyed on the OWNING PACKAGE, not # a path prefix: root npm workspaces outside packages/** @@ -3144,7 +3153,6 @@ jobs: rounds_done=0 timed_out=false infra_exits=0 - uncollected=0 round=1 # Reset BEFORE round 1 too: PR lifecycle scripts (npm ci/build, # run as node) may have mutated the tree since the list was @@ -3189,7 +3197,6 @@ jobs: infra_exits=$((infra_exits + 1)) elif [ "$status" -ne 0 ] && grep -q 'No test files found' "$out"; then mark='N' - uncollected=$((uncollected + 1)) elif [ "$status" -ne 0 ]; then mark='F' fi @@ -3861,11 +3868,24 @@ jobs: # victim, or copy INTO a planted directory so the log vanishes # from the pinned publisher path. pkill -KILL -u node 2>/dev/null || true + for _ in 1 2 3; do + [ -n "$(ps -o pid=,stat= -u node 2>/dev/null | awk '$2 !~ /^Z/')" ] || break + sleep 1 + pkill -KILL -u node 2>/dev/null || true + done if [ -L "$RUNNER_TEMP/verify-results" ] || { [ -e "$RUNNER_TEMP/verify-results" ] && [ ! -d "$RUNNER_TEMP/verify-results" ]; }; then rm -rf -- "$RUNNER_TEMP/verify-results" fi mkdir -p "$RUNNER_TEMP/verify-results" + # Revoke the replant premise outright: the agent era owned this + # directory as the build user. Once root owns the names, even a + # kill-race survivor (setsid daemon, continuous forker — pkill + # scans /proc once) can no longer unlink the staged file or + # replant a symlink between this copy and upload-artifact's + # enumeration, which FOLLOWS links — that window was a + # root-readable-file exfiltration into the public comment. + chown -R root:root "$RUNNER_TEMP/verify-results" rm -rf -- "$RUNNER_TEMP/verify-results/flake-gate.log" cp -f "$RUNNER_TEMP/flake-gate.log" "$RUNNER_TEMP/verify-results/flake-gate.log" fi @@ -4430,26 +4450,51 @@ jobs: # report — they only replace this run's own "running" status (see # the upsert below). Real outcomes (report, prepare-fail) upsert. WEAK_BODY=false + # The gate verdict travels via job outputs: it survives a later + # agent-step failure, a job timeout, and a cancellation. A + # recorded `flaky` must demote the headline in EVERY terminal + # branch — these two used to post the neutral notice and silently + # drop the demotion the download-failure branch already honors. if [ "${VERIFY_RESULT:-}" = "cancelled" ]; then WEAK_BODY=true { printf '%s\n\n' '' - printf '**Sandboxed verification: ⚠️ incomplete — cancelled** - [workflow run](%s)\n\n' "$RUN_URL" - printf 'The verification job was cancelled before producing a report.\n\n' - printf '
\n中文 — 判定：⚠️ 未完成 · 已取消\n\n' - printf '验证作业在生成报告前被取消。\n\n' - printf '
\n\n' + if [ "${FLAKE_VERDICT:-}" = 'flaky' ]; then + printf '**Sandboxed verification: ❌ not passed — non-deterministic tests (flakiness gate); job cancelled before the report** - [workflow run](%s)\n\n' "$RUN_URL" + printf 'Flakiness gate: ❌ %s\n\n' "${FLAKE_SUMMARY:-changed test files returned different results across identical re-runs}" + printf 'The job was cancelled before producing a report, but the deterministic flakiness gate had already observed run-to-run divergence — the per-round matrix is in the gate step output of the workflow run log.\n\n' + printf '
\n中文 — 判定：❌ 不通过 · 测试结果不确定（抖动门）；作业在报告前被取消\n\n' + printf '抖动门：❌ %s\n\n' "${FLAKE_SUMMARY:-changed test files returned different results across identical re-runs}" + printf '作业在生成报告前被取消，但确定性抖动门已观测到轮间分歧——各轮矩阵见工作流运行日志中 gate step 的输出。\n\n' + printf '
\n\n' + else + printf '**Sandboxed verification: ⚠️ incomplete — cancelled** - [workflow run](%s)\n\n' "$RUN_URL" + printf 'The verification job was cancelled before producing a report.\n\n' + printf '
\n中文 — 判定：⚠️ 未完成 · 已取消\n\n' + printf '验证作业在生成报告前被取消。\n\n' + printf '
\n\n' + fi printf '%s\n' '— _Qwen Code · sandboxed verification_' } > "$BODY_FILE" elif [ "${VERIFY_RESULT:-}" != "success" ] || [ -z "${VERDICT:-}" ]; then WEAK_BODY=true { printf '%s\n\n' '' - printf '**Sandboxed verification: ⚠️ incomplete — infrastructure failure** - [workflow run](%s)\n\n' "$RUN_URL" - printf 'The verification job did not complete (checkout, runner, or setup error) and produced no report. See the workflow run for details.\n\n' - printf '
\n中文 — 判定：⚠️ 未完成 · 基础设施故障\n\n' - printf '验证作业未完成（检出、runner 或初始化错误），未生成报告。详见工作流运行日志。\n\n' - printf '
\n\n' + if [ "${FLAKE_VERDICT:-}" = 'flaky' ]; then + printf '**Sandboxed verification: ❌ not passed — non-deterministic tests (flakiness gate); the job then failed before the report** - [workflow run](%s)\n\n' "$RUN_URL" + printf 'Flakiness gate: ❌ %s\n\n' "${FLAKE_SUMMARY:-changed test files returned different results across identical re-runs}" + printf 'The verification job did not complete, but the deterministic flakiness gate had already observed run-to-run divergence — the per-round matrix is in the gate step output of the workflow run log.\n\n' + printf '
\n中文 — 判定：❌ 不通过 · 测试结果不确定（抖动门）；作业随后在报告前失败\n\n' + printf '抖动门：❌ %s\n\n' "${FLAKE_SUMMARY:-changed test files returned different results across identical re-runs}" + printf '验证作业未完成，但确定性抖动门已观测到轮间分歧——各轮矩阵见工作流运行日志中 gate step 的输出。\n\n' + printf '
\n\n' + else + printf '**Sandboxed verification: ⚠️ incomplete — infrastructure failure** - [workflow run](%s)\n\n' "$RUN_URL" + printf 'The verification job did not complete (checkout, runner, or setup error) and produced no report. See the workflow run for details.\n\n' + printf '
\n中文 — 判定：⚠️ 未完成 · 基础设施故障\n\n' + printf '验证作业未完成（检出、runner 或初始化错误），未生成报告。详见工作流运行日志。\n\n' + printf '
\n\n' + fi printf '%s\n' '— _Qwen Code · sandboxed verification_' } > "$BODY_FILE" elif [ "${VERDICT:-}" = "skipped" ] || [ "${VERDICT:-}" = "n/a" ]; then From 1f2b666d7e8b685b6bdbf33acc2cd2ede69ad187 Mon Sep 17 00:00:00 2001 From: wenshao Date: Sat, 15 Aug 2026 08:28:46 +0000 Subject: [PATCH 08/18] fix(triage): per-sample reset equivalence, 125-127/N-transition classification, intake+staging hardening Round-7 Critical cluster on the flakiness gate: - Reset runs before EVERY invocation (not once per round) so file i never samples what files 1..i-1 left this round; HOME joins TMPDIR in the per-invocation isolation; `git checkout HEAD -- .` restores from the root-pinned commit instead of the index; `git clean -ffd` also drops nested-.git dirs plain -fd refuses; both git calls gain the lane's runner-injection strip and a `timeout -k 30 120` wrapper, and a failed reset fails open to the fixed error verdict instead of sampling dirty. - Exits 125-127 (timeout's own failure modes) classify as infrastructure like 124/128+N; a per-file collection-state transition (N next to P/F) is divergence, no longer collapsed to pass/consistent-fail. - Record step: diff-filter gains T (typechange); a grep error (status 2) fails the step loudly instead of starving the gate to n/a; the owning-package walk hands its result through a variable, never a `$( )` capture that strips trailing newlines. - Staging: chown preserves modes, so the root re-own is completed by `chmod -R go-rwx` and a post-revoke sweep before the copy. Behavioral scenarios pin each defect (all fail pre-round, measured): per-sample equivalence across residue/nested-repo/staged/HOME classes, exit-127 infra classification, and N-transition divergence. --- .github/scripts/qwen-triage-workflow.test.mjs | 200 +++++++++++++++--- .github/workflows/qwen-triage.yml | 110 +++++++--- scripts/tests/qwen-triage-workflow.test.js | 16 +- 3 files changed, 266 insertions(+), 60 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index 5b4afca0ee2..ee656a88b94 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -853,9 +853,11 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // Adjacency-pinned as WHOLE statements (round 6): pinning the git line // and its redirect as independent shapes let an interposed pipeline // stage satisfy both. + // T included: a typechange (symlink→regular) changes what the runner + // executes, so it is a changed test file exactly like M. assert.match( recordStep.run, - /^\s*git -c core\.quotePath=false diff -z --name-only --diff-filter=ACMR 'HEAD\^1' HEAD \\\n\s*> "\$\{RUNNER_TEMP:\?\}\/flake-gate-files-all"$/m, + /^\s*git -c core\.quotePath=false diff -z --name-only --diff-filter=ACMRT 'HEAD\^1' HEAD \\\n\s*> "\$\{RUNNER_TEMP:\?\}\/flake-gate-files-all"$/m, 'the NUL diff must flow straight into its file — $( ) strips NUL bytes, a pipeline swallows the exit status', ); assert.ok( @@ -866,9 +868,17 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( recordStep.run, - /^\s*grep -zE '[^']+' \\\n\s*"\$RUNNER_TEMP\/flake-gate-files-all" \\\n\s*> "\$RUNNER_TEMP\/flake-gate-files" \|\| true$/m, + /^\s*grep_status=0\n\s*grep -zE '[^']+' \\\n\s*"\$RUNNER_TEMP\/flake-gate-files-all" \\\n\s*> "\$RUNNER_TEMP\/flake-gate-files" \|\| grep_status=\$\?$/m, 'the grep must read the raw file and only a no-match may yield an empty list', ); + // A grep ERROR (status 2 — e.g. ENOSPC opening the output) is + // infrastructure: swallowing it narrows the gate to zero files and + // starves it into n/a, so it must fail the record step loudly. + assert.match( + recordStep.run, + /^\s*if \[ "\$grep_status" -gt 1 \]; then\n\s*echo "[^"]*" >&2\n\s*exit 1\n\s*fi$/m, + 'a grep error must fail the record step loudly — never narrow the gate silently', + ); // Continuation-collapsed (round 6): a `\⏎|` split pipeline is still a // pipeline. assert.doesNotMatch( @@ -885,6 +895,20 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /read -r -d '' f/, 'the gate must consume the list NUL-delimited — the one framing a filename cannot break', ); + // The owning-package walk must hand its result back through a + // variable, never a `$( )` capture: command substitution strips + // trailing newlines, corrupting a package dir that ends in one (the + // NUL intake admits such names). + assert.doesNotMatch( + flakeStep.run, + /\$\(owning_pkg_dir/, + 'the owning-package walk must not be captured through $( )', + ); + assert.match( + flakeStep.run, + /^\s*pkg="\$OWNING_PKG_DIR"$/m, + 'the walk result must flow through a variable, not stdout', + ); // Word-based and continuation-proof (round 5): `git -c … diff`, a // backslash-continued `git \⏎ diff`, and log/show/whatchanged // --name-only are all re-derivations. The gate's own `git checkout`/ @@ -955,8 +979,8 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( flakeStep.run, - /NODE_OPTIONS='--max-old-space-size=3072' CI=true TMPDIR="\$inv_tmp"/, - 'child env must pin CI parity, the heap limit, and the per-invocation TMPDIR', + /NODE_OPTIONS='--max-old-space-size=3072' CI=true HOME="\$inv_tmp" TMPDIR="\$inv_tmp"/, + 'child env must pin CI parity, the heap limit, and the per-invocation HOME/TMPDIR — dotfile/XDG state must not leak across samples', ); // The per-invocation temp dir must be recreated fresh and handed to // the build user — a shared or stale TMPDIR is exactly the cross-round @@ -1004,18 +1028,37 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // Round-4 Critical: the reset must run AS THE BUILD USER — a root // checkout restores node-mutated tracked files as new root-owned // inodes that later node rounds cannot write (EACCES divergence) — - // and must also drop untracked residue (`git clean -fd`, no -x so + // and must also drop untracked residue (`git clean -ffd`, no -x so // gitignored node_modules/dist survive). Line-anchored: a comment // cannot satisfy these. + // Round-7 hardening of both calls: `-ffd` because plain -fd by + // documented git behavior refuses untracked dirs holding a nested + // .git; `checkout HEAD -- .` restores from the root-pinned commit, + // not the index (PR lifecycle code can stage mutations a + // checkout-from-index would preserve); the lane's runner-injection + // strip (git filters run from PR-owned .git metadata as node); a + // timeout wrapper (a planted filter can hang them, and the reset runs + // outside the invocation loop's deadline check); and a failed reset + // fails OPEN to the fixed error verdict — dirty samples carry no + // signal. + const resetCheckoutRe = + /^\s*timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*git checkout HEAD -- \. 2>\/dev\/null \|\| reset_rc=\$\?$/m; + const resetCleanRe = + /^\s*timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*git clean -ffd 2>\/dev\/null \|\| reset_rc=\$\?$/m; + assert.match( + flakeStep.run, + resetCheckoutRe, + 'the tracked-file restore must run as the build user, stripped, bounded, and from the pinned commit', + ); assert.match( flakeStep.run, - /^\s*runuser -u node -- git checkout -- \. 2>\/dev\/null \|\| true$/m, - 'the tracked-file restore must run as the build user', + resetCleanRe, + 'untracked round residue — including nested-.git dirs — must be cleaned without touching gitignored build outputs', ); assert.match( flakeStep.run, - /^\s*runuser -u node -- git clean -fd 2>\/dev\/null \|\| true$/m, - 'untracked round residue must be cleaned without touching gitignored build outputs', + /^\s*if \[ "\$reset_rc" -ne 0 \]; then\n\s*finish error "[^"]*"\n\s*fi$/m, + 'a failed reset must fail open to the fixed error verdict — never sample a dirty tree', ); // Kill FIRST (a live daemon can re-dirty the tree after the checkout), // with SIGKILL + a bounded wait — one-shot SIGTERM races slow-draining @@ -1023,9 +1066,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { const resetKill = flakeStep.run.search( /^\s*pkill -KILL -u node 2>\/dev\/null \|\| true$/m, ); - const resetCheckout = flakeStep.run.search( - /^\s*runuser -u node -- git checkout/m, - ); + const resetCheckout = flakeStep.run.search(resetCheckoutRe); assert.ok( resetKill !== -1 && resetCheckout !== -1 && resetKill < resetCheckout, 'the reset must SIGKILL leftover test-user processes BEFORE restoring the tree', @@ -1039,15 +1080,21 @@ describe('qwen-triage: flakiness gate (#9125)', () => { resetWait !== -1 && resetKill < resetWait && resetWait < resetCheckout, 'the kill must wait out survivors (zombies disregarded) before the restore', ); - // And the reset must also run BEFORE round 1: lifecycle scripts (npm - // ci/build, run as node) mutate the tree between list-record and the - // first sample, so round 1 must sample the same restored tree as - // rounds 2..N. - const firstReset = flakeStep.run.search(/^\s*reset_round_state$/m); + // And the reset must precede EVERY invocation, not just rounds: + // lifecycle scripts (npm ci/build, run as node) mutate the tree + // before the first sample, and a between-rounds reset leaves file i + // seeing the residue, staged mutations, and HOME state files 1..i-1 + // left THIS round — equivalence is per sample, not per round. + const resetCall = flakeStep.run.search(/^\s*reset_round_state$/m); const loopStart = flakeStep.run.indexOf('while [ "$round" -le "$ROUNDS" ]'); + const invFlush = flakeStep.run.search(/^\s*rm -rf "\$inv_tmp"$/m); assert.ok( - firstReset !== -1 && loopStart !== -1 && firstReset < loopStart, - 'one reset must precede round 1, not only the between-round resets', + resetCall !== -1 && + loopStart !== -1 && + invFlush !== -1 && + loopStart < resetCall && + resetCall < invFlush, + 'the reset must run inside the round loop, before every invocation', ); assert.equal( flakeStep.if, @@ -1159,6 +1206,17 @@ describe('qwen-triage: flakiness gate (#9125)', () => { const iChown = sr.search( /^\s*chown -R root:root "\$RUNNER_TEMP\/verify-results"$/m, ); + // Round 7: chown PRESERVES modes — a tree PR code set 0777 stays + // world-writable under root ownership, so a survivor could still + // create/unlink names in it. The revoke is only complete after the + // group/other bits are stripped and the post-revoke sweep drops + // anything planted between the agent-step sweep and the chown. + const iChmod = sr.search( + /^\s*chmod -R go-rwx "\$RUNNER_TEMP\/verify-results"$/m, + ); + const iSweep = sr.search( + /^\s*find "\$RUNNER_TEMP\/verify-results" \\\( -type l -o -type p -o -type s -o -type b -o -type c \\\) -delete 2>\/dev\/null \|\| true$/m, + ); const iRmDst = sr.search( /^\s*rm -rf -- "\$RUNNER_TEMP\/verify-results\/flake-gate\.log"$/m, ); @@ -1185,6 +1243,8 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ['directory-level symlink/non-dir guard', iDirGuard], ['mkdir -p', iMkdir], ['root re-own', iChown], + ['mode revoke', iChmod], + ['post-revoke sweep', iSweep], ['destination unlink', iRmDst], ['copy', iCp], ]) { @@ -1195,9 +1255,11 @@ describe('qwen-triage: flakiness gate (#9125)', () => { iWait < iDirGuard && iDirGuard < iMkdir && iMkdir < iChown && - iChown < iRmDst && + iChown < iChmod && + iChmod < iSweep && + iSweep < iRmDst && iRmDst < iCp, - 'staging order must be: kill+wait, dir guard, recreate, root re-own, unlink destination, copy', + 'staging order must be: kill+wait, dir guard, recreate, root re-own, mode revoke, sweep, unlink destination, copy', ); assert.doesNotMatch( agentStep.run, @@ -1324,7 +1386,9 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp // outcome sequence lives at $FLAKE_SEQ_DIR/, consumed one // letter per invocation and cycled (missing sequence file = always // pass). Letters: P=exit 0, F=exit 1, T=exit 124 (timeout), K=exit 137 - // (signal kill) — T/K model infrastructure exits, not test failures. + // (signal kill), M=exit 127 (runner binary missing), N=exit 1 printing + // the no-collection marker — T/K/M model infrastructure exits and N a + // runner include-set rejection; none are test failures. const STUB_TESTRUNNER = [ '#!/bin/bash', // Round-5 guards baked into EVERY default-runner scenario: PR test code @@ -1352,6 +1416,8 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp '[ "$m" = P ] && exit 0', '[ "$m" = T ] && exit 124', '[ "$m" = K ] && exit 137', + '[ "$m" = M ] && exit 127', + '[ "$m" = N ] && { echo "No test files found, exiting with code 1"; exit 1; }', 'echo "stub failure for $key run $((n+1))"', 'exit 1', '', @@ -1388,7 +1454,7 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp sequences = {}, stubs = {}, env: envOverrides = {}, - git = false, + git = true, mutate, }) => { const root = mkdtempSync(join(scenarioRoot, 'case-')); @@ -1416,8 +1482,10 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp writeFileSync(join(seqDir, k), v); } if (git) { - // A committed tree, so the gate's between-round `git checkout -- .` - // reset has something to restore (round-state isolation scenarios). + // Default on: production always samples a checkout, the gate's + // per-invocation `git checkout HEAD -- .` reset needs a committed + // tree to restore, and a reset that fails on a missing repo would + // fail the gate open to `error`. for (const args of [ ['init', '-q'], ['add', '-A'], @@ -1614,6 +1682,40 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.doesNotMatch(outputs.flake_summary, /a\.test\.js|b\.test\.js/); }); + it('exit 125-127 are timeout failure modes, never F marks or fake flakiness', () => { + // 124 is the cap and 128+N a signal kill, but 125-127 are timeout's + // OWN failure modes (it failed, or the runner binary was + // unrunnable/missing) — recorded as F they published a fake `flaky` + // next to any pass (round-7 Critical probe: exit 127 → FPPPP read as + // flaky). + const { res, outputs, log } = runGate({ + layout: { 'scripts/tests/a.test.js': '' }, + list: 'scripts/tests/a.test.js\n', + sequences: { 'a.test.js': 'MPPPP' }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'timeout'); + assert.match(log, /a\.test\.js: IPPPP/); + assert.match(log, /exit 127/); + assert.doesNotMatch(outputs.flake_summary, /a\.test\.js/); + }); + + it('a collection-state transition (N next to P or F) is divergence', () => { + // An identical tree that COLLECTS a file in some rounds and rejects + // it in others is itself non-determinism; collapsing NPNPN to `pass` + // published a verdict about samples that never all executed (round-7 + // Critical). Faking N can only add demotions a PR earns — one-way + // authority holds. + const { res, outputs, log } = runGate({ + layout: { 'scripts/tests/a.test.js': '' }, + list: 'scripts/tests/a.test.js\n', + sequences: { 'a.test.js': 'NPNPN' }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'flaky'); + assert.match(log, /a\.test\.js: NPNPN/); + }); + it('real divergence still outranks an infra exit in another file', () => { const { res, outputs } = runGate({ layout: UNIT, @@ -1679,6 +1781,54 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.match(log, /untracked\.test\.js: PPPPP/); }); + it('every sample starts from equivalent state: per-file reset, isolated HOME, restore from the pinned commit', () => { + // Round-7 Critical probe: a between-rounds reset left file b sampling + // what file a left THIS round — untracked residue, a nested fixture + // repo (plain `git clean -fd` refuses dirs holding a nested .git), a + // staged tracked mutation (`checkout -- .` restores from the index, + // preserving it), and $HOME state shared by every invocation. Each + // class alone turned a deterministic pair into b: FFFFF. + const homeDir = mkdtempSync(join(scenarioRoot, 'home-')); + const STUB_CROSSFILE = [ + '#!/bin/bash', + 'f="${@: -1}"', + 'case "$f" in', + ' ./scripts/tests/a.test.js)', + ' mkdir -p residue-dir', + ' git init -q nested-repo', + ' mkdir -p "$HOME/.cache"', + ' touch "$HOME/.cache/marker"', + ' echo dirt >> scripts/tests/b.test.js', + ' git add scripts/tests/b.test.js', + ' exit 0', + ' ;;', + ' ./scripts/tests/b.test.js)', + ' if [ -e residue-dir ] || [ -e nested-repo ] || [ -e "$HOME/.cache/marker" ] || grep -q dirt scripts/tests/b.test.js; then', + ' echo "sampled state an earlier sample left behind"', + ' exit 1', + ' fi', + ' exit 0', + ' ;;', + 'esac', + 'echo "unexpected operand: $f"', + 'exit 1', + '', + ].join('\n'); + const { res, outputs, log } = runGate({ + layout: { + 'scripts/tests/a.test.js': '', + 'scripts/tests/b.test.js': 'pristine-b\n', + }, + list: 'scripts/tests/a.test.js\nscripts/tests/b.test.js\n', + stubs: { npx: STUB_CROSSFILE }, + env: { HOME: homeDir }, + }); + assert.equal(res.status, 0, `gate died under the wrapper: ${res.stderr}`); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match(log, /a\.test\.js: PPPPP/); + assert.match(log, /b\.test\.js: PPPPP/); + }); + it('a pre-gate tree mutation cannot make round 1 sample different content than rounds 2..N', () => { // Round-4 Critical mechanism c: PR lifecycle scripts run as node // between list-record and round 1. Without a reset BEFORE round 1 the diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index ce5d8ab72a9..bdf37e728f9 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2764,12 +2764,24 @@ jobs: # line grep with no skip-log entry. NUL is the one byte a path # cannot contain. NOTE: the git output must never pass through a # command substitution — `$( )` strips NUL bytes. - git -c core.quotePath=false diff -z --name-only --diff-filter=ACMR 'HEAD^1' HEAD \ + # T (typechange) included: a symlink->regular flip changes what + # the runner executes, so it is a changed test file exactly + # like M — excluding it silently drops the file from the gate. + git -c core.quotePath=false diff -z --name-only --diff-filter=ACMRT 'HEAD^1' HEAD \ > "${RUNNER_TEMP:?}/flake-gate-files-all" # .mts/.cts included: vitest's default include set collects them. + # Only a no-match (status 1) may yield an empty list: a grep + # error (status 2, e.g. ENOSPC opening the output) is + # infrastructure and must fail the step loudly — swallowing it + # would narrow the gate to zero files and starve it into n/a. + grep_status=0 grep -zE '\.(test|spec)\.(ts|tsx|js|jsx|mjs|cjs|mts|cts)$' \ "$RUNNER_TEMP/flake-gate-files-all" \ - > "$RUNNER_TEMP/flake-gate-files" || true + > "$RUNNER_TEMP/flake-gate-files" || grep_status=$? + if [ "$grep_status" -gt 1 ]; then + echo "flake-gate intake: grep failed with status ${grep_status}" >&2 + exit 1 + fi rm -f "$RUNNER_TEMP/flake-gate-files-all" echo "Recorded $(tr -cd '\0' < "$RUNNER_TEMP/flake-gate-files" | wc -c) changed test file(s) for the flakiness gate." @@ -3008,12 +3020,16 @@ jobs: owning_pkg_dir() { # Nearest ancestor directory carrying package.json — nested # workspaces (packages/channels/base) own their runner and must - # be entered themselves, never their parent. + # be entered themselves, never their parent. Result flows via + # OWNING_PKG_DIR, never stdout: a `$( )` capture strips + # trailing newlines, corrupting a directory name that ends in + # one (the NUL intake admits such paths). local d="$1" + OWNING_PKG_DIR='' d="${d%/*}" while [ -n "$d" ] && [ "$d" != '.' ]; do if [ -f "$d/package.json" ]; then - printf '%s' "$d" + OWNING_PKG_DIR="$d" return 0 fi case "$d" in @@ -3089,10 +3105,11 @@ jobs: # a path prefix: root npm workspaces outside packages/** # (integrations/*) are CI-tested too and must be re-run # through their own entry point like any nested workspace. - if ! pkg="$(owning_pkg_dir "$f")"; then + if ! owning_pkg_dir "$f"; then add_skip "$f" 'no owning package.json, skipped' continue fi + pkg="$OWNING_PKG_DIR" if ! has_vitest_config "$pkg"; then # e.g. packages/desktop runs `bun test` — an unsupported # runner family stays explicitly out of scope rather than @@ -3130,9 +3147,20 @@ jobs: # checkout restores node-mutated tracked files as new root-owned # inodes that later node rounds cannot write (EACCES), # manufacturing the very divergence the reset exists to prevent. - # `git clean -fd` (no -x: gitignored node_modules/dist must - # survive) drops the untracked residue `checkout -- .` never - # touches — lock files, output dirs a test creates in round 1. + # `git clean -ffd` (no -x: gitignored node_modules/dist must + # survive) drops the untracked residue `checkout` never touches + # — lock files, output dirs a test creates — INCLUDING dirs + # holding a nested .git, which plain -fd by documented git + # behavior refuses to remove. `checkout HEAD -- .` restores from + # the root-pinned commit, not the index: PR lifecycle code can + # stage mutations, and restoring from the index would preserve + # them. Both git calls carry the lane's runner-injection strip + # (git filters run from PR-owned .git metadata as node) and a + # timeout wrapper (a planted filter can hang them, and the reset + # runs outside the invocation loop's deadline check). A failed + # reset means samples would not start from equivalent state, so + # the gate fails open to the fixed error verdict instead of + # sampling a dirty tree. reset_round_state() { # Kill FIRST, then restore — a live daemon can re-dirty the tree # after the checkout. One-shot SIGTERM races slow-draining @@ -3144,8 +3172,14 @@ jobs: sleep 1 pkill -KILL -u node 2>/dev/null || true done - runuser -u node -- git checkout -- . 2>/dev/null || true - runuser -u node -- git clean -fd 2>/dev/null || true + local reset_rc=0 + timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ + git checkout HEAD -- . 2>/dev/null || reset_rc=$? + timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ + git clean -ffd 2>/dev/null || reset_rc=$? + if [ "$reset_rc" -ne 0 ]; then + finish error "workspace reset failed (exit ${reset_rc}) — samples would not start from equivalent state" + fi } declare -a results=() @@ -3154,20 +3188,23 @@ jobs: timed_out=false infra_exits=0 round=1 - # Reset BEFORE round 1 too: PR lifecycle scripts (npm ci/build, - # run as node) may have mutated the tree since the list was - # recorded — without this, round 1 samples mutated content while - # rounds 2..N sample restored content, and the difference reads - # as divergence. - reset_round_state while [ "$round" -le "$ROUNDS" ]; do for i in "${!group_labels[@]}"; do if [ "$(date +%s)" -ge "$deadline" ]; then timed_out=true break fi - # Fresh per-invocation temp/cache dir: samples must not share - # caches any more than they share the tree or processes. + # Reset before EVERY invocation, not once per round: samples + # must start from equivalent state per file. A between-rounds + # reset leaves file i seeing what files 1..i-1 left THIS + # round — residue, staged mutations, HOME state — and round + # 1 must sample the same restored tree as rounds 2..N (PR + # lifecycle scripts, npm ci/build run as node, may have + # mutated the tree since the list was recorded). + reset_round_state + # Fresh per-invocation HOME and temp/cache dirs: samples must + # not share dotfile/XDG/cache state or caches any more than + # they share the tree or processes. inv_tmp="$RUNNER_TEMP/flake-inv-tmp" rm -rf "$inv_tmp" mkdir -p "$inv_tmp" @@ -3177,13 +3214,16 @@ jobs: cd "${group_dirs[$i]}" && timeout -k 30 600 runuser -u node -- \ env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ - NODE_OPTIONS='--max-old-space-size=3072' CI=true TMPDIR="$inv_tmp" \ + NODE_OPTIONS='--max-old-space-size=3072' CI=true HOME="$inv_tmp" TMPDIR="$inv_tmp" \ bash -c "${group_cmds[$i]}" ) > "$out" 2>&1 status=$? - # Timeout/signal exits (124, 128+N) are infrastructure events — - # a runner cap or an OOM kill — not test outcomes: recorded as - # F they would publish a fake `flaky` next to any pass. + # Timeout/signal exits (124-127, 128+N) are infrastructure + # events, not test outcomes: 124 is the cap itself, 125-127 + # are timeout's own failure modes (it failed, or the runner + # binary was unrunnable/missing), 128+N is a signal kill — + # recorded as F they would publish a fake `flaky` next to any + # pass. # Zero-collection is its own class (`N`): a file the runner's # include set rejects fails every round without one test # executing — reporting that as consistent-fail would publish @@ -3192,7 +3232,7 @@ jobs: # can only SUPPRESS a demotion the PR could already dodge by # deleting its tests — one-way authority holds. mark='P' - if [ "$status" -eq 124 ] || [ "$status" -gt 128 ]; then + if [ "$status" -ge 124 ]; then mark='I' infra_exits=$((infra_exits + 1)) elif [ "$status" -ne 0 ] && grep -q 'No test files found' "$out"; then @@ -3210,15 +3250,14 @@ jobs: } >> "$DETAIL" fi done - # Restore the equivalent of a clean checkout before the next - # sample — a deterministic test that mutates a fixture or leaves - # a daemon/port/untracked file behind must not fail later rounds - # on its own residue and fake a divergence (PFF). - reset_round_state [ "$timed_out" = true ] && break rounds_done="$round" round=$((round + 1)) done + # And leave the agent the same clean tree it would have seen + # before the gate ran: the last invocation's residue must not be + # handed to the verifier. + reset_round_state flaky=0 failing=0 @@ -3226,6 +3265,12 @@ jobs: for i in "${!group_labels[@]}"; do case "${results[$i]:-}" in *P*F*|*F*P*) flaky=$((flaky + 1)) ;; + # A collection-state TRANSITION for one file (N next to P or + # F) is itself non-determinism: identical trees collected it + # in some rounds and rejected it in others. Reducing it to + # pass/consistent-fail would publish a verdict about samples + # that never all executed. + *N*P*|*P*N*|*N*F*|*F*N*) flaky=$((flaky + 1)) ;; *P*) : ;; *F*) failing=$((failing + 1)) ;; *N*) ncoll=$((ncoll + 1)) ;; @@ -3885,7 +3930,16 @@ jobs: # replant a symlink between this copy and upload-artifact's # enumeration, which FOLLOWS links — that window was a # root-readable-file exfiltration into the public comment. + # Ownership alone is NOT the revoke: chown PRESERVES modes, so + # a tree PR code set 0777 stays world-writable under root + # ownership and a survivor can still create/unlink names in it. + # Strip group/other bits entirely, THEN sweep non-regular + # entries again — anything planted between the agent-step sweep + # and the chown dies here, and after the mode revoke nothing + # can replant before upload enumeration. chown -R root:root "$RUNNER_TEMP/verify-results" + chmod -R go-rwx "$RUNNER_TEMP/verify-results" + find "$RUNNER_TEMP/verify-results" \( -type l -o -type p -o -type s -o -type b -o -type c \) -delete 2>/dev/null || true rm -rf -- "$RUNNER_TEMP/verify-results/flake-gate.log" cp -f "$RUNNER_TEMP/flake-gate.log" "$RUNNER_TEMP/verify-results/flake-gate.log" fi diff --git a/scripts/tests/qwen-triage-workflow.test.js b/scripts/tests/qwen-triage-workflow.test.js index fae9ab5e14f..f788fc835cd 100644 --- a/scripts/tests/qwen-triage-workflow.test.js +++ b/scripts/tests/qwen-triage-workflow.test.js @@ -2141,18 +2141,20 @@ describe('qwen-triage verify hardening', () => { // job's own commands: a bare step() lookup returns the tmux job's // identically named step, so verify-side regressions would pass silently. it('strips GitHub command files from every node-run verify command', () => { - // Bound to the lifecycle commands that run as node before the agent: - // npm ci and npm run build in the prepare step, plus the evidence - // browser download. The slice stops at the agent step, whose own - // `runuser` launches qwen under `env -i` and needs no per-variable - // stripping. Covering all three by construction (not enumeration) is - // what catches a future node-run command added without the strip. + // Bound to the commands that run as node before the agent: npm ci and + // npm run build in the prepare step, the evidence browser download, + // and the flake gate's two reset git invocations (git filters run + // from PR-owned .git metadata). The slice stops at the agent step, + // whose own `runuser` launches qwen under `env -i` and needs no + // per-variable stripping. Covering all five by construction (not + // enumeration) is what catches a future node-run command added + // without the strip. const prepare = verifyJob.slice( verifyJob.indexOf('Install and build PR app'), verifyJob.indexOf('Run verification agent'), ); const commands = prepare.match(/runuser -u node -- env[\s\S]*?\n/g) ?? []; - expect(commands.length).toBe(3); + expect(commands.length).toBe(5); expect(step('Run verification agent')).toContain( 'runuser -u node -- env -i', ); From 1c86bc7562341f65ed805f40b0118d92c5e1b20f Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 16 Aug 2026 01:52:47 +0800 Subject: [PATCH 09/18] fix(triage): give the gate a root-only home and rebuild the upload tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-7 R7-8 (Critical) and the R1-5 line it supersedes, one root cause: $RUNNER_TEMP's top level is uid-1000 mode-755 on this pool and the job container's `node` is uid 1000, so PR-controlled code held OWNER WRITE on the directory holding every gate working file — and each root-side consumer followed symlinks. Four probe-confirmed exfiltration routes: the staging source, the recorded list (swapped for /proc//environ, whose NUL-framed KEY=VALUE is exactly the intake framing, landing ACTIONS_RUNTIME_TOKEN in the published log), the fixed-path round output, and the destination entry itself (renameable even after the in-place hardening, so a kill-race survivor could swap the whole tree for a symlink farm that upload-artifact follows). Removing the capability rather than patching each consumer: - All gate working files (list, log, detail, round output) now live in $RUNNER_TEMP/flake-gate, created 0700 root:root by the record step (rm -rf first: the entry may be a plant from an earlier run on the persistent pool). A directory node cannot enter is one whose entries it can neither create, unlink, nor rename — routes (a)(b)(c) close by construction. - The gate verifies that home fail-closed before reading anything: not a symlink, a directory, owned by the EFFECTIVE user (root in production; keeps the extracted script runnable under a harness), and mode 700 — otherwise the fixed `error` verdict, exit 0. - Staging BUILDS a trusted upload tree in that home instead of hardening the agent-era one: kill+wait unconditionally (node can unlink the log, which must not skip the rebuild), then copy only regular files with --no-dereference out of verify-results, then the authoritative log last. The artifact now uploads from the rebuilt tree, so the entry the enumeration walks was never in a PR-writable directory — route (d). The publisher's paths are unchanged (same inner layout). Behavioral scenario pins the fail-closed refusal; the staging pin chain follows the rebuild order. --- .github/scripts/qwen-triage-workflow.test.mjs | 140 +++++++++------ .github/workflows/qwen-triage.yml | 169 ++++++++++++------ 2 files changed, 200 insertions(+), 109 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index ee656a88b94..1c50f7ace34 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -857,7 +857,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // executes, so it is a changed test file exactly like M. assert.match( recordStep.run, - /^\s*git -c core\.quotePath=false diff -z --name-only --diff-filter=ACMRT 'HEAD\^1' HEAD \\\n\s*> "\$\{RUNNER_TEMP:\?\}\/flake-gate-files-all"$/m, + /^\s*git -c core\.quotePath=false diff -z --name-only --diff-filter=ACMRT 'HEAD\^1' HEAD \\\n\s*> "\$RUNNER_TEMP\/flake-gate\/files-all"$/m, 'the NUL diff must flow straight into its file — $( ) strips NUL bytes, a pipeline swallows the exit status', ); assert.ok( @@ -868,9 +868,23 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( recordStep.run, - /^\s*grep_status=0\n\s*grep -zE '[^']+' \\\n\s*"\$RUNNER_TEMP\/flake-gate-files-all" \\\n\s*> "\$RUNNER_TEMP\/flake-gate-files" \|\| grep_status=\$\?$/m, + /^\s*grep_status=0\n\s*grep -zE '[^']+' \\\n\s*"\$RUNNER_TEMP\/flake-gate\/files-all" \\\n\s*> "\$RUNNER_TEMP\/flake-gate\/files" \|\| grep_status=\$\?$/m, 'the grep must read the raw file and only a no-match may yield an empty list', ); + // Every gate working file must live in the root-only home, because + // $RUNNER_TEMP's top level is uid-1000 writable on this pool and the + // container's `node` is uid 1000: files there can be unlinked and + // replaced with symlinks that root-side consumers follow. + assert.match( + recordStep.run, + /^\s*install -d -m 0700 -o root -g root "\$RUNNER_TEMP\/flake-gate"$/m, + 'the record step must create the root-only home', + ); + assert.match( + recordStep.run, + /^\s*rm -rf -- "\$\{RUNNER_TEMP:\?\}\/flake-gate"$/m, + 'a plant left by an earlier run on the persistent pool must be removed first', + ); // A grep ERROR (status 2 — e.g. ENOSPC opening the output) is // infrastructure: swallowing it narrows the gate to zero files and // starves it into n/a, so it must fail the record step loudly. @@ -1159,8 +1173,8 @@ describe('qwen-triage: flakiness gate (#9125)', () => { const uploadStep = verifyJob.steps[uploadIdx]; assert.equal( uploadStep.with.path, - '${{ runner.temp }}/verify-results/', - 'the artifact must ship the staged directory', + '${{ runner.temp }}/flake-gate/upload/', + 'the artifact must ship the REBUILT tree, never the agent-era directory', ); assert.match( uploadStep.with.name, @@ -1188,78 +1202,75 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // comments, and the guard's REACTION (the directory-level rm) must be // part of the chain — an inert guard body lets mkdir/cp write through // a planted symlink. + // Round 7 rewrite: the step no longer HARDENS the agent-era tree (its + // entry lived in the uid-1000-writable $RUNNER_TEMP, so a kill-race + // survivor could rename the whole hardened tree and replant a symlink + // farm for upload-artifact to follow). It BUILDS a trusted tree inside + // the 0700 root-only home instead, copying regular files only. const sr = stageStep.run; const iPkill = sr.search(/^\s*pkill -KILL -u node/m); - const iDirGuard = sr.search( - /^\s*if \[ -L "\$RUNNER_TEMP\/verify-results" \] \|\|$/m, - ); - const iMkdir = sr.search(/^\s*mkdir -p "\$RUNNER_TEMP\/verify-results"$/m); - // Round 6: a one-shot pkill loses the race against setsid daemons and - // continuous forkers; the kill needs the bounded survivor wait, and the - // directory must be re-owned by root before the copy — a survivor that - // cannot create or unlink names cannot replant a symlink for - // upload-artifact to follow (root-readable exfiltration into the - // public comment). const iWait = sr.search( /^\s*\[ -n "\$\(ps -o pid=,stat= -u node 2>\/dev\/null \| awk '\$2 !~ \/\^Z\/'\)" \] \|\| break$/m, ); - const iChown = sr.search( - /^\s*chown -R root:root "\$RUNNER_TEMP\/verify-results"$/m, + const iHomeCheck = sr.search( + /^\s*if \[ ! -L "\$GATE_DIR" \] && \[ -d "\$GATE_DIR" \] && \[ -O "\$GATE_DIR" \] &&$/m, ); - // Round 7: chown PRESERVES modes — a tree PR code set 0777 stays - // world-writable under root ownership, so a survivor could still - // create/unlink names in it. The revoke is only complete after the - // group/other bits are stripped and the post-revoke sweep drops - // anything planted between the agent-step sweep and the chown. - const iChmod = sr.search( - /^\s*chmod -R go-rwx "\$RUNNER_TEMP\/verify-results"$/m, + const iFresh = sr.search( + /^\s*install -d -m 0700 -o root -g root "\$UPLOAD_DIR"$/m, ); - const iSweep = sr.search( - /^\s*find "\$RUNNER_TEMP\/verify-results" \\\( -type l -o -type p -o -type s -o -type b -o -type c \\\) -delete 2>\/dev\/null \|\| true$/m, + const iCopyRegular = sr.search( + /^\s*find \. -type f -exec cp -f --no-dereference --parents \{\} "\$UPLOAD_DIR\/" \\;$/m, ); - const iRmDst = sr.search( - /^\s*rm -rf -- "\$RUNNER_TEMP\/verify-results\/flake-gate\.log"$/m, + const iCopyLog = sr.search( + /^\s*cp -f --no-dereference "\$GATE_DIR\/log" "\$UPLOAD_DIR\/flake-gate\.log"$/m, ); - const iCp = sr.search( - /^\s*cp -f "\$RUNNER_TEMP\/flake-gate\.log" "\$RUNNER_TEMP\/verify-results\/flake-gate\.log"$/m, + const iChown = sr.search(/^\s*chown -R root:root "\$UPLOAD_DIR"$/m); + const iChmod = sr.search(/^\s*chmod -R go-rwx "\$UPLOAD_DIR"$/m); + // The kill must NOT be gated on the log existing: node can unlink the + // log, and that must not skip the rebuild for the agent's own report. + assert.ok( + iPkill < sr.search(/^\s*GATE_DIR=/m), + 'the kill must run before (and independently of) the home lookup', ); - // The guard must cover BOTH halves (symlink and non-directory) and - // must actually remove the entry it detects. + assert.doesNotMatch( + sr, + /^\s*if \[ -f "\$\{?RUNNER_TEMP:?\??\}?\/flake-gate\.log" \]; then$/m, + 'the rebuild must not be gated on the log file existing', + ); + // Only regular files cross the boundary, and nothing is resolved: + // -type f excludes links/FIFOs/sockets/devices at selection time and + // --no-dereference never opens a target that wins a race after it. assert.match( sr, - /\[ -L "\$RUNNER_TEMP\/verify-results" \] \|\|\s*\n\s*\{ \[ -e "\$RUNNER_TEMP\/verify-results" \] && \[ ! -d "\$RUNNER_TEMP\/verify-results" \]; \}/, - 'the guard must catch symlinks AND non-directory plants', + /-type f -exec cp -f --no-dereference --parents/, + 'only regular files may be copied out of the untrusted tree', ); - // The reaction is pinned as the guard's own then-body (round 6), not - // by loose proximity to any -L mention. assert.match( sr, - /^\s*if \[ -L "\$RUNNER_TEMP\/verify-results" \] \|\|\n\s*\{ \[ -e "\$RUNNER_TEMP\/verify-results" \] && \[ ! -d "\$RUNNER_TEMP\/verify-results" \]; \}; then\n\s*rm -rf -- "\$RUNNER_TEMP\/verify-results"\n\s*fi$/m, - 'the guard must REMOVE the planted entry in its own then-body', + /\[ -d "\$RUNNER_TEMP\/verify-results" \] && \[ ! -L "\$RUNNER_TEMP\/verify-results" \]/, + 'a symlinked verify-results must not be traversed at all', ); for (const [label, idx] of [ ['pkill', iPkill], ['bounded survivor wait', iWait], - ['directory-level symlink/non-dir guard', iDirGuard], - ['mkdir -p', iMkdir], + ['root-only home integrity check', iHomeCheck], + ['fresh 0700 upload dir', iFresh], + ['regular-file-only copy', iCopyRegular], + ['authoritative log copy', iCopyLog], ['root re-own', iChown], ['mode revoke', iChmod], - ['post-revoke sweep', iSweep], - ['destination unlink', iRmDst], - ['copy', iCp], ]) { assert.ok(idx !== -1, `staging must contain the ${label}`); } assert.ok( iPkill < iWait && - iWait < iDirGuard && - iDirGuard < iMkdir && - iMkdir < iChown && - iChown < iChmod && - iChmod < iSweep && - iSweep < iRmDst && - iRmDst < iCp, - 'staging order must be: kill+wait, dir guard, recreate, root re-own, mode revoke, sweep, unlink destination, copy', + iWait < iHomeCheck && + iHomeCheck < iFresh && + iFresh < iCopyRegular && + iCopyRegular < iCopyLog && + iCopyLog < iChown && + iChown < iChmod, + 'staging order must be: kill+wait, home check, fresh dir, regular-file copy, authoritative log last, re-own, mode revoke', ); assert.doesNotMatch( agentStep.run, @@ -1456,6 +1467,7 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp env: envOverrides = {}, git = true, mutate, + gateHomeMode = 0o700, }) => { const root = mkdtempSync(join(scenarioRoot, 'case-')); const ws = join(root, 'ws'); @@ -1467,6 +1479,12 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp mkdirSync(dirname(join(ws, p)), { recursive: true }); writeFileSync(join(ws, p), content); } + // The gate's working home: 0700 and owned by whoever runs the suite — + // the same integrity premise the workflow asserts with -O (root in + // production). Scenarios that need to defeat it override the mode. + const gateDir = join(rt, 'flake-gate'); + mkdirSync(gateDir, { recursive: true }); + chmodSync(gateDir, gateHomeMode); if (list !== null) { // Scenarios describe lists as newline text; the wire format is // NUL-delimited (the record step emits `git diff -z` through @@ -1476,7 +1494,7 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp .filter(Boolean) .map((f) => `${f}\u0000`) .join(''); - writeFileSync(join(rt, 'flake-gate-files'), framed); + writeFileSync(join(gateDir, 'files'), framed); } for (const [k, v] of Object.entries(sequences)) { writeFileSync(join(seqDir, k), v); @@ -1556,7 +1574,7 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp ); let log = ''; try { - log = readFileSync(join(rt, 'flake-gate.log'), 'utf8'); + log = readFileSync(join(gateDir, 'log'), 'utf8'); } catch { // a scenario may legitimately abort before creating the log } @@ -2085,6 +2103,24 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp ); }); + it('a working home that is not exclusively ours fails closed to `error`', () => { + // R7-8: $RUNNER_TEMP's top level is uid-1000 writable and the + // container's `node` is uid 1000, so a group/other-accessible home is + // one a PR could have planted files in — the gate must refuse to read + // it rather than sample attacker-chosen bytes, and must still exit 0. + const { res, outputs } = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\n', + gateHomeMode: 0o777, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'error'); + assert.match( + outputs.flake_summary, + /not root-owned 0700|working directory/, + ); + }); + it('a missing recorded list degrades to the `error` verdict, exit 0', () => { const { res, outputs } = runGate({ layout: UNIT, list: null }); assert.equal(res.status, 0, res.stderr); diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index bdf37e728f9..dc3cbfd89c1 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2751,6 +2751,23 @@ jobs: if: "steps.pr.outputs.decision == 'run'" run: |- set -euo pipefail + # Root-only home for every gate working file. $RUNNER_TEMP itself + # is uid 1000 mode 755 on this pool, and `node` in the job's + # container is uid 1000 — so PR-controlled code (lifecycle + # scripts during install/build, the agent era afterwards) has + # OWNER WRITE on the directory that used to hold the gate's + # root-owned files. It could unlink them and replant symlinks + # that every root-side consumer would follow: the recorded list + # swapped for /proc//environ (NUL-framed KEY=VALUE is + # exactly the gate's intake framing, so ACTIONS_RUNTIME_TOKEN + # lands verbatim in the published log), the round output or the + # staged log swapped for any root-readable file. 0700 root:root + # removes the capability itself — a directory node cannot enter + # is one whose entries it can neither create, unlink, nor + # rename. rm -rf first: the entry may already be a plant from an + # earlier run on this persistent pool. + rm -rf -- "${RUNNER_TEMP:?}/flake-gate" + install -d -m 0700 -o root -g root "$RUNNER_TEMP/flake-gate" # Two statements, not a pipeline: `git diff | grep || true` would # swallow a git failure as "no changed test files", silently # narrowing the gate to n/a. A git failure here is pre-build @@ -2768,7 +2785,7 @@ jobs: # the runner executes, so it is a changed test file exactly # like M — excluding it silently drops the file from the gate. git -c core.quotePath=false diff -z --name-only --diff-filter=ACMRT 'HEAD^1' HEAD \ - > "${RUNNER_TEMP:?}/flake-gate-files-all" + > "$RUNNER_TEMP/flake-gate/files-all" # .mts/.cts included: vitest's default include set collects them. # Only a no-match (status 1) may yield an empty list: a grep # error (status 2, e.g. ENOSPC opening the output) is @@ -2776,14 +2793,14 @@ jobs: # would narrow the gate to zero files and starve it into n/a. grep_status=0 grep -zE '\.(test|spec)\.(ts|tsx|js|jsx|mjs|cjs|mts|cts)$' \ - "$RUNNER_TEMP/flake-gate-files-all" \ - > "$RUNNER_TEMP/flake-gate-files" || grep_status=$? + "$RUNNER_TEMP/flake-gate/files-all" \ + > "$RUNNER_TEMP/flake-gate/files" || grep_status=$? if [ "$grep_status" -gt 1 ]; then echo "flake-gate intake: grep failed with status ${grep_status}" >&2 exit 1 fi - rm -f "$RUNNER_TEMP/flake-gate-files-all" - echo "Recorded $(tr -cd '\0' < "$RUNNER_TEMP/flake-gate-files" | wc -c) changed test file(s) for the flakiness gate." + rm -f "$RUNNER_TEMP/flake-gate/files-all" + echo "Recorded $(tr -cd '\0' < "$RUNNER_TEMP/flake-gate/files" | wc -c) changed test file(s) for the flakiness gate." - name: 'Clear stale npm cache' if: "steps.pr.outputs.decision == 'run'" @@ -2969,13 +2986,35 @@ jobs: } trap on_gate_exit EXIT unset ACTIONS_RUNTIME_TOKEN ACTIONS_RUNTIME_URL ACTIONS_CACHE_URL - LOG="${RUNNER_TEMP:?}/flake-gate.log" - LIST="$RUNNER_TEMP/flake-gate-files" + # Every working file lives inside the root-only home the record + # step created. Verify it fail-closed BEFORE trusting anything in + # it: a plant (symlink, node-owned dir, loosened mode) means the + # integrity premise never held, and the gate must degrade to the + # fixed error verdict rather than read attacker-chosen bytes. + GATE_DIR="${RUNNER_TEMP:?}/flake-gate" + # -O is the load-bearing test: "owned by the EFFECTIVE user" is + # root in production, so a directory a PR planted (owned by node) + # fails it, while the same code stays runnable under a harness. + # 0700 then means only that owner can traverse it. + gate_dir_ok() { + [ ! -L "$GATE_DIR" ] || return 1 + [ -d "$GATE_DIR" ] || return 1 + [ -O "$GATE_DIR" ] || return 1 + [ "$(stat -c '%a' "$GATE_DIR" 2>/dev/null)" = '700' ] || return 1 + } + if ! gate_dir_ok; then + echo "flake_verdict=error" >> "$GITHUB_OUTPUT" + echo "flake_summary=the gate working directory is missing or not root-owned 0700 — refusing to read files a PR could have planted" >> "$GITHUB_OUTPUT" + GATE_DONE=1 + exit 0 + fi + LOG="$GATE_DIR/log" + LIST="$GATE_DIR/files" # Per-invocation detail (round lines, failure tails) goes to a # SEPARATE file and is appended after the verdict: the publisher # embeds the FIRST 10,000 chars of the log, so the per-file matrix # and verdict must sit ahead of detail that can outgrow the cap. - DETAIL="$RUNNER_TEMP/flake-gate-detail" + DETAIL="$GATE_DIR/detail" : > "$LOG" : > "$DETAIL" finish() { @@ -3002,7 +3041,12 @@ jobs: [ "$ROUNDS" -ge 2 ] || ROUNDS=2 [ "$ROUNDS" -le 10 ] || ROUNDS=10 - [ -f "$LIST" ] || finish error 'the recorded changed-test list is missing' + # -f AND not -L: inside a 0700 root home a plant is impossible, + # but the check is the cheap half of a defence in depth — a + # future relocation out of the root-only home must not silently + # reintroduce "follow whatever symlink is there". + { [ -f "$LIST" ] && [ ! -L "$LIST" ]; } || + finish error 'the recorded changed-test list is missing or not a regular file' # Partition into per-FILE groups. Every skipped file is logged # with its reason — a silently narrowed gate would read as @@ -3209,7 +3253,7 @@ jobs: rm -rf "$inv_tmp" mkdir -p "$inv_tmp" chown node:node "$inv_tmp" - out="$RUNNER_TEMP/flake-gate-round-out" + out="$GATE_DIR/round-out" ( cd "${group_dirs[$i]}" && timeout -k 30 600 runuser -u node -- \ @@ -3883,15 +3927,16 @@ jobs: echo "agent_verdict=$AGENT_VERDICT" >> "$GITHUB_OUTPUT" echo "verify verdict: $VERDICT agent: ${AGENT_VERDICT:-none} (exit $EXIT_CODE)" >> "$GITHUB_STEP_SUMMARY" - # The gate log's authoritative copy lives root-owned at - # $RUNNER_TEMP/flake-gate.log, outside every agent-writable directory. - # It is staged into verify-results HERE — after the agent exits, from - # an always() root step — as the LAST write to that filename: an early - # agent-step abort cannot lose it, and agent-era PR code (which owns a - # chowned verify-results while it runs) cannot control what is - # uploaded under this name. The publisher pins this exact root-level - # path, so a nested same-named file planted in a collected artifact - # dir cannot shadow it either. + # The gate log's authoritative copy lives root-owned inside the 0700 + # root-only home ($RUNNER_TEMP/flake-gate/log), which no PR-controlled + # process can enter. This step runs AFTER the agent exits, from an + # always() root step, and assembles the upload tree there too: an + # early agent-step abort cannot lose the log, and agent-era PR code + # (which owns the chowned verify-results while it runs) can neither + # control what is uploaded under that name nor rename the tree the + # upload enumerates. The publisher pins the exact root-level path, so + # a same-named file nested in a collected artifact dir cannot shadow + # it either. - name: 'Stage flakiness gate log for upload' if: "always() && steps.pr.outputs.decision == 'run'" # Evidence-copying only, and the verdict outputs are already @@ -3903,45 +3948,52 @@ jobs: continue-on-error: true run: |- set -euo pipefail - if [ -f "${RUNNER_TEMP:?}/flake-gate.log" ]; then - # verify-results was chowned to the build user while PR code ran, - # and node daemons can outlive the agent step: kill leftover - # test-user processes first so nothing races this copy. Never - # OPEN a planted destination either — rm unlinks the entry - # itself, while `cp -f` would open a planted FIFO O_WRONLY and - # block until the job timeout, follow a symlink to rewrite a - # victim, or copy INTO a planted directory so the log vanishes - # from the pinned publisher path. + # BUILD a trusted upload tree; never harden an attacker's. + # $RUNNER_TEMP is uid-1000 mode-755 on this pool and `node` is + # uid 1000, so verify-results — chowned to the build user for the + # agent era — sits in a directory PR code can write: hardening it + # in place always left the ENTRY itself renameable, so a kill-race + # survivor could swap the whole hardened tree for a symlink farm + # that upload-artifact (which follows links) would publish. The + # upload now reads from $RUNNER_TEMP/flake-gate/upload, inside the + # 0700 root-only home: a directory node cannot enter is one whose + # entries it can neither create, unlink, nor rename. + # + # Unconditional, not gated on the log existing: node can unlink + # the log, and that must not skip the kill/rebuild for the report + # the agent wrote. + pkill -KILL -u node 2>/dev/null || true + for _ in 1 2 3; do + [ -n "$(ps -o pid=,stat= -u node 2>/dev/null | awk '$2 !~ /^Z/')" ] || break + sleep 1 pkill -KILL -u node 2>/dev/null || true - for _ in 1 2 3; do - [ -n "$(ps -o pid=,stat= -u node 2>/dev/null | awk '$2 !~ /^Z/')" ] || break - sleep 1 - pkill -KILL -u node 2>/dev/null || true - done - if [ -L "$RUNNER_TEMP/verify-results" ] || - { [ -e "$RUNNER_TEMP/verify-results" ] && [ ! -d "$RUNNER_TEMP/verify-results" ]; }; then - rm -rf -- "$RUNNER_TEMP/verify-results" + done + GATE_DIR="${RUNNER_TEMP:?}/flake-gate" + UPLOAD_DIR="$GATE_DIR/upload" + if [ ! -L "$GATE_DIR" ] && [ -d "$GATE_DIR" ] && [ -O "$GATE_DIR" ] && + [ "$(stat -c '%a' "$GATE_DIR" 2>/dev/null)" = '700' ]; then + rm -rf -- "$UPLOAD_DIR" + install -d -m 0700 -o root -g root "$UPLOAD_DIR" + # Copy only REGULAR files out of the untrusted tree, resolving + # nothing: -type f excludes symlinks/FIFOs/sockets/devices at + # selection time, and `cp -f --no-dereference` never opens a + # link target even if one wins a race between find and cp. + if [ -d "$RUNNER_TEMP/verify-results" ] && [ ! -L "$RUNNER_TEMP/verify-results" ]; then + ( + cd "$RUNNER_TEMP/verify-results" && + find . -type f -exec cp -f --no-dereference --parents {} "$UPLOAD_DIR/" \; + ) 2>/dev/null || true fi - mkdir -p "$RUNNER_TEMP/verify-results" - # Revoke the replant premise outright: the agent era owned this - # directory as the build user. Once root owns the names, even a - # kill-race survivor (setsid daemon, continuous forker — pkill - # scans /proc once) can no longer unlink the staged file or - # replant a symlink between this copy and upload-artifact's - # enumeration, which FOLLOWS links — that window was a - # root-readable-file exfiltration into the public comment. - # Ownership alone is NOT the revoke: chown PRESERVES modes, so - # a tree PR code set 0777 stays world-writable under root - # ownership and a survivor can still create/unlink names in it. - # Strip group/other bits entirely, THEN sweep non-regular - # entries again — anything planted between the agent-step sweep - # and the chown dies here, and after the mode revoke nothing - # can replant before upload enumeration. - chown -R root:root "$RUNNER_TEMP/verify-results" - chmod -R go-rwx "$RUNNER_TEMP/verify-results" - find "$RUNNER_TEMP/verify-results" \( -type l -o -type p -o -type s -o -type b -o -type c \) -delete 2>/dev/null || true - rm -rf -- "$RUNNER_TEMP/verify-results/flake-gate.log" - cp -f "$RUNNER_TEMP/flake-gate.log" "$RUNNER_TEMP/verify-results/flake-gate.log" + # The gate's own log is authoritative and root-owned inside the + # home — it is copied LAST so nothing in the untrusted tree can + # shadow the name the publisher pins. + if [ -f "$GATE_DIR/log" ] && [ ! -L "$GATE_DIR/log" ]; then + cp -f --no-dereference "$GATE_DIR/log" "$UPLOAD_DIR/flake-gate.log" + fi + chown -R root:root "$UPLOAD_DIR" + chmod -R go-rwx "$UPLOAD_DIR" + else + echo "::warning::flake-gate home missing or not root-owned 0700; skipping the trusted upload rebuild." fi - name: 'Upload verify results' @@ -3952,7 +4004,10 @@ jobs: uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # v4.6.2 with: name: 'verify-results-${{ steps.pr.outputs.pr_number }}-${{ github.run_id }}-${{ github.run_attempt }}' - path: '${{ runner.temp }}/verify-results/' + # The rebuilt, root-only tree — not the agent-era directory the + # build user owned. Same inner layout, so the publisher's paths + # are unchanged. + path: '${{ runner.temp }}/flake-gate/upload/' retention-days: 7 - name: 'Clean up runner workspace' From 890b09c7f29321ca5fdc36e933010942a2bbf44a Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot Date: Sun, 16 Aug 2026 10:49:34 +0000 Subject: [PATCH 10/18] fix(ci): close flake-gate TOCTOU, reset-integrity, and verdict-discard routes (#9130) --- .github/scripts/qwen-triage-workflow.test.mjs | 396 ++++++++++++++++-- .github/workflows/qwen-triage.yml | 192 +++++++-- 2 files changed, 518 insertions(+), 70 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index f9d5c3b3294..b5fbbd78b8a 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -935,6 +935,13 @@ describe('qwen-triage: flakiness gate (#9125)', () => { const publishStep = doc.jobs['publish-verify'].steps.find( (s) => s.name === 'Post verification report comment', ); + // Round 11 (R8-35): the runner applies file commands a step's PR code + // wrote to the uid-1000-owned backing files at step end, so a + // root-side block's inherited PATH may be attacker-poisoned; each + // block this PR adds pins a root-only-writable one before resolving + // bare binaries. The EUID gate keeps the harness's stub PATH intact. + const pathPinRe = + /^\s*if \[ "\$\{EUID:-1\}" -eq 0 \]; then\n\s*export PATH='\/usr\/local\/sbin:\/usr\/local\/bin:\/usr\/sbin:\/usr\/bin:\/sbin:\/bin'\n\s*fi$/m; it('records the changed-test list BEFORE the workspace is handed to the build user', () => { assert.ok(recordStep, 'record step must exist'); @@ -1009,6 +1016,11 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /^\s*rm -rf -- "\$\{RUNNER_TEMP:\?\}\/flake-gate"$/m, 'a plant left by an earlier run on the persistent pool must be removed first', ); + assert.match( + recordStep.run, + pathPinRe, + 'the record step must pin a root-only-writable PATH — its inherited one may be poisoned through the file-command backing files', + ); // A grep ERROR (status 2 — e.g. ENOSPC opening the output) is // infrastructure: swallowing it narrows the gate to zero files and // starves it into n/a, so it must fail the record step loudly. @@ -1135,8 +1147,8 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( flakeStep.run, - /^\s*chown node:node "\$inv_tmp"$/m, - 'inv_tmp must be writable by the build user', + /^\s*chown -h node:node "\$inv_tmp"$/m, + 'inv_tmp must be writable by the build user, and -h must never dereference a planted symlink into an ownership takeover of its target', ); assert.match( flakeStep.run, @@ -1171,22 +1183,31 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // cannot satisfy these. // Round-7 hardening of both calls: `-ffd` because plain -fd by // documented git behavior refuses untracked dirs holding a nested - // .git; `checkout HEAD -- .` restores from the root-pinned commit, - // not the index (PR lifecycle code can stage mutations a - // checkout-from-index would preserve); the lane's runner-injection - // strip (git filters run from PR-owned .git metadata as node); a - // timeout wrapper (a planted filter can hang them, and the reset runs - // outside the invocation loop's deadline check); and a failed reset - // fails OPEN to the fixed error verdict — dirty samples carry no - // signal. - const resetCheckoutRe = - /^\s*timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*git checkout HEAD -- \. 2>\/dev\/null \|\| reset_rc=\$\?$/m; + // .git, and the lane's runner-injection strip plus a timeout + // wrapper (a planted filter can hang them, and the reset runs + // outside the invocation loop's deadline check). + // Round 11: the restore is `git reset --hard` to the OID pinned + // before the loop (R4-1: a test can commit mid-invocation and move + // HEAD; restoring from HEAD would make the committed mutation the + // baseline, and a pathspec checkout would keep files the moved HEAD + // added); the reset sanitizes .git's execution vectors first (R4-2: + // a planted smudge filter otherwise runs inside the restore + // itself); it returns its exit code to the callers (R8-9: a + // failure after samples must stop the sampling, not discard the + // verdict); and the kill runs again after the git calls (R8-10: + // they respawn PR-planted filters/hooks as node, and nothing + // node-owned may be alive when root touches $RUNNER_TEMP paths + // afterwards). + const resetRestoreRe = + /^\s*timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*git reset --hard "\$PINNED_OID" 2>\/dev\/null \|\| reset_rc=\$\?$/m; const resetCleanRe = /^\s*timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*git clean -ffd 2>\/dev\/null \|\| reset_rc=\$\?$/m; + const sanitizeRe = + /^\s*timeout -k 10 30 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*bash -c 'rm -f \.git\/info\/attributes; git config --local --list --name-only 2>\/dev\/null \| grep -o "\^filter\\\.\[\^\.\]\*" \| sort -u \| while IFS= read -r s; do git config --local --remove-section "\$s" 2>\/dev\/null \|\| true; done; git config --local --unset core\.fsmonitor 2>\/dev\/null \|\| true; git config --local --unset core\.hooksPath 2>\/dev\/null \|\| true; git config --local --unset core\.attributesFile 2>\/dev\/null \|\| true' \|\| reset_rc=\$\?$/m; assert.match( flakeStep.run, - resetCheckoutRe, - 'the tracked-file restore must run as the build user, stripped, bounded, and from the pinned commit', + resetRestoreRe, + 'the tracked-file restore must run as the build user, stripped, bounded, and from the pinned OID', ); assert.match( flakeStep.run, @@ -1195,28 +1216,62 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( flakeStep.run, - /^\s*if \[ "\$reset_rc" -ne 0 \]; then\n\s*finish error "[^"]*"\n\s*fi$/m, - 'a failed reset must fail open to the fixed error verdict — never sample a dirty tree', + sanitizeRe, + 'the reset must drop .git execution vectors (filters, fsmonitor, hooksPath, attributesFile) BEFORE the restore runs through them', + ); + assert.match( + flakeStep.run, + /^\s*return "\$reset_rc"$/m, + 'reset_round_state must hand its exit code back to the callers — they decide what a failure means', ); - // Kill FIRST (a live daemon can re-dirty the tree after the checkout), - // with SIGKILL + a bounded wait — one-shot SIGTERM races slow-draining - // daemons (round 5), the same reasoning as the agent step's guard. - const resetKill = flakeStep.run.search( - /^\s*pkill -KILL -u node 2>\/dev\/null \|\| true$/m, + assert.match( + flakeStep.run, + /^\s*if \[ "\$samples" -eq 0 \]; then\n\s*finish error "workspace reset failed \(exit \$\{reset_rc\}\) — samples would not start from equivalent state"\n\s*fi$/m, + 'a failed reset before ANY sample must fail open to the fixed error verdict — never sample a dirty tree', ); - const resetCheckout = flakeStep.run.search(resetCheckoutRe); + assert.match( + flakeStep.run, + /^\s*break 2$/m, + 'a failed reset after samples exist must stop the sampling — collected results are honest and must reach classification', + ); + assert.match( + flakeStep.run, + /^\s*reset_round_state \|\| echo "::warning::post-gate workspace reset failed \(exit \$\?\); continuing with the sampled verdict"$/m, + 'the post-loop cleanup must be best-effort — a fully sampled verdict must never be discarded by a cleanup failure', + ); + // Kill FIRST (a live daemon can re-dirty the tree after the + // checkout), with SIGKILL + a bounded wait — one-shot SIGTERM + // races slow-draining daemons (round 5), the same reasoning as the + // agent step's guard — and kill AGAIN after the git calls (round + // 11): they execute PR-planted filters/hooks as node. + const resetKillFn = flakeStep.run.search( + /^\s*kill_node_processes\(\) \{$/m, + ); + const killCallRe = /^\s*kill_node_processes$/gm; + const firstKillCall = killCallRe.exec(flakeStep.run)?.index ?? -1; + const secondKillCall = killCallRe.exec(flakeStep.run)?.index ?? -1; + const sanitizeIdx = flakeStep.run.search(sanitizeRe); + const resetRestore = flakeStep.run.search(resetRestoreRe); + const resetClean = flakeStep.run.search(resetCleanRe); assert.ok( - resetKill !== -1 && resetCheckout !== -1 && resetKill < resetCheckout, - 'the reset must SIGKILL leftover test-user processes BEFORE restoring the tree', + resetKillFn !== -1 && firstKillCall !== -1 && secondKillCall !== -1, + 'the kill must exist as one function, called before AND after the reset git calls', ); - // Line-anchored and position-pinned (round 6): the survivor wait must - // sit between the kill and the restore, not merely exist somewhere. + // Line-anchored and position-pinned (round 6): the survivor wait + // must sit inside the kill, not merely exist somewhere. const resetWait = flakeStep.run.search( /^\s*\[ -n "\$\(ps -o pid=,stat= -u node 2>\/dev\/null \| awk '\$2 !~ \/\^Z\/'\)" \] \|\| break$/m, ); assert.ok( - resetWait !== -1 && resetKill < resetWait && resetWait < resetCheckout, - 'the kill must wait out survivors (zombies disregarded) before the restore', + resetWait !== -1 && resetKillFn < resetWait, + 'the kill must wait out survivors (zombies disregarded)', + ); + assert.ok( + firstKillCall < sanitizeIdx && + sanitizeIdx < resetRestore && + resetRestore < resetClean && + resetClean < secondKillCall, + 'reset order must be: kill, sanitize .git execution vectors, restore from the pinned OID, clean, kill again', ); // And the reset must precede EVERY invocation, not just rounds: // lifecycle scripts (npm ci/build, run as node) mutate the tree @@ -1234,6 +1289,107 @@ describe('qwen-triage: flakiness gate (#9125)', () => { resetCall < invFlush, 'the reset must run inside the round loop, before every invocation', ); + // Round 11 (R4-1): the restore target must be pinned by OID ONCE, + // before the loop — a test can `git commit` mid-invocation (an + // explicit -c identity defeats the fresh-HOME block) and move HEAD; + // restoring from the moved HEAD would make the committed mutation + // the pristine baseline. reset --hard also drops files the moved + // HEAD added, which a pathspec checkout would keep. + const pinnedOidRe = + /^\s*PINNED_OID="\$\(timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY git rev-parse HEAD 2>\/dev\/null\)"$/m; + const pinnedOid = flakeStep.run.search(pinnedOidRe); + assert.ok( + pinnedOid !== -1, + 'the restore target must be pinned by OID before the round loop', + ); + assert.ok( + pinnedOid < loopStart, + 'the OID must be pinned before any sample runs', + ); + assert.match( + flakeStep.run, + /^\s*\[0-9a-f\]\[0-9a-f\]\[0-9a-f\]\[0-9a-f\]\[0-9a-f\]\[0-9a-f\]\[0-9a-f\]\[0-9a-f\]\*\) ;;$/m, + 'the pinned OID must be shape-validated before use', + ); + assert.match( + flakeStep.run, + /^\s*samples=0$/m, + 'the gate must count collected samples — they decide what a later reset failure means', + ); + // Round 11 (R4-3): one reused output path left the PREVIOUS + // invocation's bytes behind when a redirect failed to OPEN (bash + // reports 1 without running the subshell) — every invocation gets + // a unique path, and a never-created output is infrastructure, + // never a test outcome. + const uniqueOut = flakeStep.run.search( + /^\s*out="\$GATE_DIR\/round-out-\$round-\$i"$/m, + ); + assert.ok( + uniqueOut !== -1 && loopStart < uniqueOut, + 'each invocation output must get a unique path — no stale bytes to misread', + ); + const neverCreated = flakeStep.run.search( + /^\s*if \[ ! -e "\$out" \]; then$/m, + ); + const exitClassify = flakeStep.run.search( + /^\s*elif \[ "\$status" -ge 124 \]; then$/m, + ); + assert.ok( + neverCreated !== -1 && exitClassify !== -1 && neverCreated < exitClassify, + 'a never-created output must route to the infra class ahead of the exit-status classifier', + ); + assert.match( + flakeStep.run, + /^\s*rm -f "\$out"$/m, + 'the sample bytes must be reclaimed after classification — ENOSPC is the named hazard of this job', + ); + // Round 11 (R8-1): rename(2) needs write on the PARENT of the + // home, which the uid-1000-writable $RUNNER_TEMP top level grants + // — the 0700 home cannot stop its own entry being swapped after + // the one-time validation. Every later path-based access + // re-verifies the identity recorded at validation time. + assert.match( + flakeStep.run, + /^\s*GATE_HOME_ID="\$\(stat -c '%d:%i' "\$GATE_DIR"\)"$/m, + 'the home identity must be recorded at validation time', + ); + assert.match( + flakeStep.run, + /^\s*gate_home_intact\(\) \{$/m, + 'an identity re-check must guard every later path-based access to the home', + ); + const intactBeforeList = flakeStep.run.search( + /^\s*gate_home_intact \|\|\n\s*finish error 'the gate working directory changed since validation/m, + ); + const listRead = flakeStep.run.search( + /^\s*\{ \[ -f "\$LIST" \] && \[ ! -L "\$LIST" \]; \} \|\|$/m, + ); + assert.ok( + intactBeforeList !== -1 && listRead !== -1 && intactBeforeList < listRead, + 'the recorded list must only be read through an intact home', + ); + const intactInLoop = flakeStep.run.search( + /^\s*gate_home_intact \|\|\n\s*finish error 'the gate working directory changed mid-run/m, + ); + assert.ok( + intactInLoop !== -1 && + loopStart < intactInLoop && + intactInLoop < uniqueOut, + 'every invocation output must open through a re-verified home', + ); + assert.match( + flakeStep.run, + /^\s*if \[ -n "\$\{GATE_HOME_ID:-\}" \] && ! gate_home_intact; then$/m, + 'finish must drop the detail rather than append bytes read through a swapped home', + ); + // Round 11 (R8-35): the gate resolves bare binaries as root; its + // inherited PATH may be poisoned through the file-command backing + // files, so pin a root-only-writable one before the first use. + const gatePathPin = flakeStep.run.search(pathPinRe); + assert.ok( + gatePathPin !== -1 && gatePathPin < firstInvocation, + 'the gate must pin a root-only-writable PATH before its first bare-binary resolution', + ); assert.equal( flakeStep.if, "steps.pr.outputs.decision == 'run' && steps.prepare.outputs.verdict == ''", @@ -1350,6 +1506,23 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); const iChown = sr.search(/^\s*chown -R root:root "\$UPLOAD_DIR"$/m); const iChmod = sr.search(/^\s*chmod -R go-rwx "\$UPLOAD_DIR"$/m); + // Round 11 (R8-36): the guard and the cd re-resolve verify-results; + // a kill-loop survivor owning the uid-1000 parent can swap the + // entry between the two — the opened directory must still BE the + // validated one, or the copy is skipped. + const iVrId = sr.search( + /^\s*vr_id="\$\(stat -c '%d:%i' "\$RUNNER_TEMP\/verify-results" 2>\/dev\/null\)"$/m, + ); + const iVrIntact = sr.search( + /^\s*\[ "\$\(stat -c '%d:%i' \. 2>\/dev\/null\)" = "\$vr_id" \] &&$/m, + ); + // Round 11 (R4-6/R8-7): the pinned log name must be reserved + // before the conditional copy — a planted file must not survive a + // missing authoritative log, and a planted directory must not + // swallow the authoritative file. + const iReserveName = sr.search( + /^\s*rm -rf -- "\$UPLOAD_DIR\/flake-gate\.log"$/m, + ); // The kill must NOT be gated on the log existing: node can unlink the // log, and that must not skip the rebuild for the agent's own report. assert.ok( @@ -1379,7 +1552,10 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ['bounded survivor wait', iWait], ['root-only home integrity check', iHomeCheck], ['fresh 0700 upload dir', iFresh], + ['verify-results identity pin', iVrId], + ['opened-directory identity re-check', iVrIntact], ['regular-file-only copy', iCopyRegular], + ['log-name reservation', iReserveName], ['authoritative log copy', iCopyLog], ['root re-own', iChown], ['mode revoke', iChmod], @@ -1390,11 +1566,19 @@ describe('qwen-triage: flakiness gate (#9125)', () => { iPkill < iWait && iWait < iHomeCheck && iHomeCheck < iFresh && - iFresh < iCopyRegular && - iCopyRegular < iCopyLog && + iFresh < iVrId && + iVrId < iVrIntact && + iVrIntact < iCopyRegular && + iCopyRegular < iReserveName && + iReserveName < iCopyLog && iCopyLog < iChown && iChown < iChmod, - 'staging order must be: kill+wait, home check, fresh dir, regular-file copy, authoritative log last, re-own, mode revoke', + 'staging order must be: kill+wait, home check, fresh dir, identity-pinned copy, reserved log name, authoritative log last, re-own, mode revoke', + ); + assert.match( + sr, + pathPinRe, + 'staging must pin a root-only-writable PATH — its inherited one may be poisoned through the file-command backing files', ); assert.doesNotMatch( agentStep.run, @@ -1625,9 +1809,9 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp } if (git) { // Default on: production always samples a checkout, the gate's - // per-invocation `git checkout HEAD -- .` reset needs a committed - // tree to restore, and a reset that fails on a missing repo would - // fail the gate open to `error`. + // per-invocation `git reset --hard` reset needs a committed tree + // to restore, and a reset that fails on a missing repo would fail + // the gate open to `error`. for (const args of [ ['init', '-q'], ['add', '-A'], @@ -2523,4 +2707,148 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp 'flaky must demote in BOTH languages — deleting either pair or status line has to fail this test', ); }); + + it('a test that commits mid-invocation cannot move the restore baseline (pinned OID)', () => { + // Round 11 (R4-1): the deterministic shape 'pass while the marker + // is absent, then commit the marker' passes round 1, and every + // later reset restores from the moved HEAD — the committed marker + // becomes the pristine baseline (PFFFF -> false flaky). reset + // --hard to the OID pinned before the loop drops it (PPPPP). + const STUB_COMMITTING = [ + '#!/bin/bash', + 'if [ -f marker.txt ]; then', + ' echo "marker survived the reset"', + ' exit 1', + 'fi', + 'echo committed > marker.txt', + 'git add marker.txt', + 'git -c user.name=att -c user.email=att@local commit -qm marker', + 'exit 0', + '', + ].join('\n'); + const { res, outputs, log } = runGate({ + layout: { 'scripts/tests/committing.test.js': '' }, + list: 'scripts/tests/committing.test.js\n', + stubs: { npx: STUB_COMMITTING }, + git: true, + }); + assert.equal(res.status, 0, `gate died under the wrapper: ${res.stderr}`); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match(log, /committing\.test\.js: PPPPP/); + }); + + it('a smudge filter planted in .git cannot rewrite restored content per round', () => { + // Round 11 (R4-2): checkout/clean never touch .git, so a filter + // planted during a round survives every reset and executes inside + // the NEXT reset's own restore — even the same round's pre-reader + // restore runs through it, so the reader sees FILTERED content in + // EVERY round (FFFFF -> bogus consistent-fail, published as + // "deterministic, CI owns that signal"). The reset sanitizes + // .git's execution vectors first (PPPPP). + const STUB_FILTER = [ + '#!/bin/bash', + 'f="${@: -1}"', + 'case "$f" in', + ' ./scripts/tests/plant.test.js)', + ' git config filter.evil.smudge "sed s/pristine/FILTERED/"', + ' mkdir -p .git/info', + ' echo "fixture.txt filter=evil" > .git/info/attributes', + ' echo noise >> fixture.txt', + ' exit 0', + ' ;;', + ' ./scripts/tests/reader.test.js)', + ' if grep -q FILTERED fixture.txt; then', + ' echo "sampled filter-rewritten content"', + ' exit 1', + ' fi', + ' exit 0', + ' ;;', + 'esac', + 'exit 1', + '', + ].join('\n'); + const { res, outputs, log } = runGate({ + layout: { + 'scripts/tests/plant.test.js': '', + 'scripts/tests/reader.test.js': '', + 'fixture.txt': 'pristine\n', + }, + list: 'scripts/tests/plant.test.js\nscripts/tests/reader.test.js\n', + stubs: { npx: STUB_FILTER }, + git: true, + }); + assert.equal(res.status, 0, `gate died under the wrapper: ${res.stderr}`); + assert.equal(outputs.flake_verdict, 'pass'); + assert.match(log, /plant\.test\.js: PPPPP/); + assert.match(log, /reader\.test\.js: PPPPP/); + }); + + it('a reset failure after samples keeps the verdict the rounds earned', () => { + // Round 11 (R8-9): divergence is sampled (P,F), then residue that + // defeats `git clean -ffd` (a chmod-500 dir) fails the next reset. + // The collected results are honest — classification must still + // land the flaky verdict; discarding them publishes `error` and + // the PR escapes its demotion. + const STUB_POISON = [ + '#!/bin/bash', + 'n_file="$FLAKE_SEQ_DIR/.count-poison"', + 'n=$(cat "$n_file" 2>/dev/null || echo 0)', + 'echo $((n+1)) > "$n_file"', + 'if [ "$n" -eq 1 ]; then', + ' mkdir -p poison', + ' touch poison/f', + ' chmod 500 poison', + ' exit 1', + 'fi', + 'exit 0', + '', + ].join('\n'); + const { res, outputs, log } = runGate({ + layout: { 'scripts/tests/poison.test.js': '' }, + list: 'scripts/tests/poison.test.js\n', + stubs: { npx: STUB_POISON }, + git: true, + }); + assert.equal(res.status, 0, `gate died under the wrapper: ${res.stderr}`); + assert.equal(outputs.flake_verdict, 'flaky'); + assert.match(log, /scripts\/tests\/poison\.test\.js: PF$/m); + assert.match(log, /sampling stopped/); + }); + + it('a home swapped mid-run fails closed instead of sampling through the plant', () => { + // Round 11 (R8-1): rename(2) needs write on the PARENT directory — + // the uid-1000 $RUNNER_TEMP top level grants it, so the 0700 home + // cannot stop its own entry being swapped after the one-time + // validation. The stub swaps the home during round 1; the identity + // re-check before the next output open must fail the gate closed + // (-O passes on the harness user, so only the recorded identity + // catches the plant). + const STUB_SWAPPER = [ + '#!/bin/bash', + 'f="${@: -1}"', + 'case "$f" in', + ' ./scripts/tests/swapper.test.js)', + ' if [ ! -e "$RUNNER_TEMP/flake-gate.real" ]; then', + ' mv "$RUNNER_TEMP/flake-gate" "$RUNNER_TEMP/flake-gate.real"', + ' mkdir "$RUNNER_TEMP/flake-gate"', + ' chmod 700 "$RUNNER_TEMP/flake-gate"', + ' fi', + ' exit 0', + ' ;;', + 'esac', + 'exit 0', + '', + ].join('\n'); + const { res, outputs } = runGate({ + layout: { + 'scripts/tests/a.test.js': '', + 'scripts/tests/swapper.test.js': '', + }, + list: 'scripts/tests/a.test.js\nscripts/tests/swapper.test.js\n', + stubs: { npx: STUB_SWAPPER }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'error'); + assert.match(outputs.flake_summary, /working directory changed/); + }); }); diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index dc3cbfd89c1..a4892f7b8cf 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2751,6 +2751,15 @@ jobs: if: "steps.pr.outputs.decision == 'run'" run: |- set -euo pipefail + # Runner file-command backing files under $RUNNER_TEMP are + # uid-1000-owned: a step's PR code can poison the job + # environment (PATH above all) that LATER steps inherit, and + # root-side blocks resolve bare binaries through it. Pin a + # root-only-writable PATH. Production runs this block as root; + # the test harness does not and keeps its stub PATH. + if [ "${EUID:-1}" -eq 0 ]; then + export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + fi # Root-only home for every gate working file. $RUNNER_TEMP itself # is uid 1000 mode 755 on this pool, and `node` in the job's # container is uid 1000 — so PR-controlled code (lifecycle @@ -2971,6 +2980,15 @@ jobs: # gate exists to classify (round-1 sandboxed verify, cells C/D). set -uo pipefail set +e + # Same poisoned-env premise as the record step: the file + # commands a PR step wrote to the uid-1000-owned backing files + # are applied at step end, so this block's inherited PATH may + # be attacker-chosen. Pin a root-only-writable one before any + # bare binary is resolved (production runs as root; the test + # harness does not and keeps its stub PATH). + if [ "${EUID:-1}" -eq 0 ]; then + export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + fi # Abnormal-exit net: `set -u` (or any unforeseen fatal) still # aborts non-zero. Convert that ending to the fixed `error` verdict # and a zero exit, so even a gate implementation bug is @@ -3008,6 +3026,17 @@ jobs: GATE_DONE=1 exit 0 fi + # The validated entry lives in the uid-1000-writable + # $RUNNER_TEMP top level, and rename(2) needs write on the + # PARENT, not the entry: the 0700 home cannot stop its own + # entry being swapped after this validation, and every later + # path-based access re-resolves the path. Record the home's + # identity now; every such access re-verifies it before use. + GATE_HOME_ID="$(stat -c '%d:%i' "$GATE_DIR")" + gate_home_intact() { + gate_dir_ok && + [ "$(stat -c '%d:%i' "$GATE_DIR" 2>/dev/null)" = "$GATE_HOME_ID" ] + } LOG="$GATE_DIR/log" LIST="$GATE_DIR/files" # Per-invocation detail (round lines, failure tails) goes to a @@ -3028,8 +3057,14 @@ jobs: # the matrix/verdict must never be truncated away behind # failure tails (full copy stays in the artifact). if [ -s "$DETAIL" ]; then - printf -- '\n--- per-invocation detail (full copy in the artifact) ---\n' >> "$LOG" - cat "$DETAIL" >> "$LOG" + if [ -n "${GATE_HOME_ID:-}" ] && ! gate_home_intact; then + # A home swapped after the samples ran would feed + # attacker-chosen bytes into the published log. + printf -- '\n(per-invocation detail dropped: the gate working directory changed mid-run)\n' >> "$LOG" + else + printf -- '\n--- per-invocation detail (full copy in the artifact) ---\n' >> "$LOG" + cat "$DETAIL" >> "$LOG" + fi fi echo "Flakiness gate: $1 — $2" >> "$GITHUB_STEP_SUMMARY" GATE_DONE=1 @@ -3045,6 +3080,8 @@ jobs: # but the check is the cheap half of a defence in depth — a # future relocation out of the root-only home must not silently # reintroduce "follow whatever symlink is there". + gate_home_intact || + finish error 'the gate working directory changed since validation — refusing to read files a PR could have planted' { [ -f "$LIST" ] && [ ! -L "$LIST" ]; } || finish error 'the recorded changed-test list is missing or not a regular file' @@ -3187,50 +3224,80 @@ jobs: # timeout budget above accounts for. Divergence needs no uniform # round count: a P and an F for the same group is non-determinism # no matter how many rounds fit the budget. - # Round-state reset — run AS THE BUILD USER, not root: a root - # checkout restores node-mutated tracked files as new root-owned - # inodes that later node rounds cannot write (EACCES), - # manufacturing the very divergence the reset exists to prevent. - # `git clean -ffd` (no -x: gitignored node_modules/dist must - # survive) drops the untracked residue `checkout` never touches - # — lock files, output dirs a test creates — INCLUDING dirs - # holding a nested .git, which plain -fd by documented git - # behavior refuses to remove. `checkout HEAD -- .` restores from - # the root-pinned commit, not the index: PR lifecycle code can - # stage mutations, and restoring from the index would preserve - # them. Both git calls carry the lane's runner-injection strip - # (git filters run from PR-owned .git metadata as node) and a - # timeout wrapper (a planted filter can hang them, and the reset - # runs outside the invocation loop's deadline check). A failed - # reset means samples would not start from equivalent state, so - # the gate fails open to the fixed error verdict instead of - # sampling a dirty tree. - reset_round_state() { - # Kill FIRST, then restore — a live daemon can re-dirty the tree - # after the checkout. One-shot SIGTERM races slow-draining - # daemons, so SIGKILL with a bounded wait, zombies disregarded - # (same reasoning as the agent step's process guard). + kill_node_processes() { + # One-shot SIGTERM races slow-draining daemons, so SIGKILL + # with a bounded wait, zombies disregarded (same reasoning + # as the agent step's process guard). pkill -KILL -u node 2>/dev/null || true for _ in 1 2 3; do [ -n "$(ps -o pid=,stat= -u node 2>/dev/null | awk '$2 !~ /^Z/')" ] || break sleep 1 pkill -KILL -u node 2>/dev/null || true done + } + # Round-state reset — the git calls run AS THE BUILD USER, not + # root: a root checkout restores node-mutated tracked files as + # new root-owned inodes that later node rounds cannot write + # (EACCES), manufacturing the very divergence the reset exists + # to prevent. `git clean -ffd` (no -x: gitignored + # node_modules/dist must survive) drops the untracked residue + # `checkout` never touches — lock files, output dirs a test + # creates — INCLUDING dirs holding a nested .git, which plain + # -fd by documented git behavior refuses to remove. The restore + # is `git reset --hard` to the OID pinned before the round + # loop: a test can `git commit` mid-invocation and move HEAD, + # after which restoring from HEAD would make the committed + # mutation the baseline, and a pathspec checkout would keep + # files the moved HEAD added. All calls carry the lane's + # runner-injection strip and a timeout wrapper (a planted + # filter can hang them, and the reset runs outside the + # invocation loop's deadline check). The calls also execute + # PR-planted git filters/hooks as node, so the kill runs AGAIN + # after them: nothing node-owned may be alive when root touches + # paths in the uid-1000-writable $RUNNER_TEMP afterwards. A + # failed reset fails open to the fixed error verdict while no + # sample exists; once samples are collected it stops the + # sampling instead — collected results are honest, and a later + # cleanup failure must not discard them. + reset_round_state() { + # Kill FIRST, then restore — a live daemon can re-dirty the + # tree after the checkout (the post-git kill covers the + # processes the restore itself re-spawns). + kill_node_processes local reset_rc=0 + # The restore below runs THROUGH the checkout's metadata: a + # smudge filter or fsmonitor hook planted in .git during a + # round survives checkout/clean (they never touch .git) and + # executes inside the next reset's own git calls, rewriting + # restored content per round — manufactured divergence. Drop + # .git's execution vectors as round state BEFORE restoring, + # as the build user (root's git trips the dubious-ownership + # guard); none of these calls runs filters or hooks itself. + timeout -k 10 30 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ + bash -c 'rm -f .git/info/attributes; git config --local --list --name-only 2>/dev/null | grep -o "^filter\.[^.]*" | sort -u | while IFS= read -r s; do git config --local --remove-section "$s" 2>/dev/null || true; done; git config --local --unset core.fsmonitor 2>/dev/null || true; git config --local --unset core.hooksPath 2>/dev/null || true; git config --local --unset core.attributesFile 2>/dev/null || true' || reset_rc=$? timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ - git checkout HEAD -- . 2>/dev/null || reset_rc=$? + git reset --hard "$PINNED_OID" 2>/dev/null || reset_rc=$? timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ git clean -ffd 2>/dev/null || reset_rc=$? - if [ "$reset_rc" -ne 0 ]; then - finish error "workspace reset failed (exit ${reset_rc}) — samples would not start from equivalent state" - fi + kill_node_processes + return "$reset_rc" } + # Pin the restore target ONCE, before any sample (see + # reset_round_state). Read as the build user, stripped — + # root's git would trip the dubious-ownership guard. + PINNED_OID="$(timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY git rev-parse HEAD 2>/dev/null)" + case "$PINNED_OID" in + [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*) ;; + *) finish error 'could not pin the baseline commit for workspace resets' ;; + esac + declare -a results=() deadline=$(( $(date +%s) + 900 )) rounds_done=0 timed_out=false infra_exits=0 + samples=0 round=1 while [ "$round" -le "$ROUNDS" ]; do for i in "${!group_labels[@]}"; do @@ -3246,14 +3313,38 @@ jobs: # lifecycle scripts, npm ci/build run as node, may have # mutated the tree since the list was recorded). reset_round_state + reset_rc=$? + if [ "$reset_rc" -ne 0 ]; then + if [ "$samples" -eq 0 ]; then + finish error "workspace reset failed (exit ${reset_rc}) — samples would not start from equivalent state" + fi + # Samples already collected are honest — a reset failure + # must not discard them (including a computed flaky). + # Stop sampling; classification decides from what + # completed. + printf 'reset failed (exit %s) before round %s · %s: sampling stopped, classifying the collected results\n' "$reset_rc" "$round" "${group_labels[$i]}" >> "$LOG" + break 2 + fi + # Re-verify the home identity before opening the output + # through it: a swapped home would redirect the sample's + # bytes and every read that follows (see GATE_HOME_ID). + gate_home_intact || + finish error 'the gate working directory changed mid-run — refusing to continue' # Fresh per-invocation HOME and temp/cache dirs: samples must # not share dotfile/XDG/cache state or caches any more than # they share the tree or processes. inv_tmp="$RUNNER_TEMP/flake-inv-tmp" rm -rf "$inv_tmp" mkdir -p "$inv_tmp" - chown node:node "$inv_tmp" - out="$GATE_DIR/round-out" + # -h: never dereference — a planted symlink at this fixed + # path in the uid-1000-writable $RUNNER_TEMP must not turn + # the chown into an ownership takeover of its target. + chown -h node:node "$inv_tmp" + # Unique per invocation: if the redirect below fails to + # OPEN (ENOSPC), bash never runs the subshell and reports + # 1 — a reused path would leave the PREVIOUS invocation's + # bytes for the classifier to misread as this one's. + out="$GATE_DIR/round-out-$round-$i" ( cd "${group_dirs[$i]}" && timeout -k 30 600 runuser -u node -- \ @@ -3276,7 +3367,13 @@ jobs: # can only SUPPRESS a demotion the PR could already dodge by # deleting its tests — one-way authority holds. mark='P' - if [ "$status" -ge 124 ]; then + if [ ! -e "$out" ]; then + # The redirect never opened: the invocation did not run. + # The unique path rules out stale bytes; a never-created + # output is infrastructure, never a test outcome. + mark='I' + infra_exits=$((infra_exits + 1)) + elif [ "$status" -ge 124 ]; then mark='I' infra_exits=$((infra_exits + 1)) elif [ "$status" -ne 0 ] && grep -q 'No test files found' "$out"; then @@ -3285,14 +3382,18 @@ jobs: mark='F' fi results[$i]="${results[$i]:-}${mark}" + samples=$((samples + 1)) printf 'round %s · %s: %s (exit %s)\n' "$round" "${group_labels[$i]}" "$mark" "$status" >> "$DETAIL" - if [ "$status" -ne 0 ]; then + if [ "$status" -ne 0 ] && [ -e "$out" ]; then { printf -- '--- output tail · round %s · %s ---\n' "$round" "${group_labels[$i]}" tail -c 8000 "$out" printf '\n' } >> "$DETAIL" fi + # Reclaim the sample's bytes: ENOSPC is the named hazard of + # this job, and the classifier already consumed the mark. + rm -f "$out" done [ "$timed_out" = true ] && break rounds_done="$round" @@ -3300,8 +3401,9 @@ jobs: done # And leave the agent the same clean tree it would have seen # before the gate ran: the last invocation's residue must not be - # handed to the verifier. - reset_round_state + # handed to the verifier. Best-effort: the samples are complete, + # so a cleanup failure must not discard their verdict. + reset_round_state || echo "::warning::post-gate workspace reset failed (exit $?); continuing with the sampled verdict" flaky=0 failing=0 @@ -3948,6 +4050,14 @@ jobs: continue-on-error: true run: |- set -euo pipefail + # Pin a root-only-writable PATH: the job env this step + # inherits may be poisoned through the uid-1000-owned + # file-command backing files (see the record step). Production + # runs as root; the test harness does not and keeps its stub + # PATH. + if [ "${EUID:-1}" -eq 0 ]; then + export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + fi # BUILD a trusted upload tree; never harden an attacker's. # $RUNNER_TEMP is uid-1000 mode-755 on this pool and `node` is # uid 1000, so verify-results — chowned to the build user for the @@ -3979,14 +4089,24 @@ jobs: # selection time, and `cp -f --no-dereference` never opens a # link target even if one wins a race between find and cp. if [ -d "$RUNNER_TEMP/verify-results" ] && [ ! -L "$RUNNER_TEMP/verify-results" ]; then + # The guard above and the cd below re-resolve the path: a + # kill-loop survivor owning the uid-1000 parent can swap + # the entry between the two. The opened directory must + # still be the validated one, or the copy is skipped. + vr_id="$(stat -c '%d:%i' "$RUNNER_TEMP/verify-results" 2>/dev/null)" ( cd "$RUNNER_TEMP/verify-results" && + [ "$(stat -c '%d:%i' . 2>/dev/null)" = "$vr_id" ] && find . -type f -exec cp -f --no-dereference --parents {} "$UPLOAD_DIR/" \; ) 2>/dev/null || true fi # The gate's own log is authoritative and root-owned inside the # home — it is copied LAST so nothing in the untrusted tree can - # shadow the name the publisher pins. + # shadow the name the publisher pins. Reserve the name first: + # when the log is absent (an ENOSPC'd gate) a planted file + # would otherwise survive the copy, and a planted DIRECTORY + # would swallow the authoritative file even when it exists. + rm -rf -- "$UPLOAD_DIR/flake-gate.log" if [ -f "$GATE_DIR/log" ] && [ ! -L "$GATE_DIR/log" ]; then cp -f --no-dereference "$GATE_DIR/log" "$UPLOAD_DIR/flake-gate.log" fi From 3905c8c6e1ea0c14ad5a53fe7ec6d0c436bc5803 Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot Date: Sun, 16 Aug 2026 22:41:27 +0000 Subject: [PATCH 11/18] fix(ci): keep sub-2-round gate stops informational and scrub startup env channels (#9130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reset-failure early stop and a mid-run home swap both reached classification with fewer than two completed rounds — one published `pass` off a single agreeing round, the other discarded samples that already encoded a computed flaky, letting a PR dodge its demotion by renaming the gate home. Both stops now degrade to the informational timeout verdict under two rounds while an observed divergence still demotes, and a swap after samples keeps the collected results. The record/gate/staging blocks also scrubbed only PATH, while the channels consumed at shell/loader startup stayed live: BASH_ENV and the LD_* loader channels are now blanked at step env (with a fail-closed check if the blank loses), BASH_FUNC_* imports are dropped by a one-shot env -i re-exec whose child marker is positional (an env sentinel would be forgeable through the same file-command channel), and the GIT_* family is stripped on the reset and sampling children. Staging additionally validates the run identity stamped by the record step, and a verify-results stat race degrades to a skipped copy instead of aborting before the authoritative log copy. The count pin covered five of the seven stripped node-run commands, which is what failed the scripts lane; the job timeout budget now includes the resets the deadline check runs ahead of (175 -> 190). --- .github/scripts/qwen-triage-workflow.test.mjs | 219 ++++++++++++++++-- .github/workflows/qwen-triage.yml | 200 ++++++++++++++-- scripts/tests/qwen-triage-workflow.test.js | 17 +- 3 files changed, 391 insertions(+), 45 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index b5fbbd78b8a..36318705f7f 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -1016,6 +1016,29 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /^\s*rm -rf -- "\$\{RUNNER_TEMP:\?\}\/flake-gate"$/m, 'a plant left by an earlier run on the persistent pool must be removed first', ); + // Run-freshness marker: a stale-but-genuine home from an earlier + // run on the persistent pool passes every ownership/mode/shape + // check — only this marker lets the always() staging step tell + // runs apart when the record step itself was skipped. + assert.match( + recordStep.run, + /^\s*printf '%s-%s' "\$\{GITHUB_RUN_ID:\?\}" "\$\{GITHUB_RUN_ATTEMPT:\?\}" > "\$RUNNER_TEMP\/flake-gate\/run-id"$/m, + 'the record step must stamp the run identity into the home it creates', + ); + // Startup-channel scrub: BASH_FUNC_* imports are dropped by a + // one-shot env -i re-exec whose child marker is POSITIONAL — an + // env-borne sentinel would be forgeable through the very + // file-command channel the scrub defends against. + assert.match( + recordStep.run, + /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, + 'the record step must re-exec through env -i with a positional child marker', + ); + assert.doesNotMatch( + recordStep.run, + /_FLAKE_CLEAN_REEXEC/, + 'the re-exec child marker must never be an env entry — env markers are forgeable via the file-command channel', + ); assert.match( recordStep.run, pathPinRe, @@ -1106,12 +1129,37 @@ describe('qwen-triage: flakiness gate (#9125)', () => { 'runner-injection files must be invisible to PR test code', ); // Whole-env pin: any future secret added to this step env reaches - // process.env of PR test code, so it must be an explicit test decision. + // process.env of PR test code, so it must be an explicit test + // decision. BASH_ENV/LD_PRELOAD/LD_AUDIT/LD_LIBRARY_PATH are the + // defensive startup-channel blanks (consumed at shell/loader startup, + // before any in-script defence) — blanks, never secrets. assert.deepEqual( Object.keys(flakeStep.env).sort(), - ['FLAKE_ROUNDS', 'GH_TOKEN', 'GITHUB_TOKEN'], + [ + 'BASH_ENV', + 'FLAKE_ROUNDS', + 'GH_TOKEN', + 'GITHUB_TOKEN', + 'LD_AUDIT', + 'LD_LIBRARY_PATH', + 'LD_PRELOAD', + ], 'the gate env must stay tokens-blanked and secret-free', ); + // Startup-channel scrub: BASH_FUNC_* imports are dropped by a + // one-shot env -i re-exec whose child marker is POSITIONAL — an + // env-borne sentinel would be forgeable through the very + // file-command channel the scrub defends against. + assert.match( + flakeStep.run, + /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, + 'the gate must re-exec through env -i with a positional child marker', + ); + assert.doesNotMatch( + flakeStep.run, + /_FLAKE_CLEAN_REEXEC/, + 'the re-exec child marker must never be an env entry — env markers are forgeable via the file-command channel', + ); // Actions merges workflow- and job-level env into every step: the // step-key pin above is only exhaustive while those levels stay empty. assert.equal(doc.env, undefined, 'no workflow-level env may appear'); @@ -1199,11 +1247,11 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // node-owned may be alive when root touches $RUNNER_TEMP paths // afterwards). const resetRestoreRe = - /^\s*timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*git reset --hard "\$PINNED_OID" 2>\/dev\/null \|\| reset_rc=\$\?$/m; + /^\s*timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*-u GIT_DIR -u GIT_WORK_TREE -u GIT_COMMON_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_SHALLOW_FILE -u GIT_EXEC_PATH -u GIT_CONFIG_COUNT -u GIT_CONFIG_PARAMETERS -u GIT_ALLOW_PROTOCOL -u GIT_PROXY_COMMAND -u GIT_SSL_NO_VERIFY -u GIT_SSL_CAINFO -u GIT_ASKPASS -u GIT_SSH -u GIT_SSH_COMMAND \\\n\s*git reset --hard "\$PINNED_OID" 2>\/dev\/null \|\| reset_rc=\$\?$/m; const resetCleanRe = - /^\s*timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*git clean -ffd 2>\/dev\/null \|\| reset_rc=\$\?$/m; + /^\s*timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*-u GIT_DIR -u GIT_WORK_TREE -u GIT_COMMON_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_SHALLOW_FILE -u GIT_EXEC_PATH -u GIT_CONFIG_COUNT -u GIT_CONFIG_PARAMETERS -u GIT_ALLOW_PROTOCOL -u GIT_PROXY_COMMAND -u GIT_SSL_NO_VERIFY -u GIT_SSL_CAINFO -u GIT_ASKPASS -u GIT_SSH -u GIT_SSH_COMMAND \\\n\s*git clean -ffd 2>\/dev\/null \|\| reset_rc=\$\?$/m; const sanitizeRe = - /^\s*timeout -k 10 30 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*bash -c 'rm -f \.git\/info\/attributes; git config --local --list --name-only 2>\/dev\/null \| grep -o "\^filter\\\.\[\^\.\]\*" \| sort -u \| while IFS= read -r s; do git config --local --remove-section "\$s" 2>\/dev\/null \|\| true; done; git config --local --unset core\.fsmonitor 2>\/dev\/null \|\| true; git config --local --unset core\.hooksPath 2>\/dev\/null \|\| true; git config --local --unset core\.attributesFile 2>\/dev\/null \|\| true' \|\| reset_rc=\$\?$/m; + /^\s*timeout -k 10 30 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*-u GIT_DIR -u GIT_WORK_TREE -u GIT_COMMON_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_SHALLOW_FILE -u GIT_EXEC_PATH -u GIT_CONFIG_COUNT -u GIT_CONFIG_PARAMETERS -u GIT_ALLOW_PROTOCOL -u GIT_PROXY_COMMAND -u GIT_SSL_NO_VERIFY -u GIT_SSL_CAINFO -u GIT_ASKPASS -u GIT_SSH -u GIT_SSH_COMMAND \\\n\s*bash -c 'rm -f \.git\/info\/attributes; git config --local --list --name-only 2>\/dev\/null \| grep -o "\^filter\\\.\[\^\.\]\*" \| sort -u \| while IFS= read -r s; do git config --local --remove-section "\$s" 2>\/dev\/null \|\| true; done; git config --local --unset core\.fsmonitor 2>\/dev\/null \|\| true; git config --local --unset core\.hooksPath 2>\/dev\/null \|\| true; git config --local --unset core\.attributesFile 2>\/dev\/null \|\| true' \|\| reset_rc=\$\?$/m; assert.match( flakeStep.run, resetRestoreRe, @@ -1296,7 +1344,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // the pristine baseline. reset --hard also drops files the moved // HEAD added, which a pathspec checkout would keep. const pinnedOidRe = - /^\s*PINNED_OID="\$\(timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY git rev-parse HEAD 2>\/dev\/null\)"$/m; + /^\s*PINNED_OID="\$\(timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \\\n\s*-u GIT_DIR -u GIT_WORK_TREE -u GIT_COMMON_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_SHALLOW_FILE -u GIT_EXEC_PATH -u GIT_CONFIG_COUNT -u GIT_CONFIG_PARAMETERS -u GIT_ALLOW_PROTOCOL -u GIT_PROXY_COMMAND -u GIT_SSL_NO_VERIFY -u GIT_SSL_CAINFO -u GIT_ASKPASS -u GIT_SSH -u GIT_SSH_COMMAND \\\n\s*git rev-parse HEAD 2>\/dev\/null\)"$/m; const pinnedOid = flakeStep.run.search(pinnedOidRe); assert.ok( pinnedOid !== -1, @@ -1369,7 +1417,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { 'the recorded list must only be read through an intact home', ); const intactInLoop = flakeStep.run.search( - /^\s*gate_home_intact \|\|\n\s*finish error 'the gate working directory changed mid-run/m, + /^\s*if ! gate_home_intact; then\n\s*if \[ "\$samples" -eq 0 \]; then\n\s*finish error 'the gate working directory changed mid-run — refusing to continue'\n\s*fi/m, ); assert.ok( intactInLoop !== -1 && @@ -1377,6 +1425,15 @@ describe('qwen-triage: flakiness gate (#9125)', () => { intactInLoop < uniqueOut, 'every invocation output must open through a re-verified home', ); + // A swap AFTER samples exist must stop and classify them, not + // discard them: publishing `error` there would let a PR dodge a + // computed demotion by renaming the home after its first divergent + // sample. + assert.match( + flakeStep.run, + /^\s*printf 'gate home changed mid-run: sampling stopped, classifying the collected results\\n' >> "\$DETAIL"\n\s*break 2$/m, + 'a swapped home after samples must keep the collected results', + ); assert.match( flakeStep.run, /^\s*if \[ -n "\$\{GATE_HOME_ID:-\}" \] && ! gate_home_intact; then$/m, @@ -1495,6 +1552,12 @@ describe('qwen-triage: flakiness gate (#9125)', () => { const iHomeCheck = sr.search( /^\s*if \[ ! -L "\$GATE_DIR" \] && \[ -d "\$GATE_DIR" \] && \[ -O "\$GATE_DIR" \] &&$/m, ); + // The RUN-identity conjunct: ownership/mode/shape all pass on a + // stale-but-genuine home an earlier run left on the persistent + // pool; only the marker the record step stamped separates runs. + const iRunId = sr.search( + /^\s*\[ "\$\(cat "\$GATE_DIR\/run-id" 2>\/dev\/null\)" = "\$\{GITHUB_RUN_ID:\?\}-\$\{GITHUB_RUN_ATTEMPT:\?\}" \]; then$/m, + ); const iFresh = sr.search( /^\s*install -d -m 0700 -o root -g root "\$UPLOAD_DIR"$/m, ); @@ -1510,8 +1573,11 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // a kill-loop survivor owning the uid-1000 parent can swap the // entry between the two — the opened directory must still BE the // validated one, or the copy is skipped. + // || true (round 14): a survivor renaming verify-results between + // the guard and this stat must degrade to a skipped copy, never a + // set -e abort that discards the authoritative gate-log copy. const iVrId = sr.search( - /^\s*vr_id="\$\(stat -c '%d:%i' "\$RUNNER_TEMP\/verify-results" 2>\/dev\/null\)"$/m, + /^\s*vr_id="\$\(stat -c '%d:%i' "\$RUNNER_TEMP\/verify-results" 2>\/dev\/null \|\| true\)"$/m, ); const iVrIntact = sr.search( /^\s*\[ "\$\(stat -c '%d:%i' \. 2>\/dev\/null\)" = "\$vr_id" \] &&$/m, @@ -1551,6 +1617,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ['pkill', iPkill], ['bounded survivor wait', iWait], ['root-only home integrity check', iHomeCheck], + ['run-identity check', iRunId], ['fresh 0700 upload dir', iFresh], ['verify-results identity pin', iVrId], ['opened-directory identity re-check', iVrIntact], @@ -1565,7 +1632,8 @@ describe('qwen-triage: flakiness gate (#9125)', () => { assert.ok( iPkill < iWait && iWait < iHomeCheck && - iHomeCheck < iFresh && + iHomeCheck < iRunId && + iRunId < iFresh && iFresh < iVrId && iVrId < iVrIntact && iVrIntact < iCopyRegular && @@ -1580,6 +1648,18 @@ describe('qwen-triage: flakiness gate (#9125)', () => { pathPinRe, 'staging must pin a root-only-writable PATH — its inherited one may be poisoned through the file-command backing files', ); + // Startup-channel scrub: same one-shot env -i re-exec as the record + // and gate blocks, positional child marker (env markers forgeable). + assert.match( + sr, + /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, + 'staging must re-exec through env -i with a positional child marker', + ); + assert.doesNotMatch( + sr, + /_FLAKE_CLEAN_REEXEC/, + 'the re-exec child marker must never be an env entry — env markers are forgeable via the file-command channel', + ); assert.doesNotMatch( agentStep.run, /flake-gate\.log/, @@ -1671,11 +1751,14 @@ describe('qwen-triage: flakiness gate (#9125)', () => { }); it('the verify job timeout still covers agent + prepare + gate', () => { - // agent 120m + install/build 15m + gate ~25m + misc ~5m — the job limit - // must stay comfortably above the sum or the container is killed mid-run - // and the ship-what-ran path is bypassed (see the budget comment). + // agent 120m + install/build 15m + gate ~40m (the 15m round budget is + // checked BEFORE each reset, so the last invocation drags its reset + // plus its -k 30 600 cap; add the OID pin and the post-gate reset) + // + misc ~5m ≈ 180m — the job limit must stay comfortably above the + // sum or the container is killed mid-run and the ship-what-ran path + // is bypassed (see the budget comment). assert.ok( - verifyJob['timeout-minutes'] >= 170, + verifyJob['timeout-minutes'] >= 190, `timeout-minutes must cover the gate budget (got ${verifyJob['timeout-minutes']})`, ); }); @@ -1765,7 +1848,14 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp const STUB_PS = ['#!/bin/bash', 'exit 0', ''].join('\n'); const scenarioRoot = mkdtempSync(join(tmpdir(), 'flake-behavioral-')); - after(() => rmSync(scenarioRoot, { recursive: true, force: true })); + after(() => { + // STUB_POISON leaves a mode-500 directory rmSync cannot delete + // (force suppresses ENOENT only, not EACCES), which marks the whole + // suite hookFailed and leaks the tree; restore owner permissions + // first. + spawnSync('chmod', ['-R', 'u+rwx', scenarioRoot]); + rmSync(scenarioRoot, { recursive: true, force: true }); + }); const runGate = ({ layout = {}, @@ -2815,14 +2905,48 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.match(log, /sampling stopped/); }); + it('a reset failure before two full rounds carries no flakiness signal either way', () => { + // Round 14 (R11-3/R13-3): the deadline path has always degraded + // sub-2-round sampling to the informational timeout verdict; the + // reset-failure early stop must do the same. Round 1 passes and + // plants residue that defeats `git clean -ffd`, failing the next + // reset — classifying one agreeing round as `pass` would certify + // a ~50% flake that happened to pass its single sample. + const STUB_POISON_PASS = [ + '#!/bin/bash', + 'n_file="$FLAKE_SEQ_DIR/.count-poisonpass"', + 'n=$(cat "$n_file" 2>/dev/null || echo 0)', + 'echo $((n+1)) > "$n_file"', + 'if [ "$n" -eq 0 ]; then', + ' mkdir -p poison', + ' touch poison/f', + ' chmod 500 poison', + 'fi', + 'exit 0', + '', + ].join('\n'); + const { res, outputs, log } = runGate({ + layout: { 'scripts/tests/poisonpass.test.js': '' }, + list: 'scripts/tests/poisonpass.test.js\n', + stubs: { npx: STUB_POISON_PASS }, + git: true, + }); + assert.equal(res.status, 0, `gate died under the wrapper: ${res.stderr}`); + assert.equal(outputs.flake_verdict, 'timeout'); + assert.match(outputs.flake_summary, /no flakiness signal either way/); + assert.match(log, /sampling stopped/); + }); + it('a home swapped mid-run fails closed instead of sampling through the plant', () => { - // Round 11 (R8-1): rename(2) needs write on the PARENT directory — - // the uid-1000 $RUNNER_TEMP top level grants it, so the 0700 home - // cannot stop its own entry being swapped after the one-time - // validation. The stub swaps the home during round 1; the identity - // re-check before the next output open must fail the gate closed - // (-O passes on the harness user, so only the recorded identity - // catches the plant). + // Round 11 (R8-1) + round 14 (R12-1): rename(2) needs write on the + // PARENT directory — the uid-1000 $RUNNER_TEMP top level grants it, + // so the 0700 home cannot stop its own entry being swapped after + // the one-time validation. The stub swaps the home during round 1; + // the identity re-check before the next output open must stop the + // sampling (-O passes on the harness user, so only the recorded + // identity catches the plant). With only agreeing samples collected + // the stop lands the sub-2-round timeout verdict — never the old + // `error`, which discarded honest samples. const STUB_SWAPPER = [ '#!/bin/bash', 'f="${@: -1}"', @@ -2848,7 +2972,56 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp stubs: { npx: STUB_SWAPPER }, }); assert.equal(res.status, 0, res.stderr); - assert.equal(outputs.flake_verdict, 'error'); - assert.match(outputs.flake_summary, /working directory changed/); + assert.equal(outputs.flake_verdict, 'timeout'); + assert.match(outputs.flake_summary, /no flakiness signal either way/); + }); + + it('a home swapped after a computed divergence still demotes', () => { + // Round 14 (R12-1): samples already collected are honest — the + // swap must stop the sampling and classify them, because + // publishing `error` instead lets a PR dodge its demotion by + // renaming the home after the first divergent sample. a.test.js + // diverges F-then-P in rounds 1-2; the home is swapped during + // round 2, after the divergence is already collected. + const STUB_SWAPPER = [ + '#!/bin/bash', + 'f="${@: -1}"', + 'case "$f" in', + ' ./scripts/tests/a.test.js)', + ' n_file="$FLAKE_SEQ_DIR/.count-a"', + ' n=$(cat "$n_file" 2>/dev/null || echo 0)', + ' echo $((n+1)) > "$n_file"', + ' seq="$(cat "$FLAKE_SEQ_DIR/a.test.js" 2>/dev/null || echo P)"', + ' [ "${seq:$((n % ${#seq})):1}" = F ] && exit 1', + ' exit 0', + ' ;;', + ' ./scripts/tests/swapper.test.js)', + ' n_file="$FLAKE_SEQ_DIR/.count-swapper"', + ' n=$(cat "$n_file" 2>/dev/null || echo 0)', + ' echo $((n+1)) > "$n_file"', + ' if [ "$n" -eq 1 ] && [ ! -e "$RUNNER_TEMP/flake-gate.real" ]; then', + ' mv "$RUNNER_TEMP/flake-gate" "$RUNNER_TEMP/flake-gate.real"', + ' mkdir "$RUNNER_TEMP/flake-gate"', + ' chmod 700 "$RUNNER_TEMP/flake-gate"', + ' fi', + ' exit 0', + ' ;;', + 'esac', + 'exit 1', + '', + ].join('\n'); + const { res, outputs, log } = runGate({ + layout: { + 'scripts/tests/a.test.js': '', + 'scripts/tests/swapper.test.js': '', + }, + list: 'scripts/tests/a.test.js\nscripts/tests/swapper.test.js\n', + sequences: { 'a.test.js': 'FP' }, + stubs: { npx: STUB_SWAPPER }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'flaky'); + assert.match(outputs.flake_summary, /returned different results/); + assert.match(log, /a\.test\.js: FP/); }); }); diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index 9ee48d7d486..ab8dd0457ec 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2151,17 +2151,22 @@ jobs: # install + build ~6m measured (run 30284341325: npm ci # 3m00 + build 2m40), budget 15m for a # cold cache or a heavier dependency tree - # flakiness gate ~25m worst case (15m round budget, - # checked before each invocation, plus one - # in-flight invocation capped at 10m) + # flakiness gate ~40m worst case: the 15m round budget + # is checked BEFORE each reset, so the + # last invocation drags its reset (≤~345s: + # sanitize -k 10 30, reset/clean -k 30 120 + # each, bounded kill waits) plus its own + # -k 30 600 cap; add the OID pin (≤150s) + # and the unconditional post-gate reset + # (≤~345s) # resolver/tools, checkout, # pin, upload, cleanup ~5m # ------------------------------------ - # worst case ~165m ⇒ 175 leaves 10m of headroom. + # worst case ~180m ⇒ 190 leaves 10m of headroom. # # Cost of the raise, stated so it is a decision and not a surprise: a # verify run now occupies one ECS slot for up to ~3h instead of 1h. - timeout-minutes: 175 + timeout-minutes: 190 runs-on: ['self-hosted', 'linux', 'x64', 'ecs-qwen'] # The job checks out and executes PR code. Run the steps in a container so # package scripts/builds cannot persist changes in the self-hosted runner's @@ -2763,8 +2768,46 @@ jobs: # execution. - name: 'Record changed test files for the flakiness gate' if: "steps.pr.outputs.decision == 'run'" + # Startup-channel scrub (see the gate step): BASH_ENV and the + # LD_* loader channels are consumed at shell/loader startup, + # before any in-script defence — blank them where the step env + # is assembled. BASH_FUNC_* imports are handled by the re-exec + # below. + env: + BASH_ENV: '' + LD_PRELOAD: '' + LD_AUDIT: '' + LD_LIBRARY_PATH: '' run: |- set -euo pipefail + # Startup-channel scrub, part two: BASH_FUNC_%% env + # entries are imported as shell functions BEFORE this body + # runs, are invisible to ${!BASH_FUNC_@}/`set`, and shadow + # every later resolution — including the PATH pin below, + # because function lookup precedes PATH. No in-script purge is + # safe (an import can shadow the purge builtin itself), so + # re-exec ONCE through an absolute-path env -i child: a + # slash-bearing word never goes through function lookup, and + # -i drops every import. The child marker is a POSITIONAL + # argument, never an env entry — the poisoned file-command + # channel this block defends against can plant any env + # variable, so an env-borne "already re-executed" sentinel + # would be forgeable; argv is set by exec, and the runner + # wrapper passes no arguments after the script path. Mirrors + # the autofix lane's env -i bootstrap; only builtins and + # expansions resolve before this guard. + if [ "${EUID:-1}" -eq 0 ] && { [ -n "${BASH_ENV:-}" ] || [ -n "${LD_PRELOAD:-}" ] || [ -n "${LD_AUDIT:-}" ] || [ -n "${LD_LIBRARY_PATH:-}" ]; }; then + echo "::error::flake-gate record: step-env scrub lost — refusing to run in a poisoned environment" + exit 1 + fi + if [ "${1:-}" != '--flake-clean-child' ]; then + [ -x /usr/bin/env ] || { echo "::error::flake-gate record: clean re-exec unavailable"; exit 1; } + LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ + PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ + GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ + GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ + bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child + fi # Runner file-command backing files under $RUNNER_TEMP are # uid-1000-owned: a step's PR code can poison the job # environment (PATH above all) that LATER steps inherit, and @@ -2791,6 +2834,14 @@ jobs: # earlier run on this persistent pool. rm -rf -- "${RUNNER_TEMP:?}/flake-gate" install -d -m 0700 -o root -g root "$RUNNER_TEMP/flake-gate" + # Run-freshness marker: staging re-validates it before trusting + # the home. A stale-but-genuine home left by an earlier run on + # the persistent pool passes every ownership/mode/shape check — + # only this marker separates runs, so the always() staging step + # can never restage a previous run's evidence under this run's + # artifact name when the record step itself was skipped (a + # cancel between steps.pr and record still runs staging). + printf '%s-%s' "${GITHUB_RUN_ID:?}" "${GITHUB_RUN_ATTEMPT:?}" > "$RUNNER_TEMP/flake-gate/run-id" # Two statements, not a pipeline: `git diff | grep || true` would # swallow a git failure as "no changed test files", silently # narrowing the gate to n/a. A git failure here is pre-build @@ -2981,6 +3032,16 @@ jobs: GITHUB_TOKEN: '' GH_TOKEN: '' FLAKE_ROUNDS: '${{ vars.QWEN_VERIFY_FLAKE_ROUNDS }}' + # Startup-channel scrub (see the re-exec below): BASH_ENV and + # the LD_* loader channels are consumed at shell/loader + # startup, before any in-script defence — blank them where the + # step env is assembled. Step env outranks the poisoned job + # env a PR step wrote to the uid-1000 file-command backing + # files; all four are defensive blanks, never secrets. + BASH_ENV: '' + LD_PRELOAD: '' + LD_AUDIT: '' + LD_LIBRARY_PATH: '' run: |- # Fail OPEN by contract: the gate is advisory-and-demoting only, so # a bug in it must degrade to the fixed `error` verdict — never @@ -2994,6 +3055,48 @@ jobs: # gate exists to classify (round-1 sandboxed verify, cells C/D). set -uo pipefail set +e + # Startup-channel scrub, part one: if the step-env blanks + # above lost to the poisoned job env, BASH_ENV's payload + # already ran at startup and the LD_* channels are live — a + # compromised root shell must not keep producing verdicts. + # EUID-gated so the test harness (which cannot apply this + # step's env block and runs non-root) stays out of it. + if [ "${EUID:-1}" -eq 0 ] && { [ -n "${BASH_ENV:-}" ] || [ -n "${LD_PRELOAD:-}" ] || [ -n "${LD_AUDIT:-}" ] || [ -n "${LD_LIBRARY_PATH:-}" ]; }; then + echo "flake_verdict=error" >> "$GITHUB_OUTPUT" + echo "flake_summary=the gate refused to sample in a poisoned environment — see the step log" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Startup-channel scrub, part two: BASH_FUNC_%% env + # entries are imported as shell functions BEFORE this body + # runs, are invisible to ${!BASH_FUNC_@}/`set`, and shadow + # every later resolution — including the PATH pin below, + # because function lookup precedes PATH. No in-script purge is + # safe (an import can shadow the purge builtin itself), so + # re-exec ONCE through an absolute-path env -i child: a + # slash-bearing word never goes through function lookup, and + # -i drops every import. The child marker is a POSITIONAL + # argument, never an env entry — the poisoned file-command + # channel this block defends against can plant any env + # variable, so an env-borne "already re-executed" sentinel + # would be forgeable; argv is set by exec, and the runner + # wrapper passes no arguments after the script path. Mirrors + # the autofix lane's env -i bootstrap; only builtins and + # expansions resolve before this guard. The gate fails OPEN + # if the re-exec is unavailable, never into a compromised + # shell. + if [ "${1:-}" != '--flake-clean-child' ]; then + if [ ! -x /usr/bin/env ]; then + echo "flake_verdict=error" >> "$GITHUB_OUTPUT" + echo "flake_summary=the gate could not start a clean shell — refusing to sample" >> "$GITHUB_OUTPUT" + exit 0 + fi + LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ + PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ + GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ + GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ + FLAKE_ROUNDS="${FLAKE_ROUNDS:-}" FLAKE_SEQ_DIR="${FLAKE_SEQ_DIR:-}" \ + bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child + fi # Same poisoned-env premise as the record step: the file # commands a PR step wrote to the uid-1000-owned backing files # are applied at step end, so this block's inherited PATH may @@ -3233,9 +3336,11 @@ jobs: printf '\n' } >> "$LOG" - # 15-minute wall budget, checked before every invocation, plus a - # 10-minute cap per invocation — worst case ~25m, which the job - # timeout budget above accounts for. Divergence needs no uniform + # 15-minute wall budget, checked before every invocation; each + # invocation drags its reset (≤~345s) and a -k 30 600 cap, and + # the OID pin plus the unconditional post-gate reset add ≤150s + # and ≤~345s — worst case ~40m, which the job timeout budget + # above accounts for. Divergence needs no uniform # round count: a P and an F for the same group is non-determinism # no matter how many rounds fit the budget. kill_node_processes() { @@ -3288,10 +3393,13 @@ jobs: # as the build user (root's git trips the dubious-ownership # guard); none of these calls runs filters or hooks itself. timeout -k 10 30 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ + -u GIT_DIR -u GIT_WORK_TREE -u GIT_COMMON_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_SHALLOW_FILE -u GIT_EXEC_PATH -u GIT_CONFIG_COUNT -u GIT_CONFIG_PARAMETERS -u GIT_ALLOW_PROTOCOL -u GIT_PROXY_COMMAND -u GIT_SSL_NO_VERIFY -u GIT_SSL_CAINFO -u GIT_ASKPASS -u GIT_SSH -u GIT_SSH_COMMAND \ bash -c 'rm -f .git/info/attributes; git config --local --list --name-only 2>/dev/null | grep -o "^filter\.[^.]*" | sort -u | while IFS= read -r s; do git config --local --remove-section "$s" 2>/dev/null || true; done; git config --local --unset core.fsmonitor 2>/dev/null || true; git config --local --unset core.hooksPath 2>/dev/null || true; git config --local --unset core.attributesFile 2>/dev/null || true' || reset_rc=$? timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ + -u GIT_DIR -u GIT_WORK_TREE -u GIT_COMMON_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_SHALLOW_FILE -u GIT_EXEC_PATH -u GIT_CONFIG_COUNT -u GIT_CONFIG_PARAMETERS -u GIT_ALLOW_PROTOCOL -u GIT_PROXY_COMMAND -u GIT_SSL_NO_VERIFY -u GIT_SSL_CAINFO -u GIT_ASKPASS -u GIT_SSH -u GIT_SSH_COMMAND \ git reset --hard "$PINNED_OID" 2>/dev/null || reset_rc=$? timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ + -u GIT_DIR -u GIT_WORK_TREE -u GIT_COMMON_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_SHALLOW_FILE -u GIT_EXEC_PATH -u GIT_CONFIG_COUNT -u GIT_CONFIG_PARAMETERS -u GIT_ALLOW_PROTOCOL -u GIT_PROXY_COMMAND -u GIT_SSL_NO_VERIFY -u GIT_SSL_CAINFO -u GIT_ASKPASS -u GIT_SSH -u GIT_SSH_COMMAND \ git clean -ffd 2>/dev/null || reset_rc=$? kill_node_processes return "$reset_rc" @@ -3300,7 +3408,9 @@ jobs: # Pin the restore target ONCE, before any sample (see # reset_round_state). Read as the build user, stripped — # root's git would trip the dubious-ownership guard. - PINNED_OID="$(timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY git rev-parse HEAD 2>/dev/null)" + PINNED_OID="$(timeout -k 30 120 runuser -u node -- env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ + -u GIT_DIR -u GIT_WORK_TREE -u GIT_COMMON_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_SHALLOW_FILE -u GIT_EXEC_PATH -u GIT_CONFIG_COUNT -u GIT_CONFIG_PARAMETERS -u GIT_ALLOW_PROTOCOL -u GIT_PROXY_COMMAND -u GIT_SSL_NO_VERIFY -u GIT_SSL_CAINFO -u GIT_ASKPASS -u GIT_SSH -u GIT_SSH_COMMAND \ + git rev-parse HEAD 2>/dev/null)" case "$PINNED_OID" in [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*) ;; *) finish error 'could not pin the baseline commit for workspace resets' ;; @@ -3342,8 +3452,18 @@ jobs: # Re-verify the home identity before opening the output # through it: a swapped home would redirect the sample's # bytes and every read that follows (see GATE_HOME_ID). - gate_home_intact || - finish error 'the gate working directory changed mid-run — refusing to continue' + # Samples already collected are honest (same rule as a + # reset failure): stop and classify them. Publishing + # `error` here instead would let a PR dodge a computed + # demotion by renaming the home after its first divergent + # sample. + if ! gate_home_intact; then + if [ "$samples" -eq 0 ]; then + finish error 'the gate working directory changed mid-run — refusing to continue' + fi + printf 'gate home changed mid-run: sampling stopped, classifying the collected results\n' >> "$DETAIL" + break 2 + fi # Fresh per-invocation HOME and temp/cache dirs: samples must # not share dotfile/XDG/cache state or caches any more than # they share the tree or processes. @@ -3363,6 +3483,7 @@ jobs: cd "${group_dirs[$i]}" && timeout -k 30 600 runuser -u node -- \ env -u GITHUB_OUTPUT -u GITHUB_STATE -u GITHUB_ENV -u GITHUB_PATH -u GITHUB_STEP_SUMMARY \ + -u GIT_DIR -u GIT_WORK_TREE -u GIT_COMMON_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_SHALLOW_FILE -u GIT_EXEC_PATH -u GIT_CONFIG_COUNT -u GIT_CONFIG_PARAMETERS -u GIT_ALLOW_PROTOCOL -u GIT_PROXY_COMMAND -u GIT_SSL_NO_VERIFY -u GIT_SSL_CAINFO -u GIT_ASKPASS -u GIT_SSH -u GIT_SSH_COMMAND \ NODE_OPTIONS='--max-old-space-size=3072' CI=true HOME="$inv_tmp" TMPDIR="$inv_tmp" \ bash -c "${group_cmds[$i]}" ) > "$out" 2>&1 @@ -3444,8 +3565,16 @@ jobs: if [ "$flaky" -gt 0 ]; then finish flaky "${flaky} of ${total} changed test file(s) returned different results across identical re-runs (${rounds_done} full round(s))" fi - if [ "$timed_out" = true ] && [ "$rounds_done" -lt 2 ]; then - finish timeout "the 15-minute budget elapsed before two full rounds completed (${rounds_done} done) — no flakiness signal either way" + if [ "$rounds_done" -lt 2 ]; then + # Classification needs two completed rounds: one round (or + # zero) cannot separate a flake from a deterministic + # outcome. BOTH early stops land here — the deadline, and a + # reset failure with samples already collected (classified + # above first: an observed divergence still demotes). + if [ "$timed_out" = true ]; then + finish timeout "the 15-minute budget elapsed before two full rounds completed (${rounds_done} done) — no flakiness signal either way" + fi + finish timeout "sampling stopped after ${rounds_done} full round(s) — no flakiness signal either way" fi if [ "$infra_exits" -gt 0 ]; then finish timeout "${infra_exits} invocation(s) ended in a timeout/signal exit — infrastructure, not test nondeterminism, so these rounds carry no flakiness signal" @@ -4062,8 +4191,38 @@ jobs: # verdict and report "infrastructure failure" instead. Same rule # the Upload step below documents. continue-on-error: true + # Startup-channel scrub (see the gate step): BASH_ENV and the + # LD_* loader channels are consumed at shell/loader startup, + # before any in-script defence — blank them where the step env + # is assembled. BASH_FUNC_* imports are handled by the re-exec + # below. + env: + BASH_ENV: '' + LD_PRELOAD: '' + LD_AUDIT: '' + LD_LIBRARY_PATH: '' run: |- set -euo pipefail + # Startup-channel scrub (same shape and rationale as the gate + # step's): fail closed if the step-env blanks lost, then + # re-exec once through an absolute-path env -i child to drop + # any BASH_FUNC_* imports before a bare binary resolves. + if [ "${EUID:-1}" -eq 0 ] && { [ -n "${BASH_ENV:-}" ] || [ -n "${LD_PRELOAD:-}" ] || [ -n "${LD_AUDIT:-}" ] || [ -n "${LD_LIBRARY_PATH:-}" ]; }; then + echo "::error::flake-gate staging: step-env scrub lost — refusing to stage evidence in a poisoned environment" + exit 1 + fi + # The child marker is positional, not an env entry: the + # poisoned file-command channel can plant any env variable, so + # an env-borne sentinel would be forgeable (see the record + # step for the full rationale). + if [ "${1:-}" != '--flake-clean-child' ]; then + [ -x /usr/bin/env ] || { echo "::error::flake-gate staging: clean re-exec unavailable"; exit 1; } + LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ + PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ + GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ + GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ + bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child + fi # Pin a root-only-writable PATH: the job env this step # inherits may be poisoned through the uid-1000-owned # file-command backing files (see the record step). Production @@ -4094,8 +4253,15 @@ jobs: done GATE_DIR="${RUNNER_TEMP:?}/flake-gate" UPLOAD_DIR="$GATE_DIR/upload" + # The run-id conjunct is the RUN-identity check: ownership, + # mode and shape all pass on a stale-but-genuine home an + # earlier run left on the persistent pool, and the always() + # staging step runs on exactly the paths where this run's + # record step (the only creator) was skipped — without it a + # previous run's evidence would ship under this run's name. if [ ! -L "$GATE_DIR" ] && [ -d "$GATE_DIR" ] && [ -O "$GATE_DIR" ] && - [ "$(stat -c '%a' "$GATE_DIR" 2>/dev/null)" = '700' ]; then + [ "$(stat -c '%a' "$GATE_DIR" 2>/dev/null)" = '700' ] && + [ "$(cat "$GATE_DIR/run-id" 2>/dev/null)" = "${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}" ]; then rm -rf -- "$UPLOAD_DIR" install -d -m 0700 -o root -g root "$UPLOAD_DIR" # Copy only REGULAR files out of the untrusted tree, resolving @@ -4107,7 +4273,11 @@ jobs: # kill-loop survivor owning the uid-1000 parent can swap # the entry between the two. The opened directory must # still be the validated one, or the copy is skipped. - vr_id="$(stat -c '%d:%i' "$RUNNER_TEMP/verify-results" 2>/dev/null)" + # || true: a survivor renaming verify-results between the + # guard above and this stat must degrade to a skipped copy + # (empty vr_id matches no opened directory), never a set -e + # abort that discards the authoritative gate-log copy below. + vr_id="$(stat -c '%d:%i' "$RUNNER_TEMP/verify-results" 2>/dev/null || true)" ( cd "$RUNNER_TEMP/verify-results" && [ "$(stat -c '%d:%i' . 2>/dev/null)" = "$vr_id" ] && diff --git a/scripts/tests/qwen-triage-workflow.test.js b/scripts/tests/qwen-triage-workflow.test.js index 7a10d9356a1..ff6f0a80d5d 100644 --- a/scripts/tests/qwen-triage-workflow.test.js +++ b/scripts/tests/qwen-triage-workflow.test.js @@ -2143,18 +2143,21 @@ describe('qwen-triage verify hardening', () => { it('strips GitHub command files from every node-run verify command', () => { // Bound to the commands that run as node before the agent: npm ci and // npm run build in the prepare step, the evidence browser download, - // and the flake gate's two reset git invocations (git filters run - // from PR-owned .git metadata). The slice stops at the agent step, - // whose own `runuser` launches qwen under `env -i` and needs no - // per-variable stripping. Covering all five by construction (not - // enumeration) is what catches a future node-run command added - // without the strip. + // and the flake gate's four pre-sample git invocations — the .git + // sanitize, git reset --hard, git clean -ffd, and the PINNED_OID + // rev-parse (git filters run from PR-owned .git metadata). The slice + // stops at the agent step, whose own `runuser` launches qwen under + // `env -i` and needs no per-variable stripping; the gate's + // per-sample invocation is a line-continuation shape this + // single-line match does not fold. Covering all seven by + // construction (not enumeration) is what catches a future node-run + // command added without the strip. const prepare = verifyJob.slice( verifyJob.indexOf('Install and build PR app'), verifyJob.indexOf('Run verification agent'), ); const commands = prepare.match(/runuser -u node -- env[\s\S]*?\n/g) ?? []; - expect(commands.length).toBe(5); + expect(commands.length).toBe(7); expect(step('Run verification agent')).toContain( 'runuser -u node -- env -i', ); From ac694b0364de8ade82849ed1ea422908a1210ff1 Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot Date: Mon, 17 Aug 2026 03:08:19 +0000 Subject: [PATCH 12/18] fix(ci): close gate re-exec startup races and stale-evidence paths (#9130) --- .github/scripts/qwen-triage-workflow.test.mjs | 177 ++++++++++++--- .github/workflows/qwen-triage.yml | 201 +++++++++++++----- 2 files changed, 301 insertions(+), 77 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index 36318705f7f..a2c0c2e5d18 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -1031,8 +1031,8 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // file-command channel the scrub defends against. assert.match( recordStep.run, - /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, - 'the record step must re-exec through env -i with a positional child marker', + /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?\/usr\/bin\/bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, + 'the record step must re-exec through env -i with a positional child marker and an absolute-path bash operand', ); assert.doesNotMatch( recordStep.run, @@ -1152,14 +1152,32 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // file-command channel the scrub defends against. assert.match( flakeStep.run, - /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, - 'the gate must re-exec through env -i with a positional child marker', + /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?\/usr\/bin\/bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, + 'the gate must re-exec through env -i with a positional child marker and an absolute-path bash operand', ); assert.doesNotMatch( flakeStep.run, /_FLAKE_CLEAN_REEXEC/, 'the re-exec child marker must never be an env entry — env markers are forgeable via the file-command channel', ); + // R14-2: the runner writes this script as a node-owned file inside + // the uid-1000-writable $RUNNER_TEMP and the re-exec RE-READS it + // from disk — a detached install/build survivor still alive at the + // re-exec overwrites the file in place and the wrapper executes + // attacker content in its full environment, every in-script defence + // living in the overwritten body. The kill must precede the + // re-exec, absolute-pathed (no PATH pin applies this early) and + // EUID-gated (the harness stays on its stubs). + const gatePreKill = flakeStep.run.search( + /^\s*if \[ "\$\{EUID:-1\}" -eq 0 \]; then\n\s*\/usr\/bin\/pkill -KILL -u node 2>\/dev\/null \|\| true\n\s*for _ in 1 2 3; do\n\s*\[ -n "\$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\)" \] \|\| break\n\s*\/usr\/bin\/sleep 1\n\s*\/usr\/bin\/pkill -KILL -u node 2>\/dev\/null \|\| true\n\s*done\n\s*fi$/m, + ); + const gateReExec = flakeStep.run.search( + /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then/, + ); + assert.ok( + gatePreKill !== -1 && gateReExec !== -1 && gatePreKill < gateReExec, + 'node survivors must be killed BEFORE the re-exec re-reads this script from disk', + ); // Actions merges workflow- and job-level env into every step: the // step-key pin above is only exhaustive while those levels stay empty. assert.equal(doc.env, undefined, 'no workflow-level env may appear'); @@ -1545,9 +1563,12 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // farm for upload-artifact to follow). It BUILDS a trusted tree inside // the 0700 root-only home instead, copying regular files only. const sr = stageStep.run; - const iPkill = sr.search(/^\s*pkill -KILL -u node/m); + const iPkill = sr.search(/^\s*\/usr\/bin\/pkill -KILL -u node/m); const iWait = sr.search( - /^\s*\[ -n "\$\(ps -o pid=,stat= -u node 2>\/dev\/null \| awk '\$2 !~ \/\^Z\/'\)" \] \|\| break$/m, + /^\s*\[ -n "\$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\)" \] \|\| break$/m, + ); + const stageReExec = sr.search( + /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then/, ); const iHomeCheck = sr.search( /^\s*if \[ ! -L "\$GATE_DIR" \] && \[ -d "\$GATE_DIR" \] && \[ -O "\$GATE_DIR" \] &&$/m, @@ -1558,17 +1579,40 @@ describe('qwen-triage: flakiness gate (#9125)', () => { const iRunId = sr.search( /^\s*\[ "\$\(cat "\$GATE_DIR\/run-id" 2>\/dev\/null\)" = "\$\{GITHUB_RUN_ID:\?\}-\$\{GITHUB_RUN_ATTEMPT:\?\}" \]; then$/m, ); - const iFresh = sr.search( - /^\s*install -d -m 0700 -o root -g root "\$UPLOAD_DIR"$/m, + // R12-2: the validated ENTRY lives in the uid-1000-writable + // $RUNNER_TEMP top level, so the rebuild cds into the home once, + // re-stats the opened directory against the validated identity, and + // runs every phase from relative paths anchored to that inode. + const iHomeId = sr.search( + /^\s*home_id="\$\(stat -c '%d:%i' "\$GATE_DIR" 2>\/dev\/null \|\| true\)"$/m, ); + const iHomeCd = sr.search(/^\s*cd "\$GATE_DIR"$/m); + const iHomeIntact = sr.search( + /^\s*\[ "\$\(stat -c '%d:%i' \. 2>\/dev\/null\)" = "\$home_id" \]$/m, + ); + const iFresh = sr.search(/^\s*install -d -m 0700 -o root -g root upload$/m); const iCopyRegular = sr.search( - /^\s*find \. -type f -exec cp -f --no-dereference --parents \{\} "\$UPLOAD_DIR\/" \\;$/m, + /^\s*timeout -k 10 60 find \. -type f -exec cp -f --no-dereference --parents \{\} "\$UPLOAD_DIR\/" \\;$/m, + ); + // R12-2: the per-entry find→cp race can still land a symlink or + // FIFO/socket/device inside the rebuilt tree; the scrub deletes + // every non-regular arrival inside the root-only home before + // upload-artifact (which follows links) can ship it. + const iScrub = sr.search( + /^\s*find upload \\\( -type l -o -type p -o -type s -o -type b -o -type c \\\) -delete$/m, ); const iCopyLog = sr.search( - /^\s*cp -f --no-dereference "\$GATE_DIR\/log" "\$UPLOAD_DIR\/flake-gate\.log"$/m, + /^\s*cp -f --no-dereference log upload\/flake-gate\.log$/m, + ); + const iChown = sr.search(/^\s*chown -R root:root upload$/m); + const iChmod = sr.search(/^\s*chmod -R go-rwx upload$/m); + // R13-21: the always() upload step enumerates the path + // unconditionally, so a home that failed validation must be removed + // — a stale tree left in place ships a previous run's evidence + // under this run's artifact name. + const iStaleRemoval = sr.search( + /^\s*rm -rf -- "\$\{RUNNER_TEMP:\?\}\/flake-gate"$/m, ); - const iChown = sr.search(/^\s*chown -R root:root "\$UPLOAD_DIR"$/m); - const iChmod = sr.search(/^\s*chmod -R go-rwx "\$UPLOAD_DIR"$/m); // Round 11 (R8-36): the guard and the cd re-resolve verify-results; // a kill-loop survivor owning the uid-1000 parent can swap the // entry between the two — the opened directory must still BE the @@ -1586,9 +1630,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // before the conditional copy — a planted file must not survive a // missing authoritative log, and a planted directory must not // swallow the authoritative file. - const iReserveName = sr.search( - /^\s*rm -rf -- "\$UPLOAD_DIR\/flake-gate\.log"$/m, - ); + const iReserveName = sr.search(/^\s*rm -rf -- upload\/flake-gate\.log$/m); // The kill must NOT be gated on the log existing: node can unlink the // log, and that must not skip the rebuild for the agent's own report. assert.ok( @@ -1616,32 +1658,44 @@ describe('qwen-triage: flakiness gate (#9125)', () => { for (const [label, idx] of [ ['pkill', iPkill], ['bounded survivor wait', iWait], + ['clean re-exec guard', stageReExec], ['root-only home integrity check', iHomeCheck], ['run-identity check', iRunId], + ['home identity pin', iHomeId], + ['home cd', iHomeCd], + ['home opened-directory re-check', iHomeIntact], ['fresh 0700 upload dir', iFresh], ['verify-results identity pin', iVrId], ['opened-directory identity re-check', iVrIntact], ['regular-file-only copy', iCopyRegular], + ['non-regular arrival scrub', iScrub], ['log-name reservation', iReserveName], ['authoritative log copy', iCopyLog], ['root re-own', iChown], ['mode revoke', iChmod], + ['stale-tree removal', iStaleRemoval], ]) { assert.ok(idx !== -1, `staging must contain the ${label}`); } assert.ok( iPkill < iWait && - iWait < iHomeCheck && + iWait < stageReExec && + stageReExec < iHomeCheck && iHomeCheck < iRunId && - iRunId < iFresh && + iRunId < iHomeId && + iHomeId < iHomeCd && + iHomeCd < iHomeIntact && + iHomeIntact < iFresh && iFresh < iVrId && iVrId < iVrIntact && iVrIntact < iCopyRegular && - iCopyRegular < iReserveName && + iCopyRegular < iScrub && + iScrub < iReserveName && iReserveName < iCopyLog && iCopyLog < iChown && - iChown < iChmod, - 'staging order must be: kill+wait, home check, fresh dir, identity-pinned copy, reserved log name, authoritative log last, re-own, mode revoke', + iChown < iChmod && + iChmod < iStaleRemoval, + 'staging order must be: kill+wait, re-exec, home check, run identity, home identity pin, fresh dir, identity-pinned copy, scrub, reserved log name, authoritative log last, re-own, mode revoke, stale-tree removal', ); assert.match( sr, @@ -1652,8 +1706,8 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // and gate blocks, positional child marker (env markers forgeable). assert.match( sr, - /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, - 'staging must re-exec through env -i with a positional child marker', + /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?\/usr\/bin\/bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, + 'staging must re-exec through env -i with a positional child marker and an absolute-path bash operand', ); assert.doesNotMatch( sr, @@ -1846,6 +1900,24 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp // reset's wait loop. const STUB_PKILL = ['#!/bin/bash', 'exit 0', ''].join('\n'); const STUB_PS = ['#!/bin/bash', 'exit 0', ''].join('\n'); + // R14-3 model: the invocation's runner binary is a detached survivor — + // while the sample runs it swaps the gate home's ENTRY in the + // uid-1000-writable $RUNNER_TEMP (rename needs write on the parent + // only), then fails the sample. The redirect fd was opened against the + // genuine home; every path-based read the mark cascade does afterwards + // must be re-validated first. + const SWAP_HOME_STUB = [ + '#!/bin/bash', + 'if [ ! -e "$RUNNER_TEMP/.flake-home-swapped" ]; then', + ' touch "$RUNNER_TEMP/.flake-home-swapped"', + ' mv "$RUNNER_TEMP/flake-gate" "$RUNNER_TEMP/flake-gate-stash"', + ' mkdir "$RUNNER_TEMP/flake-gate"', + ' chmod 700 "$RUNNER_TEMP/flake-gate"', + 'fi', + 'echo "stub test failure (home-swap scenario)"', + 'exit 1', + '', + ].join('\n'); const scenarioRoot = mkdtempSync(join(tmpdir(), 'flake-behavioral-')); after(() => { @@ -1897,6 +1969,13 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp for (const [k, v] of Object.entries(sequences)) { writeFileSync(join(seqDir, k), v); } + // An ambient GIT_DIR redirects init/add/commit at the AMBIENT + // repository (and clobbers its index) while ws gets no .git — scrub + // the GIT_* keys the way the gate strips them from its own git + // calls. + const fixtureGitEnv = Object.fromEntries( + Object.entries(process.env).filter(([k]) => !k.startsWith('GIT_')), + ); if (git) { // Default on: production always samples a checkout, the gate's // per-invocation `git reset --hard` reset needs a committed tree @@ -1915,7 +1994,11 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp 'fixture', ], ]) { - const g = spawnSync('git', args, { cwd: ws, encoding: 'utf8' }); + const g = spawnSync('git', args, { + cwd: ws, + encoding: 'utf8', + env: fixtureGitEnv, + }); assert.equal(g.status, 0, `git ${args.join(' ')}: ${g.stderr}`); } } @@ -1942,7 +2025,12 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp writeFileSync(join(rt, 'github-summary'), ''); const env = { ...process.env, - PATH: `${bin}:${process.env.PATH}`, + // Hermetic PATH: the gate's env -i re-exec scrubs the environment + // while the (non-root) harness skips the EUID-gated PATH pin, so + // any env-dependent git wrapper in the ambient PATH (e.g. a shim + // exec'ing a variable the scrub drops) breaks every git-backed + // scenario with misleading `error` verdicts. + PATH: `${bin}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`, RUNNER_TEMP: rt, GITHUB_OUTPUT: out, GITHUB_STEP_SUMMARY: join(rt, 'github-summary'), @@ -2132,6 +2220,47 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.match(log, /a\.test\.js: NPNPN/); }); + it('an N↔F transition is divergence too — the *N*F*|*F*N* arm is behaviorally pinned', () => { + // R14-4: the divergence arm's `*N*F*|*F*N*` half had no behavioral + // fixture — deleting it kept every suite green (mutant run) while a + // file alternating between a real failure and a collection refusal + // (e.g. a flaky test intermittently crashing the runner's own + // collection) silently lost its earned flaky demotion. + const { res, outputs, log } = runGate({ + layout: { 'scripts/tests/a.test.js': '' }, + list: 'scripts/tests/a.test.js\n', + sequences: { 'a.test.js': 'NF' }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'flaky'); + assert.match(log, /a\.test\.js: NFNFN/); + }); + + it('a home swapped mid-invocation is detected after the subshell returns — the mark comes from the exit status, never from swapped bytes', () => { + // R14-3: the cascade after `status=$?` read $out path-based with no + // intact re-check — a survivor swapping the home's entry during the + // invocation erased recorded F marks into I (the redirect fd opened + // against the genuine home, but `[ ! -e "$out" ]` re-resolved + // through the swapped entry), dodging the very demotion the gate + // exists to apply. Honest bound: a compromised home stops sampling — + // the mark comes from the exit status alone and the detection is + // visible; preserving the demotion itself would price a mid-run + // swap as demoting evidence, which stays a design call. + const { res, outputs, log } = runGate({ + layout: { 'scripts/tests/a.test.js': '' }, + list: 'scripts/tests/a.test.js\n', + sequences: { 'a.test.js': 'F' }, + stubs: { npx: SWAP_HOME_STUB, node: SWAP_HOME_STUB }, + }); + assert.equal(res.status, 0, res.stderr); + // Sub-2-round stop: one honest sample cannot separate a flake from + // a deterministic outcome — the verdict stays informational, but + // the mark must be the honest F, never the erased I. + assert.equal(outputs.flake_verdict, 'timeout'); + assert.match(log, /^\s*scripts\/tests\/a\.test\.js: F$/m); + assert.doesNotMatch(log, /^\s*scripts\/tests\/a\.test\.js: I$/m); + }); + it('real divergence still outranks an infra exit in another file', () => { const { res, outputs } = runGate({ layout: UNIT, diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index ab8dd0457ec..5f99356b45d 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2795,7 +2795,10 @@ jobs: # would be forgeable; argv is set by exec, and the runner # wrapper passes no arguments after the script path. Mirrors # the autofix lane's env -i bootstrap; only builtins and - # expansions resolve before this guard. + # expansions resolve before this guard. The bash operand is + # absolute-pathed too: env applies the forwarded environment + # FIRST, then resolves the operand via execvp against its PATH — + # the inherited one, which may already be attacker-chosen. if [ "${EUID:-1}" -eq 0 ] && { [ -n "${BASH_ENV:-}" ] || [ -n "${LD_PRELOAD:-}" ] || [ -n "${LD_AUDIT:-}" ] || [ -n "${LD_LIBRARY_PATH:-}" ]; }; then echo "::error::flake-gate record: step-env scrub lost — refusing to run in a poisoned environment" exit 1 @@ -2806,7 +2809,7 @@ jobs: PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ - bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child + /usr/bin/bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child fi # Runner file-command backing files under $RUNNER_TEMP are # uid-1000-owned: a step's PR code can poison the job @@ -3066,6 +3069,24 @@ jobs: echo "flake_summary=the gate refused to sample in a poisoned environment — see the step log" >> "$GITHUB_OUTPUT" exit 0 fi + # The runner wrote THIS script as a node-owned file inside the + # uid-1000-writable $RUNNER_TEMP, and the re-exec below RE-READS + # it from disk: install/build ran PR lifecycle code as node, so + # a detached survivor still alive when the re-exec opens the + # file overwrites it in place, and the wrapper executes attacker + # content in its full environment — every in-script defence lives + # in the overwritten body. Kill BEFORE the re-exec; absolute- + # pathed (no PATH pin applies this early, and a BASH_FUNC import + # shadows every bare word) and EUID-gated (production runs as + # root; the test harness does not and keeps its stubs). + if [ "${EUID:-1}" -eq 0 ]; then + /usr/bin/pkill -KILL -u node 2>/dev/null || true + for _ in 1 2 3; do + [ -n "$(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/')" ] || break + /usr/bin/sleep 1 + /usr/bin/pkill -KILL -u node 2>/dev/null || true + done + fi # Startup-channel scrub, part two: BASH_FUNC_%% env # entries are imported as shell functions BEFORE this body # runs, are invisible to ${!BASH_FUNC_@}/`set`, and shadow @@ -3081,9 +3102,12 @@ jobs: # would be forgeable; argv is set by exec, and the runner # wrapper passes no arguments after the script path. Mirrors # the autofix lane's env -i bootstrap; only builtins and - # expansions resolve before this guard. The gate fails OPEN - # if the re-exec is unavailable, never into a compromised - # shell. + # expansions resolve before this guard. The bash operand is + # absolute-pathed too: env applies the forwarded environment + # FIRST, then resolves the operand via execvp against its PATH — + # the inherited one, which may already be attacker-chosen. The + # gate fails OPEN if the re-exec is unavailable, never into a + # compromised shell. if [ "${1:-}" != '--flake-clean-child' ]; then if [ ! -x /usr/bin/env ]; then echo "flake_verdict=error" >> "$GITHUB_OUTPUT" @@ -3095,7 +3119,7 @@ jobs: GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ FLAKE_ROUNDS="${FLAKE_ROUNDS:-}" FLAKE_SEQ_DIR="${FLAKE_SEQ_DIR:-}" \ - bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child + /usr/bin/bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child fi # Same poisoned-env premise as the record step: the file # commands a PR step wrote to the uid-1000-owned backing files @@ -3488,6 +3512,30 @@ jobs: bash -c "${group_cmds[$i]}" ) > "$out" 2>&1 status=$? + # Re-verify the home BEFORE the mark cascade re-resolves + # $out: the redirect fd was opened against the validated + # home, but every test below re-resolves the path — a home + # swapped during the invocation would turn the cascade into + # a reader of attacker bytes, and a root-side tail/grep + # through a planted symlink is a bounded arbitrary read. + # The exit status is the one honest signal left: record the + # mark from it alone — N needs output evidence that is no + # longer safely readable, so a non-zero status stays a + # failure — and stop sampling: a compromised home must not + # keep producing marks. + if ! gate_home_intact; then + mark='P' + if [ "$status" -ge 124 ]; then + mark='I' + infra_exits=$((infra_exits + 1)) + elif [ "$status" -ne 0 ]; then + mark='F' + fi + results[$i]="${results[$i]:-}${mark}" + samples=$((samples + 1)) + printf 'gate home changed during round %s · %s (exit %s): mark recorded from the exit status alone, sampling stopped\n' "$round" "${group_labels[$i]}" "$status" >> "$DETAIL" + break 2 + fi # Timeout/signal exits (124-127, 128+N) are infrastructure # events, not test outcomes: 124 is the cap itself, 125-127 # are timeout's own failure modes (it failed, or the runner @@ -4211,17 +4259,37 @@ jobs: echo "::error::flake-gate staging: step-env scrub lost — refusing to stage evidence in a poisoned environment" exit 1 fi + # The runner wrote THIS script as a node-owned file inside the + # uid-1000-writable $RUNNER_TEMP, and the re-exec below RE-READS + # it from disk: the agent era just ran PR-controlled node code, + # so a detached survivor still alive when the re-exec opens the + # file overwrites it in place, and the wrapper executes attacker + # content in its full environment. Kill BEFORE the re-exec — + # unconditional, not gated on the log existing: node can unlink + # the log, and that must not skip the kill/rebuild for the + # report the agent wrote. Absolute-pathed (no PATH pin applies + # this early) and EUID-gated (production runs as root; the test + # harness does not and keeps its stubs). + if [ "${EUID:-1}" -eq 0 ]; then + /usr/bin/pkill -KILL -u node 2>/dev/null || true + for _ in 1 2 3; do + [ -n "$(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/')" ] || break + /usr/bin/sleep 1 + /usr/bin/pkill -KILL -u node 2>/dev/null || true + done + fi # The child marker is positional, not an env entry: the # poisoned file-command channel can plant any env variable, so - # an env-borne sentinel would be forgeable (see the record - # step for the full rationale). + # an env-borne sentinel would be forgeable; the bash operand is + # absolute-pathed for the same reason as the PATH pin (see the + # record step for the full rationale). if [ "${1:-}" != '--flake-clean-child' ]; then [ -x /usr/bin/env ] || { echo "::error::flake-gate staging: clean re-exec unavailable"; exit 1; } LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ - bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child + /usr/bin/bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child fi # Pin a root-only-writable PATH: the job env this step # inherits may be poisoned through the uid-1000-owned @@ -4241,16 +4309,6 @@ jobs: # upload now reads from $RUNNER_TEMP/flake-gate/upload, inside the # 0700 root-only home: a directory node cannot enter is one whose # entries it can neither create, unlink, nor rename. - # - # Unconditional, not gated on the log existing: node can unlink - # the log, and that must not skip the kill/rebuild for the report - # the agent wrote. - pkill -KILL -u node 2>/dev/null || true - for _ in 1 2 3; do - [ -n "$(ps -o pid=,stat= -u node 2>/dev/null | awk '$2 !~ /^Z/')" ] || break - sleep 1 - pkill -KILL -u node 2>/dev/null || true - done GATE_DIR="${RUNNER_TEMP:?}/flake-gate" UPLOAD_DIR="$GATE_DIR/upload" # The run-id conjunct is the RUN-identity check: ownership, @@ -4262,42 +4320,79 @@ jobs: if [ ! -L "$GATE_DIR" ] && [ -d "$GATE_DIR" ] && [ -O "$GATE_DIR" ] && [ "$(stat -c '%a' "$GATE_DIR" 2>/dev/null)" = '700' ] && [ "$(cat "$GATE_DIR/run-id" 2>/dev/null)" = "${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}" ]; then - rm -rf -- "$UPLOAD_DIR" - install -d -m 0700 -o root -g root "$UPLOAD_DIR" - # Copy only REGULAR files out of the untrusted tree, resolving - # nothing: -type f excludes symlinks/FIFOs/sockets/devices at - # selection time, and `cp -f --no-dereference` never opens a - # link target even if one wins a race between find and cp. - if [ -d "$RUNNER_TEMP/verify-results" ] && [ ! -L "$RUNNER_TEMP/verify-results" ]; then - # The guard above and the cd below re-resolve the path: a - # kill-loop survivor owning the uid-1000 parent can swap - # the entry between the two. The opened directory must - # still be the validated one, or the copy is skipped. - # || true: a survivor renaming verify-results between the - # guard above and this stat must degrade to a skipped copy - # (empty vr_id matches no opened directory), never a set -e - # abort that discards the authoritative gate-log copy below. - vr_id="$(stat -c '%d:%i' "$RUNNER_TEMP/verify-results" 2>/dev/null || true)" - ( - cd "$RUNNER_TEMP/verify-results" && - [ "$(stat -c '%d:%i' . 2>/dev/null)" = "$vr_id" ] && - find . -type f -exec cp -f --no-dereference --parents {} "$UPLOAD_DIR/" \; - ) 2>/dev/null || true - fi - # The gate's own log is authoritative and root-owned inside the - # home — it is copied LAST so nothing in the untrusted tree can - # shadow the name the publisher pins. Reserve the name first: - # when the log is absent (an ENOSPC'd gate) a planted file - # would otherwise survive the copy, and a planted DIRECTORY - # would swallow the authoritative file even when it exists. - rm -rf -- "$UPLOAD_DIR/flake-gate.log" - if [ -f "$GATE_DIR/log" ] && [ ! -L "$GATE_DIR/log" ]; then - cp -f --no-dereference "$GATE_DIR/log" "$UPLOAD_DIR/flake-gate.log" - fi - chown -R root:root "$UPLOAD_DIR" - chmod -R go-rwx "$UPLOAD_DIR" + # The validated ENTRY still lives in the uid-1000-writable + # $RUNNER_TEMP top level, and every absolute-path operation + # re-resolves it through the writable parent: cd in ONCE, + # re-stat the opened directory against the validated identity + # (the same discipline as vr_id below), and run every phase + # from relative paths — once cd'd, the phases stay anchored + # to the opened inode no matter how the entry above it is + # swapped. The one exception is the copy target, which must + # cross trees; a race there can only misdirect OUR bytes, + # never inject foreign ones into the anchored tree. + home_id="$(stat -c '%d:%i' "$GATE_DIR" 2>/dev/null || true)" + ( + cd "$GATE_DIR" + [ "$(stat -c '%d:%i' . 2>/dev/null)" = "$home_id" ] + rm -rf -- upload + install -d -m 0700 -o root -g root upload + # Copy only REGULAR files out of the untrusted tree, + # resolving nothing: -type f excludes symlinks/FIFOs/ + # sockets/devices at selection time, and + # `cp -f --no-dereference` never opens a link target even + # if one wins the per-entry race between find and cp. The + # timeout bounds that race's worst arrival — a file + # swapped for a FIFO between lstat and open blocks cp + # forever and would hang this otherwise timeout-less step. + if [ -d "$RUNNER_TEMP/verify-results" ] && [ ! -L "$RUNNER_TEMP/verify-results" ]; then + # The guard above and the cd below re-resolve the path: + # a kill-loop survivor owning the uid-1000 parent can + # swap the entry between the two. The opened directory + # must still be the validated one, or the copy is + # skipped. || true: a survivor renaming verify-results + # between the guard above and this stat must degrade to + # a skipped copy (empty vr_id matches no opened + # directory), never a set -e abort that discards the + # authoritative gate-log copy below. + vr_id="$(stat -c '%d:%i' "$RUNNER_TEMP/verify-results" 2>/dev/null || true)" + ( + cd "$RUNNER_TEMP/verify-results" && + [ "$(stat -c '%d:%i' . 2>/dev/null)" = "$vr_id" ] && + timeout -k 10 60 find . -type f -exec cp -f --no-dereference --parents {} "$UPLOAD_DIR/" \; + ) 2>/dev/null || true + fi + # Scrub the rebuilt tree INSIDE the root-only home, where + # a survivor cannot re-enter: the per-entry find→cp race + # can still land a symlink (find lstat'd a regular file, + # cp saw the swapped link and --no-dereference copied the + # link itself) or a FIFO/socket/device — and + # upload-artifact follows links, so any non-regular + # arrival is a root-readable-content primitive into the + # public artifact and must not ship. + find upload \( -type l -o -type p -o -type s -o -type b -o -type c \) -delete + # The gate's own log is authoritative and root-owned inside + # the home — it is copied LAST so nothing in the untrusted + # tree can shadow the name the publisher pins. Reserve the + # name first: when the log is absent (an ENOSPC'd gate) a + # planted file would otherwise survive the copy, and a + # planted DIRECTORY would swallow the authoritative file + # even when it exists. + rm -rf -- upload/flake-gate.log + if [ -f log ] && [ ! -L log ]; then + cp -f --no-dereference log upload/flake-gate.log + fi + chown -R root:root upload + chmod -R go-rwx upload + ) else echo "::warning::flake-gate home missing or not root-owned 0700; skipping the trusted upload rebuild." + # Remove the rejected tree anyway: the always() upload step + # below enumerates this path unconditionally, so a stale tree + # left in place ships a PREVIOUS run's evidence under this + # run's artifact name — the exact outcome the run-id conjunct + # above exists to prevent. rm -rf removes a symlink operand + # itself, never following it. + rm -rf -- "${RUNNER_TEMP:?}/flake-gate" fi - name: 'Upload verify results' From 1afadec3aad5fb8b45254b01df6913274b644c27 Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot Date: Mon, 17 Aug 2026 10:27:48 +0000 Subject: [PATCH 13/18] fix(ci): harden flake-gate startup decisions and close staging swap windows (#9130) Co-authored-by: Qwen-Coder --- .github/scripts/qwen-triage-workflow.test.mjs | 459 ++++++++++++++++-- .github/workflows/qwen-triage.yml | 397 +++++++++------ 2 files changed, 682 insertions(+), 174 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index a2c0c2e5d18..d23af7f5ecd 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -9,6 +9,7 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; import { chmodSync, + existsSync, mkdirSync, mkdtempSync, readFileSync, @@ -941,7 +942,20 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // block this PR adds pins a root-only-writable one before resolving // bare binaries. The EUID gate keeps the harness's stub PATH intact. const pathPinRe = - /^\s*if \[ "\$\{EUID:-1\}" -eq 0 \]; then\n\s*export PATH='\/usr\/local\/sbin:\/usr\/local\/bin:\/usr\/sbin:\/usr\/bin:\/sbin:\/bin'\n\s*fi$/m; + /^\s*if \[\[ \$\{EUID:-1\} -eq 0 \]\]; then\n\s*export PATH='\/usr\/local\/sbin:\/usr\/local\/bin:\/usr\/sbin:\/usr\/bin:\/sbin:\/bin'\n\s*fi$/m; + // R15-1: the pre-exec decisions are reserved words — bash imports a + // BASH_FUNC_[%% env entry as a FUNCTION named `[`, and function lookup + // precedes builtins, so a `[`-shaped guard is itself hijackable + // (probe-verified on this pool's bash). R14-2: the child runs the body + // the parent snapshotted — the runner wrote the script node-owned in + // the uid-1000-writable $RUNNER_TEMP, so a second open by the child is + // a plant window. + const scrubRefusalRe = + /^\s*if \[\[ \$\{EUID:-1\} -eq 0 \]\] && \[\[ -n \$\{BASH_ENV:-\} \|\| -n \$\{LD_PRELOAD:-\} \|\| -n \$\{LD_AUDIT:-\} \|\| -n \$\{LD_LIBRARY_PATH:-\} \]\]; then$/m; + const reExecRe = + /case "\$\{1:-\}" in[\s\S]*?--flake-clean-child\) ;;[\s\S]*?_flake_body="\$\(<"\$\{BASH_SOURCE\[0\]\}"\)"[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?\/usr\/bin\/bash --noprofile --norc -e -o pipefail -c "\$_flake_body" \S+ --flake-clean-child/; + const reExecMarkerRe = /case "\$\{1:-\}" in/; + const pathChildRe = /"\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/; it('records the changed-test list BEFORE the workspace is handed to the build user', () => { assert.ok(recordStep, 'record step must exist'); @@ -1031,8 +1045,29 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // file-command channel the scrub defends against. assert.match( recordStep.run, - /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?\/usr\/bin\/bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, - 'the record step must re-exec through env -i with a positional child marker and an absolute-path bash operand', + reExecRe, + 'the record step must re-exec through env -i with a positional child marker, an absolute-path bash operand, and the parent-snapshotted body', + ); + assert.doesNotMatch( + recordStep.run, + pathChildRe, + 'the re-exec child must never re-open the script by path — the second open is the plant window', + ); + assert.match( + recordStep.run, + scrubRefusalRe, + 'the record step scrub refusal must be reserved-word shaped — `[` is shadowable by a BASH_FUNC import', + ); + assert.match( + recordStep.run, + /survivors="\$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\)" \|\| true\n\s*if \[\[ -n \$survivors \]\]; then\n\s*echo "::error::flake-gate record:[\s\S]*?exit 1/, + 'the record step kill must be liveness-verified — the budget loop alone is out-forked between sweeps', + ); + const recordKill = recordStep.run.search(/survivors="\$\(\/usr\/bin\/ps/); + const recordReExec = recordStep.run.search(reExecMarkerRe); + assert.ok( + recordKill !== -1 && recordReExec !== -1 && recordKill < recordReExec, + 'node survivors must be killed BEFORE the re-exec snapshot re-reads this script from disk', ); assert.doesNotMatch( recordStep.run, @@ -1152,31 +1187,41 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // file-command channel the scrub defends against. assert.match( flakeStep.run, - /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?\/usr\/bin\/bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, - 'the gate must re-exec through env -i with a positional child marker and an absolute-path bash operand', + reExecRe, + 'the gate must re-exec through env -i with a positional child marker, an absolute-path bash operand, and the parent-snapshotted body', + ); + assert.doesNotMatch( + flakeStep.run, + pathChildRe, + 'the re-exec child must never re-open the script by path — the second open is the plant window', ); assert.doesNotMatch( flakeStep.run, /_FLAKE_CLEAN_REEXEC/, 'the re-exec child marker must never be an env entry — env markers are forgeable via the file-command channel', ); - // R14-2: the runner writes this script as a node-owned file inside - // the uid-1000-writable $RUNNER_TEMP and the re-exec RE-READS it - // from disk — a detached install/build survivor still alive at the - // re-exec overwrites the file in place and the wrapper executes - // attacker content in its full environment, every in-script defence - // living in the overwritten body. The kill must precede the - // re-exec, absolute-pathed (no PATH pin applies this early) and - // EUID-gated (the harness stays on its stubs). + assert.match( + flakeStep.run, + scrubRefusalRe, + 'the gate scrub refusal must be reserved-word shaped — `[` is shadowable by a BASH_FUNC import', + ); + // R14-2/R12-2: the runner writes this script as a node-owned file + // inside the uid-1000-writable $RUNNER_TEMP — a detached + // install/build survivor still alive at the re-exec snapshot + // overwrites the file in place and the wrapper executes attacker + // content in its full environment. The kill must precede the + // snapshot, absolute-pathed (no PATH pin applies this early), + // EUID-gated (the harness stays on its stubs), and liveness- + // verified with a fail-open refusal (the agent-step guard shape): + // the budget loop alone is out-forked by a plant repopulating + // between sweeps. const gatePreKill = flakeStep.run.search( - /^\s*if \[ "\$\{EUID:-1\}" -eq 0 \]; then\n\s*\/usr\/bin\/pkill -KILL -u node 2>\/dev\/null \|\| true\n\s*for _ in 1 2 3; do\n\s*\[ -n "\$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\)" \] \|\| break\n\s*\/usr\/bin\/sleep 1\n\s*\/usr\/bin\/pkill -KILL -u node 2>\/dev\/null \|\| true\n\s*done\n\s*fi$/m, - ); - const gateReExec = flakeStep.run.search( - /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then/, + /^\s*if \[\[ \$\{EUID:-1\} -eq 0 \]\]; then\n\s*\/usr\/bin\/pkill -KILL -u node 2>\/dev\/null \|\| true\n\s*for _ in 1 2 3; do\n\s*\[\[ -n \$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\) \]\] \|\| break\n\s*\/usr\/bin\/sleep 1\n\s*\/usr\/bin\/pkill -KILL -u node 2>\/dev\/null \|\| true\n\s*done\n\s*survivors="\$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\)" \|\| true\n\s*if \[\[ -n \$survivors \]\]; then\n\s*echo "flake_verdict=error" >> "\$GITHUB_OUTPUT"\n\s*echo "flake_summary=node-owned processes survived SIGKILL — the gate refused to sample" >> "\$GITHUB_OUTPUT"\n\s*exit 0\n\s*fi\n\s*fi$/m, ); + const gateReExec = flakeStep.run.search(reExecMarkerRe); assert.ok( gatePreKill !== -1 && gateReExec !== -1 && gatePreKill < gateReExec, - 'node survivors must be killed BEFORE the re-exec re-reads this script from disk', + 'node survivors must be killed (and their absence verified) BEFORE the re-exec snapshot re-reads this script from disk', ); // Actions merges workflow- and job-level env into every step: the // step-key pin above is only exhaustive while those levels stay empty. @@ -1434,6 +1479,13 @@ describe('qwen-triage: flakiness gate (#9125)', () => { intactBeforeList !== -1 && listRead !== -1 && intactBeforeList < listRead, 'the recorded list must only be read through an intact home', ); + // R12-2 entrance 5: `: >` opens with O_TRUNC through any symlink — + // the truncate must never run ahead of the first identity re-check. + const truncLog = flakeStep.run.search(/^\s*: > "\$LOG"$/m); + assert.ok( + intactBeforeList !== -1 && truncLog !== -1 && intactBeforeList < truncLog, + 'the log truncate must run only through the verified home', + ); const intactInLoop = flakeStep.run.search( /^\s*if ! gate_home_intact; then\n\s*if \[ "\$samples" -eq 0 \]; then\n\s*finish error 'the gate working directory changed mid-run — refusing to continue'\n\s*fi/m, ); @@ -1454,8 +1506,8 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( flakeStep.run, - /^\s*if \[ -n "\$\{GATE_HOME_ID:-\}" \] && ! gate_home_intact; then$/m, - 'finish must drop the detail rather than append bytes read through a swapped home', + /^\s*if gate_home_intact; then\n\s*printf '\\nverdict: %s\\nsummary: %s\\n' "\$1" "\$2" >> "\$LOG"$/m, + 'finish must write the verdict to the log only through a home re-verified at call time — a swapped home must not receive the bytes through a planted `log` symlink', ); // Round 11 (R8-35): the gate resolves bare binaries as root; its // inherited PATH may be poisoned through the file-command backing @@ -1531,6 +1583,44 @@ describe('qwen-triage: flakiness gate (#9125)', () => { '${{ runner.temp }}/flake-gate/upload/', 'the artifact must ship the REBUILT tree, never the agent-era directory', ); + // R12-2 entrance 3: staging's anchoring expires at its exit, and the + // upload re-resolves the path in a later step — a re-check step must + // re-validate the entry identity immediately before the enumeration + // and gate the upload on it. + const recheckStep = verifyJob.steps.find( + (s) => s.id === 'flake-upload-check', + ); + assert.ok(recheckStep, 'the pre-upload re-check step must exist'); + assert.equal( + recheckStep.if, + "always() && steps.pr.outputs.decision == 'run'", + 'the re-check must run whenever staging can', + ); + assert.equal( + recheckStep['continue-on-error'], + true, + 'the re-check must not be able to fail the job', + ); + const recheckIdx = verifyJob.steps.indexOf(recheckStep); + assert.ok( + stageIdx < recheckIdx && recheckIdx < uploadIdx, + 'the re-check must sit between staging and the upload', + ); + assert.equal( + uploadStep.if, + "always() && steps.pr.outputs.decision == 'run' && steps.flake-upload-check.outputs.upload_ok == 'true'", + 'the upload must only enumerate a home the re-check step just validated', + ); + assert.match( + recheckStep.run, + /upload_ok=true/, + 'the re-check must publish its decision as a step output', + ); + assert.match( + recheckStep.run, + /\/usr\/bin\/rm -rf -- "\$\{RUNNER_TEMP:\?\}\/flake-gate"/, + 'a home that fails the re-check must be removed before the upload enumerates the path', + ); assert.match( uploadStep.with.name, /^verify-results-/, @@ -1564,20 +1654,27 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // the 0700 root-only home instead, copying regular files only. const sr = stageStep.run; const iPkill = sr.search(/^\s*\/usr\/bin\/pkill -KILL -u node/m); + // R12-2 entrance 1: one cleanup for both refusals — a detected + // mid-staging swap must be removed, not merely aborted on. + const iStagedOk = sr.search(/^\s*staged_ok=''$/m); const iWait = sr.search( - /^\s*\[ -n "\$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\)" \] \|\| break$/m, + /^\s*\[\[ -n \$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\) \]\] \|\| break$/m, ); - const stageReExec = sr.search( - /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then/, + // R12-2 entrance 4: the budget loop alone is out-forked by a plant + // repopulating between sweeps — the kill ends in a liveness check + // and a fail-closed refusal matching the agent-step guard. + const iSurvivors = sr.search( + /^\s*survivors="\$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\)" \|\| true$/m, ); + const stageReExec = sr.search(reExecMarkerRe); const iHomeCheck = sr.search( - /^\s*if \[ ! -L "\$GATE_DIR" \] && \[ -d "\$GATE_DIR" \] && \[ -O "\$GATE_DIR" \] &&$/m, + /^\s*if \[\[ ! -L \$GATE_DIR \]\] && \[\[ -d \$GATE_DIR \]\] && \[\[ -O \$GATE_DIR \]\] &&$/m, ); // The RUN-identity conjunct: ownership/mode/shape all pass on a // stale-but-genuine home an earlier run left on the persistent // pool; only the marker the record step stamped separates runs. const iRunId = sr.search( - /^\s*\[ "\$\(cat "\$GATE_DIR\/run-id" 2>\/dev\/null\)" = "\$\{GITHUB_RUN_ID:\?\}-\$\{GITHUB_RUN_ATTEMPT:\?\}" \]; then$/m, + /^\s*\[\[ \$\(cat "\$GATE_DIR\/run-id" 2>\/dev\/null\) == "\$\{GITHUB_RUN_ID:\?\}-\$\{GITHUB_RUN_ATTEMPT:\?\}" \]\]; then$/m, ); // R12-2: the validated ENTRY lives in the uid-1000-writable // $RUNNER_TEMP top level, so the rebuild cds into the home once, @@ -1586,11 +1683,27 @@ describe('qwen-triage: flakiness gate (#9125)', () => { const iHomeId = sr.search( /^\s*home_id="\$\(stat -c '%d:%i' "\$GATE_DIR" 2>\/dev\/null \|\| true\)"$/m, ); - const iHomeCd = sr.search(/^\s*cd "\$GATE_DIR"$/m); + const iHomeCd = sr.search(/^\s*cd "\$GATE_DIR" \|\| exit 1$/m); const iHomeIntact = sr.search( - /^\s*\[ "\$\(stat -c '%d:%i' \. 2>\/dev\/null\)" = "\$home_id" \]$/m, + /^\s*\[ "\$\(stat -c '%d:%i' \. 2>\/dev\/null\)" = "\$home_id" \] \|\| exit 1$/m, + ); + // R12-2 entrance 2: the outer conjuncts are six separate path + // resolutions a swap can thread — the attribute half must re-run + // against the OPENED directory. + // The guards are explicit `|| exit 1`, not bare statements: the + // subshell is an `if` condition, where bash suppresses errexit — + // an unguarded failing check would fall through into the copy + // phases instead of refusing. + const iInnerOwner = sr.search(/^\s*\[ -O \. \] \|\| exit 1$/m); + const iInnerMode = sr.search( + /^\s*\[ "\$\(stat -c '%a' \. 2>\/dev\/null\)" = '700' \] \|\| exit 1$/m, + ); + const iInnerRunId = sr.search( + /^\s*\[ "\$\(cat \.\/run-id 2>\/dev\/null\)" = "\$\{GITHUB_RUN_ID:\?\}-\$\{GITHUB_RUN_ATTEMPT:\?\}" \] \|\| exit 1$/m, + ); + const iFresh = sr.search( + /^\s*install -d -m 0700 -o root -g root upload \|\| exit 1$/m, ); - const iFresh = sr.search(/^\s*install -d -m 0700 -o root -g root upload$/m); const iCopyRegular = sr.search( /^\s*timeout -k 10 60 find \. -type f -exec cp -f --no-dereference --parents \{\} "\$UPLOAD_DIR\/" \\;$/m, ); @@ -1599,13 +1712,13 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // every non-regular arrival inside the root-only home before // upload-artifact (which follows links) can ship it. const iScrub = sr.search( - /^\s*find upload \\\( -type l -o -type p -o -type s -o -type b -o -type c \\\) -delete$/m, + /^\s*find upload \\\( -type l -o -type p -o -type s -o -type b -o -type c \\\) -delete \|\| exit 1$/m, ); const iCopyLog = sr.search( - /^\s*cp -f --no-dereference log upload\/flake-gate\.log$/m, + /^\s*cp -f --no-dereference log upload\/flake-gate\.log \|\| exit 1$/m, ); - const iChown = sr.search(/^\s*chown -R root:root upload$/m); - const iChmod = sr.search(/^\s*chmod -R go-rwx upload$/m); + const iChown = sr.search(/^\s*chown -R root:root upload \|\| exit 1$/m); + const iChmod = sr.search(/^\s*chmod -R go-rwx upload \|\| exit 1$/m); // R13-21: the always() upload step enumerates the path // unconditionally, so a home that failed validation must be removed // — a stale tree left in place ships a previous run's evidence @@ -1630,7 +1743,9 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // before the conditional copy — a planted file must not survive a // missing authoritative log, and a planted directory must not // swallow the authoritative file. - const iReserveName = sr.search(/^\s*rm -rf -- upload\/flake-gate\.log$/m); + const iReserveName = sr.search( + /^\s*rm -rf -- upload\/flake-gate\.log \|\| exit 1$/m, + ); // The kill must NOT be gated on the log existing: node can unlink the // log, and that must not skip the rebuild for the agent's own report. assert.ok( @@ -1658,12 +1773,17 @@ describe('qwen-triage: flakiness gate (#9125)', () => { for (const [label, idx] of [ ['pkill', iPkill], ['bounded survivor wait', iWait], + ['liveness refusal', iSurvivors], ['clean re-exec guard', stageReExec], + ['staging outcome flag', iStagedOk], ['root-only home integrity check', iHomeCheck], ['run-identity check', iRunId], ['home identity pin', iHomeId], ['home cd', iHomeCd], ['home opened-directory re-check', iHomeIntact], + ['opened-directory owner re-check', iInnerOwner], + ['opened-directory mode re-check', iInnerMode], + ['opened-directory run-identity re-check', iInnerRunId], ['fresh 0700 upload dir', iFresh], ['verify-results identity pin', iVrId], ['opened-directory identity re-check', iVrIntact], @@ -1679,13 +1799,18 @@ describe('qwen-triage: flakiness gate (#9125)', () => { } assert.ok( iPkill < iWait && - iWait < stageReExec && - stageReExec < iHomeCheck && + iWait < iSurvivors && + iSurvivors < stageReExec && + stageReExec < iStagedOk && + iStagedOk < iHomeCheck && iHomeCheck < iRunId && iRunId < iHomeId && iHomeId < iHomeCd && iHomeCd < iHomeIntact && - iHomeIntact < iFresh && + iHomeIntact < iInnerOwner && + iInnerOwner < iInnerMode && + iInnerMode < iInnerRunId && + iInnerRunId < iFresh && iFresh < iVrId && iVrId < iVrIntact && iVrIntact < iCopyRegular && @@ -1695,7 +1820,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { iCopyLog < iChown && iChown < iChmod && iChmod < iStaleRemoval, - 'staging order must be: kill+wait, re-exec, home check, run identity, home identity pin, fresh dir, identity-pinned copy, scrub, reserved log name, authoritative log last, re-own, mode revoke, stale-tree removal', + 'staging order must be: kill+wait+liveness, re-exec, home check, run identity, home identity pin, opened-directory attribute re-checks, fresh dir, identity-pinned copy, scrub, reserved log name, authoritative log last, re-own, mode revoke, stale-tree removal', ); assert.match( sr, @@ -1706,8 +1831,18 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // and gate blocks, positional child marker (env markers forgeable). assert.match( sr, - /if \[ "\$\{1:-\}" != '--flake-clean-child' \]; then[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?\/usr\/bin\/bash --noprofile --norc -e -o pipefail "\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/, - 'staging must re-exec through env -i with a positional child marker and an absolute-path bash operand', + reExecRe, + 'staging must re-exec through env -i with a positional child marker, an absolute-path bash operand, and the parent-snapshotted body', + ); + assert.doesNotMatch( + sr, + pathChildRe, + 'the re-exec child must never re-open the script by path — the second open is the plant window', + ); + assert.match( + sr, + scrubRefusalRe, + 'the staging scrub refusal must be reserved-word shaped — `[` is shadowable by a BASH_FUNC import', ); assert.doesNotMatch( sr, @@ -3153,4 +3288,250 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.match(outputs.flake_summary, /returned different results/); assert.match(log, /a\.test\.js: FP/); }); + + it('a BASH_FUNC_[%% import cannot flip the pre-exec decisions — a poisoned `[` still fails closed on a plant', () => { + // bash imports a BASH_FUNC_[%% env entry as a FUNCTION named `[` — + // function lookup precedes builtins, so a poisoned step environment + // flips every `[`-shaped guard, including the decision whether to + // take the env -i re-exec itself (probe-verified on this pool's + // bash: `type -t [` reports `function` under the poison, `[[`/ + // `case` stay immune). The poison's first three calls defeat the + // scrub/re-exec/PATH-pin trio; every later call returns true so + // the hijacked home checks would pass the 0777 plant. Reserved-word + // decisions plus the immune re-exec must still fail closed. + const { res, outputs } = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\n', + gateHomeMode: 0o777, + env: { 'BASH_FUNC_[%%': '() { ((_p_n=${_p_n:-0}+1)); (( _p_n > 3 )); }' }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'error'); + assert.match( + outputs.flake_summary, + /not root-owned 0700|working directory/, + ); + }); + + it('a same-stem sibling (X.test.tsx next to changed X.test.ts) runs in ONE merged group, never attributed separately', () => { + // vitest's positional filters are lowercase SUBSTRING matches on + // root-relative paths — verified against the installed vitest: one + // filter collects both files of the same-stem pairs in this repo, + // so the sibling's outcome rides in the changed file's invocation + // (manufactured divergence, or masked flakiness). + const pair = { + 'packages/pkga/package.json': '{}', + 'packages/pkga/vitest.config.ts': '', + 'packages/pkga/src/x.test.ts': '', + 'packages/pkga/src/x.test.tsx': '', + }; + const both = runGate({ + layout: pair, + list: 'packages/pkga/src/x.test.ts\npackages/pkga/src/x.test.tsx\n', + }); + assert.equal(both.res.status, 0, both.res.stderr); + assert.equal(both.outputs.flake_verdict, 'pass'); + assert.match( + both.log, + /file packages\/pkga\/src\/x\.test\.ts \+ packages\/pkga\/src\/x\.test\.tsx:/, + 'the merged group label must name both files', + ); + assert.match( + both.log, + /substring-colliding sibling[\s\S]*?: packages\/pkga\/src\/x\.test\.tsx/, + 'the sibling must be skip-logged when the list reaches it', + ); + assert.match( + both.outputs.flake_summary, + /^1 changed test file\(s\)/, + 'the summary must count one merged group, not two files', + ); + assert.equal( + both.counts('x.test.ts'), + 5, + 'the merged group ran five rounds under the changed file operand', + ); + assert.equal( + both.counts('x.test.tsx'), + 0, + 'the sibling never ran under its own operand', + ); + // A changed .tsx with an unchanged .ts twin collects no sibling — + // it stays its own group with no merge. + const tsxOnly = runGate({ + layout: pair, + list: 'packages/pkga/src/x.test.tsx\n', + }); + assert.equal(tsxOnly.res.status, 0, tsxOnly.res.stderr); + assert.equal(tsxOnly.outputs.flake_verdict, 'pass'); + assert.doesNotMatch(tsxOnly.log, /substring-colliding sibling/); + assert.doesNotMatch(tsxOnly.log, /x\.test\.ts \+/); + assert.equal(tsxOnly.counts('x.test.tsx'), 5); + }); +}); +describe('qwen-triage: flakiness gate staging/upload — behavioral, under the production wrapper', () => { + // The structural pins cannot observe whether a DETECTED swap is also + // CLEANED UP — a set -e abort used to leave the swapped-in tree for + // the always() upload to enumerate (R12-2 entrances 1+3). + const stageStep = verifyJob.steps.find( + (s) => s.name === 'Stage flakiness gate log for upload', + ); + const recheckStep = verifyJob.steps.find( + (s) => s.id === 'flake-upload-check', + ); + + const stageRoot = mkdtempSync(join(tmpdir(), 'flake-staging-')); + after(() => rmSync(stageRoot, { recursive: true, force: true })); + + const makeHome = (rt, { runId = '777-1', files = {} } = {}) => { + const home = join(rt, 'flake-gate'); + mkdirSync(home, { recursive: true }); + chmodSync(home, 0o700); + writeFileSync(join(home, 'run-id'), runId); + for (const [name, content] of Object.entries(files)) { + writeFileSync(join(home, name), content); + } + return home; + }; + + const runStaging = (rt, bin) => { + const scriptFile = join(rt, 'staging.sh'); + writeFileSync(scriptFile, stageStep.run); + const out = join(rt, 'github-output'); + writeFileSync(out, ''); + return spawnSync( + 'bash', + ['--noprofile', '--norc', '-e', '-o', 'pipefail', scriptFile], + { + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + RUNNER_TEMP: rt, + GITHUB_OUTPUT: out, + GITHUB_STEP_SUMMARY: join(rt, 'github-summary'), + GITHUB_RUN_ID: '777', + GITHUB_RUN_ATTEMPT: '1', + }, + encoding: 'utf8', + timeout: 30_000, + }, + ); + }; + + // Models the kill-race survivor deterministically: the plant lands in + // the window AFTER the home_id capture (the 2nd stat call reads the + // genuine state first) and BEFORE the cd opens the directory — the + // inner re-stat must detect the mismatch, and the detection must + // REMOVE the plant, not merely abort on it. + const STUB_STAT_SWAP = [ + '#!/bin/bash', + 'n_file="$RUNNER_TEMP/.stat-count"', + 'n=$(cat "$n_file" 2>/dev/null || echo 0)', + 'echo $((n+1)) > "$n_file"', + 'out="$(/usr/bin/stat "$@")"', + 'if [ "$n" -eq 1 ] && [ ! -e "$RUNNER_TEMP/flake-gate.real" ]; then', + ' mv "$RUNNER_TEMP/flake-gate" "$RUNNER_TEMP/flake-gate.real"', + ' mkdir "$RUNNER_TEMP/flake-gate"', + ' chmod 700 "$RUNNER_TEMP/flake-gate"', + ' echo PLANT-MARKER > "$RUNNER_TEMP/flake-gate/plant-marker"', + ' echo 777-1 > "$RUNNER_TEMP/flake-gate/run-id"', + 'fi', + 'printf "%s\\n" "$out"', + '', + ].join('\n'); + + it('a swap detected by the opened-directory re-check is removed, never left to the always() upload', () => { + const rt = mkdtempSync(join(stageRoot, 'swap-')); + const bin = join(stageRoot, 'bin-swap'); + mkdirSync(bin, { recursive: true }); + writeFileSync(join(bin, 'stat'), STUB_STAT_SWAP); + chmodSync(join(bin, 'stat'), 0o755); + makeHome(rt, { files: { log: 'genuine gate log\n' } }); + const res = runStaging(rt, bin); + assert.equal( + res.status, + 0, + `staging must survive a detected swap: ${res.stderr}`, + ); + assert.ok( + !existsSync(join(rt, 'flake-gate')), + 'the swapped-in tree must be removed — the always() upload enumerates this path unconditionally', + ); + assert.ok( + existsSync(join(rt, 'flake-gate.real')), + 'sanity: the stub stashed the genuine home', + ); + }); + + it('a genuine home still rebuilds the upload tree with the authoritative log', () => { + const rt = mkdtempSync(join(stageRoot, 'clean-')); + const bin = join(stageRoot, 'bin-clean'); + mkdirSync(bin, { recursive: true }); + // install/chown/chmod need root in production; the harness stubs the + // ownership plumbing and keeps the real directory creation. + for (const [name, body] of [ + ['install', '#!/bin/bash\nmkdir -p "${@: -1}"\n'], + ['chown', '#!/bin/bash\nexit 0\n'], + ['chmod', '#!/bin/bash\nexit 0\n'], + ]) { + writeFileSync(join(bin, name), body); + chmodSync(join(bin, name), 0o755); + } + makeHome(rt, { files: { log: 'genuine gate log\n' } }); + const res = runStaging(rt, bin); + assert.equal(res.status, 0, res.stderr); + assert.equal( + readFileSync(join(rt, 'flake-gate', 'upload', 'flake-gate.log'), 'utf8'), + 'genuine gate log\n', + 'the authoritative log must land in the rebuilt upload tree', + ); + }); + + const runRecheck = (rt) => { + const out = join(rt, 'github-output'); + writeFileSync(out, ''); + const res = spawnSync( + 'bash', + ['--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', recheckStep.run], + { + env: { + ...process.env, + RUNNER_TEMP: rt, + GITHUB_OUTPUT: out, + GITHUB_RUN_ID: '777', + GITHUB_RUN_ATTEMPT: '1', + }, + cwd: rt, + encoding: 'utf8', + timeout: 30_000, + }, + ); + const outputs = Object.fromEntries( + readFileSync(out, 'utf8') + .split('\n') + .filter((l) => l.includes('=')) + .map((l) => [l.slice(0, l.indexOf('=')), l.slice(l.indexOf('=') + 1)]), + ); + return { res, outputs }; + }; + + it('the pre-upload re-check removes a stale or planted home and gates the upload', () => { + const bad = mkdtempSync(join(stageRoot, 'recheck-bad-')); + makeHome(bad, { runId: '666-9' }); + mkdirSync(join(bad, 'flake-gate', 'upload'), { recursive: true }); + const rejected = runRecheck(bad); + assert.equal(rejected.res.status, 0, rejected.res.stderr); + assert.equal(rejected.outputs.upload_ok, 'false'); + assert.ok( + !existsSync(join(bad, 'flake-gate')), + 'a home that fails the re-check must be removed before the upload enumerates it', + ); + const good = mkdtempSync(join(stageRoot, 'recheck-good-')); + makeHome(good); + mkdirSync(join(good, 'flake-gate', 'upload'), { recursive: true }); + const accepted = runRecheck(good); + assert.equal(accepted.res.status, 0, accepted.res.stderr); + assert.equal(accepted.outputs.upload_ok, 'true'); + assert.ok(existsSync(join(good, 'flake-gate', 'upload'))); + }); }); diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index 5f99356b45d..d91f0ff4478 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2784,40 +2784,69 @@ jobs: # entries are imported as shell functions BEFORE this body # runs, are invisible to ${!BASH_FUNC_@}/`set`, and shadow # every later resolution — including the PATH pin below, - # because function lookup precedes PATH. No in-script purge is - # safe (an import can shadow the purge builtin itself), so - # re-exec ONCE through an absolute-path env -i child: a - # slash-bearing word never goes through function lookup, and - # -i drops every import. The child marker is a POSITIONAL - # argument, never an env entry — the poisoned file-command - # channel this block defends against can plant any env - # variable, so an env-borne "already re-executed" sentinel - # would be forgeable; argv is set by exec, and the runner - # wrapper passes no arguments after the script path. Mirrors - # the autofix lane's env -i bootstrap; only builtins and - # expansions resolve before this guard. The bash operand is - # absolute-pathed too: env applies the forwarded environment - # FIRST, then resolves the operand via execvp against its PATH — - # the inherited one, which may already be attacker-chosen. - if [ "${EUID:-1}" -eq 0 ] && { [ -n "${BASH_ENV:-}" ] || [ -n "${LD_PRELOAD:-}" ] || [ -n "${LD_AUDIT:-}" ] || [ -n "${LD_LIBRARY_PATH:-}" ]; }; then + # because function lookup precedes PATH. An import can even be + # NAMED `[` (bash accepts BASH_FUNC_[%%), and function lookup + # precedes builtins — so every decision up to and including + # the re-exec is a reserved word (`[[`, `case`), recognized + # at parse time: through a `[`-shaped guard the poison skips + # the re-exec that is the whole defense. No in-script purge + # is safe (an import can shadow the purge builtin itself). + # The child marker is a POSITIONAL argument, never an env + # entry — the poisoned file-command channel this block + # defends against can plant any env variable, so an env-borne + # "already re-executed" sentinel would be forgeable; argv is + # set by exec, and the runner wrapper passes no arguments + # after the script path. Mirrors the autofix lane's env -i + # bootstrap; only builtins and expansions resolve before this + # guard. The bash operand is absolute-pathed too: env applies + # the forwarded environment FIRST, then resolves the operand + # via execvp against its PATH — the inherited one, which may + # already be attacker-chosen. The child runs the body the + # PARENT snapshots into a variable — never the path again: + # the runner wrote this script node-owned and 644 inside the + # uid-1000-writable $RUNNER_TEMP, so a second open by the + # child is a deterministic window a kill-race survivor can + # rename-plant into. + if [[ ${EUID:-1} -eq 0 ]] && [[ -n ${BASH_ENV:-} || -n ${LD_PRELOAD:-} || -n ${LD_AUDIT:-} || -n ${LD_LIBRARY_PATH:-} ]]; then echo "::error::flake-gate record: step-env scrub lost — refusing to run in a poisoned environment" exit 1 fi - if [ "${1:-}" != '--flake-clean-child' ]; then - [ -x /usr/bin/env ] || { echo "::error::flake-gate record: clean re-exec unavailable"; exit 1; } - LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ - PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ - GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ - GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ - /usr/bin/bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child + # Kill BEFORE the snapshot, liveness-verified (the gate + # step's shape): a live node writer is all the snapshot race + # needs, even at this pre-build point. + if [[ ${EUID:-1} -eq 0 ]]; then + /usr/bin/pkill -KILL -u node 2>/dev/null || true + for _ in 1 2 3; do + [[ -n $(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/') ]] || break + /usr/bin/sleep 1 + /usr/bin/pkill -KILL -u node 2>/dev/null || true + done + survivors="$(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/')" || true + if [[ -n $survivors ]]; then + echo "::error::flake-gate record: node-owned processes survived SIGKILL — refusing to record through a contested environment" + exit 1 + fi fi + case "${1:-}" in + --flake-clean-child) ;; + *) + [[ -x /usr/bin/env ]] || { echo "::error::flake-gate record: clean re-exec unavailable"; exit 1; } + _flake_body="$(<"${BASH_SOURCE[0]}")" + [[ -n $_flake_body ]] || { echo "::error::flake-gate record: clean re-exec snapshot empty"; exit 1; } + LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ + PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ + GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ + GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ + /usr/bin/bash --noprofile --norc -e -o pipefail -c "$_flake_body" flake-record --flake-clean-child + ;; + esac # Runner file-command backing files under $RUNNER_TEMP are # uid-1000-owned: a step's PR code can poison the job # environment (PATH above all) that LATER steps inherit, and # root-side blocks resolve bare binaries through it. Pin a # root-only-writable PATH. Production runs this block as root; # the test harness does not and keeps its stub PATH. - if [ "${EUID:-1}" -eq 0 ]; then + if [[ ${EUID:-1} -eq 0 ]]; then export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' fi # Root-only home for every gate working file. $RUNNER_TEMP itself @@ -3064,70 +3093,69 @@ jobs: # compromised root shell must not keep producing verdicts. # EUID-gated so the test harness (which cannot apply this # step's env block and runs non-root) stays out of it. - if [ "${EUID:-1}" -eq 0 ] && { [ -n "${BASH_ENV:-}" ] || [ -n "${LD_PRELOAD:-}" ] || [ -n "${LD_AUDIT:-}" ] || [ -n "${LD_LIBRARY_PATH:-}" ]; }; then + if [[ ${EUID:-1} -eq 0 ]] && [[ -n ${BASH_ENV:-} || -n ${LD_PRELOAD:-} || -n ${LD_AUDIT:-} || -n ${LD_LIBRARY_PATH:-} ]]; then echo "flake_verdict=error" >> "$GITHUB_OUTPUT" echo "flake_summary=the gate refused to sample in a poisoned environment — see the step log" >> "$GITHUB_OUTPUT" exit 0 fi - # The runner wrote THIS script as a node-owned file inside the - # uid-1000-writable $RUNNER_TEMP, and the re-exec below RE-READS - # it from disk: install/build ran PR lifecycle code as node, so - # a detached survivor still alive when the re-exec opens the - # file overwrites it in place, and the wrapper executes attacker - # content in its full environment — every in-script defence lives - # in the overwritten body. Kill BEFORE the re-exec; absolute- - # pathed (no PATH pin applies this early, and a BASH_FUNC import - # shadows every bare word) and EUID-gated (production runs as - # root; the test harness does not and keeps its stubs). - if [ "${EUID:-1}" -eq 0 ]; then + # Kill BEFORE the re-exec snapshot; absolute-pathed (no PATH + # pin applies this early, and a BASH_FUNC import shadows + # every bare word), EUID-gated (the harness keeps its + # stubs). The liveness refusal mirrors the agent step's + # guard: the budget loop alone is out-forked by a plant + # repopulating between sweeps, and the snapshot must not + # read this node-owned script through a contested + # environment (install/build ran PR lifecycle code). + if [[ ${EUID:-1} -eq 0 ]]; then /usr/bin/pkill -KILL -u node 2>/dev/null || true for _ in 1 2 3; do - [ -n "$(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/')" ] || break + [[ -n $(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/') ]] || break /usr/bin/sleep 1 /usr/bin/pkill -KILL -u node 2>/dev/null || true done - fi - # Startup-channel scrub, part two: BASH_FUNC_%% env - # entries are imported as shell functions BEFORE this body - # runs, are invisible to ${!BASH_FUNC_@}/`set`, and shadow - # every later resolution — including the PATH pin below, - # because function lookup precedes PATH. No in-script purge is - # safe (an import can shadow the purge builtin itself), so - # re-exec ONCE through an absolute-path env -i child: a - # slash-bearing word never goes through function lookup, and - # -i drops every import. The child marker is a POSITIONAL - # argument, never an env entry — the poisoned file-command - # channel this block defends against can plant any env - # variable, so an env-borne "already re-executed" sentinel - # would be forgeable; argv is set by exec, and the runner - # wrapper passes no arguments after the script path. Mirrors - # the autofix lane's env -i bootstrap; only builtins and - # expansions resolve before this guard. The bash operand is - # absolute-pathed too: env applies the forwarded environment - # FIRST, then resolves the operand via execvp against its PATH — - # the inherited one, which may already be attacker-chosen. The - # gate fails OPEN if the re-exec is unavailable, never into a - # compromised shell. - if [ "${1:-}" != '--flake-clean-child' ]; then - if [ ! -x /usr/bin/env ]; then + survivors="$(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/')" || true + if [[ -n $survivors ]]; then echo "flake_verdict=error" >> "$GITHUB_OUTPUT" - echo "flake_summary=the gate could not start a clean shell — refusing to sample" >> "$GITHUB_OUTPUT" + echo "flake_summary=node-owned processes survived SIGKILL — the gate refused to sample" >> "$GITHUB_OUTPUT" exit 0 fi - LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ - PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ - GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ - GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ - FLAKE_ROUNDS="${FLAKE_ROUNDS:-}" FLAKE_SEQ_DIR="${FLAKE_SEQ_DIR:-}" \ - /usr/bin/bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child fi + # Startup-channel scrub, part two — the record step carries + # the full rationale: reserved-word decisions (a BASH_FUNC + # import can be named `[`), one-shot env -i re-exec, + # positional child marker, and the snapshot body (the child + # never re-opens this node-owned script by path). The gate + # fails OPEN if the re-exec is unavailable, never into a + # compromised shell. + case "${1:-}" in + --flake-clean-child) ;; + *) + if [[ ! -x /usr/bin/env ]]; then + echo "flake_verdict=error" >> "$GITHUB_OUTPUT" + echo "flake_summary=the gate could not start a clean shell — refusing to sample" >> "$GITHUB_OUTPUT" + exit 0 + fi + _flake_body="$(<"${BASH_SOURCE[0]}")" + if [[ -z $_flake_body ]]; then + echo "flake_verdict=error" >> "$GITHUB_OUTPUT" + echo "flake_summary=the gate could not snapshot its own body — refusing to sample" >> "$GITHUB_OUTPUT" + exit 0 + fi + LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ + PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ + GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ + GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ + FLAKE_ROUNDS="${FLAKE_ROUNDS:-}" FLAKE_SEQ_DIR="${FLAKE_SEQ_DIR:-}" \ + /usr/bin/bash --noprofile --norc -e -o pipefail -c "$_flake_body" flake-gate --flake-clean-child + ;; + esac # Same poisoned-env premise as the record step: the file # commands a PR step wrote to the uid-1000-owned backing files # are applied at step end, so this block's inherited PATH may # be attacker-chosen. Pin a root-only-writable one before any # bare binary is resolved (production runs as root; the test # harness does not and keeps its stub PATH). - if [ "${EUID:-1}" -eq 0 ]; then + if [[ ${EUID:-1} -eq 0 ]]; then export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' fi # Abnormal-exit net: `set -u` (or any unforeseen fatal) still @@ -3185,24 +3213,22 @@ jobs: # embeds the FIRST 10,000 chars of the log, so the per-file matrix # and verdict must sit ahead of detail that can outgrow the cap. DETAIL="$GATE_DIR/detail" - : > "$LOG" - : > "$DETAIL" finish() { # Outputs carry FIXED strings and counters only — file paths are # PR-controlled text and belong in the log, which the publisher # HTML-escapes before embedding. echo "flake_verdict=$1" >> "$GITHUB_OUTPUT" echo "flake_summary=$2" >> "$GITHUB_OUTPUT" - printf '\nverdict: %s\nsummary: %s\n' "$1" "$2" >> "$LOG" - # Detail LAST: the publisher embeds the first 10,000 chars, and - # the matrix/verdict must never be truncated away behind - # failure tails (full copy stays in the artifact). - if [ -s "$DETAIL" ]; then - if [ -n "${GATE_HOME_ID:-}" ] && ! gate_home_intact; then - # A home swapped after the samples ran would feed - # attacker-chosen bytes into the published log. - printf -- '\n(per-invocation detail dropped: the gate working directory changed mid-run)\n' >> "$LOG" - else + # Log appends re-resolve the home path, and finish() is + # callable from any point: write them only through a home + # re-verified at call time — a swapped home must not receive + # the verdict bytes through a planted `log` symlink. + if gate_home_intact; then + printf '\nverdict: %s\nsummary: %s\n' "$1" "$2" >> "$LOG" + # Detail LAST: the publisher embeds the first 10,000 chars, and + # the matrix/verdict must never be truncated away behind + # failure tails (full copy stays in the artifact). + if [ -s "$DETAIL" ]; then printf -- '\n--- per-invocation detail (full copy in the artifact) ---\n' >> "$LOG" cat "$DETAIL" >> "$LOG" fi @@ -3223,9 +3249,29 @@ jobs: # reintroduce "follow whatever symlink is there". gate_home_intact || finish error 'the gate working directory changed since validation — refusing to read files a PR could have planted' + # Truncate ONLY through the verified home: `: >` opens with + # O_TRUNC through any symlink, so it must never run ahead of + # the first identity re-check. + : > "$LOG" + : > "$DETAIL" { [ -f "$LIST" ] && [ ! -L "$LIST" ]; } || finish error 'the recorded changed-test list is missing or not a regular file' + # vitest's positional filters are lowercase SUBSTRING matches on + # root-relative paths: the operand for X.test.ts also collects a + # same-stem X.test.tsx sibling, and the one invocation's outcome + # would be attributed to the changed file alone — manufacturing + # divergence from a deterministic PR, or masking real divergence. + # Merge each colliding pair into ONE group and skip the sibling + # when the list reaches it. `${f}x` is the live shape: a + # collected superstring of a changed test path can only append + # the `x` of a .tsx/.jsx twin. + declare -A sibling_owner=() + while IFS= read -r -d '' f; do + [ -n "$f" ] || continue + [ -f "${f}x" ] && sibling_owner["${f}x"]="$f" + done < "$LIST" + # Partition into per-FILE groups. Every skipped file is logged # with its reason — a silently narrowed gate would read as # "covered" when it was not. Operands are `./`-prefixed BEFORE %q, @@ -3282,6 +3328,10 @@ jobs: add_skip "$f" 'not present in the merge tree, skipped' continue fi + if [ -n "${sibling_owner["$f"]:-}" ]; then + add_skip "$f" "substring-colliding sibling — one vitest filter collects it together with ${sibling_owner["$f"]}, whose group it runs in" + continue + fi case "$f" in integration-tests/*) # E2E suites need sandbox/model plumbing this gate does not have. @@ -3340,8 +3390,13 @@ jobs: continue fi # From the owning package CWD (vitest configs assume it); - # npx resolves the binary by walking up node_modules. - group_labels+=("$f") + # npx resolves the binary by walking up node_modules. The + # label names a substring-colliding sibling too: one + # filter collects both, so the group's outcome belongs to + # both (see sibling_owner). + label="$f" + [ -f "${f}x" ] && label="$f + ${f}x" + group_labels+=("$label") group_dirs+=("$pkg") group_cmds+=("npx --no-install vitest run $(printf '%q' "./${f#"$pkg"/}")") ;; @@ -4255,48 +4310,56 @@ jobs: # step's): fail closed if the step-env blanks lost, then # re-exec once through an absolute-path env -i child to drop # any BASH_FUNC_* imports before a bare binary resolves. - if [ "${EUID:-1}" -eq 0 ] && { [ -n "${BASH_ENV:-}" ] || [ -n "${LD_PRELOAD:-}" ] || [ -n "${LD_AUDIT:-}" ] || [ -n "${LD_LIBRARY_PATH:-}" ]; }; then + if [[ ${EUID:-1} -eq 0 ]] && [[ -n ${BASH_ENV:-} || -n ${LD_PRELOAD:-} || -n ${LD_AUDIT:-} || -n ${LD_LIBRARY_PATH:-} ]]; then echo "::error::flake-gate staging: step-env scrub lost — refusing to stage evidence in a poisoned environment" exit 1 fi - # The runner wrote THIS script as a node-owned file inside the - # uid-1000-writable $RUNNER_TEMP, and the re-exec below RE-READS - # it from disk: the agent era just ran PR-controlled node code, - # so a detached survivor still alive when the re-exec opens the - # file overwrites it in place, and the wrapper executes attacker - # content in its full environment. Kill BEFORE the re-exec — - # unconditional, not gated on the log existing: node can unlink - # the log, and that must not skip the kill/rebuild for the - # report the agent wrote. Absolute-pathed (no PATH pin applies - # this early) and EUID-gated (production runs as root; the test - # harness does not and keeps its stubs). - if [ "${EUID:-1}" -eq 0 ]; then + # Kill BEFORE the re-exec snapshot — unconditional, not gated + # on the log existing: node can unlink the log, and that must + # not skip the kill/rebuild for the report the agent wrote. + # Absolute-pathed, EUID-gated (the harness keeps its stubs). + # The liveness refusal mirrors the agent step's guard: the + # budget loop alone is out-forked by a plant repopulating + # between sweeps, and the snapshot must not read this + # node-owned script through a contested environment (the + # agent era just ran PR-controlled node code). + if [[ ${EUID:-1} -eq 0 ]]; then /usr/bin/pkill -KILL -u node 2>/dev/null || true for _ in 1 2 3; do - [ -n "$(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/')" ] || break + [[ -n $(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/') ]] || break /usr/bin/sleep 1 /usr/bin/pkill -KILL -u node 2>/dev/null || true done + survivors="$(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/')" || true + if [[ -n $survivors ]]; then + echo "::error::flake-gate staging: node-owned processes survived SIGKILL — refusing to stage evidence through a contested environment" + exit 1 + fi fi - # The child marker is positional, not an env entry: the - # poisoned file-command channel can plant any env variable, so - # an env-borne sentinel would be forgeable; the bash operand is - # absolute-pathed for the same reason as the PATH pin (see the - # record step for the full rationale). - if [ "${1:-}" != '--flake-clean-child' ]; then - [ -x /usr/bin/env ] || { echo "::error::flake-gate staging: clean re-exec unavailable"; exit 1; } - LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ - PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ - GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ - GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ - /usr/bin/bash --noprofile --norc -e -o pipefail "${BASH_SOURCE[0]}" --flake-clean-child - fi + # The record step carries the full scrub/re-exec rationale: + # reserved-word decisions (a BASH_FUNC import can be named + # `[`), positional child marker (env markers forgeable), + # snapshot body (the child never re-opens this node-owned + # script by path), absolute-path bash operand. + case "${1:-}" in + --flake-clean-child) ;; + *) + [[ -x /usr/bin/env ]] || { echo "::error::flake-gate staging: clean re-exec unavailable"; exit 1; } + _flake_body="$(<"${BASH_SOURCE[0]}")" + [[ -n $_flake_body ]] || { echo "::error::flake-gate staging: clean re-exec snapshot empty"; exit 1; } + LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ + PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ + GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ + GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-}" \ + /usr/bin/bash --noprofile --norc -e -o pipefail -c "$_flake_body" flake-stage --flake-clean-child + ;; + esac # Pin a root-only-writable PATH: the job env this step # inherits may be poisoned through the uid-1000-owned # file-command backing files (see the record step). Production # runs as root; the test harness does not and keeps its stub # PATH. - if [ "${EUID:-1}" -eq 0 ]; then + if [[ ${EUID:-1} -eq 0 ]]; then export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' fi # BUILD a trusted upload tree; never harden an attacker's. @@ -4317,9 +4380,10 @@ jobs: # staging step runs on exactly the paths where this run's # record step (the only creator) was skipped — without it a # previous run's evidence would ship under this run's name. - if [ ! -L "$GATE_DIR" ] && [ -d "$GATE_DIR" ] && [ -O "$GATE_DIR" ] && - [ "$(stat -c '%a' "$GATE_DIR" 2>/dev/null)" = '700' ] && - [ "$(cat "$GATE_DIR/run-id" 2>/dev/null)" = "${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}" ]; then + staged_ok='' + if [[ ! -L $GATE_DIR ]] && [[ -d $GATE_DIR ]] && [[ -O $GATE_DIR ]] && + [[ $(stat -c '%a' "$GATE_DIR" 2>/dev/null) == '700' ]] && + [[ $(cat "$GATE_DIR/run-id" 2>/dev/null) == "${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}" ]]; then # The validated ENTRY still lives in the uid-1000-writable # $RUNNER_TEMP top level, and every absolute-path operation # re-resolves it through the writable parent: cd in ONCE, @@ -4329,13 +4393,26 @@ jobs: # to the opened inode no matter how the entry above it is # swapped. The one exception is the copy target, which must # cross trees; a race there can only misdirect OUR bytes, - # never inject foreign ones into the anchored tree. + # never inject foreign ones into the anchored tree. The + # outer conjuncts above are six separate path resolutions a + # swap can thread, so the ATTRIBUTE half re-runs inside the + # opened directory: a plant the cd resolved into must still + # be owned by the effective user, 0700, and carry this + # run's marker — and node cannot create the root-owned + # directory -O demands. home_id="$(stat -c '%d:%i' "$GATE_DIR" 2>/dev/null || true)" - ( - cd "$GATE_DIR" - [ "$(stat -c '%d:%i' . 2>/dev/null)" = "$home_id" ] - rm -rf -- upload - install -d -m 0700 -o root -g root upload + # Explicit `|| exit 1` on every check and phase: this + # subshell is an `if` condition, and bash suppresses errexit + # there — an unguarded failure would fall through into the + # copy phases instead of refusing. + if ( + cd "$GATE_DIR" || exit 1 + [ "$(stat -c '%d:%i' . 2>/dev/null)" = "$home_id" ] || exit 1 + [ -O . ] || exit 1 + [ "$(stat -c '%a' . 2>/dev/null)" = '700' ] || exit 1 + [ "$(cat ./run-id 2>/dev/null)" = "${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}" ] || exit 1 + rm -rf -- upload || exit 1 + install -d -m 0700 -o root -g root upload || exit 1 # Copy only REGULAR files out of the untrusted tree, # resolving nothing: -type f excludes symlinks/FIFOs/ # sockets/devices at selection time, and @@ -4369,7 +4446,7 @@ jobs: # upload-artifact follows links, so any non-regular # arrival is a root-readable-content primitive into the # public artifact and must not ship. - find upload \( -type l -o -type p -o -type s -o -type b -o -type c \) -delete + find upload \( -type l -o -type p -o -type s -o -type b -o -type c \) -delete || exit 1 # The gate's own log is authoritative and root-owned inside # the home — it is copied LAST so nothing in the untrusted # tree can shadow the name the publisher pins. Reserve the @@ -4377,26 +4454,76 @@ jobs: # planted file would otherwise survive the copy, and a # planted DIRECTORY would swallow the authoritative file # even when it exists. - rm -rf -- upload/flake-gate.log + rm -rf -- upload/flake-gate.log || exit 1 if [ -f log ] && [ ! -L log ]; then - cp -f --no-dereference log upload/flake-gate.log + cp -f --no-dereference log upload/flake-gate.log || exit 1 fi - chown -R root:root upload - chmod -R go-rwx upload - ) - else - echo "::warning::flake-gate home missing or not root-owned 0700; skipping the trusted upload rebuild." - # Remove the rejected tree anyway: the always() upload step - # below enumerates this path unconditionally, so a stale tree - # left in place ships a PREVIOUS run's evidence under this - # run's artifact name — the exact outcome the run-id conjunct - # above exists to prevent. rm -rf removes a symlink operand - # itself, never following it. + chown -R root:root upload || exit 1 + chmod -R go-rwx upload || exit 1 + ); then + staged_ok=1 + fi + fi + if [ -z "$staged_ok" ]; then + # ONE cleanup for BOTH refusals — a home that failed the + # outer validation, and one whose opened directory failed + # the identity/attribute re-checks mid-staging. Detection + # alone is not enough: a set -e abort used to leave the + # swapped-in tree for the always() upload to enumerate, + # shipping a plant — or a PREVIOUS run's evidence under + # this run's artifact name. rm -rf removes a symlink + # operand itself, never following it. + echo "::warning::flake-gate home missing, invalid, or swapped mid-staging; skipping the trusted upload rebuild." rm -rf -- "${RUNNER_TEMP:?}/flake-gate" fi - - name: 'Upload verify results' + # The always() upload below re-resolves $RUNNER_TEMP/flake-gate in a + # fresh step — staging's anchoring expires at its exit, so a swap + # between the two hands the upload a plant. Re-validate the entry's + # identity immediately before the upload and remove it on any + # mismatch; the upload runs only on an affirmative output. Every + # decision is a reserved word and every external absolute-pathed: + # this step inherits the same poisoned job environment the + # scrub/re-exec blocks defend against, and a hijacked check here + # fails CLOSED — never into a shipped plant. + - name: 'Re-check flake-gate home before upload' + id: 'flake-upload-check' if: "always() && steps.pr.outputs.decision == 'run'" + # Same posture as the staging step: evidence-copying only. + continue-on-error: true + env: + BASH_ENV: '' + LD_PRELOAD: '' + LD_AUDIT: '' + LD_LIBRARY_PATH: '' + run: |- + set -uo pipefail + if [[ ${EUID:-1} -eq 0 ]]; then + export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + fi + upload_ok=false + GATE_DIR="${RUNNER_TEMP:?}/flake-gate" + if [[ ! -L $GATE_DIR ]] && [[ -d $GATE_DIR ]] && [[ -O $GATE_DIR ]] && + [[ $(/usr/bin/stat -c '%a' "$GATE_DIR" 2>/dev/null) == '700' ]] && + [[ $(/usr/bin/cat "$GATE_DIR/run-id" 2>/dev/null) == "${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}" ]]; then + home_id="$(/usr/bin/stat -c '%d:%i' "$GATE_DIR" 2>/dev/null || true)" + if ( cd "$GATE_DIR" && + [ "$(stat -c '%d:%i' . 2>/dev/null)" = "$home_id" ] && + [ -O . ] && [ "$(stat -c '%a' . 2>/dev/null)" = '700' ] && + [ -d upload ] ); then + upload_ok=true + fi + fi + if [[ $upload_ok != true ]]; then + # Whatever occupies the entry now is not this run's + # validated home — remove it so the upload cannot + # enumerate a plant. + /usr/bin/rm -rf -- "${RUNNER_TEMP:?}/flake-gate" + fi + echo "upload_ok=$upload_ok" >> "$GITHUB_OUTPUT" + + - name: 'Upload verify results' + if: "always() && steps.pr.outputs.decision == 'run' && steps.flake-upload-check.outputs.upload_ok == 'true'" # Don't let a missing/empty results dir (qwen crashed before writing # any) fail the job and mask the original error. continue-on-error: true From 387a8434341a7adf04125452f4ae3138e91bb473 Mon Sep 17 00:00:00 2001 From: wenshao Date: Tue, 18 Aug 2026 04:07:43 +0800 Subject: [PATCH 14/18] fix(ci): move the flake-gate home out of PR-writable space; pin git metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rounds 8-16 kept re-reporting one class (R8-1, R8-36, R12-2 and its re-reports, R13-21, R15-1, R16-1) because every fix was a re-validation of a path that could still be swapped underneath it. The premise, not the consumers, was wrong: rename(2) and unlink(2) need write permission on the PARENT directory, never on the entry, and $RUNNER_TEMP's top level is uid-1000 mode-755 on this pool while the container's node is uid 1000. A 0700 root:root home there could always be renamed away wholesale and replaced — so each added check (ownership, mode, inode anchor, run-id marker, opened-directory re-stat) only narrowed the window, and the inter-step window before a `uses:` upload could not be closed at all, because that step cannot run shell. - The home moves to /flake-gate. `/` is root:root 755 in this container: node can neither create, unlink nor rename entries in it, so the entry — and everything reached through it — is outside PR-controlled reach by construction, with no window left to re-check. The upload ships that tree directly. No env override: $GITHUB_ENV is uid-1000 writable, so a configurable home would be a PR-reachable channel, and the record step rm -rf's whatever the home names. The extracted-script harness relocates that one constant into its scratch tree instead, and the production value is pinned structurally. - inv_tmp stays under RUNNER_TEMP by necessity (the build user cannot enter the root-only home) and gains the :? guard. - .git is re-owned to root after prepare's workspace chown (R4-1/R4-2): with it node-owned, a lifecycle script could plant a smudge filter and have root's per-invocation reset execute it every round, or rewrite HEAD so the restore installs a tree of its choosing. Non-fatal, since the reset already runs git with GIT_* stripped, under a timeout, against an OID pinned before any PR code ran. - The scripts/tests arm gets the same substring-collision labelling the generic arm already had, so a same-stem sibling collected by vitest's positional filter is named in the attribution (R8-31/R13-28 lineage). Suites: helper 112/112, twin 134/134 — both run as a NON-ROOT user, which is what CI does; as root the gate pins its own PATH and the harness stubs are bypassed by design, so a root run reports false failures. --- .github/scripts/qwen-triage-workflow.test.mjs | 159 ++++++++++++++++-- .github/workflows/qwen-triage.yml | 89 +++++++--- 2 files changed, 210 insertions(+), 38 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index d23af7f5ecd..6c47491df60 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -1002,7 +1002,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // executes, so it is a changed test file exactly like M. assert.match( recordStep.run, - /^\s*git -c core\.quotePath=false diff -z --name-only --diff-filter=ACMRT 'HEAD\^1' HEAD \\\n\s*> "\$RUNNER_TEMP\/flake-gate\/files-all"$/m, + /^\s*git -c core\.quotePath=false diff -z --name-only --diff-filter=ACMRT 'HEAD\^1' HEAD \\\n\s*> "\$GATE_HOME\/files-all"$/m, 'the NUL diff must flow straight into its file — $( ) strips NUL bytes, a pipeline swallows the exit status', ); assert.ok( @@ -1013,7 +1013,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( recordStep.run, - /^\s*grep_status=0\n\s*grep -zE '[^']+' \\\n\s*"\$RUNNER_TEMP\/flake-gate\/files-all" \\\n\s*> "\$RUNNER_TEMP\/flake-gate\/files" \|\| grep_status=\$\?$/m, + /^\s*grep_status=0\n\s*grep -zE '[^']+' \\\n\s*"\$GATE_HOME\/files-all" \\\n\s*> "\$GATE_HOME\/files" \|\| grep_status=\$\?$/m, 'the grep must read the raw file and only a no-match may yield an empty list', ); // Every gate working file must live in the root-only home, because @@ -1022,12 +1022,12 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // replaced with symlinks that root-side consumers follow. assert.match( recordStep.run, - /^\s*install -d -m 0700 -o root -g root "\$RUNNER_TEMP\/flake-gate"$/m, + /^\s*install -d -m 0700 -o root -g root "\$GATE_HOME"$/m, 'the record step must create the root-only home', ); assert.match( recordStep.run, - /^\s*rm -rf -- "\$\{RUNNER_TEMP:\?\}\/flake-gate"$/m, + /^\s*rm -rf -- "\$GATE_HOME"$/m, 'a plant left by an earlier run on the persistent pool must be removed first', ); // Run-freshness marker: a stale-but-genuine home from an earlier @@ -1036,7 +1036,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // runs apart when the record step itself was skipped. assert.match( recordStep.run, - /^\s*printf '%s-%s' "\$\{GITHUB_RUN_ID:\?\}" "\$\{GITHUB_RUN_ATTEMPT:\?\}" > "\$RUNNER_TEMP\/flake-gate\/run-id"$/m, + /^\s*printf '%s-%s' "\$\{GITHUB_RUN_ID:\?\}" "\$\{GITHUB_RUN_ATTEMPT:\?\}" > "\$GATE_HOME\/run-id"$/m, 'the record step must stamp the run identity into the home it creates', ); // Startup-channel scrub: BASH_FUNC_* imports are dropped by a @@ -1580,7 +1580,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { const uploadStep = verifyJob.steps[uploadIdx]; assert.equal( uploadStep.with.path, - '${{ runner.temp }}/flake-gate/upload/', + '/flake-gate/upload/', 'the artifact must ship the REBUILT tree, never the agent-era directory', ); // R12-2 entrance 3: staging's anchoring expires at its exit, and the @@ -1618,7 +1618,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( recheckStep.run, - /\/usr\/bin\/rm -rf -- "\$\{RUNNER_TEMP:\?\}\/flake-gate"/, + /\/usr\/bin\/rm -rf -- "\$GATE_DIR"/, 'a home that fails the re-check must be removed before the upload enumerates the path', ); assert.match( @@ -1723,9 +1723,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // unconditionally, so a home that failed validation must be removed // — a stale tree left in place ships a previous run's evidence // under this run's artifact name. - const iStaleRemoval = sr.search( - /^\s*rm -rf -- "\$\{RUNNER_TEMP:\?\}\/flake-gate"$/m, - ); + const iStaleRemoval = sr.search(/^\s*rm -rf -- "\$GATE_DIR"$/m); // Round 11 (R8-36): the guard and the cd re-resolve verify-results; // a kill-loop survivor owning the uid-1000 parent can swap the // entry between the two — the opened directory must still BE the @@ -1939,6 +1937,101 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); }); + it('the gate home sits at the container root, not in the writable RUNNER_TEMP', () => { + // The whole TOCTOU class (R8-1, R8-36, R12-2 and their re-reports) + // rests on one property of the PARENT, not the entry: rename(2) and + // unlink(2) need write permission on the directory holding the entry. + // $RUNNER_TEMP's top level is uid-1000 writable and the container's + // node is uid 1000, so a 0700 root home there could always be + // renamed away wholesale — every added re-validation only narrowed + // the window. `/` is root:root 755, so entries in it are outside + // PR-controlled reach with no window to re-check. + for (const [label, step] of [ + ['record', recordStep], + ['gate', flakeStep], + [ + 'staging', + verifyJob.steps.find( + (x) => x.name === 'Stage flakiness gate log for upload', + ), + ], + [ + 'upload re-check', + verifyJob.steps.find((x) => x.id === 'flake-upload-check'), + ], + ]) { + assert.doesNotMatch( + step.run, + /\$\{?RUNNER_TEMP:?\??\}?\/flake-gate/, + `${label} must not place the gate home under RUNNER_TEMP`, + ); + } + assert.match( + recordStep.run, + /^\s*GATE_HOME=\/flake-gate$/m, + 'the record step must create the home at the container root', + ); + for (const [label, step] of [ + ['gate', flakeStep], + [ + 'staging', + verifyJob.steps.find( + (x) => x.name === 'Stage flakiness gate log for upload', + ), + ], + [ + 'upload re-check', + verifyJob.steps.find((x) => x.id === 'flake-upload-check'), + ], + ]) { + assert.match( + step.run, + /^\s*GATE_DIR=\/flake-gate$/m, + `${label} must resolve the home to the container-root constant`, + ); + } + const uploadStep = verifyJob.steps.find( + (x) => x.name === 'Upload verify results', + ); + assert.equal( + uploadStep.with.path, + '/flake-gate/upload/', + 'the artifact must ship the tree that lives outside PR-writable space', + ); + // No env knob: $GITHUB_ENV is uid-1000 writable, so an overridable + // home would be a PR-reachable channel — and the record step rm -rf's + // whatever the home names. + for (const step of verifyJob.steps) { + assert.doesNotMatch( + String(step.run ?? ''), + /FLAKE_GATE_HOME/, + 'the gate home must not be overridable through the environment', + ); + } + }); + + it('git metadata stays root-owned across the build so the reset cannot be steered', () => { + // With .git node-owned, a lifecycle script could plant a smudge + // filter plus info/attributes and have ROOT's per-invocation reset + // execute it every round, or rewrite HEAD so the "restore" installs + // a tree of its choosing (R4-1/R4-2). + assert.match( + prepareStep.run, + /^\s*chown -R root:root "\$GITHUB_WORKSPACE\/\.git"/m, + 'prepare must re-own .git to root after chowning the workspace', + ); + const wsChown = prepareStep.run.indexOf( + 'chown -R node:node "$GITHUB_WORKSPACE"', + ); + const gitChown = prepareStep.run.indexOf( + 'chown -R root:root "$GITHUB_WORKSPACE/.git"', + ); + assert.ok( + wsChown !== -1 && gitChown > wsChown, + 'the .git re-own must come after the workspace chown that would otherwise hand it over', + ); + }); + it('the verify job timeout still covers agent + prepare + gate', () => { // agent 120m + install/build 15m + gate ~40m (the 15m round budget is // checked BEFORE each reset, so the last invocation drags its reset @@ -1961,7 +2054,18 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp // does NOT clear that inherited -e. That exact blind spot shipped the // round-1 blocker — the first failing test invocation killed the step — // so every scenario here runs under the wrapper, not under a bare bash. - const flakeRun = verifyJob.steps.find((s) => s.id === 'flake').run; + const flakeRunVerbatim = verifyJob.steps.find((s) => s.id === 'flake').run; + // The gate's home is a hard-coded container-root constant on purpose: an + // env-overridable home would be a PR-reachable channel ($GITHUB_ENV is + // uid-1000 writable and the record step rm -rf's whatever the home names). + // The harness therefore relocates that one constant into its scratch tree + // — a fixture substitution, not a production knob. The structural suite + // pins the production value separately. + const PROD_GATE_HOME = '/flake-gate'; + assert.ok( + flakeRunVerbatim.includes(`GATE_DIR=${PROD_GATE_HOME}`), + 'the gate must define its home as the container-root constant', + ); const publishRun = doc.jobs['publish-verify'].steps.find( (s) => s.name === 'Post verification report comment', ).run; @@ -2154,7 +2258,13 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp chmodSync(join(bin, name), 0o755); } const gateFile = join(root, 'gate.sh'); - writeFileSync(gateFile, flakeRun); + writeFileSync( + gateFile, + flakeRunVerbatim.replaceAll( + `GATE_DIR=${PROD_GATE_HOME}`, + `GATE_DIR=${gateDir}`, + ), + ); const out = join(rt, 'github-output'); writeFileSync(out, ''); writeFileSync(join(rt, 'github-summary'), ''); @@ -3396,7 +3506,17 @@ describe('qwen-triage: flakiness gate staging/upload — behavioral, under the p const runStaging = (rt, bin) => { const scriptFile = join(rt, 'staging.sh'); - writeFileSync(scriptFile, stageStep.run); + // Same fixture relocation as the gate harness: the home is a + // hard-coded container-root constant in production (an env knob there + // would be PR-reachable), so the suite moves that one constant into + // its scratch tree and pins the production value structurally. + writeFileSync( + scriptFile, + stageStep.run.replaceAll( + 'GATE_DIR=/flake-gate', + `GATE_DIR=${join(rt, 'flake-gate')}`, + ), + ); const out = join(rt, 'github-output'); writeFileSync(out, ''); return spawnSync( @@ -3492,7 +3612,18 @@ describe('qwen-triage: flakiness gate staging/upload — behavioral, under the p writeFileSync(out, ''); const res = spawnSync( 'bash', - ['--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', recheckStep.run], + [ + '--noprofile', + '--norc', + '-e', + '-o', + 'pipefail', + '-c', + recheckStep.run.replaceAll( + 'GATE_DIR=/flake-gate', + `GATE_DIR=${join(rt, 'flake-gate')}`, + ), + ], { env: { ...process.env, diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index d91f0ff4478..c6e5772da6c 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2859,13 +2859,28 @@ jobs: # swapped for /proc//environ (NUL-framed KEY=VALUE is # exactly the gate's intake framing, so ACTIONS_RUNTIME_TOKEN # lands verbatim in the published log), the round output or the - # staged log swapped for any root-readable file. 0700 root:root - # removes the capability itself — a directory node cannot enter - # is one whose entries it can neither create, unlink, nor - # rename. rm -rf first: the entry may already be a plant from an - # earlier run on this persistent pool. - rm -rf -- "${RUNNER_TEMP:?}/flake-gate" - install -d -m 0700 -o root -g root "$RUNNER_TEMP/flake-gate" + # staged log swapped for any root-readable file. + # + # The home lives at the CONTAINER ROOT, not under $RUNNER_TEMP. + # That is the whole defence, and it is a property of the PARENT: + # rename(2) and unlink(2) need write permission on the directory + # holding the entry, never on the entry itself — so a 0700 + # root:root home inside the uid-1000-writable $RUNNER_TEMP could + # always be renamed away wholesale and replaced with a + # node-owned lookalike, no matter how the gate validated it + # afterwards. Every re-validation we added there (ownership, + # mode, inode anchor, run-id marker) only narrowed the window; + # each one left the next TOCTOU gap, including the inter-step + # window before a `uses:` upload that cannot run shell at all. + # `/` is root:root 755 in this container: node can neither + # create, unlink, nor rename entries in it, so /flake-gate's + # entry — and therefore everything reached through it — is + # outside PR-controlled reach by construction, with no window to + # re-check. rm -rf first: the container is fresh per job, but a + # retried job on a warm image must not adopt an earlier tree. + GATE_HOME=/flake-gate + rm -rf -- "$GATE_HOME" + install -d -m 0700 -o root -g root "$GATE_HOME" # Run-freshness marker: staging re-validates it before trusting # the home. A stale-but-genuine home left by an earlier run on # the persistent pool passes every ownership/mode/shape check — @@ -2873,7 +2888,7 @@ jobs: # can never restage a previous run's evidence under this run's # artifact name when the record step itself was skipped (a # cancel between steps.pr and record still runs staging). - printf '%s-%s' "${GITHUB_RUN_ID:?}" "${GITHUB_RUN_ATTEMPT:?}" > "$RUNNER_TEMP/flake-gate/run-id" + printf '%s-%s' "${GITHUB_RUN_ID:?}" "${GITHUB_RUN_ATTEMPT:?}" > "$GATE_HOME/run-id" # Two statements, not a pipeline: `git diff | grep || true` would # swallow a git failure as "no changed test files", silently # narrowing the gate to n/a. A git failure here is pre-build @@ -2891,7 +2906,7 @@ jobs: # the runner executes, so it is a changed test file exactly # like M — excluding it silently drops the file from the gate. git -c core.quotePath=false diff -z --name-only --diff-filter=ACMRT 'HEAD^1' HEAD \ - > "$RUNNER_TEMP/flake-gate/files-all" + > "$GATE_HOME/files-all" # .mts/.cts included: vitest's default include set collects them. # Only a no-match (status 1) may yield an empty list: a grep # error (status 2, e.g. ENOSPC opening the output) is @@ -2899,14 +2914,14 @@ jobs: # would narrow the gate to zero files and starve it into n/a. grep_status=0 grep -zE '\.(test|spec)\.(ts|tsx|js|jsx|mjs|cjs|mts|cts)$' \ - "$RUNNER_TEMP/flake-gate/files-all" \ - > "$RUNNER_TEMP/flake-gate/files" || grep_status=$? + "$GATE_HOME/files-all" \ + > "$GATE_HOME/files" || grep_status=$? if [ "$grep_status" -gt 1 ]; then echo "flake-gate intake: grep failed with status ${grep_status}" >&2 exit 1 fi - rm -f "$RUNNER_TEMP/flake-gate/files-all" - echo "Recorded $(tr -cd '\0' < "$RUNNER_TEMP/flake-gate/files" | wc -c) changed test file(s) for the flakiness gate." + rm -f "$GATE_HOME/files-all" + echo "Recorded $(tr -cd '\0' < "$GATE_HOME/files" | wc -c) changed test file(s) for the flakiness gate." - name: 'Clear stale npm cache' if: "steps.pr.outputs.decision == 'run'" @@ -2944,6 +2959,24 @@ jobs: rm -f "$RUNNER_TEMP/verify-chromium-path" mkdir -p "$RUNNER_TEMP/verify-results" chown -R node:node "$GITHUB_WORKSPACE" + # ...but NOT .git. The gate's per-invocation reset restores the + # tree with `git checkout HEAD -- .`, and git reads its config, + # hooks and attributes from this directory as it does so: with + # .git node-owned, a lifecycle script could plant a + # `filter..smudge` command plus `info/attributes` and have + # ROOT's reset execute it on every round, or rewrite HEAD so the + # "restore" installs a tree of its choosing. Read access is all + # the build needs (git describe and friends); write access is + # what turns the reset into an execution and content primitive. + # Defence in depth, not the only line: the reset already runs + # git with the GIT_* env stripped, under a timeout, against an + # OID pinned before any PR code ran. Non-fatal so a fixture or + # a non-root harness cannot abort the build over it. + if [ -d "$GITHUB_WORKSPACE/.git" ]; then + chown -R root:root "$GITHUB_WORKSPACE/.git" 2>/dev/null || + echo "::warning::could not re-own .git to root; the gate reset falls back to its env/timeout/pinned-OID defences" + chmod -R go-w "$GITHUB_WORKSPACE/.git" 2>/dev/null || true + fi # Make the npm cache readable+writable by the build user so # `npm ci --prefer-offline --cache …` can use it without # touching the Actions cache API (whose credentials were @@ -3178,7 +3211,7 @@ jobs: # it: a plant (symlink, node-owned dir, loosened mode) means the # integrity premise never held, and the gate must degrade to the # fixed error verdict rather than read attacker-chosen bytes. - GATE_DIR="${RUNNER_TEMP:?}/flake-gate" + GATE_DIR=/flake-gate # -O is the load-bearing test: "owned by the EFFECTIVE user" is # root in production, so a directory a PR planted (owned by node) # fails it, while the same code stays runnable under a harness. @@ -3348,7 +3381,11 @@ jobs: esac case "$f" in scripts/tests/*.test.js|scripts/tests/*.test.ts) - group_labels+=("$f") + # Same substring-collision honesty as the generic arm: one + # positional filter collects every path containing it. + label="$f" + [ -f "${f}x" ] && label="$f + ${f}x" + group_labels+=("$label") group_dirs+=('.') group_cmds+=("npx --no-install vitest run --config ./scripts/tests/vitest.config.ts $(printf '%q' "./$f")") ;; @@ -3546,7 +3583,11 @@ jobs: # Fresh per-invocation HOME and temp/cache dirs: samples must # not share dotfile/XDG/cache state or caches any more than # they share the tree or processes. - inv_tmp="$RUNNER_TEMP/flake-inv-tmp" + # Node-writable by necessity (the build user cannot enter + # the root-only home), so it stays under RUNNER_TEMP — and + # :? keeps a missing RUNNER_TEMP from silently relocating + # it to the container root. + inv_tmp="${RUNNER_TEMP:?}/flake-inv-tmp" rm -rf "$inv_tmp" mkdir -p "$inv_tmp" # -h: never dereference — a planted symlink at this fixed @@ -4276,7 +4317,7 @@ jobs: echo "verify verdict: $VERDICT agent: ${AGENT_VERDICT:-none} (exit $EXIT_CODE)" >> "$GITHUB_STEP_SUMMARY" # The gate log's authoritative copy lives root-owned inside the 0700 - # root-only home ($RUNNER_TEMP/flake-gate/log), which no PR-controlled + # root-only home (/flake-gate/log), which no PR-controlled # process can enter. This step runs AFTER the agent exits, from an # always() root step, and assembles the upload tree there too: an # early agent-step abort cannot lose the log, and agent-era PR code @@ -4369,10 +4410,10 @@ jobs: # in place always left the ENTRY itself renameable, so a kill-race # survivor could swap the whole hardened tree for a symlink farm # that upload-artifact (which follows links) would publish. The - # upload now reads from $RUNNER_TEMP/flake-gate/upload, inside the + # upload now reads from /flake-gate/upload, inside the # 0700 root-only home: a directory node cannot enter is one whose # entries it can neither create, unlink, nor rename. - GATE_DIR="${RUNNER_TEMP:?}/flake-gate" + GATE_DIR=/flake-gate UPLOAD_DIR="$GATE_DIR/upload" # The run-id conjunct is the RUN-identity check: ownership, # mode and shape all pass on a stale-but-genuine home an @@ -4474,10 +4515,10 @@ jobs: # this run's artifact name. rm -rf removes a symlink # operand itself, never following it. echo "::warning::flake-gate home missing, invalid, or swapped mid-staging; skipping the trusted upload rebuild." - rm -rf -- "${RUNNER_TEMP:?}/flake-gate" + rm -rf -- "$GATE_DIR" fi - # The always() upload below re-resolves $RUNNER_TEMP/flake-gate in a + # The always() upload below re-resolves /flake-gate in a # fresh step — staging's anchoring expires at its exit, so a swap # between the two hands the upload a plant. Re-validate the entry's # identity immediately before the upload and remove it on any @@ -4502,7 +4543,7 @@ jobs: export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' fi upload_ok=false - GATE_DIR="${RUNNER_TEMP:?}/flake-gate" + GATE_DIR=/flake-gate if [[ ! -L $GATE_DIR ]] && [[ -d $GATE_DIR ]] && [[ -O $GATE_DIR ]] && [[ $(/usr/bin/stat -c '%a' "$GATE_DIR" 2>/dev/null) == '700' ]] && [[ $(/usr/bin/cat "$GATE_DIR/run-id" 2>/dev/null) == "${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}" ]]; then @@ -4518,7 +4559,7 @@ jobs: # Whatever occupies the entry now is not this run's # validated home — remove it so the upload cannot # enumerate a plant. - /usr/bin/rm -rf -- "${RUNNER_TEMP:?}/flake-gate" + /usr/bin/rm -rf -- "$GATE_DIR" fi echo "upload_ok=$upload_ok" >> "$GITHUB_OUTPUT" @@ -4533,7 +4574,7 @@ jobs: # The rebuilt, root-only tree — not the agent-era directory the # build user owned. Same inner layout, so the publisher's paths # are unchanged. - path: '${{ runner.temp }}/flake-gate/upload/' + path: '/flake-gate/upload/' retention-days: 7 - name: 'Clean up runner workspace' From 2795443bd52411e20f1b8830bfd5f46b75afed63 Mon Sep 17 00:00:00 2001 From: wenshao Date: Tue, 18 Aug 2026 04:11:16 +0800 Subject: [PATCH 15/18] fix(ci): clear the publisher's downloaded results before the download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R16-3: publish-verify runs on the persistent ECS pool and downloads the artifact into a workspace-relative `verify-results`, which the runner does not clean between jobs. The publisher treats the presence of `verify-results/flake-gate.log` as proof that THIS run staged it, so a previous run's log — possibly from another PR — could be embedded as this run's evidence. The verify side already applies the same rm-first rule to its own $RUNNER_TEMP tree; this brings the publisher in line. Pinned by a structural test asserting the clear step exists and precedes the download. --- .github/scripts/qwen-triage-workflow.test.mjs | 18 ++++++++++++++++++ .github/workflows/qwen-triage.yml | 10 ++++++++++ 2 files changed, 28 insertions(+) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index 6c47491df60..96d4f3337ce 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -2010,6 +2010,24 @@ describe('qwen-triage: flakiness gate (#9125)', () => { } }); + it('the publisher clears its downloaded results before the download', () => { + // publish-verify runs on the persistent pool and downloads into a + // workspace-relative dir the runner never cleans: a previous run's + // flake-gate.log would otherwise survive and be embedded as this + // run's evidence, since the publisher treats presence as proof. + const publishJob = doc.jobs['publish-verify']; + const clearIdx = publishJob.steps.findIndex( + (x) => x.name === 'Clear stale downloaded results', + ); + const downloadIdx = publishJob.steps.findIndex((x) => x.id === 'download'); + assert.ok(clearIdx !== -1, 'the publisher must clear stale results'); + assert.ok( + downloadIdx !== -1 && clearIdx < downloadIdx, + 'the clear must precede the download', + ); + assert.match(publishJob.steps[clearIdx].run, /rm -rf verify-results/); + }); + it('git metadata stays root-owned across the build so the reset cannot be steered', () => { // With .git node-owned, a lifecycle script could plant a smudge // filter plus info/attributes and have ROOT's per-invocation reset diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index c6e5772da6c..531fe674420 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -4684,6 +4684,16 @@ jobs: fi chmod -R u+rwX "$GITHUB_WORKSPACE" 2>/dev/null || sudo -n chmod -R u+rwX "$GITHUB_WORKSPACE" || echo "::warning::could not restore workspace write permissions; artifact download may fail on leftover read-only files" + # The publish job runs on the persistent ECS pool and downloads into + # a WORKSPACE-relative directory, which the runner does not clean + # between jobs: without this, a previous run's (possibly a different + # PR's) flake-gate.log survives and the publisher — which treats the + # file's presence as proof THIS run staged it — embeds it as this + # run's evidence. The verify side already applies the same rm-first + # rule to its own $RUNNER_TEMP tree. + - name: 'Clear stale downloaded results' + run: 'rm -rf verify-results' + - name: 'Download verify results' id: 'download' uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v5.0.0 From 4a927a1b3b7bc920fd74f7264b99ea21cba28efd Mon Sep 17 00:00:00 2001 From: wenshao Date: Tue, 18 Aug 2026 04:15:41 +0800 Subject: [PATCH 16/18] =?UTF-8?q?fix(ci):=20drop=20the=20.git=20re-own=20?= =?UTF-8?q?=E2=80=94=20it=20breaks=20the=20build=20user's=20own=20reset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverting the .git hardening from 387a843434 after measuring it: the per-invocation reset runs as `node` (root's git trips the dubious-ownership guard), and a root-owned .git makes it fail at `Unable to create '.git/index.lock'` — every round's reset would abort, which the gate correctly reports as `error`, i.e. the gate would stop working entirely. Probe: root:root + go-w on .git, reset as the build user → 'Permission denied' on index.lock. The R4-1/R4-2/R16-1 surface it aimed at (metadata-steered resets) keeps its existing defences — OID pinned before sampling, GIT_* stripped from the reset's environment, timeouts, and the per-round strip of .git execution vectors — and the residual (that strip is a denylist, so include/includeIf indirection can still reach it) is tracked as follow-up rather than closed by a change that disables the gate. --- .github/scripts/qwen-triage-workflow.test.mjs | 22 ------------------- .github/workflows/qwen-triage.yml | 18 --------------- 2 files changed, 40 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index 96d4f3337ce..5546aa0904d 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -2028,28 +2028,6 @@ describe('qwen-triage: flakiness gate (#9125)', () => { assert.match(publishJob.steps[clearIdx].run, /rm -rf verify-results/); }); - it('git metadata stays root-owned across the build so the reset cannot be steered', () => { - // With .git node-owned, a lifecycle script could plant a smudge - // filter plus info/attributes and have ROOT's per-invocation reset - // execute it every round, or rewrite HEAD so the "restore" installs - // a tree of its choosing (R4-1/R4-2). - assert.match( - prepareStep.run, - /^\s*chown -R root:root "\$GITHUB_WORKSPACE\/\.git"/m, - 'prepare must re-own .git to root after chowning the workspace', - ); - const wsChown = prepareStep.run.indexOf( - 'chown -R node:node "$GITHUB_WORKSPACE"', - ); - const gitChown = prepareStep.run.indexOf( - 'chown -R root:root "$GITHUB_WORKSPACE/.git"', - ); - assert.ok( - wsChown !== -1 && gitChown > wsChown, - 'the .git re-own must come after the workspace chown that would otherwise hand it over', - ); - }); - it('the verify job timeout still covers agent + prepare + gate', () => { // agent 120m + install/build 15m + gate ~40m (the 15m round budget is // checked BEFORE each reset, so the last invocation drags its reset diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index 531fe674420..fb9bad09ba0 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2959,24 +2959,6 @@ jobs: rm -f "$RUNNER_TEMP/verify-chromium-path" mkdir -p "$RUNNER_TEMP/verify-results" chown -R node:node "$GITHUB_WORKSPACE" - # ...but NOT .git. The gate's per-invocation reset restores the - # tree with `git checkout HEAD -- .`, and git reads its config, - # hooks and attributes from this directory as it does so: with - # .git node-owned, a lifecycle script could plant a - # `filter..smudge` command plus `info/attributes` and have - # ROOT's reset execute it on every round, or rewrite HEAD so the - # "restore" installs a tree of its choosing. Read access is all - # the build needs (git describe and friends); write access is - # what turns the reset into an execution and content primitive. - # Defence in depth, not the only line: the reset already runs - # git with the GIT_* env stripped, under a timeout, against an - # OID pinned before any PR code ran. Non-fatal so a fixture or - # a non-root harness cannot abort the build over it. - if [ -d "$GITHUB_WORKSPACE/.git" ]; then - chown -R root:root "$GITHUB_WORKSPACE/.git" 2>/dev/null || - echo "::warning::could not re-own .git to root; the gate reset falls back to its env/timeout/pinned-OID defences" - chmod -R go-w "$GITHUB_WORKSPACE/.git" 2>/dev/null || true - fi # Make the npm cache readable+writable by the build user so # `npm ci --prefer-offline --cache …` can use it without # touching the Actions cache API (whose credentials were From 495955fdf46b5aaf6a85082cf0ee4accd7dd43dc Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot Date: Tue, 18 Aug 2026 01:28:40 +0000 Subject: [PATCH 17/18] fix(ci): close flake-gate startup-channel shadowing; blank the upload loader env (#9130) --- .github/scripts/qwen-triage-workflow.test.mjs | 128 +++++++++++++++++- .github/workflows/qwen-triage.yml | 96 +++++++++++-- 2 files changed, 214 insertions(+), 10 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index 5546aa0904d..5b12b60516d 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -956,6 +956,14 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /case "\$\{1:-\}" in[\s\S]*?--flake-clean-child\) ;;[\s\S]*?_flake_body="\$\(<"\$\{BASH_SOURCE\[0\]\}"\)"[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?\/usr\/bin\/bash --noprofile --norc -e -o pipefail -c "\$_flake_body" \S+ --flake-clean-child/; const reExecMarkerRe = /case "\$\{1:-\}" in/; const pathChildRe = /"\$\{BASH_SOURCE\[0\]\}" --flake-clean-child/; + // R15-1: POSIX mode resolves special builtins before functions, so the + // re-exec's `exec` and every refusal's `exit` cannot be shadowed by a + // BASH_FUNC_* import the way bare builtins can (probe-verified on this + // pool's bash). `set` is itself shadowable, so the switch is verified + // with a reserved word, and the refusal ends in a slash-pathed kill of + // last resort for the case where `exit` is shadowed too. + const posixSwitchRe = /^\s*set -o posix\n\s*if \[\[ ! -o posix \]\]; then$/m; + const posixKillRe = /exit [01]\n\s*\/usr\/bin\/kill -9 \$\$\n\s*fi/; it('records the changed-test list BEFORE the workspace is handed to the build user', () => { assert.ok(recordStep, 'record step must exist'); @@ -1079,6 +1087,29 @@ describe('qwen-triage: flakiness gate (#9125)', () => { pathPinRe, 'the record step must pin a root-only-writable PATH — its inherited one may be poisoned through the file-command backing files', ); + // R15-1: `exec` and `exit` are builtins and function lookup precedes + // builtins — a BASH_FUNC_exec%% import shadows the re-exec keyword and + // the poisoned parent falls through with every import alive + // (probe-verified on this pool's bash). POSIX mode resolves special + // builtins before functions, closing the class for `exec`, `exit` and + // every refusal; the switch must precede the first refusal it + // immunizes. + assert.match( + recordStep.run, + posixSwitchRe, + 'the record step must enter POSIX mode so exec/exit cannot be shadowed by a BASH_FUNC import', + ); + assert.match( + recordStep.run, + posixKillRe, + 'the posix refusal must end in a slash-pathed kill — exit itself may be the shadowed builtin', + ); + const recordPosix = recordStep.run.search(/^\s*set -o posix$/m); + const recordScrub = recordStep.run.search(scrubRefusalRe); + assert.ok( + recordPosix !== -1 && recordScrub !== -1 && recordPosix < recordScrub, + 'the POSIX switch must precede the first refusal whose exit it immunizes', + ); // A grep ERROR (status 2 — e.g. ENOSPC opening the output) is // infrastructure: swallowing it narrows the gate to zero files and // starves it into n/a, so it must fail the record step loudly. @@ -1205,6 +1236,24 @@ describe('qwen-triage: flakiness gate (#9125)', () => { scrubRefusalRe, 'the gate scrub refusal must be reserved-word shaped — `[` is shadowable by a BASH_FUNC import', ); + // R15-1: same discipline as the record step — the gate's fail-open + // refusals ride the same exit channel. + assert.match( + flakeStep.run, + posixSwitchRe, + 'the gate must enter POSIX mode so exec/exit cannot be shadowed by a BASH_FUNC import', + ); + assert.match( + flakeStep.run, + posixKillRe, + 'the posix refusal must end in a slash-pathed kill — exit itself may be the shadowed builtin', + ); + const gatePosix = flakeStep.run.search(/^\s*set -o posix$/m); + const gateScrub = flakeStep.run.search(scrubRefusalRe); + assert.ok( + gatePosix !== -1 && gateScrub !== -1 && gatePosix < gateScrub, + 'the POSIX switch must precede the first refusal whose exit it immunizes', + ); // R14-2/R12-2: the runner writes this script as a node-owned file // inside the uid-1000-writable $RUNNER_TEMP — a detached // install/build survivor still alive at the re-exec snapshot @@ -1621,6 +1670,32 @@ describe('qwen-triage: flakiness gate (#9125)', () => { /\/usr\/bin\/rm -rf -- "\$GATE_DIR"/, 'a home that fails the re-check must be removed before the upload enumerates the path', ); + // R15-1: the re-check has no env -i re-exec — it runs in the + // inherited job environment. POSIX mode immunizes set/export/exit; + // every remaining decision must stay a reserved word and every + // external absolute-pathed, so no shadowable command word stands + // between a poisoned env and the verdict (a bare cd/[/stat subshell + // or a bare echo verdict write would re-open exactly that surface). + assert.match( + recheckStep.run, + posixSwitchRe, + 'the re-check must enter POSIX mode so set/export/exit cannot be shadowed by a BASH_FUNC import', + ); + assert.match( + recheckStep.run, + posixKillRe, + 'the posix refusal must end in a slash-pathed kill — exit itself may be the shadowed builtin', + ); + assert.match( + recheckStep.run, + /\/usr\/bin\/printf 'upload_ok=%s\\n' "\$upload_ok" >> "\$GITHUB_OUTPUT"/, + 'the verdict write must be slash-pathed — echo is shadowable by a BASH_FUNC import', + ); + assert.doesNotMatch( + recheckStep.run, + /\(\s*cd /, + 'the re-check must not run a cd-anchored subshell — bare cd/[/stat are shadowable command words', + ); assert.match( uploadStep.with.name, /^verify-results-/, @@ -1631,6 +1706,14 @@ describe('qwen-triage: flakiness gate (#9125)', () => { true, 'a missing/empty results dir must not fail the job and mask the original error', ); + // R16-2: the loader channel reaches the final consumer of the defense + // chain — this uses: step's node process inherits the job env the + // run: steps blank at their own blocks. + assert.deepEqual( + Object.keys(uploadStep.env).sort(), + ['LD_AUDIT', 'LD_LIBRARY_PATH', 'LD_PRELOAD'], + 'the upload step must blank the LD_* loader channels like its run: siblings', + ); // Evidence-copying only, after the verdict outputs are written: a // staging failure (ENOSPC, hostile mount) must not flip the job red or // the publisher discards the recorded verdict as "infrastructure". @@ -1842,6 +1925,17 @@ describe('qwen-triage: flakiness gate (#9125)', () => { scrubRefusalRe, 'the staging scrub refusal must be reserved-word shaped — `[` is shadowable by a BASH_FUNC import', ); + // R15-1: same discipline as the record step. + assert.match( + sr, + posixSwitchRe, + 'staging must enter POSIX mode so exec/exit cannot be shadowed by a BASH_FUNC import', + ); + assert.match( + sr, + posixKillRe, + 'the posix refusal must end in a slash-pathed kill — exit itself may be the shadowed builtin', + ); assert.doesNotMatch( sr, /_FLAKE_CLEAN_REEXEC/, @@ -3419,6 +3513,31 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp ); }); + it('a BASH_FUNC_exec%% import cannot skip the env -i re-exec — the body runs exactly once', () => { + // `exec` is a builtin and function lookup precedes builtins: a + // BASH_FUNC_exec%% import shadows it, the re-exec line runs the + // function instead of replacing the shell, and the poisoned parent + // falls through into the body with every import alive + // (probe-verified on this pool's bash: the env-i child never runs). + // POSIX mode resolves special builtins before functions, so the + // transition is immune — observable as an honest `pass` and exactly + // one run of the rounds: against the pre-round code this scenario + // fell through into the poisoned parent and corrupted the verdict. + const { res, outputs, counts } = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\n', + sequences: { 'a.test.js': 'PPPPP' }, + env: { 'BASH_FUNC_exec%%': '() { return 0; }' }, + }); + assert.equal(res.status, 0, res.stderr); + assert.equal(outputs.flake_verdict, 'pass'); + assert.equal( + counts('a.test.js'), + 5, + 'the re-exec must run exactly once — a shadowed exec falls through and re-runs the body', + ); + }); + it('a same-stem sibling (X.test.tsx next to changed X.test.ts) runs in ONE merged group, never attributed separately', () => { // vitest's positional filters are lowercase SUBSTRING matches on // root-relative paths — verified against the installed vitest: one @@ -3487,7 +3606,14 @@ describe('qwen-triage: flakiness gate staging/upload — behavioral, under the p ); const stageRoot = mkdtempSync(join(tmpdir(), 'flake-staging-')); - after(() => rmSync(stageRoot, { recursive: true, force: true })); + after(() => { + // Same safeguard as the gate suite's hook: a scenario that leaves a + // mode-500 directory behind makes rmSync throw EACCES (force + // suppresses ENOENT only), marking the whole suite hookFailed and + // leaking the tree; restore owner permissions first. + spawnSync('chmod', ['-R', 'u+rwx', stageRoot]); + rmSync(stageRoot, { recursive: true, force: true }); + }); const makeHome = (rt, { runId = '777-1', files = {} } = {}) => { const home = join(rt, 'flake-gate'); diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index fb9bad09ba0..1d971d603c1 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2780,6 +2780,21 @@ jobs: LD_LIBRARY_PATH: '' run: |- set -euo pipefail + # Startup-channel scrub, part zero: POSIX mode resolves special + # builtins BEFORE functions, so the `exec` below — and every + # later `exit` — cannot be shadowed by a BASH_FUNC_* import the + # way bare builtins can. A shadowed `exec` would skip the env -i + # re-exec and let the poisoned parent continue with every import + # alive; a shadowed `exit` would let it fall through after. + # `set` itself is a builtin and shadowable, so verify with a + # reserved word that the mode took, and refuse if not (the kill + # is the stop of last resort when `exit` is shadowed too). + set -o posix + if [[ ! -o posix ]]; then + echo "::error::flake-gate record: startup-channel scrub unavailable — refusing to record in a poisoned environment" + exit 1 + /usr/bin/kill -9 $$ + fi # Startup-channel scrub, part two: BASH_FUNC_%% env # entries are imported as shell functions BEFORE this body # runs, are invisible to ${!BASH_FUNC_@}/`set`, and shadow @@ -3102,6 +3117,23 @@ jobs: # gate exists to classify (round-1 sandboxed verify, cells C/D). set -uo pipefail set +e + # Startup-channel scrub, part zero: POSIX mode resolves special + # builtins BEFORE functions, so the `exec` below — and every + # later `exit` — cannot be shadowed by a BASH_FUNC_* import the + # way bare builtins can (a shadowed `exec` would skip the env -i + # re-exec and let the poisoned parent continue with every import + # alive; a shadowed `exit` would let it fall through after). + # `set` itself is a builtin and shadowable, so verify with a + # reserved word that the mode took; the refusal keeps the gate's + # fail-open verdict shape, and the kill only fires when `exit` + # is shadowed too. + set -o posix + if [[ ! -o posix ]]; then + echo "flake_verdict=error" >> "$GITHUB_OUTPUT" + echo "flake_summary=the gate could not enter a scrub-safe shell mode — refusing to sample" >> "$GITHUB_OUTPUT" + exit 0 + /usr/bin/kill -9 $$ + fi # Startup-channel scrub, part one: if the step-env blanks # above lost to the poisoned job env, BASH_ENV's payload # already ran at startup and the LD_* channels are live — a @@ -3183,6 +3215,12 @@ jobs: if [ -z "$GATE_DONE" ] && [ -n "${GITHUB_OUTPUT:-}" ]; then echo "flake_verdict=error" >> "$GITHUB_OUTPUT" echo "flake_summary=the gate aborted before reaching a verdict — see the step log" >> "$GITHUB_OUTPUT" + # Keep the two channels agreeing (finish() writes both): an + # abnormal abort otherwise leaves the step summary empty + # while the step output carries the verdict. + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + echo "Flakiness gate: error — the gate aborted before reaching a verdict — see the step log" >> "$GITHUB_STEP_SUMMARY" + fi fi exit 0 } @@ -4329,6 +4367,21 @@ jobs: LD_LIBRARY_PATH: '' run: |- set -euo pipefail + # Startup-channel scrub, part zero: POSIX mode resolves special + # builtins BEFORE functions, so the `exec` below — and every + # later `exit` — cannot be shadowed by a BASH_FUNC_* import the + # way bare builtins can. A shadowed `exec` would skip the env -i + # re-exec and let the poisoned parent continue with every import + # alive; a shadowed `exit` would let it fall through after. + # `set` itself is a builtin and shadowable, so verify with a + # reserved word that the mode took, and refuse if not (the kill + # is the stop of last resort when `exit` is shadowed too). + set -o posix + if [[ ! -o posix ]]; then + echo "::error::flake-gate staging: startup-channel scrub unavailable — refusing to stage evidence in a poisoned environment" + exit 1 + /usr/bin/kill -9 $$ + fi # Startup-channel scrub (same shape and rationale as the gate # step's): fail closed if the step-env blanks lost, then # re-exec once through an absolute-path env -i child to drop @@ -4521,21 +4574,37 @@ jobs: LD_LIBRARY_PATH: '' run: |- set -uo pipefail + # Startup-channel scrub (see the record step): this step has no + # env -i re-exec — it runs in the inherited job environment. + # POSIX mode resolves special builtins before functions, so + # set/export/exit cannot be shadowed by a BASH_FUNC_* import; + # every remaining decision below is a reserved word and every + # external absolute-pathed, so no shadowable command word stands + # between the poisoned environment this step may inherit and the + # verdict it writes. Verify with a reserved word that the mode + # took, and refuse if not (the kill is the stop of last resort + # when `exit` is shadowed too). + set -o posix + if [[ ! -o posix ]]; then + echo "::error::flake-gate re-check: startup-channel scrub unavailable — refusing to validate in a poisoned environment" + exit 1 + /usr/bin/kill -9 $$ + fi if [[ ${EUID:-1} -eq 0 ]]; then export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' fi upload_ok=false GATE_DIR=/flake-gate + # No cd-anchored subshell: the home sits directly under the + # root-only container root, where no PR-controlled process can + # rename the entry, so re-resolving $GATE_DIR cannot land + # anywhere else — and bare cd/[/stat would re-introduce exactly + # the shadowable command words this step must not run. if [[ ! -L $GATE_DIR ]] && [[ -d $GATE_DIR ]] && [[ -O $GATE_DIR ]] && [[ $(/usr/bin/stat -c '%a' "$GATE_DIR" 2>/dev/null) == '700' ]] && - [[ $(/usr/bin/cat "$GATE_DIR/run-id" 2>/dev/null) == "${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}" ]]; then - home_id="$(/usr/bin/stat -c '%d:%i' "$GATE_DIR" 2>/dev/null || true)" - if ( cd "$GATE_DIR" && - [ "$(stat -c '%d:%i' . 2>/dev/null)" = "$home_id" ] && - [ -O . ] && [ "$(stat -c '%a' . 2>/dev/null)" = '700' ] && - [ -d upload ] ); then - upload_ok=true - fi + [[ $(/usr/bin/cat "$GATE_DIR/run-id" 2>/dev/null) == "${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}" ]] && + [[ -d $GATE_DIR/upload ]]; then + upload_ok=true fi if [[ $upload_ok != true ]]; then # Whatever occupies the entry now is not this run's @@ -4543,13 +4612,22 @@ jobs: # enumerate a plant. /usr/bin/rm -rf -- "$GATE_DIR" fi - echo "upload_ok=$upload_ok" >> "$GITHUB_OUTPUT" + /usr/bin/printf 'upload_ok=%s\n' "$upload_ok" >> "$GITHUB_OUTPUT" - name: 'Upload verify results' if: "always() && steps.pr.outputs.decision == 'run' && steps.flake-upload-check.outputs.upload_ok == 'true'" # Don't let a missing/empty results dir (qwen crashed before writing # any) fail the job and mask the original error. continue-on-error: true + # Startup-channel scrub (see the gate step): the run: steps this PR + # adds blank the LD_* loader channels in their step env; this uses: + # step's node process inherits the same job env, and the loader + # channel reaches the final consumer of the chain — the enumeration + # of the staged tree — unless it is blanked here too. + env: + LD_PRELOAD: '' + LD_AUDIT: '' + LD_LIBRARY_PATH: '' uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # v4.6.2 with: name: 'verify-results-${{ steps.pr.outputs.pr_number }}-${{ github.run_id }}-${{ github.run_attempt }}' From 54b2494acbfdffdf652d07afb076554fdfbdd902 Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot Date: Tue, 18 Aug 2026 06:58:59 +0000 Subject: [PATCH 18/18] fix(ci): close flake-gate startup-window races in the re-exec and verdict path (#9130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R18-1: anchor the record/gate/staging re-exec snapshots to the inode bash is executing (fd 255) and re-verify the path against it before the exec — a swap that lands between bash's open of the runner-written step script and the snapshot is filesystem state the kill cannot un-land. R18-2: POSIXLY_CORRECT in all four gate-family step envs — bash then refuses BASH_FUNC imports named after special builtins at startup, so a poisoned `set` cannot run attacker code on the body's first command (the in-script POSIX switch arrives one command late); a poisoned startup fails the step red, and the abort is the refusal. R18-3: every pre-re-exec refusal writes through /usr/bin/printf — echo is a regular builtin, shadowable by a BASH_FUNC_echo%% import even in POSIX mode (probe-verified). R18-4: parent-side identity gates query the kernel via /usr/bin/id -u instead of reading $EUID, which bash imports from the process environment — one planted EUID line could skip the kill sweeps and the poisoned-env refusals in every later step. R16-4 (interim): the publisher's full-report branch maps a missing or unrecognized FLAKE_VERDICT to a visible fixed-text error line instead of silently dropping it — on that branch the gate ran and owes a verdict, so absence means the uid-1000-writable $GITHUB_OUTPUT backing channel corrupted it in transit. Fixed text only: the raw value is attacker-influenced on this path and is never embedded. The behavioral harness now applies the step env block (production parity), and the two round-15 poison scenarios are re-pinned to the stronger startup refusal. --- .github/scripts/qwen-triage-workflow.test.mjs | 380 ++++++++++++++++-- .github/workflows/qwen-triage.yml | 157 ++++++-- 2 files changed, 457 insertions(+), 80 deletions(-) diff --git a/.github/scripts/qwen-triage-workflow.test.mjs b/.github/scripts/qwen-triage-workflow.test.mjs index 5b12b60516d..f21b2c562bf 100644 --- a/.github/scripts/qwen-triage-workflow.test.mjs +++ b/.github/scripts/qwen-triage-workflow.test.mjs @@ -950,8 +950,12 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // the parent snapshotted — the runner wrote the script node-owned in // the uid-1000-writable $RUNNER_TEMP, so a second open by the child is // a plant window. + // R18-4: the identity conjunct queries the kernel through an absolute + // path — bash imports $EUID from the process environment, overriding + // the native readonly variable, so an EUID line planted through the + // uid-1000 file-command channel would skip every root-gated defence. const scrubRefusalRe = - /^\s*if \[\[ \$\{EUID:-1\} -eq 0 \]\] && \[\[ -n \$\{BASH_ENV:-\} \|\| -n \$\{LD_PRELOAD:-\} \|\| -n \$\{LD_AUDIT:-\} \|\| -n \$\{LD_LIBRARY_PATH:-\} \]\]; then$/m; + /^\s*if \[\[ \$\(\/usr\/bin\/id -u\) -eq 0 \]\] && \[\[ -n \$\{BASH_ENV:-\} \|\| -n \$\{LD_PRELOAD:-\} \|\| -n \$\{LD_AUDIT:-\} \|\| -n \$\{LD_LIBRARY_PATH:-\} \]\]; then$/m; const reExecRe = /case "\$\{1:-\}" in[\s\S]*?--flake-clean-child\) ;;[\s\S]*?_flake_body="\$\(<"\$\{BASH_SOURCE\[0\]\}"\)"[\s\S]*?LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec \/usr\/bin\/env -i[\s\S]*?\/usr\/bin\/bash --noprofile --norc -e -o pipefail -c "\$_flake_body" \S+ --flake-clean-child/; const reExecMarkerRe = /case "\$\{1:-\}" in/; @@ -1068,7 +1072,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); assert.match( recordStep.run, - /survivors="\$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\)" \|\| true\n\s*if \[\[ -n \$survivors \]\]; then\n\s*echo "::error::flake-gate record:[\s\S]*?exit 1/, + /survivors="\$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\)" \|\| true\n\s*if \[\[ -n \$survivors \]\]; then\n\s*\/usr\/bin\/printf '::error::flake-gate record:[\s\S]*?\\n'\n\s*exit 1/, 'the record step kill must be liveness-verified — the budget loop alone is out-forked between sweeps', ); const recordKill = recordStep.run.search(/survivors="\$\(\/usr\/bin\/ps/); @@ -1164,6 +1168,107 @@ describe('qwen-triage: flakiness gate (#9125)', () => { ); }); + it('round-19 startup hardening: kernel identity, POSIX-from-invocation, pathed refusal writes, inode-anchored snapshot', () => { + const stageStep = verifyJob.steps.find( + (s) => s.name === 'Stage flakiness gate log for upload', + ); + const recheckStep = verifyJob.steps.find( + (s) => s.id === 'flake-upload-check', + ); + const scrubbedSteps = [ + ['record', recordStep], + ['gate', flakeStep], + ['staging', stageStep], + ['re-check', recheckStep], + ]; + // R18-2: POSIXLY_CORRECT in the step env puts bash in POSIX mode at + // INVOCATION — a BASH_FUNC_* import named after a special builtin is + // refused at import (probe-verified: without it, a poisoned `set` + // runs attacker code on the body's first command and can even enable + // posix itself to slip past the reserved-word refusal). + for (const [label, step] of scrubbedSteps) { + assert.equal( + step.env.POSIXLY_CORRECT, + '1', + `${label}: POSIXLY_CORRECT must make bash POSIX-mode before the body's first shadowable command`, + ); + } + // R18-4: parent-side identity comes from the kernel — $EUID is + // imported from the process environment (probe-verified: a planted + // EUID skips or fires the gate at will; blanking cannot restore it, + // set-but-empty reads as unset). The record/gate/staging PATH pins + // run INSIDE the env -i child where imports are wiped and EUID is + // the native readonly — only the parent-side gates are pinned here. + const preReExec = (run) => run.slice(0, run.search(reExecMarkerRe)); + for (const [label, step] of [ + ['record', recordStep], + ['gate', flakeStep], + ['staging', stageStep], + ]) { + assert.doesNotMatch( + preReExec(step.run), + /\$\{EUID/, + `${label}: parent-side identity must not read $EUID — the poisoned file-command channel can import it`, + ); + } + assert.doesNotMatch( + recheckStep.run, + /\$\{EUID/, + 're-check identity must not read $EUID — the step has no env -i re-exec, so it runs in the inherited job environment', + ); + assert.match( + recheckStep.run, + /^\s*if \[\[ \$\(\/usr\/bin\/id -u\) -eq 0 \]\]; then\n\s*export PATH='/m, + 'the re-check PATH pin must key on the kernel identity too', + ); + // R18-3: every pre-re-exec refusal write goes through slash-pathed + // printf — echo is a REGULAR builtin, shadowable by a BASH_FUNC + // import even in POSIX mode (probe-verified; the mechanism pin lives + // in the behavioral suite), and the refusal path runs in exactly the + // poisoned environment the refusals detect. + for (const [label, section] of [ + ['record', preReExec(recordStep.run)], + ['gate', preReExec(flakeStep.run)], + ['staging', preReExec(stageStep.run)], + ['re-check', recheckStep.run], + ]) { + assert.doesNotMatch( + section, + /^\s*echo /m, + `${label}: no bare echo before the env -i re-exec — BASH_FUNC_echo%% shadows it even in POSIX mode`, + ); + } + // R18-1: the re-exec snapshot is anchored to the inode bash is + // executing (fd 255) — a swap that lands between bash's open of the + // runner-written step script and the snapshot is filesystem state a + // kill cannot un-land, and the path re-open would read the plant. + // Capture precedes the snapshot, the check precedes the exec. + const inodeAnchorRe = + /_flake_self_id="\$\(\/usr\/bin\/stat -L -c '%d:%i' "\/proc\/\$\$\/fd\/255" 2>\/dev\/null\)" \|\| _flake_self_id=''[\s\S]*?_flake_body="\$\(<"\$\{BASH_SOURCE\[0\]\}"\)"[\s\S]*?if \[\[ -z \$_flake_self_id \]\] \|\|\n\s*\[\[ "\$\(\/usr\/bin\/stat -L -c '%d:%i' "\$\{BASH_SOURCE\[0\]\}" 2>\/dev\/null\)" != "\$_flake_self_id" \]\]; then/; + for (const [label, step] of [ + ['record', recordStep], + ['gate', flakeStep], + ['staging', stageStep], + ]) { + assert.match( + step.run, + inodeAnchorRe, + `${label}: the re-exec snapshot must be anchored to the inode bash is executing, re-verified after the snapshot, before the exec`, + ); + const killAt = step.run.search(/\/usr\/bin\/pkill -KILL -u node/); + const reExecAt = step.run.search(reExecMarkerRe); + const anchorAt = step.run.search(/_flake_self_id="\$\(\/usr\/bin\/stat/); + assert.ok( + killAt !== -1 && + reExecAt !== -1 && + anchorAt !== -1 && + killAt < reExecAt && + reExecAt < anchorAt, + `${label}: the anchor sits inside the parent re-exec arm, after the kill`, + ); + } + }); + it('runs PR test code as the build user with no tokens, and fails open', () => { assert.equal(flakeStep.env.GITHUB_TOKEN, '', 'no GitHub token in the gate'); assert.equal(flakeStep.env.GH_TOKEN, '', 'no gh token in the gate'); @@ -1199,6 +1304,10 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // decision. BASH_ENV/LD_PRELOAD/LD_AUDIT/LD_LIBRARY_PATH are the // defensive startup-channel blanks (consumed at shell/loader startup, // before any in-script defence) — blanks, never secrets. + // POSIXLY_CORRECT is the round-19 startup defence: POSIX mode at + // INVOCATION refuses BASH_FUNC_* imports named after special + // builtins, so a poisoned `set` cannot run on the body's first + // command (see the behavioral poison scenario below). assert.deepEqual( Object.keys(flakeStep.env).sort(), [ @@ -1209,6 +1318,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { 'LD_AUDIT', 'LD_LIBRARY_PATH', 'LD_PRELOAD', + 'POSIXLY_CORRECT', ], 'the gate env must stay tokens-blanked and secret-free', ); @@ -1265,7 +1375,7 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // the budget loop alone is out-forked by a plant repopulating // between sweeps. const gatePreKill = flakeStep.run.search( - /^\s*if \[\[ \$\{EUID:-1\} -eq 0 \]\]; then\n\s*\/usr\/bin\/pkill -KILL -u node 2>\/dev\/null \|\| true\n\s*for _ in 1 2 3; do\n\s*\[\[ -n \$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\) \]\] \|\| break\n\s*\/usr\/bin\/sleep 1\n\s*\/usr\/bin\/pkill -KILL -u node 2>\/dev\/null \|\| true\n\s*done\n\s*survivors="\$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\)" \|\| true\n\s*if \[\[ -n \$survivors \]\]; then\n\s*echo "flake_verdict=error" >> "\$GITHUB_OUTPUT"\n\s*echo "flake_summary=node-owned processes survived SIGKILL — the gate refused to sample" >> "\$GITHUB_OUTPUT"\n\s*exit 0\n\s*fi\n\s*fi$/m, + /^\s*if \[\[ \$\(\/usr\/bin\/id -u\) -eq 0 \]\]; then\n\s*\/usr\/bin\/pkill -KILL -u node 2>\/dev\/null \|\| true\n\s*for _ in 1 2 3; do\n\s*\[\[ -n \$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\) \]\] \|\| break\n\s*\/usr\/bin\/sleep 1\n\s*\/usr\/bin\/pkill -KILL -u node 2>\/dev\/null \|\| true\n\s*done\n\s*survivors="\$\(\/usr\/bin\/ps -o pid=,stat= -u node 2>\/dev\/null \| \/usr\/bin\/awk '\$2 !~ \/\^Z\/'\)" \|\| true\n\s*if \[\[ -n \$survivors \]\]; then\n\s*\/usr\/bin\/printf 'flake_verdict=%s\\n' error >> "\$GITHUB_OUTPUT"\n\s*\/usr\/bin\/printf 'flake_summary=%s\\n' 'node-owned processes survived SIGKILL — the gate refused to sample' >> "\$GITHUB_OUTPUT"\n\s*exit 0\n\s*fi\n\s*fi$/m, ); const gateReExec = flakeStep.run.search(reExecMarkerRe); assert.ok( @@ -1574,7 +1684,8 @@ describe('qwen-triage: flakiness gate (#9125)', () => { // The message above is only true while both conditions stay identical: // if the agent's `if:` drifts wider, the agent publishes a verdict on // runs the gate never sampled, and the empty FLAKE_VERDICT leaves the - // headline untouched (the behavioral drive test proves '' is a no-op). + // headline untouched (the behavioral drive test proves '' can never + // touch it — the most it can produce is the visible gate-error line). assert.equal( agentStep.if, flakeStep.if, @@ -2144,7 +2255,8 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp // does NOT clear that inherited -e. That exact blind spot shipped the // round-1 blocker — the first failing test invocation killed the step — // so every scenario here runs under the wrapper, not under a bare bash. - const flakeRunVerbatim = verifyJob.steps.find((s) => s.id === 'flake').run; + const flakeStep = verifyJob.steps.find((s) => s.id === 'flake'); + const flakeRunVerbatim = flakeStep.run; // The gate's home is a hard-coded container-root constant on purpose: an // env-overridable home would be a PR-reachable channel ($GITHUB_ENV is // uid-1000 writable and the record step rm -rf's whatever the home names). @@ -2360,6 +2472,16 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp writeFileSync(join(rt, 'github-summary'), ''); const env = { ...process.env, + // The runner assembles the step's env: block around run:, which this + // harness used to skip — that dropped the POSIXLY_CORRECT startup + // defence out of every behavioral scenario. Apply its literal + // values (expression entries stay out — the harness owns those); + // harness keys win on collisions, scenarios can still override. + ...Object.fromEntries( + Object.entries(flakeStep.env).filter( + ([, v]) => typeof v === 'string' && !v.includes('${{'), + ), + ), // Hermetic PATH: the gate's env -i re-exec scrubs the environment // while the (non-root) harness skips the EUID-gated PATH pin, so // any env-dependent git wrapper in the ambient PATH (e.g. a shim @@ -3173,6 +3295,159 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.equal(floored.outputs.flake_verdict, 'flaky'); }); + describe('round-19 startup-channel hardening (behavioral)', () => { + it('a BASH_FUNC_set%% import never runs — POSIXLY_CORRECT refuses it at bash startup', () => { + // R18-2: without POSIX mode at INVOCATION the import runs attacker + // code as root on the step's first command, and can even enable + // posix itself so the reserved-word refusal never fires + // (probe-verified). The harness applies the step env block, so the + // step's POSIXLY_CORRECT reaches bash exactly as in production. + const marker = join(scenarioRoot, 'set-poison-marker'); + const { res, outputs } = runGate({ + layout: UNIT, + list: 'scripts/tests/a.test.js\n', + env: { + 'BASH_FUNC_set%%': `() { command touch "${marker}"; command set "$@"; command set -o posix; }`, + }, + }); + assert.notEqual( + res.status, + 0, + 'a poisoned startup must fail the step red at bash startup', + ); + assert.ok( + !existsSync(marker), + 'the poisoned set function must never execute — the import is refused before the body starts', + ); + assert.equal( + outputs.flake_verdict, + undefined, + 'bash aborts at import (exit 2), before the fail-open verdict path — the abort IS the refusal', + ); + }); + + it('echo stays shadowable even in POSIX mode — refusal writes must stay slash-pathed', () => { + // R18-3 mechanism pin: function lookup precedes REGULAR builtins; + // only SPECIAL builtins outrank functions (probe-verified). This is + // why every pre-re-exec refusal writes through /usr/bin/printf. + const poisonEnv = { + ...process.env, + 'BASH_FUNC_echo%%': '() { builtin printf "FORGED\\n"; }', + }; + const shadowed = spawnSync( + 'bash', + ['--noprofile', '--norc', '--posix', '-c', 'echo first; echo second'], + { env: poisonEnv, encoding: 'utf8' }, + ); + assert.equal(shadowed.stdout, 'FORGED\nFORGED\n'); + const pathed = spawnSync( + 'bash', + [ + '--noprofile', + '--norc', + '--posix', + '-c', + '/usr/bin/printf "%s\\n" honest', + ], + { env: poisonEnv, encoding: 'utf8' }, + ); + assert.equal(pathed.stdout, 'honest\n'); + }); + + it('a planted EUID cannot move the root identity gates (kernel-queried)', () => { + // R18-4: extract the gate's poisoned-env refusal condition verbatim + // and drive it under spoofed EUID values from a non-root process. + // The kernel query must ignore the spoof; the pre-round $EUID shape + // fired on a planted EUID=0 and skipped on a planted EUID=1000. + const cond = flakeRunVerbatim.match( + /^\s*if (\[\[ [^\n]*\/usr\/bin\/id -u[^\n]*\]\] && \[\[ -n \$\{BASH_ENV:-\}[^\n]*\]\]); then$/m, + ); + assert.ok( + cond, + 'the gate poisoned-env refusal must key its identity conjunct on /usr/bin/id -u', + ); + const drive = (extra) => + spawnSync( + 'bash', + [ + '--noprofile', + '--norc', + '-c', + `if ${cond[1]}; then printf FIRED; else printf SKIPPED; fi`, + ], + { + env: { ...process.env, BASH_ENV: '/dev/null', ...extra }, + encoding: 'utf8', + }, + ); + assert.equal( + drive({ EUID: '0' }).stdout, + 'SKIPPED', + 'a planted EUID=0 must not fire a root-gated refusal for a non-root process', + ); + assert.equal( + drive({ EUID: '1000' }).stdout, + 'SKIPPED', + 'a planted EUID=1000 changes nothing either — identity comes from the kernel', + ); + }); + + it('a script swap between bash open and the re-exec snapshot is refused by the inode anchor', () => { + // R18-1: model the window deterministically by swapping the script + // from its own first line — fd 255 already holds the genuine inode + // when the swap lands, exactly the state an external watcher + // produces by racing the kill (a swap before bash opens is the one + // case no step-level defence can catch: bash would execute the + // plant directly). Pre-round, the snapshot re-open read the plant + // and the re-exec ran it. + const caseBlock = flakeRunVerbatim.match( + /^\s*case "\$\{1:-\}" in\n[\s\S]*?\n\s*esac$/m, + ); + assert.ok(caseBlock, 'the gate re-exec case block must exist'); + const root = mkdtempSync(join(scenarioRoot, 'anchor-')); + const script = join(root, 'gate-arm.sh'); + const plantMarker = join(root, 'plant-marker'); + const swapLines = [ + 'mv -- "$0" "$0.genuine"', + `printf '%s\\n' 'printf "PLANT-EXECUTED\\n" > "${plantMarker}"' > "$0"`, + ]; + writeFileSync(script, `${swapLines.join('\n')}\n${caseBlock[0]}\n`); + const out = join(root, 'github-output'); + writeFileSync(out, ''); + const res = spawnSync( + 'bash', + ['--noprofile', '--norc', '-e', '-o', 'pipefail', script], + { + cwd: root, + env: { + ...process.env, + GITHUB_OUTPUT: out, + RUNNER_TEMP: root, + GITHUB_STEP_SUMMARY: join(root, 'summary'), + }, + encoding: 'utf8', + timeout: 30_000, + }, + ); + assert.equal(res.status, 0, `the gate refusal is fail-open: ${res.stderr}`); + const outputs = Object.fromEntries( + readFileSync(out, 'utf8') + .split('\n') + .filter((l) => l.includes('=')) + .map((l) => [l.slice(0, l.indexOf('=')), l.slice(l.indexOf('=') + 1)]), + ); + assert.equal(outputs.flake_verdict, 'error'); + assert.match( + outputs.flake_summary, + /step script changed between open and re-exec snapshot/, + ); + assert.ok( + !existsSync(plantMarker), + 'the swapped-in script body must never execute', + ); + }); + }); + it('publisher demotion executes one-way: only `flaky` demotes, and it MUST demote', () => { const block = publishRun.match( /^\s*case "\$\{FLAKE_VERDICT:-\}" in[\s\S]*?^\s*esac$/m, @@ -3221,8 +3496,19 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp // headline, the flaky lines must carry the ❌, and the Chinese line — // the one verdict a collapsed-details reader sees — must never drop // out while the English one keeps the suite green. + // R16-4 interim: this case block only executes on the full-report + // path, where the gate step ran and owes a verdict — an empty or + // unrecognized value means the uid-1000-writable $GITHUB_OUTPUT + // backing channel corrupted it in transit, and must render a visible + // fixed-text error line instead of dropping silently. The exact-equality + // assertion below also proves a planted value never lands in the body. + const CHANNEL_ERROR_LINE = + 'Flakiness gate: ⚠️ error — the gate verdict was missing or unrecognized at publish time; treating the gate as errored'; + const CHANNEL_ERROR_LINE_ZH = + '抖动门：⚠️ error — 发布时门判定缺失或无法识别，按 error 处理'; for (const [v, line, zh] of [ - ['', '', ''], + ['', CHANNEL_ERROR_LINE, CHANNEL_ERROR_LINE_ZH], + ['planted-garbage', CHANNEL_ERROR_LINE, CHANNEL_ERROR_LINE_ZH], [ 'pass', 'Flakiness gate: ✅ 1 of 2 changed test file(s) diverged', @@ -3489,53 +3775,61 @@ describe('qwen-triage: flakiness gate — behavioral, under the production wrapp assert.match(log, /a\.test\.js: FP/); }); - it('a BASH_FUNC_[%% import cannot flip the pre-exec decisions — a poisoned `[` still fails closed on a plant', () => { - // bash imports a BASH_FUNC_[%% env entry as a FUNCTION named `[` — - // function lookup precedes builtins, so a poisoned step environment - // flips every `[`-shaped guard, including the decision whether to - // take the env -i re-exec itself (probe-verified on this pool's - // bash: `type -t [` reports `function` under the poison, `[[`/ - // `case` stay immune). The poison's first three calls defeat the - // scrub/re-exec/PATH-pin trio; every later call returns true so - // the hijacked home checks would pass the 0777 plant. Reserved-word - // decisions plus the immune re-exec must still fail closed. - const { res, outputs } = runGate({ + it('a BASH_FUNC_[%% import never reaches the pre-exec decisions — POSIXLY_CORRECT kills it at startup', () => { + // Round 15 pinned the second layer: a poisoned `[` that survived + // import still failed closed on the reserved-word decisions. Round + // 19's POSIXLY_CORRECT closes the class one layer earlier: POSIX + // mode at INVOCATION refuses the `[` import (probe-verified), and + // under the runner wrapper's `-e` the step then dies red before its + // first command — the poison never runs, no round is sampled, no + // verdict is forged. The second layer is unreachable by construction + // while the step env carries POSIXLY_CORRECT (pinned above). + const { res, outputs, counts } = runGate({ layout: UNIT, list: 'scripts/tests/a.test.js\n', gateHomeMode: 0o777, env: { 'BASH_FUNC_[%%': '() { ((_p_n=${_p_n:-0}+1)); (( _p_n > 3 )); }' }, }); - assert.equal(res.status, 0, res.stderr); - assert.equal(outputs.flake_verdict, 'error'); + assert.notEqual(res.status, 0, 'a poisoned startup must fail the step red'); assert.match( - outputs.flake_summary, - /not root-owned 0700|working directory/, + res.stderr, + /error importing function definition for `\['/, + 'bash must refuse the `[` import at startup', + ); + assert.equal( + outputs.flake_verdict, + undefined, + 'no verdict may be written on a poisoned startup', ); + assert.equal(counts('a.test.js'), 0, 'no round may run on a poisoned startup'); }); - it('a BASH_FUNC_exec%% import cannot skip the env -i re-exec — the body runs exactly once', () => { - // `exec` is a builtin and function lookup precedes builtins: a - // BASH_FUNC_exec%% import shadows it, the re-exec line runs the - // function instead of replacing the shell, and the poisoned parent - // falls through into the body with every import alive - // (probe-verified on this pool's bash: the env-i child never runs). - // POSIX mode resolves special builtins before functions, so the - // transition is immune — observable as an honest `pass` and exactly - // one run of the rounds: against the pre-round code this scenario - // fell through into the poisoned parent and corrupted the verdict. + it('a BASH_FUNC_exec%% import cannot skip the env -i re-exec — bash refuses it at startup', () => { + // Round 15 pinned the second layer: POSIX mode resolves the SPECIAL + // builtin `exec` before functions. Round 19's POSIXLY_CORRECT closes + // the class one layer earlier: `exec` being special, bash refuses + // the import outright at startup (probe-verified: exit 2, body never + // runs) — the poisoned parent fall-through is unreachable by + // construction while the step env carries POSIXLY_CORRECT (pinned + // above). const { res, outputs, counts } = runGate({ layout: UNIT, list: 'scripts/tests/a.test.js\n', sequences: { 'a.test.js': 'PPPPP' }, env: { 'BASH_FUNC_exec%%': '() { return 0; }' }, }); - assert.equal(res.status, 0, res.stderr); - assert.equal(outputs.flake_verdict, 'pass'); + assert.notEqual(res.status, 0, 'a poisoned startup must fail the step red'); + assert.match( + res.stderr, + /`exec': is a special builtin/, + 'bash must refuse the special-builtin import at startup', + ); assert.equal( - counts('a.test.js'), - 5, - 'the re-exec must run exactly once — a shadowed exec falls through and re-runs the body', + outputs.flake_verdict, + undefined, + 'no verdict may be written on a poisoned startup', ); + assert.equal(counts('a.test.js'), 0, 'the body must never run on a poisoned startup'); }); it('a same-stem sibling (X.test.tsx next to changed X.test.ts) runs in ONE merged group, never attributed separately', () => { @@ -3647,6 +3941,13 @@ describe('qwen-triage: flakiness gate staging/upload — behavioral, under the p { env: { ...process.env, + // Step-env parity with the gate harness (POSIXLY_CORRECT + // startup defence included) — literal values only. + ...Object.fromEntries( + Object.entries(stageStep.env).filter( + ([, v]) => typeof v === 'string' && !v.includes('${{'), + ), + ), PATH: `${bin}:${process.env.PATH}`, RUNNER_TEMP: rt, GITHUB_OUTPUT: out, @@ -3749,6 +4050,13 @@ describe('qwen-triage: flakiness gate staging/upload — behavioral, under the p { env: { ...process.env, + // Step-env parity with the gate harness (POSIXLY_CORRECT + // startup defence included) — literal values only. + ...Object.fromEntries( + Object.entries(recheckStep.env).filter( + ([, v]) => typeof v === 'string' && !v.includes('${{'), + ), + ), RUNNER_TEMP: rt, GITHUB_OUTPUT: out, GITHUB_RUN_ID: '777', diff --git a/.github/workflows/qwen-triage.yml b/.github/workflows/qwen-triage.yml index 1d971d603c1..2da60b1a83e 100644 --- a/.github/workflows/qwen-triage.yml +++ b/.github/workflows/qwen-triage.yml @@ -2772,12 +2772,14 @@ jobs: # LD_* loader channels are consumed at shell/loader startup, # before any in-script defence — blank them where the step env # is assembled. BASH_FUNC_* imports are handled by the re-exec - # below. + # below; POSIXLY_CORRECT refuses the special-builtin-named ones + # at bash startup (see the gate step's env block). env: BASH_ENV: '' LD_PRELOAD: '' LD_AUDIT: '' LD_LIBRARY_PATH: '' + POSIXLY_CORRECT: '1' run: |- set -euo pipefail # Startup-channel scrub, part zero: POSIX mode resolves special @@ -2791,7 +2793,7 @@ jobs: # is the stop of last resort when `exit` is shadowed too). set -o posix if [[ ! -o posix ]]; then - echo "::error::flake-gate record: startup-channel scrub unavailable — refusing to record in a poisoned environment" + /usr/bin/printf '::error::flake-gate record: startup-channel scrub unavailable — refusing to record in a poisoned environment\n' exit 1 /usr/bin/kill -9 $$ fi @@ -2822,14 +2824,20 @@ jobs: # uid-1000-writable $RUNNER_TEMP, so a second open by the # child is a deterministic window a kill-race survivor can # rename-plant into. - if [[ ${EUID:-1} -eq 0 ]] && [[ -n ${BASH_ENV:-} || -n ${LD_PRELOAD:-} || -n ${LD_AUDIT:-} || -n ${LD_LIBRARY_PATH:-} ]]; then - echo "::error::flake-gate record: step-env scrub lost — refusing to run in a poisoned environment" + # Identity comes from the kernel (/usr/bin/id -u), never $EUID: + # bash imports EUID from the process environment, overriding + # the native readonly variable, so one planted EUID line could + # silently skip every root-gated defence below in this step and + # in every later step (probe-verified; the runner's set-env + # blocklist does not cover EUID). + if [[ $(/usr/bin/id -u) -eq 0 ]] && [[ -n ${BASH_ENV:-} || -n ${LD_PRELOAD:-} || -n ${LD_AUDIT:-} || -n ${LD_LIBRARY_PATH:-} ]]; then + /usr/bin/printf '::error::flake-gate record: step-env scrub lost — refusing to run in a poisoned environment\n' exit 1 fi # Kill BEFORE the snapshot, liveness-verified (the gate # step's shape): a live node writer is all the snapshot race # needs, even at this pre-build point. - if [[ ${EUID:-1} -eq 0 ]]; then + if [[ $(/usr/bin/id -u) -eq 0 ]]; then /usr/bin/pkill -KILL -u node 2>/dev/null || true for _ in 1 2 3; do [[ -n $(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/') ]] || break @@ -2838,16 +2846,28 @@ jobs: done survivors="$(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/')" || true if [[ -n $survivors ]]; then - echo "::error::flake-gate record: node-owned processes survived SIGKILL — refusing to record through a contested environment" + /usr/bin/printf '::error::flake-gate record: node-owned processes survived SIGKILL — refusing to record through a contested environment\n' exit 1 fi fi case "${1:-}" in --flake-clean-child) ;; *) - [[ -x /usr/bin/env ]] || { echo "::error::flake-gate record: clean re-exec unavailable"; exit 1; } + [[ -x /usr/bin/env ]] || { /usr/bin/printf '::error::flake-gate record: clean re-exec unavailable\n'; exit 1; } + # Inode anchor: bash reads this script through fd 255, so + # the fd's identity IS the content bash is executing, while + # the snapshot below RE-OPENS the path — a swap that lands + # between bash's open and this point is filesystem state a + # kill cannot un-land. Require the path to still resolve to + # the opened inode; refuse otherwise. + _flake_self_id="$(/usr/bin/stat -L -c '%d:%i' "/proc/$$/fd/255" 2>/dev/null)" || _flake_self_id='' _flake_body="$(<"${BASH_SOURCE[0]}")" - [[ -n $_flake_body ]] || { echo "::error::flake-gate record: clean re-exec snapshot empty"; exit 1; } + [[ -n $_flake_body ]] || { /usr/bin/printf '::error::flake-gate record: clean re-exec snapshot empty\n'; exit 1; } + if [[ -z $_flake_self_id ]] || + [[ "$(/usr/bin/stat -L -c '%d:%i' "${BASH_SOURCE[0]}" 2>/dev/null)" != "$_flake_self_id" ]]; then + /usr/bin/printf '::error::flake-gate record: step script changed between open and re-exec snapshot — refusing to run\n' + exit 1 + fi LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ @@ -3100,10 +3120,20 @@ jobs: # step env is assembled. Step env outranks the poisoned job # env a PR step wrote to the uid-1000 file-command backing # files; all four are defensive blanks, never secrets. + # POSIXLY_CORRECT enters POSIX mode at INVOCATION, so a + # BASH_FUNC_* import named after a special builtin (set, exit, + # exec ...) is refused at import — bash aborts red (exit 2) + # before the body's first command — instead of the import + # running attacker code as root on the first shadowable `set`, + # one command before the in-script POSIX switch can take + # effect. A poisoned startup therefore fails the step red + # rather than landing the fail-open error verdict: the abort + # IS the refusal. BASH_ENV: '' LD_PRELOAD: '' LD_AUDIT: '' LD_LIBRARY_PATH: '' + POSIXLY_CORRECT: '1' run: |- # Fail OPEN by contract: the gate is advisory-and-demoting only, so # a bug in it must degrade to the fixed `error` verdict — never @@ -3129,8 +3159,8 @@ jobs: # is shadowed too. set -o posix if [[ ! -o posix ]]; then - echo "flake_verdict=error" >> "$GITHUB_OUTPUT" - echo "flake_summary=the gate could not enter a scrub-safe shell mode — refusing to sample" >> "$GITHUB_OUTPUT" + /usr/bin/printf 'flake_verdict=%s\n' error >> "$GITHUB_OUTPUT" + /usr/bin/printf 'flake_summary=%s\n' 'the gate could not enter a scrub-safe shell mode — refusing to sample' >> "$GITHUB_OUTPUT" exit 0 /usr/bin/kill -9 $$ fi @@ -3138,22 +3168,25 @@ jobs: # above lost to the poisoned job env, BASH_ENV's payload # already ran at startup and the LD_* channels are live — a # compromised root shell must not keep producing verdicts. - # EUID-gated so the test harness (which cannot apply this - # step's env block and runs non-root) stays out of it. - if [[ ${EUID:-1} -eq 0 ]] && [[ -n ${BASH_ENV:-} || -n ${LD_PRELOAD:-} || -n ${LD_AUDIT:-} || -n ${LD_LIBRARY_PATH:-} ]]; then - echo "flake_verdict=error" >> "$GITHUB_OUTPUT" - echo "flake_summary=the gate refused to sample in a poisoned environment — see the step log" >> "$GITHUB_OUTPUT" + # Root-gated via /usr/bin/id -u — never $EUID, which the + # poisoned file-command channel can import (the record step + # carries the rationale); the test harness runs non-root, so + # it stays out. + if [[ $(/usr/bin/id -u) -eq 0 ]] && [[ -n ${BASH_ENV:-} || -n ${LD_PRELOAD:-} || -n ${LD_AUDIT:-} || -n ${LD_LIBRARY_PATH:-} ]]; then + /usr/bin/printf 'flake_verdict=%s\n' error >> "$GITHUB_OUTPUT" + /usr/bin/printf 'flake_summary=%s\n' 'the gate refused to sample in a poisoned environment — see the step log' >> "$GITHUB_OUTPUT" exit 0 fi # Kill BEFORE the re-exec snapshot; absolute-pathed (no PATH # pin applies this early, and a BASH_FUNC import shadows - # every bare word), EUID-gated (the harness keeps its - # stubs). The liveness refusal mirrors the agent step's - # guard: the budget loop alone is out-forked by a plant - # repopulating between sweeps, and the snapshot must not - # read this node-owned script through a contested - # environment (install/build ran PR lifecycle code). - if [[ ${EUID:-1} -eq 0 ]]; then + # every bare word), root-gated via /usr/bin/id -u ($EUID is + # importable; the harness keeps its stubs). The liveness + # refusal mirrors the agent step's guard: the budget loop + # alone is out-forked by a plant repopulating between sweeps, + # and the snapshot must not read this node-owned script + # through a contested environment (install/build ran PR + # lifecycle code). + if [[ $(/usr/bin/id -u) -eq 0 ]]; then /usr/bin/pkill -KILL -u node 2>/dev/null || true for _ in 1 2 3; do [[ -n $(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/') ]] || break @@ -3162,8 +3195,8 @@ jobs: done survivors="$(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/')" || true if [[ -n $survivors ]]; then - echo "flake_verdict=error" >> "$GITHUB_OUTPUT" - echo "flake_summary=node-owned processes survived SIGKILL — the gate refused to sample" >> "$GITHUB_OUTPUT" + /usr/bin/printf 'flake_verdict=%s\n' error >> "$GITHUB_OUTPUT" + /usr/bin/printf 'flake_summary=%s\n' 'node-owned processes survived SIGKILL — the gate refused to sample' >> "$GITHUB_OUTPUT" exit 0 fi fi @@ -3178,14 +3211,24 @@ jobs: --flake-clean-child) ;; *) if [[ ! -x /usr/bin/env ]]; then - echo "flake_verdict=error" >> "$GITHUB_OUTPUT" - echo "flake_summary=the gate could not start a clean shell — refusing to sample" >> "$GITHUB_OUTPUT" + /usr/bin/printf 'flake_verdict=%s\n' error >> "$GITHUB_OUTPUT" + /usr/bin/printf 'flake_summary=%s\n' 'the gate could not start a clean shell — refusing to sample' >> "$GITHUB_OUTPUT" exit 0 fi + # Inode anchor (the record step carries the rationale): + # the snapshot re-opens the path, so it must still resolve + # to the inode bash is executing through fd 255. + _flake_self_id="$(/usr/bin/stat -L -c '%d:%i' "/proc/$$/fd/255" 2>/dev/null)" || _flake_self_id='' _flake_body="$(<"${BASH_SOURCE[0]}")" if [[ -z $_flake_body ]]; then - echo "flake_verdict=error" >> "$GITHUB_OUTPUT" - echo "flake_summary=the gate could not snapshot its own body — refusing to sample" >> "$GITHUB_OUTPUT" + /usr/bin/printf 'flake_verdict=%s\n' error >> "$GITHUB_OUTPUT" + /usr/bin/printf 'flake_summary=%s\n' 'the gate could not snapshot its own body — refusing to sample' >> "$GITHUB_OUTPUT" + exit 0 + fi + if [[ -z $_flake_self_id ]] || + [[ "$(/usr/bin/stat -L -c '%d:%i' "${BASH_SOURCE[0]}" 2>/dev/null)" != "$_flake_self_id" ]]; then + /usr/bin/printf 'flake_verdict=%s\n' error >> "$GITHUB_OUTPUT" + /usr/bin/printf 'flake_summary=%s\n' 'the gate step script changed between open and re-exec snapshot — refusing to sample' >> "$GITHUB_OUTPUT" exit 0 fi LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ @@ -4359,12 +4402,14 @@ jobs: # LD_* loader channels are consumed at shell/loader startup, # before any in-script defence — blank them where the step env # is assembled. BASH_FUNC_* imports are handled by the re-exec - # below. + # below; POSIXLY_CORRECT refuses the special-builtin-named ones + # at bash startup (see the gate step's env block). env: BASH_ENV: '' LD_PRELOAD: '' LD_AUDIT: '' LD_LIBRARY_PATH: '' + POSIXLY_CORRECT: '1' run: |- set -euo pipefail # Startup-channel scrub, part zero: POSIX mode resolves special @@ -4378,7 +4423,7 @@ jobs: # is the stop of last resort when `exit` is shadowed too). set -o posix if [[ ! -o posix ]]; then - echo "::error::flake-gate staging: startup-channel scrub unavailable — refusing to stage evidence in a poisoned environment" + /usr/bin/printf '::error::flake-gate staging: startup-channel scrub unavailable — refusing to stage evidence in a poisoned environment\n' exit 1 /usr/bin/kill -9 $$ fi @@ -4386,20 +4431,21 @@ jobs: # step's): fail closed if the step-env blanks lost, then # re-exec once through an absolute-path env -i child to drop # any BASH_FUNC_* imports before a bare binary resolves. - if [[ ${EUID:-1} -eq 0 ]] && [[ -n ${BASH_ENV:-} || -n ${LD_PRELOAD:-} || -n ${LD_AUDIT:-} || -n ${LD_LIBRARY_PATH:-} ]]; then - echo "::error::flake-gate staging: step-env scrub lost — refusing to stage evidence in a poisoned environment" + if [[ $(/usr/bin/id -u) -eq 0 ]] && [[ -n ${BASH_ENV:-} || -n ${LD_PRELOAD:-} || -n ${LD_AUDIT:-} || -n ${LD_LIBRARY_PATH:-} ]]; then + /usr/bin/printf '::error::flake-gate staging: step-env scrub lost — refusing to stage evidence in a poisoned environment\n' exit 1 fi # Kill BEFORE the re-exec snapshot — unconditional, not gated # on the log existing: node can unlink the log, and that must # not skip the kill/rebuild for the report the agent wrote. - # Absolute-pathed, EUID-gated (the harness keeps its stubs). - # The liveness refusal mirrors the agent step's guard: the - # budget loop alone is out-forked by a plant repopulating - # between sweeps, and the snapshot must not read this - # node-owned script through a contested environment (the - # agent era just ran PR-controlled node code). - if [[ ${EUID:-1} -eq 0 ]]; then + # Absolute-pathed, root-gated via /usr/bin/id -u ($EUID is + # importable; the harness keeps its stubs). The liveness + # refusal mirrors the agent step's guard: the budget loop + # alone is out-forked by a plant repopulating between sweeps, + # and the snapshot must not read this node-owned script + # through a contested environment (the agent era just ran + # PR-controlled node code). + if [[ $(/usr/bin/id -u) -eq 0 ]]; then /usr/bin/pkill -KILL -u node 2>/dev/null || true for _ in 1 2 3; do [[ -n $(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/') ]] || break @@ -4408,7 +4454,7 @@ jobs: done survivors="$(/usr/bin/ps -o pid=,stat= -u node 2>/dev/null | /usr/bin/awk '$2 !~ /^Z/')" || true if [[ -n $survivors ]]; then - echo "::error::flake-gate staging: node-owned processes survived SIGKILL — refusing to stage evidence through a contested environment" + /usr/bin/printf '::error::flake-gate staging: node-owned processes survived SIGKILL — refusing to stage evidence through a contested environment\n' exit 1 fi fi @@ -4420,9 +4466,18 @@ jobs: case "${1:-}" in --flake-clean-child) ;; *) - [[ -x /usr/bin/env ]] || { echo "::error::flake-gate staging: clean re-exec unavailable"; exit 1; } + [[ -x /usr/bin/env ]] || { /usr/bin/printf '::error::flake-gate staging: clean re-exec unavailable\n'; exit 1; } + # Inode anchor (the record step carries the rationale): + # the snapshot re-opens the path, so it must still resolve + # to the inode bash is executing through fd 255. + _flake_self_id="$(/usr/bin/stat -L -c '%d:%i' "/proc/$$/fd/255" 2>/dev/null)" || _flake_self_id='' _flake_body="$(<"${BASH_SOURCE[0]}")" - [[ -n $_flake_body ]] || { echo "::error::flake-gate staging: clean re-exec snapshot empty"; exit 1; } + [[ -n $_flake_body ]] || { /usr/bin/printf '::error::flake-gate staging: clean re-exec snapshot empty\n'; exit 1; } + if [[ -z $_flake_self_id ]] || + [[ "$(/usr/bin/stat -L -c '%d:%i' "${BASH_SOURCE[0]}" 2>/dev/null)" != "$_flake_self_id" ]]; then + /usr/bin/printf '::error::flake-gate staging: step script changed between open and re-exec snapshot — refusing to stage\n' + exit 1 + fi LD_PRELOAD= LD_AUDIT= LD_LIBRARY_PATH= exec /usr/bin/env -i \ PATH="$PATH" RUNNER_TEMP="${RUNNER_TEMP:-}" \ GITHUB_OUTPUT="${GITHUB_OUTPUT:-}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-}" \ @@ -4572,6 +4627,7 @@ jobs: LD_PRELOAD: '' LD_AUDIT: '' LD_LIBRARY_PATH: '' + POSIXLY_CORRECT: '1' run: |- set -uo pipefail # Startup-channel scrub (see the record step): this step has no @@ -4586,11 +4642,14 @@ jobs: # when `exit` is shadowed too). set -o posix if [[ ! -o posix ]]; then - echo "::error::flake-gate re-check: startup-channel scrub unavailable — refusing to validate in a poisoned environment" + /usr/bin/printf '::error::flake-gate re-check: startup-channel scrub unavailable — refusing to validate in a poisoned environment\n' exit 1 /usr/bin/kill -9 $$ fi - if [[ ${EUID:-1} -eq 0 ]]; then + # /usr/bin/id -u, never $EUID (the record step carries the + # rationale): this step has no env -i re-exec, so this gate + # runs in the inherited job environment. + if [[ $(/usr/bin/id -u) -eq 0 ]]; then export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' fi upload_ok=false @@ -5478,6 +5537,16 @@ jobs: FLAKE_LINE="Flakiness gate: ⚠️ ${FLAKE_VERDICT} — ${FLAKE_SUMMARY:-see flake-gate.log in the run artifacts}" FLAKE_LINE_ZH="抖动门：⚠️ ${FLAKE_VERDICT} — ${FLAKE_SUMMARY:-see flake-gate.log in the run artifacts}" ;; + *) + # This branch only runs after a successful prepare, when + # the gate step ran and owes a verdict: an empty or + # unrecognized value means the uid-1000-writable + # $GITHUB_OUTPUT backing channel corrupted it in transit. + # Fail visible, never silent — and never embed the raw + # value, which is attacker-influenced on this path. + FLAKE_LINE='Flakiness gate: ⚠️ error — the gate verdict was missing or unrecognized at publish time; treating the gate as errored' + FLAKE_LINE_ZH='抖动门：⚠️ error — 发布时门判定缺失或无法识别，按 error 处理' + ;; esac if [ -z "$REPORT" ]; then MISSING_REPORT_NOTE='No report.md was found in the run artifacts, so the report section is omitted — see the workflow run output.'