fix(ci): watchdog silent sandbox hangs and reap the containers they leak

.github/scripts/run-autofix-review-verification.sh

@@ -43,25 +43,154 @@ git checkout "${BRANCH}"
 GATE_LOG="${WORKDIR}/gate-output.log"
 : > "${GATE_LOG}"
 reject_fix() {
-  echo "❌ ${1}"
+  local label="${1}"
+  local preexisting="${2:-false}"
+  echo "❌ ${label}"
   # Declare the verdict before writing its detail. An empty outcome on a failed
   # step means the gate itself crashed, so losing the detail file must not turn
   # a deterministic rejection into an infrastructure retry.
   echo "outcome=failed" >> "${GITHUB_OUTPUT}"
-  echo "retryable=true" >> "${GITHUB_OUTPUT}"
+  if [[ "${preexisting}" == 'true' ]]; then
+    # NOT retryable: the repair agent is only allowed to amend this round's
+    # fix, and a failure that exists without the fix is outside that boundary
+    # by definition — the 18-minute repair budget cannot reach it. The remedy
+    # is a base update (merge main into the branch), not a repair.
+    echo "preexisting=true" >> "${GITHUB_OUTPUT}"
+  else
+    echo "retryable=true" >> "${GITHUB_OUTPUT}"
+  fi
+  # The evidence tail flexes so the WHOLE document stays under the report
+  # step's head -c 3900 render cap: truncating the finished document from
+  # the outside cuts the closing fence and malforms everything after it in
+  # the posted comment. Budget = 3300 minus the preamble, floored at 500.
+  local preamble tail_budget
+  preamble="**${label}**"
+  if [[ "${preexisting}" == 'true' ]]; then
+    preamble+="$(printf '\n\nMeasured fact: the same check also fails at \`origin/%s\` (the branch as pushed, before this round) in this environment, with a matching failure signature. The repair pass may only amend the round'"'"'s own fix, so it cannot reach this failure. If the branch is behind \`main\`, a base update (merge main) is the usual cure; otherwise the failure lives in the branch'"'"'s own pre-round commits.' "${BRANCH}")"
+  fi
+  tail_budget=$(( 3300 - ${#preamble} ))
+  (( tail_budget < 500 )) && tail_budget=500
   {
-    echo "**${1}**"
+    printf '%s\n' "${preamble}"
     echo
     # Captured output can contain triple-backtick fences.
     echo '````'
-    tail -c 3000 "${GATE_LOG}" 2> /dev/null
+    tail -c "${tail_budget}" "${GATE_LOG}" 2> /dev/null
     echo '````'
   } > "${WORKDIR}/gate-rejection.md" ||
     echo "::warning::could not write the gate rejection detail; the verdict stands."
   exit 1
 }
+baseline_also_fails() {
+  # A deterministic rejection is only chargeable to this round if the same
+  # check passes WITHOUT the round's commits. Measured counterexample, run
+  # 31276008548: PR #8614's branch predated #8693's tsconfig guard while
+  # node_modules came from the post-#8693 trusted base, so `npm run build`
+  # was just as red at origin/<branch> — 63 minutes of accepted agent work
+  # were discarded and an 18-minute repair burned on a failure the repair
+  # agent is forbidden to touch, thirteen rounds in a row.
+  # Returns 0 (pre-existing) only when the SAME command demonstrably fails
+  # at the pre-round ref; any A/B infrastructure problem returns 1 so the
+  # rejection keeps today's semantics (fail closed toward "charge the fix").
+  local current baseline rc
+  current="$(git rev-parse HEAD)" || return 1
+  baseline="$(git rev-parse --quiet --verify "origin/${BRANCH}^{commit}")" ||
+    return 1
+  # No round commit (the core-rebuild check runs before the commit gate and
+  # is A/B-eligible) — the baseline IS the tree under test; nothing to
+  # compare.
+  [[ "${baseline}" != "${current}" ]] || return 1
+  echo "🔁 Baseline A/B: re-running the failed check at origin/${BRANCH}" \
+    "(${baseline})" | tee -a "${GATE_LOG}"
+  git checkout --quiet --detach "${baseline}" 2>> "${GATE_LOG}" || return 1
+  # The baseline transcript goes to a SIDE log: gate-rejection.md renders
+  # `tail -c 3000` of GATE_LOG as the evidence window, and on a green
+  # baseline a chatty success transcript would fill it and push the actual
+  # failure text out — misdirecting the repair agent, the PR comment, and
+  # the next round's LAST_REJECTION block all at once.
+  local ab_log="${GATE_LOG}.baseline"
+  : > "${ab_log}"
+  rc=0
+  if ! "$@" >> "${ab_log}" 2>&1; then
+    rc=1
+  fi
+  if ! git checkout --quiet "${BRANCH}" 2>> "${GATE_LOG}"; then
+    # The tree is no longer the one under verification and nothing after
+    # this point may trust it.
+    reject_fix 'could not restore the verification tree after the baseline check'
+  fi
+  if [[ "${rc}" -ne 1 ]]; then
+    echo "🔁 baseline is green — the failure belongs to this round" \
+      | tee -a "${GATE_LOG}"
+    return 1
+  fi
+  # A nonzero baseline is NOT enough: the branch can fail there for reason A
+  # while the round fails for reason B, and an infrastructure hiccup in the
+  # baseline leg is a nonzero exit too. Pre-existing requires the SAME
+  # failure identity on both legs — compiler diagnostics normalized to
+  # file + error code (line/column shift with the round's edits). No
+  # diagnostics on either side means identity cannot be established, and
+  # the rejection stays charged to the round (fail closed).
+  local sig_head sig_base
+  # `|| true`: grep exits 1 on the NORMAL no-match case, and these
+  # assignments only survive `set -e` today because this function is called
+  # from an `if` condition (which suspends errexit). A future unconditional
+  # call site would otherwise turn the documented fail-closed path into a
+  # verdict-less gate crash.
+  sig_head="$(fail_signature "${GATE_LOG}.check")" || true
+  sig_base="$(fail_signature "${ab_log}")" || true
+  if [[ -z "${sig_head}" || -z "${sig_base}" ]] ||
+    ! comm -12 <(printf '%s\n' "${sig_head}") <(printf '%s\n' "${sig_base}") \
+      | grep -q .; then
+    echo "🔁 baseline fails for a DIFFERENT reason — charged to the round" \
+      | tee -a "${GATE_LOG}"
+    return 1
+  fi
+  # Only a FAILING baseline transcript with a matching signature is
+  # evidence — merge its tail into the window, where it backs the label.
+  tail -c 1500 "${ab_log}" >> "${GATE_LOG}" 2> /dev/null || true
+  return 0
+}
+fail_signature() {
+  # Stable identity of a failed check: tsc-style diagnostics with the
+  # position stripped but the MESSAGE kept ("src/a.ts: error TS2504: …").
+  # Position strips because line/column shift with the round's edits; the
+  # message stays because file + code alone collide — two unrelated defects
+  # in one file sharing a common code (TS2339 is everywhere) would compare
+  # as "the same failure" and skip a repair that could have worked. A
+  # message naming a round-renamed identifier then under-matches — the
+  # fail-closed direction. Sorted unique so two transcripts compare with
+  # comm(1). KNOWN LIMIT: only tsc diagnostics carry identity; vite/esbuild
+  # failures yield an empty signature and deliberately fail closed (charged
+  # to the round) — widening needs their position formats normalized first.
+  grep -oE "[^ '\"]+\([0-9]+,[0-9]+\): error TS[0-9]+[^\n]*" "${1}" 2> /dev/null \
+    | sed -E 's/\([0-9]+,[0-9]+\)//' | sort -u
+}
 run_check() {
-  # pipefail makes the pipeline carry the command's status, not tee's.
+  # pipefail makes the pipeline carry the command's status, not tee's. The
+  # side copy holds THIS check's transcript alone — the identity comparison
+  # must not match diagnostics an earlier check left in the shared log.
+  local label="${1}"
+  shift
+  : > "${GATE_LOG}.check"
+  if ! "$@" 2>&1 | tee -a "${GATE_LOG}" "${GATE_LOG}.check"; then
+    if baseline_also_fails "$@"; then
+      reject_fix "${label} (pre-existing: also fails without this round's commit)" 'true'
+    fi
+    reject_fix "${label}"
+  fi
+}
+run_check_no_ab() {
+  # A/B-exempt: for checks whose baseline re-run would compare a DIFFERENT
+  # computation than the one that failed, so a baseline verdict proves
+  # nothing. The contracts check consumes its file list from stdin, which
+  # the first run drains — the baseline leg would re-check an empty list
+  # and pass vacuously. The schema check reads packages/core/dist, which
+  # the core-rebuild guard built from the ROUND's sources and which,
+  # being gitignored, survives the detach and confounds the baseline. Their
+  # rejections stay charged to the round — which is also where the repair
+  # agent can actually act on them (generate:settings-schema is in its
+  # allowlist).
   local label="${1}"
   shift
   if ! "$@" 2>&1 | tee -a "${GATE_LOG}"; then
@@ -112,10 +241,10 @@ fi
 # that predates the script does not contain it (bash would exit 127
 # and kill the gate with no outcome), and the gate logic must come
 # from the trusted base, not the branch under verification.
-run_check 'settings schema is stale on the agent-committed fix' \
+run_check_no_ab 'settings schema is stale on the agent-committed fix' \
   bash "${RUNNER_TEMP}/check-settings-schema.sh"
 CHANGED_FILES="$(git diff --name-only "origin/main...${BRANCH}")"
-run_check 'cross-package contract verification failed' \
+run_check_no_ab 'cross-package contract verification failed' \
   bash "${RUNNER_TEMP}/check-autofix-contracts.sh" <<< "${CHANGED_FILES}"
 assert_verification_tree
 
@@ -140,8 +269,14 @@ fi
 
 echo '🔬 Re-running deterministic checks (independent of the agent)...'
 run_check 'build failed on the agent-committed fix' npm run build
-run_check 'typecheck failed on the agent-committed fix' npm run typecheck
-run_check 'lint failed on the agent-committed fix' npm run lint
+# Typecheck consumes core's dist (sdk-typescript resolves
+# @qwen-code/qwen-code-core through the package exports to ./dist/*.d.ts),
+# and dist is gitignored — it survives the baseline detach carrying the
+# ROUND's build, so a baseline typecheck would run reverted sources against
+# round-built declarations. Probe-verified three-arm flip on this tree. Same
+# class as the schema check: A/B-exempt.
+run_check_no_ab 'typecheck failed on the agent-committed fix' npm run typecheck
+run_check_no_ab 'lint failed on the agent-committed fix' npm run lint
 
 # Test changed/related files for the packages this PR touches.
 # --changed follows the import graph so transitive breakage is caught.
@@ -172,7 +307,14 @@ else
       continue
     fi
     echo "🧪 Testing ${p} (changed files only)..."
-    run_check "tests failed in ${p}" \
+    # A/B-exempt: package tests resolve sibling workspaces through their
+    # dist exports (channels/github -> @qwen-code/channel-base/dist), and
+    # dist survives the baseline detach carrying the ROUND's build — a
+    # baseline leg would test reverted sources against round-built
+    # dependencies. (A round-ADDED workspace also has no baseline at all:
+    # npm exits 1 there with "No workspaces found".) Their rejections stay
+    # charged to the round, where the repair agent can act.
+    run_check_no_ab "tests failed in ${p}" \
       npm run test --workspace "${p}" --if-present -- --changed origin/main --passWithNoTests
   done
 fi

.github/workflows/qwen-autofix.yml

@@ -825,6 +825,24 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
+      - name: 'Remove stale sandbox containers'
+        run: |-
+          # A budget kill (SIGTERM/SIGKILL from run-agent.mjs or the job
+          # timeout) reaps the HOST-side docker client, not the container:
+          # every killed sandbox keeps running on this persistent runner.
+          # Observed directly — a hung leg's container name counter found
+          # qwen-code-0.21.8-0 already occupied and picked -1. A runner
+          # executes one job at a time, so any qwen-code-* container alive
+          # at job start is stale by definition; reap it before the sandbox
+          # picks a name (and before the leftovers can wedge the daemon).
+          command -v docker > /dev/null || exit 0
+          STALE="$(docker ps -aq --filter 'name=qwen-code-')"
+          if [ -n "${STALE}" ]; then
+            echo "removing stale sandbox containers:"
+            docker ps -a --filter 'name=qwen-code-' --format '  {{.Names}} ({{.Status}})'
+            printf '%s\n' "${STALE}" | xargs -r docker rm -f > /dev/null
+          fi
+
       - name: 'Reset autofix workspace'
         run: |-
           rm -rf "${WORKDIR}"
@@ -3471,6 +3489,24 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
+      - name: 'Remove stale sandbox containers'
+        run: |-
+          # A budget kill (SIGTERM/SIGKILL from run-agent.mjs or the job
+          # timeout) reaps the HOST-side docker client, not the container:
+          # every killed sandbox keeps running on this persistent runner.
+          # Observed directly — a hung leg's container name counter found
+          # qwen-code-0.21.8-0 already occupied and picked -1. A runner
+          # executes one job at a time, so any qwen-code-* container alive
+          # at job start is stale by definition; reap it before the sandbox
+          # picks a name (and before the leftovers can wedge the daemon).
+          command -v docker > /dev/null || exit 0
+          STALE="$(docker ps -aq --filter 'name=qwen-code-')"
+          if [ -n "${STALE}" ]; then
+            echo "removing stale sandbox containers:"
+            docker ps -a --filter 'name=qwen-code-' --format '  {{.Names}} ({{.Status}})'
+            printf '%s\n' "${STALE}" | xargs -r docker rm -f > /dev/null
+          fi
+
       - name: 'Reset autofix workspace'
         run: |-
           rm -rf "${WORKDIR}"
@@ -4549,10 +4585,25 @@ jobs:
           REPAIR_OUTCOME: '${{ steps.verify_repair.outputs.outcome }}'
           REPAIR_COMMITTED: '${{ steps.verify_repair.outputs.committed }}'
           REPAIR_VERIFIED_HEAD: '${{ steps.verify_repair.outputs.verified_head }}'
+          FIRST_PREEXISTING: '${{ steps.verify.outputs.preexisting }}'
+          REPAIR_PREEXISTING: '${{ steps.verify_repair.outputs.preexisting }}'
         run: |-
           OUTCOME="${FIRST_OUTCOME}"
           COMMITTED="${FIRST_COMMITTED}"
           VERIFIED_HEAD="${FIRST_VERIFIED_HEAD}"
+          # The flag travels WITH the attempt whose outcome is selected: the
+          # first pass can fail a round-caused check, the repair fixes it, and
+          # the repair verification can then hit a pre-existing failure — that
+          # final classification is the one the report must render. Forwarded
+          # so the failure report can say "base update needed" instead of the
+          # generic gate-rejection clause.
+          PREEXISTING="${FIRST_PREEXISTING}"
+          if [[ "${REPAIR_ATTEMPTED}" == 'true' ]]; then
+            PREEXISTING="${REPAIR_PREEXISTING}"
+          fi
+          if [[ "${PREEXISTING}" == 'true' ]]; then
+            echo "preexisting=true" >> "${GITHUB_OUTPUT}"
+          fi
           if [[ "${REPAIR_ATTEMPTED}" == 'true' ]]; then
             OUTCOME="${REPAIR_OUTCOME}"
             COMMITTED="${REPAIR_COMMITTED:-${FIRST_COMMITTED}}"
@@ -5009,6 +5060,7 @@ jobs:
         env:
           OUTCOME: '${{ steps.final_verify.outputs.outcome }}'
           COMMITTED: '${{ steps.final_verify.outputs.committed }}'
+          PREEXISTING: '${{ steps.final_verify.outputs.preexisting }}'
           CONFLICT: '${{ steps.prepare.outputs.conflict }}'
           DRY_RUN: '${{ needs.route.outputs.dry_run }}'
           GITHUB_TOKEN: '${{ secrets.CI_DEV_BOT_PAT }}'
@@ -5280,6 +5332,20 @@ jobs:
                   # wording-doesn't-match-behaviour bug this PR fixes.
                   GATE_CLAUSE=''
                   [[ -s "${WORKDIR}/gate-rejection.md" ]] && GATE_CLAUSE=' — the verification gate rejected the attempt'
+                  # Pre-existing failures get the honest clause: the rejection
+                  # is not the agent's and the repair was deliberately skipped.
+                  # The remedy depends on WHY it pre-exists, and this branch
+                  # only renders when the stale-base auto-update above did NOT
+                  # fire — which includes a branch current with main whose own
+                  # pre-round commits carry the failure, where "merge main"
+                  # changes nothing. The compare already ran; use it.
+                  if [[ "${PREEXISTING}" == 'true' ]]; then
+                    if [[ "${CMP_R:-}" == 'behind' || "${CMP_R:-}" == 'diverged' ]]; then
+                      GATE_CLAUSE=' — the verification gate hit a PRE-EXISTING failure (present without this round'"'"'s commit); the repair pass may only amend the round'"'"'s own fix, so it cannot reach it — the branch needs a base update (merge main)'
+                    else
+                      GATE_CLAUSE=' — the verification gate hit a PRE-EXISTING failure (present without this round'"'"'s commit); the repair pass may only amend the round'"'"'s own fix, so it cannot reach it — the branch'"'"'s own pre-round code needs attention'
+                    fi
+                  fi
                   HEADLINE="🤖 Could not produce a passing fix for this feedback (round ${MARK_ROUND}/${MAX_ROUNDS})${GATE_CLAUSE}. This item now needs a human; the loop stays engaged and still picks up new feedback and base conflicts, but will not retry this item on its own."
                 fi
               fi
@@ -5455,9 +5521,13 @@ jobs:
                 echo '<!-- autofix-gate-rejection-start -->'
                 echo "**Why it was not pushed:**"
                 echo
-                # Must stay >= reject_fix's tail -c 3000 + label + two four-backtick
-                # fences (~3.1 KB), or the closing fence is silently truncated.
-                head -c 3500 "${WORKDIR}/gate-rejection.md" | iconv -f utf-8 -t utf-8 -c | sed 's/<!--/<!\\-\\-/g' || true
+                # reject_fix sizes its own document: the evidence tail is
+                # dynamic (budget 3300 minus the preamble, floored at 500),
+                # so the finished file tops out ≈3.3 KB. 3900 is headroom,
+                # not arithmetic — retune it only together with the
+                # script's tail_budget formula, or the closing fence gets
+                # silently truncated again.
+                head -c 3900 "${WORKDIR}/gate-rejection.md" | iconv -f utf-8 -t utf-8 -c | sed 's/<!--/<!\\-\\-/g' || true
                 echo '<!-- autofix-gate-rejection-end -->'
               fi
               echo