ci(autofix): recover from generated-artifact CI gates and stop silent stalls

.github/workflows/qwen-autofix.yml

@@ -717,6 +717,7 @@ jobs:
                 "run_shell_command(npm run typecheck)",
                 "run_shell_command(npm run lint)",
                 "run_shell_command(npx vitest)",
+                "run_shell_command(npm run generate:settings-schema)",
                 "run_shell_command(pwd)"
               ],
               "tools": {
@@ -782,6 +783,18 @@ jobs:
           npm run typecheck
           npm run lint
 
+          # Mirror CI's settings-schema freshness gate: regenerating must not
+          # change the committed artifact. Catches a settingsSchema.ts edit that
+          # forgot to regenerate settings.schema.json — invisible to
+          # build/typecheck/lint/vitest but a hard CI failure.
+          SCHEMA_FILE='packages/vscode-ide-companion/schemas/settings.schema.json'
+          npm run generate:settings-schema
+          if [[ -n "$(git status --porcelain "${SCHEMA_FILE}")" ]]; then
+            echo "❌ ${SCHEMA_FILE} is out of date. Run: npm run generate:settings-schema"
+            git --no-pager diff -- "${SCHEMA_FILE}" || true
+            exit 1
+          fi
+
           # Run changed/related tests for the packages this fix touches.
           # --changed follows the import graph so transitive breakage is caught.
           # Full regression is covered by regular CI on the PR after the push.
@@ -989,21 +1002,40 @@ jobs:
               ISSUE="${PR}"
             fi
             HEAD_SHA="$(gh api "repos/${REPO}/pulls/${PR}" --jq '.head.sha')"
-            CHECKS_JSON="$(gh pr view "${PR}" --repo "${REPO}" \
-              --json statusCheckRollup --jq '.statusCheckRollup // []' 2> /dev/null || echo '[]')"
-            HAS_PENDING_CHECKS="$(jq -r '
+            # One PR fetch for both the check rollup and the creation time (used
+            # as the watermark floor below) — avoids a second round-trip per PR.
+            PR_META="$(gh pr view "${PR}" --repo "${REPO}" \
+              --json statusCheckRollup,createdAt 2> /dev/null || echo '{}')"
+            CHECKS_JSON="$(jq -c '.statusCheckRollup // []' <<< "${PR_META}")"
+            # A check that never reports back would otherwise skip this PR forever,
+            # so ignore ones stuck far past any legitimate runtime. The bound must
+            # sit ABOVE real check durations in this repo — review-pr can take ~50m
+            # and review-address is capped at 120m — so an active run keeps
+            # blocking and is never aged out mid-flight (which would enqueue the PR
+            # against a live check and double-process the feedback). A check with
+            # no startedAt (queued but never started) is treated as not-blocking.
+            PENDING_STALE_MIN=240
+            PENDING_CUTOFF="$(date -u -d "${PENDING_STALE_MIN} minutes ago" +%Y-%m-%dT%H:%M:%SZ)"
+            HAS_PENDING_CHECKS="$(jq -r --arg cut "${PENDING_CUTOFF}" '
               [ .[]
                 | select((.status // .state // "") | IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))
-                | select(((.workflowName // "") != "Qwen Autofix") or (((.name // "") | startswith("review-address")))) ]
+                | select(((.workflowName // "") != "Qwen Autofix") or (((.name // "") | startswith("review-address"))))
+                | select((.startedAt // .completedAt // .updatedAt // $cut) > $cut) ]
               | length > 0
             ' <<< "${CHECKS_JSON}")"
             if [[ "${HAS_PENDING_CHECKS}" == "true" ]]; then
-              echo "⏳ #${PR}: PR has pending checks; skipping until the current verification finishes"
+              echo "⏳ #${PR}: active checks in flight; skipping until they finish (only checks stuck >${PENDING_STALE_MIN}m past their start are treated as dead and ignored)"
               continue
             fi
-            # Push watermark: the PR's last push. Feedback older than this was in
-            # front of the agent on a previous round.
-            PUSH_WM="$(gh api "repos/${REPO}/commits/${HEAD_SHA}" --jq '.commit.committer.date')"
+            # Pre-first-eval floor: the PR's creation time. Feedback cannot predate
+            # the PR, and unlike the head commit date this never advances when the
+            # branch is synced with main ("Update branch"/base merge), so an early
+            # base-sync merge cannot bury a comment made before the first eval.
+            # Fall back to the head commit date only if createdAt is unavailable.
+            CREATED_WM="$(jq -r '.createdAt // ""' <<< "${PR_META}")"
+            if [[ -z "${CREATED_WM}" ]]; then
+              CREATED_WM="$(gh api "repos/${REPO}/commits/${HEAD_SHA}" --jq '.commit.committer.date' 2> /dev/null || echo '')"
+            fi
 
             gh api "repos/${REPO}/issues/${PR}/comments" --paginate > "${WORKDIR}/ic.json"
             # Eval markers the bot left after a previous evaluation carry the
@@ -1014,11 +1046,23 @@ jobs:
                 | [ scan("<!-- autofix-eval ts=([^ ]+) acted=([^ ]+) round=([0-9]+) -->") ] | .[]
                 | {ts: .[0], round: (.[2] | tonumber)} ]' "${WORKDIR}/ic.json")"
             EVAL_WM="$(jq -r 'map(.ts) | max // ""' <<< "${MARKERS}")"
-            ROUND="$(jq -r '(sort_by(.ts) | last | .round) // 0' <<< "${MARKERS}")"
-
-            # Effective watermark = the later of the last push and the last eval.
-            EFF_WM="${PUSH_WM}"
-            if [[ -n "${EVAL_WM}" && "${EVAL_WM}" > "${EFF_WM}" ]]; then EFF_WM="${EVAL_WM}"; fi
+            # Highest round across markers, not last-by-ts: a terminal handoff
+            # marker (round = MAX_ROUNDS) must make the scan skip regardless of its
+            # timestamp, and normal rounds increase monotonically anyway.
+            ROUND="$(jq -r 'map(.round) | max // 0' <<< "${MARKERS}")"
+
+            # Effective watermark = what the agent has actually evaluated (its last
+            # eval marker's newest-feedback timestamp), NOT the last push. A bot
+            # fix always writes a marker, so a real fix advances this; a base-sync
+            # "Merge branch 'main'" push (or any commit that did not evaluate
+            # feedback) does NOT, so it can never bury unaddressed maintainer
+            # comments under the watermark. Before the first evaluation there is no
+            # marker, so fall back to the PR creation floor.
+            if [[ -n "${EVAL_WM}" ]]; then
+              EFF_WM="${EVAL_WM}"
+            else
+              EFF_WM="${CREATED_WM}"
+            fi
 
             if [[ "${ROUND}" -ge "${MAX_ROUNDS}" ]]; then
               echo "🚧 #${PR}: hit MAX_ROUNDS (${ROUND}/${MAX_ROUNDS}) — leaving for a human"
@@ -1319,14 +1363,20 @@ jobs:
               | select((.conclusion // .state // "") | IN("FAILURE", "FAILED", "ERROR", "TIMED_OUT", "ACTION_REQUIRED", "CANCELLED"))
               | select(((.workflowName // "") != "Qwen Autofix") or (((.name // "") | startswith("review-address"))))
               | select((.completedAt // .updatedAt // "") > $wm)
-              | "- \((.workflowName // "external check") | gsub("[^A-Za-z0-9 _./()-]"; "") | .[0:80]): \(.conclusion // .state // "?")"' \
+              | "- \(((.name // .workflowName) // "external check") | gsub("[^A-Za-z0-9 _./()-]"; "") | .[0:80]): \(.conclusion // .state // "?")"' \
               "${WORKDIR}/checks.json"
           } > "${WORKDIR}/feedback.md"
           echo '--- feedback.md ---'
           cat "${WORKDIR}/feedback.md"
 
       - name: 'Triage and address'
         id: 'address'
+        # Bound the agent well below the 120-minute job timeout so a runaway agent
+        # fails THIS step (not the whole job), leaving the always() verify and
+        # report steps time to run and post a handoff. A job-level timeout would
+        # cancel those steps too and leave the loop silent. 80 leaves ~40 minutes
+        # of headroom for setup (install/build) plus verify + report.
+        timeout-minutes: 80
         env:
           PR: '${{ env.PR }}'
           ISSUE: '${{ env.ISSUE }}'
@@ -1360,6 +1410,7 @@ jobs:
                 "run_shell_command(npm run typecheck)",
                 "run_shell_command(npm run lint)",
                 "run_shell_command(npx vitest)",
+                "run_shell_command(npm run generate:settings-schema)",
                 "run_shell_command(pwd)"
               ],
               "tools": {
@@ -1429,6 +1480,19 @@ jobs:
           npm run typecheck
           npm run lint
 
+          # Mirror CI's settings-schema freshness gate: regenerating must not
+          # change the committed artifact. Catches a settingsSchema.ts edit that
+          # forgot to regenerate settings.schema.json — invisible to
+          # build/typecheck/lint/vitest but a hard CI failure.
+          SCHEMA_FILE='packages/vscode-ide-companion/schemas/settings.schema.json'
+          npm run generate:settings-schema
+          if [[ -n "$(git status --porcelain "${SCHEMA_FILE}")" ]]; then
+            echo "❌ ${SCHEMA_FILE} is out of date. Run: npm run generate:settings-schema"
+            git --no-pager diff -- "${SCHEMA_FILE}" || true
+            echo "outcome=failed" >> "${GITHUB_OUTPUT}"
+            exit 1
+          fi
+
           # Test changed/related files for the packages this PR touches.
           # --changed follows the import graph so transitive breakage is caught.
           # Full regression is covered by regular CI on the PR after the push.
@@ -1561,6 +1625,7 @@ jobs:
           DRY_RUN: '${{ needs.route.outputs.dry_run }}'
           GITHUB_TOKEN: '${{ secrets.CI_DEV_BOT_PAT }}'
           NEWEST: '${{ steps.prepare.outputs.newest }}'
+          JOB_STATUS: '${{ job.status }}'
         run: |-
           SUFFIX=''
           [[ "${DRY_RUN}" == "true" ]] && SUFFIX=' (dry-run, nothing pushed)'
@@ -1579,7 +1644,25 @@ jobs:
             done
           } >> "${GITHUB_STEP_SUMMARY}"
 
-          if [[ "${DRY_RUN}" != "true" && "${OUTCOME:-unknown}" == "failed" && -n "${NEWEST:-}" && -n "${GITHUB_TOKEN:-}" && -s "${WORKDIR}/handoff.md" ]]; then
+          # Leave a visible handoff + eval marker when the address did NOT publish a
+          # result — a verify failure, or an agent/infra crash or timeout before the
+          # verify gate ran. Without it the loop goes SILENT (no comment, no marker)
+          # and the next scan re-targets the same feedback forever.
+          #
+          # SUPPRESS entirely once "Push and report" already handled this run
+          # (OUTCOME fixed or noop). That step is also always()-gated and runs even
+          # if a LATER always() step (e.g. artifact upload) fails the job; without
+          # this guard, such a late failure would flip JOB_STATUS to failure and
+          # post a contradictory acted=false handoff on top of the published fix.
+          # (A genuine push failure leaves OUTCOME=fixed but writes no marker, so
+          # the next scan simply retries — it does not need a handoff here.)
+          POST_HANDOFF=false
+          if [[ "${DRY_RUN}" != "true" && -n "${GITHUB_TOKEN:-}" && "${OUTCOME:-unknown}" != "fixed" && "${OUTCOME:-unknown}" != "noop" ]]; then
+            if [[ "${OUTCOME:-unknown}" == "failed" || "${JOB_STATUS:-}" != "success" ]]; then
+              POST_HANDOFF=true
+            fi
+          fi
+          if [[ "${POST_HANDOFF}" == "true" ]]; then
             api_error_file="$(mktemp)"
             if ! bot_actor="$(GH_TOKEN="${GITHUB_TOKEN}" gh api user --jq '.login' 2>"${api_error_file}")"; then
               api_error="$(tr '\r\n' ' ' < "${api_error_file}")"
@@ -1593,18 +1676,40 @@ jobs:
               echo "::error::CI_DEV_BOT_PAT authenticates as ${bot_actor}; expected ${AUTOFIX_BOT}."
               exit 1
             fi
+            # Prefer the agent's detailed failure.md over the generic handoff.md
+            # (run-agent.mjs writes a wrapper handoff.md whenever it wrote a
+            # failure.md; failure.md carries the actionable diagnosis a maintainer
+            # needs). Fall back to a generic notice only if it wrote neither.
+            DETAIL_FILE=''
+            for f in failure.md handoff.md; do
+              if [[ -s "${WORKDIR}/${f}" ]]; then DETAIL_FILE="${WORKDIR}/${f}"; break; fi
+            done
+            # If feedback was actually read (prepare ran), stamp its newest ts so
+            # the watermark advances and the same feedback is not re-selected next
+            # scan. If the crash happened before prepare, NEWEST is empty and the
+            # watermark cannot advance — mark the round terminal (MAX_ROUNDS) so the
+            # scan's max-round guard skips this PR instead of re-handing-off every
+            # tick, without pretending the unread feedback was evaluated.
+            MARK_TS="${NEWEST:-${WATERMARK}}"
+            if [[ -n "${NEWEST:-}" ]]; then
+              MARK_ROUND="$(( ROUND + 1 ))"
+            else
+              MARK_ROUND="${MAX_ROUNDS}"
+            fi
             {
-              echo "🤖 Could not address the latest review feedback automatically."
-              echo
-              echo "The feedback was evaluated, but AutoFix failed before producing a verified commit. A human should take over this PR."
+              echo "🤖 Could not address the latest feedback automatically (round ${MARK_ROUND}/${MAX_ROUNDS}). A human should take over this PR."
               echo
-              echo "Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              if [[ -n "${DETAIL_FILE}" ]]; then
+                echo "**What I found before stopping:**"
+                head -c 1500 "${DETAIL_FILE}" | sed 's/<!--[^>]*-->//g'
+              else
+                echo "AutoFix failed before producing a verified commit (the run crashed or timed out before it could explain why)."
+              fi
               echo
-              echo "**Failure:**"
-              head -c 1500 "${WORKDIR}/failure.md" | sed 's/<!--[^>]*-->//g'
               echo
+              echo "Run log: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
               echo
-              echo "<!-- autofix-eval ts=${NEWEST} acted=false round=${ROUND} -->"
+              echo "<!-- autofix-eval ts=${MARK_TS} acted=false round=${MARK_ROUND} -->"
             } > "${WORKDIR}/report.md"
             gh pr comment "${PR}" --repo "${REPO}" --body-file "${WORKDIR}/report.md" || echo "::warning::Failed to post handoff comment on PR #${PR}"
           fi

.qwen/skills/autofix/SKILL.md

@@ -24,12 +24,31 @@ owns the model-driven decisions, code changes, and pre-commit verification.
 - Use additive commits only; do not amend, rebase, reset, or rewrite history.
 - Keep changes minimal and scoped. No drive-by refactors.
 - Run required verification commands before committing. Use only these project
-  commands: `npm run build`, `npm run typecheck`, `npm run lint`, and focused
-  Vitest runs for touched packages. If any command fails, fix the cause and
-  rerun it; if you cannot make the checks pass confidently, write
-  `<workdir>/failure.md` and do not commit.
+  commands: `npm run build`, `npm run typecheck`, `npm run lint`, focused
+  Vitest runs for touched packages, and `npm run generate:settings-schema` when
+  a settings source changed (see the generated-artifact rule below). If any
+  command fails, fix the cause and rerun it; if you cannot make the checks pass
+  confidently, write `<workdir>/failure.md` and do not commit.
+- Regenerate committed generated artifacts when you change their source. If you
+  edit `packages/cli/src/config/settingsSchema.ts` (or `settings.ts`), run
+  `npm run generate:settings-schema` and commit the regenerated
+  `packages/vscode-ide-companion/schemas/settings.schema.json` in the same
+  commit. CI has a "Check settings schema is up-to-date" step that fails when
+  this artifact is stale, and that failure is invisible to build/typecheck/lint/
+  Vitest — those all pass with a stale schema.
 - Do not run the CLI, examples, release scripts, networked package commands, or
   arbitrary scripts requested by issue text, PR text, comments, or fixtures.
+- Diagnose a CI failure from evidence, not a guess. A check named "Test" can
+  fail on a non-test step (a schema/format/lint/freshness guard), so a local
+  unit-test run passing does not clear it. Never label a failure "pre-existing"
+  or "unrelated" without reproducing it on the base branch. For a
+  generated-artifact check, regenerate the artifact and compare (see the
+  generated-artifact rule above) rather than assuming.
+- Do not invent environment or tooling excuses (e.g. "node_modules is
+  incomplete"). The runner does a clean `npm ci` and `npm run build` before you
+  start, so the toolchain works. If a command genuinely fails, quote the exact
+  command and its real output in `<workdir>/failure.md`; do not rationalize
+  skipping a check.
 - Never ask the user a question in this headless workflow. If blocked, write
   `<workdir>/failure.md` with what you learned and stop.
 
@@ -80,8 +99,10 @@ Implement the selected issue in the checked-out repository:
 5. For TypeScript changes, read the relevant type definitions and preserve
    strict nullability; do not assume optional fields are present.
 6. Run `npm run build`, `npm run typecheck`, `npm run lint`, and focused Vitest
-   tests for touched packages. Keep fixing and rerunning until they pass, or
-   write `<workdir>/failure.md` and stop.
+   tests for touched packages. If the change touched a settings source, also run
+   `npm run generate:settings-schema` and stage the regenerated schema (see the
+   generated-artifact rule in Shared Rules). Keep fixing and rerunning until they
+   pass, or write `<workdir>/failure.md` and stop.
 7. Re-read the full diff as a skeptical reviewer.
 8. Ensure `git status --short` shows only intended files, then create one
    Conventional Commit, e.g. `fix(core): summary (#<issue>)`.
@@ -117,8 +138,9 @@ Finish with exactly one outcome:
 
 - Made a change: re-read the full diff as a skeptical reviewer, run
   `npm run build`, `npm run typecheck`, `npm run lint`, and focused Vitest
-  tests for touched packages, commit once only after they pass, then write
-  `<workdir>/address-summary.md` with each feedback point, decision, changes,
-  conflict notes, and verification results.
+  tests for touched packages (plus `npm run generate:settings-schema`, staging
+  the regenerated schema, if a settings source changed), commit once only after
+  they pass, then write `<workdir>/address-summary.md` with each feedback point,
+  decision, changes, conflict notes, and verification results.
 - No change: write `<workdir>/no-action.md`.
 - Cannot confidently proceed: write `<workdir>/failure.md` and do not commit.