From 5fcabe27d6d83e8d1c1aa9f281fe9ebfa3c63175 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Wed, 12 Aug 2026 07:52:15 +0800 Subject: [PATCH 01/17] feat(autofix): escalate stopped takeover PRs and age out unanswered pauses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Takeover PRs that hit the round cap (or a circuit breaker) went silent: no label, no dashboard entry, no escalation — five PRs had been paused for days. The fleet shepherd only tracked bot-authored PRs, so the whole 35-PR human takeover pool was invisible. The autofix scan now applies an autofix/needs-human label whenever a PR reaches its cap (the write rides every cap detection, so already-paused PRs backfill on the regular scan rotation), and removes it wherever management resumes or a human releases the PR. The fleet shepherd enumerates the takeover pool onto its dashboard (state, stop reason, pause age, plus an awaiting-human section for released PRs) and gains a single bounded lever: a takeover whose pause went unanswered for AUTO_RELEASE_DAYS days gets its takeover label removed with a bilingual summary, keeping the needs-human label as the filterable TODO. Resume evidence newer than the pause notice — bot markers, trusted re-arm commands, fresh labeled events — vetoes the release; every read fails closed and a per-tick cap bounds blast radius. Co-authored-by: Qwen-Coder --- .github/workflows/qwen-autofix.yml | 74 +++- .github/workflows/qwen-fleet-shepherd.yml | 281 ++++++++++++++- docs/design/takeover-fleet-visibility.md | 140 ++++++++ scripts/tests/qwen-autofix-workflow.test.js | 61 +++- .../qwen-fleet-shepherd-workflow.test.js | 324 +++++++++++++++++- 5 files changed, 853 insertions(+), 27 deletions(-) create mode 100644 docs/design/takeover-fleet-visibility.md diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 3c81ac64bc6..1d00dedb4cc 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -197,6 +197,12 @@ env: # Comment-command sugar over TAKEOVER_LABEL ('' applies it, ' # stop' removes it). Matched EXACTLY against the trimmed comment body. TAKEOVER_COMMAND: '@qwen-code /takeover' + # Escalation label: applied when the loop STOPS on a PR (round cap, + # consecutive-failure or time-budget breaker) and a human must act — + # re-arm, split, merge, or close. It makes paused PRs filterable and lets + # the fleet shepherd age out unanswered stops. Removed wherever management + # resumes (engage/re-arm) or a human releases the PR. + NEEDS_HUMAN_LABEL: 'autofix/needs-human' # Re-arm sugar. Recovering a stranded PR previously meant DELETING the # bot's autofix-eval marker comment by hand (undiscoverable, destructive, # and it erases the audit trail). This command instead posts an @@ -1762,6 +1768,11 @@ jobs: # already addressed is never replayed. gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '🔄 Takeover re-armed: the round counter starts a fresh window (previous rounds no longer count toward the cap); management continues.\n\n
\n中文说明\n\n🔄 已重新武装：轮次计数开启新窗口（此前轮次不再计入上限），托管继续。\n\n
\n\n')" echo "🔄 re-armed ${TAKEOVER_LABEL} window on #${PR}" + # Management resumed — the escalation label is stale. 404 is + # the common case (the PR was never paused). + if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + fi else # REST for consistency and runner-version independence: `gh pr # edit`'s GraphQL lookup requests @@ -1799,6 +1810,11 @@ jobs: fi gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '🤝 Takeover engaged: the autofix loop now manages this PR — it will address new review feedback and resolve base conflicts until the label is removed or the round cap is reached.%s Remove the `%s` label (or comment `%s stop`) to release.\n\n
\n中文说明\n\n🤝 已接管：autofix 循环现在管理此 PR —— 将持续处理新的评审反馈与 base 冲突，直到移除标签或达到轮次上限。%s移除 `%s` 标签（或评论 `%s stop`）即可释放。\n\n
\n\n' "${FORK_NOTE}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}" "${FORK_NOTE_ZH}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}")" \ || echo "::warning::engage ack comment failed on #${PR}; the scan's first-pickup ack heals it" + # Engaged (possibly re-engaging an auto-released PR) — the + # escalation label is stale. 404 is the common case. + if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + fi fi else if [[ "${HAS}" != 'true' ]]; then @@ -1817,6 +1833,11 @@ jobs: [[ "${REMOVE_ERR}" == *404* ]] || echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}" fi echo "🏷️ removed ${TAKEOVER_LABEL} from #${PR}" + # Released means a human is driving — the escalation label is + # stale. 404 is the common case (never paused). + if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + fi # Release ack, direct from the command — the exact mirror of # the engage side above, for the same reason: the unlabeled # round-trip is the thing we no longer trust, fork unlabeled @@ -1894,6 +1915,11 @@ jobs: fi gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '\U0001F504 AutoFix re-armed. The next scan re-reads this PR'"'"'s feedback from the start and the round counter resets. Nothing was deleted — this marker supersedes the evaluation markers above it.\n\n
\n中文说明\n\n\U0001F504 已重新武装 AutoFix。下一次扫描会从头重新读取本 PR 的反馈，轮次计数也已重置。未删除任何内容 —— 本标记使其上方的评估标记失效。\n\n
\n\n')" echo "🔄 re-armed PR #${PR}" + # Management resumed — the escalation label is stale. 404 is the + # common case (the PR was never paused). + if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + fi takeover-ack: needs: 'route' @@ -1967,6 +1993,19 @@ jobs: else BODY="$(printf '👋 Takeover released: the autofix loop will no longer engage this PR (an in-flight round, if any, completes its bounded work). Re-apply `%s` (or comment `%s`) to re-engage.\n\n
\n中文说明\n\n👋 已释放：autofix 循环不再介入此 PR（在飞的一轮如有，将完成其有界工作）。重新打上 `%s` 标签（或评论 `%s`）即可再次接管。\n\n
\n\n' "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}")" fi + # The escalation label goes stale on a real engage or any release + # (a human is driving again). NOT on base-refused (nothing changed) + # and NOT on skip-blocked (management never resumed). 404 is the + # common case (the PR was never paused). Runs BEFORE the ack + # comment: the state change already happened (the human toggled + # the label — the comment is purely informational), so a transient + # comment failure aborting this step under set -e must not strand + # the stale label. + if [[ "${ACK}" == 'released' || ( "${ACK}" == 'engaged' && "${HAS_SKIP}" != 'true' ) ]]; then + if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + fi + fi gh pr comment "${PR}" --repo "${REPO}" --body "${BODY}" # =========================================================================== @@ -2697,6 +2736,11 @@ jobs: fi if [[ "${SCAN_BOT_ACTOR}" == "${AUTOFIX_BOT}" ]]; then if gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '🤝 Takeover engaged: the autofix loop now manages this PR — it will address new review feedback and resolve base conflicts until the label is removed or the round cap is reached. Remove the `%s` label (or comment `%s stop`) to release.\n\n
\n中文说明\n\n🤝 已接管：autofix 循环现在管理此 PR —— 将持续处理新的评审反馈与 base 冲突，直到移除标签或达到轮次上限。移除 `%s` 标签（或评论 `%s stop`）即可释放。\n\n
\n\n' "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}")"; then + # Engaged — the escalation label is stale. 404 is the + # common case (the PR was never paused). + if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + fi # Atomic re-fetch: ic.json already holds a successful full # fetch from above; a truncated stream must not leave it 0 # bytes (jq -s fails only AFTER the redirect truncates) — @@ -2872,8 +2916,8 @@ jobs: | select((.body // "") | contains("")) | select((.created_at // "") > $rt) ] | length' "${WORKDIR}/ic.json")" if [[ "${DRY_RUN}" == "true" ]]; then - echo "🧪 DRY-RUN: would post cap-paused notice on #${PR}" - elif [[ "${CAP_NOTICED}" == "0" ]]; then + echo "🧪 DRY-RUN: would post cap-paused notice and apply ${NEEDS_HUMAN_LABEL} on #${PR}" + else # Consent may have moved since PR_META: skip wins everywhere, # and a takeover notice additionally requires the label to # still be present — a label removed (or skip added) moments @@ -2894,13 +2938,27 @@ jobs: if [[ "${SCAN_BOT_ACTOR}" != "${AUTOFIX_BOT}" ]]; then echo "::warning::cap-paused notice skipped: PAT authenticates as '${SCAN_BOT_ACTOR}', expected ${AUTOFIX_BOT}" else - if [[ "${HAS_TAKEOVER}" == "true" ]]; then - CAP_BODY="$(printf '⏸️ Takeover paused: this PR reached its round cap (%s/%s). Comment `%s` to re-arm a fresh window and continue management, or `%s stop` to release.\n\n
\n中文说明\n\n⏸️ 托管已暂停：本 PR 达到轮次上限（%s/%s）。评论 `%s` 可重新武装、开启新窗口继续托管；或评论 `%s stop` 释放。\n\n
\n\n' "${ROUND}" "${EFF_MAX_ROUNDS}" "${TAKEOVER_COMMAND}" "${TAKEOVER_COMMAND}" "${ROUND}" "${EFF_MAX_ROUNDS}" "${TAKEOVER_COMMAND}" "${TAKEOVER_COMMAND}")" - else - CAP_BODY="$(printf '⏸️ AutoFix paused: this PR reached its automatic round cap (%s/%s) and the loop will not manage it further — new feedback and base conflicts stay unhandled. Comment `%s` to re-arm a fresh window under the same cap, or `%s` to take it over with the raised cap.\n\n
\n中文说明\n\n⏸️ AutoFix 已暂停：本 PR 达到自动轮次上限（%s/%s），循环不再管理——新反馈与 base 冲突将无人处理。评论 `%s` 可在同一上限下重置计数窗口，或评论 `%s` 以更高上限接管。\n\n
\n\n' "${ROUND}" "${EFF_MAX_ROUNDS}" "${RETRY_COMMAND}" "${TAKEOVER_COMMAND}" "${ROUND}" "${EFF_MAX_ROUNDS}" "${RETRY_COMMAND}" "${TAKEOVER_COMMAND}")" + # The escalation label rides EVERY cap detection, noticed + # or not: the once-per-window dedup suppresses repeat + # comments, but the label is what makes a paused PR + # filterable and what the shepherd's auto-release ages + # from — and applying it unconditionally backfills the + # already-paused fleet on the first scan after this ships. + gh label create "${NEEDS_HUMAN_LABEL}" --repo "${REPO}" --color 'D93F0B' \ + --description 'The autofix loop stopped on this PR — a human must re-arm, split, merge, or close it' \ + 2> /dev/null || true + if ! gh api -X POST "repos/${REPO}/issues/${PR}/labels" -f "labels[]=${NEEDS_HUMAN_LABEL}" > /dev/null; then + echo "::warning::${NEEDS_HUMAN_LABEL} add failed for #${PR}; will retry next scan" fi - if ! gh pr comment "${PR}" --repo "${REPO}" --body "${CAP_BODY}"; then - echo "::warning::cap-paused notice failed for #${PR}; will retry next scan" + if [[ "${CAP_NOTICED}" == "0" ]]; then + if [[ "${HAS_TAKEOVER}" == "true" ]]; then + CAP_BODY="$(printf '⏸️ Takeover paused: this PR reached its round cap (%s/%s). Comment `%s` to re-arm a fresh window and continue management, or `%s stop` to release.\n\n
\n中文说明\n\n⏸️ 托管已暂停：本 PR 达到轮次上限（%s/%s）。评论 `%s` 可重新武装、开启新窗口继续托管；或评论 `%s stop` 释放。\n\n
\n\n' "${ROUND}" "${EFF_MAX_ROUNDS}" "${TAKEOVER_COMMAND}" "${TAKEOVER_COMMAND}" "${ROUND}" "${EFF_MAX_ROUNDS}" "${TAKEOVER_COMMAND}" "${TAKEOVER_COMMAND}")" + else + CAP_BODY="$(printf '⏸️ AutoFix paused: this PR reached its automatic round cap (%s/%s) and the loop will not manage it further — new feedback and base conflicts stay unhandled. Comment `%s` to re-arm a fresh window under the same cap, or `%s` to take it over with the raised cap.\n\n
\n中文说明\n\n⏸️ AutoFix 已暂停：本 PR 达到自动轮次上限（%s/%s），循环不再管理——新反馈与 base 冲突将无人处理。评论 `%s` 可在同一上限下重置计数窗口，或评论 `%s` 以更高上限接管。\n\n
\n\n' "${ROUND}" "${EFF_MAX_ROUNDS}" "${RETRY_COMMAND}" "${TAKEOVER_COMMAND}" "${ROUND}" "${EFF_MAX_ROUNDS}" "${RETRY_COMMAND}" "${TAKEOVER_COMMAND}")" + fi + if ! gh pr comment "${PR}" --repo "${REPO}" --body "${CAP_BODY}"; then + echo "::warning::cap-paused notice failed for #${PR}; will retry next scan" + fi fi fi fi diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index b646f2f1569..329f3ce78b4 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -14,7 +14,20 @@ name: 'Fleet Shepherd' # recently, dispatch one (GitHub cron is unreliable) # # It also maintains a single "Fleet Shepherd Dashboard" issue (edited in -# place, never comment spam) so fleet state is observable at a glance. +# place, never comment spam) so fleet state is observable at a glance. The +# dashboard additionally lists the TAKEOVER pool (open PRs carrying +# autofix/takeover, forks included) — the autofix loop manages those, but +# without a row here a paused takeover PR was invisible until someone opened +# the PR page. +# +# One lever exists for the takeover pool: AUTO-RELEASE. A PR carrying both +# autofix/takeover and autofix/needs-human whose cap pause went unanswered +# for AUTO_RELEASE_DAYS days gets its takeover label removed plus one +# bilingual summary comment (merge / close / split + re-takeover). The +# needs-human label stays as the filterable TODO. No marker is needed for +# idempotency: the lever's scope condition (both labels) is false once the +# release lands. Every other takeover state — conflicts, red CI, new +# feedback — remains the autofix scan's job; the shepherd only reports it. # # NON-GOAL: rerunning flaky-failed CI. That is owned by the CI Failure Patrol # (.github/workflows/qwen-ci-flaky-rerun.yml), which has its own markers, @@ -22,15 +35,16 @@ name: 'Fleet Shepherd' # rerun owner here raced it (observed live as rerun-vs-rerun cancellation # storms), so the shepherd only REPORTS red CI on the dashboard. # -# Safety rails: bot-authored main-targeting in-repo PRs only; per-action -# markers make every write idempotent; per-tick action caps bound blast -# radius; every remote read fails CLOSED (an unreadable snapshot skips the -# levers it feeds — it never masquerades as empty state); DRY-RUN via -# dispatch input; global kill switch via the FLEET_SHEPHERD_DISABLED -# repository variable. Dispatches use the workflow's own token -# (actions: write); comments, update-branch, and the dashboard use -# CI_DEV_BOT_PAT so synced branches still trigger CI and all writes carry -# the bot identity. +# Safety rails: the conflict/sync levers touch bot-authored main-targeting +# in-repo PRs only (the takeover pool gets visibility plus the label-only +# auto-release); per-action markers make every write idempotent; per-tick +# action caps bound blast radius; every remote read fails CLOSED (an +# unreadable snapshot skips the levers it feeds — it never masquerades as +# empty state); DRY-RUN via dispatch input; global kill switch via the +# FLEET_SHEPHERD_DISABLED repository variable. Dispatches use the workflow's +# own token (actions: write); comments, label edits, update-branch, and the +# dashboard use CI_DEV_BOT_PAT so synced branches still trigger CI and all +# writes carry the bot identity. on: schedule: - cron: '*/15 * * * *' @@ -60,6 +74,17 @@ env: # Maintainer opt-out label, honored at every engagement path (mirrors # qwen-autofix.yml's SKIP_LABEL). SKIP_LABEL: 'autofix/skip' + # The takeover pool these two labels define (both mirror qwen-autofix.yml): + # TAKEOVER opts a PR into the loop; NEEDS_HUMAN marks a PR the loop STOPPED + # on (round cap or a breaker) until a human re-arms, splits, merges, or + # closes it. + TAKEOVER_LABEL: 'autofix/takeover' + NEEDS_HUMAN_LABEL: 'autofix/needs-human' + # Days a takeover PR may sit paused (both labels present, no re-arm since + # the cap notice) before the shepherd releases the takeover label. Tunable + # without a deploy via the repository variable. + AUTO_RELEASE_DAYS: "${{ vars.QWEN_SHEPHERD_AUTO_RELEASE_DAYS || '3' }}" + MAX_RELEASES_PER_TICK: '3' jobs: shepherd: @@ -287,19 +312,23 @@ jobs: # maintainer adding autofix/skip mid-tick must still win before a # dispatch or branch sync. Fail closed: an unreadable label state # counts as skipped. + # The fetched payload is exported as LIVE_LABELS_JSON so a caller + # with follow-up label checks (the auto-release scope condition) + # rides the SAME read instead of spending a second one. # Returns 0 when the mutation must NOT proceed, with the reason in # LIVE_SKIP_REASON: 'label' (consent withdrawn) vs 'unreadable' # (fail closed on an API failure) — callers word their notes # accordingly so an outage is never reported as a maintainer # decision. live_skip() { - local pr="$1" labels + local pr="$1" LIVE_SKIP_REASON='' - if ! labels="$(gh pr view "${pr}" --repo "${REPO}" --json labels 2> /dev/null)"; then + LIVE_LABELS_JSON='' + if ! LIVE_LABELS_JSON="$(gh pr view "${pr}" --repo "${REPO}" --json labels 2> /dev/null)"; then LIVE_SKIP_REASON='unreadable' return 0 fi - if [[ "$(jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${labels}")" == "true" ]]; then + if [[ "$(jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${LIVE_LABELS_JSON}")" == "true" ]]; then LIVE_SKIP_REASON='label' return 0 fi @@ -352,6 +381,13 @@ jobs: STATUS_NOTE='idle' ACTION_NOTE='—' + # A bot PR the loop stopped on (round cap or a breaker) carries + # the escalation label — prefix its state so the pause is visible + # on the dashboard instead of only on the PR page. + NH_PREFIX='' + if [[ "$(jq -r --arg l "${NEEDS_HUMAN_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" == "true" ]]; then + NH_PREFIX='🛑 ' + fi # 1) conflict → dispatch the loop for this PR, once per head SHA. # The dedup marker is posted ONLY when the dispatch succeeded — a @@ -452,18 +488,233 @@ jobs: fi echo "🐑 #${PR} [${STATUS_NOTE}] → ${ACTION_NOTE}" - DASH_ROWS="${DASH_ROWS}| #${PR} | \`${HEAD:0:9}\` | ${STATUS_NOTE} | ${ACTION_NOTE} |\n" + DASH_ROWS="${DASH_ROWS}| #${PR} | \`${HEAD:0:9}\` | ${NH_PREFIX}${STATUS_NOTE} | ${ACTION_NOTE} |\n" done < <(jq -c '.[]' /tmp/fleet.json) + # ---- takeover pool: dashboard rows + the auto-release lever ----- + # These PRs are managed by the autofix loop, not the shepherd — the + # only lever here is releasing a takeover whose pause went + # unanswered. A failed enumeration keeps the PREVIOUS dashboard + # body: an unreadable snapshot never masquerades as an empty pool. + RELEASES=0 + TAKEOVER_ROWS='' + HUMAN_ROWS='' + # The variable is operator-tunable, so it is also operator- + # breakable: a non-numeric value would fail the -ge comparison + # under set -e mid-tick. Fall back to the default instead. + if [[ ! "${AUTO_RELEASE_DAYS}" =~ ^[0-9]+$ ]]; then + echo "::warning::AUTO_RELEASE_DAYS '${AUTO_RELEASE_DAYS}' is not numeric; using 3" + AUTO_RELEASE_DAYS=3 + fi + # sort:updated-asc so the STALE tail — paused PRs, the release + # lever's whole target population — stays inside the window when + # the pool outgrows the limit; freshly-active takeover PRs are + # watched by the loop and their authors anyway. The saturation + # warning keeps the truncation loud when the window still fills. + if ! gh pr list --repo "${REPO}" --state open --label "${TAKEOVER_LABEL}" \ + --search 'sort:updated-asc' \ + --limit 50 --json number,author,updatedAt,mergeable,statusCheckRollup,labels \ + > /tmp/takeover-raw.json 2> /dev/null; then + echo "::warning::takeover enumeration failed; skipping the takeover table, auto-release, and dashboard update this tick" + exit 0 + fi + if [[ "$(jq length /tmp/takeover-raw.json)" -ge 50 ]]; then + echo "::warning::takeover pool at the 50-PR enumeration limit — the freshest PRs are dropped from this tick's table" + fi + # The released-but-unresolved tail: needs-human WITHOUT takeover is + # the deliberate post-release TODO state, but the PR is in neither + # enumeration above — a second, read-only list keeps it visible. + if ! gh pr list --repo "${REPO}" --state open --label "${NEEDS_HUMAN_LABEL}" \ + --search 'sort:updated-asc' \ + --limit 50 --json number,author,updatedAt,labels \ + > /tmp/human-raw.json 2> /dev/null; then + echo "::warning::needs-human enumeration failed; skipping the takeover table, auto-release, and dashboard update this tick" + exit 0 + fi + jq --arg skip "${SKIP_LABEL}" \ + '[.[] | select([.labels[]?.name] | index($skip) | not)]' \ + /tmp/takeover-raw.json > /tmp/takeover.json + + while IFS= read -r ROW; do + [[ -z "${ROW}" ]] && continue + PR="$(jq -r '.number' <<< "${ROW}")" + AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" + UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" + MERGEABLE="$(jq -r '.mergeable // "UNKNOWN"' <<< "${ROW}")" + PENDING="$(jq -r '[.statusCheckRollup[]? | select((.status // "") | IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))] | length' <<< "${ROW}")" + FAILED_TEST_URL="$(jq -r '[.statusCheckRollup[]? | select(.conclusion == "FAILURE") | select(.name | startswith("Test (")) | .detailsUrl][0] // ""' <<< "${ROW}")" + HAS_NH="$(jq -r --arg l "${NEEDS_HUMAN_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" + UPD_AGO='?' + if [[ -n "${UPDATED}" ]]; then + UPD_AGO="$(( (NOW_EPOCH - $(date -u -d "${UPDATED}" +%s)) / 86400 ))d" + fi + + STATE='managed · idle' + NOTE='—' + if [[ "${HAS_NH}" == "true" ]]; then + STATE='🛑 needs-human' + # Paused PRs are few, so the paginated comment/event reads live + # only in this branch. Fail closed everywhere: an unreadable + # history defers the release instead of acting on partial state. + if ! gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ic.json; then + NOTE='comment read failed — release deferred this tick' + elif ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev.json; then + NOTE='event read failed — release deferred this tick (fail closed)' + else + # The pause clock starts at the LATEST cap notice. + TERM_TS="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | contains("")) + | .created_at ] | max // ""' /tmp/tk-ic.json)" + # Resume evidence, widest first — anything newer than + # TERM_TS means a human already acted and the release must + # never fire: + # 1. the bot's resume markers (re-arm / engage ack); + # 2. a TRUSTED human's re-arm COMMAND comment — the + # command's own ack rides a queued job, so counting + # only the ack would let a shepherd tick release a PR + # minutes after the human re-armed it; + # 3. a fresh `labeled` event (below) — a UI re-apply is + # timestamped immediately, while its ack rides the same + # queue (or, on a lost event, the scan's idle-backoff + # pickup — an hours-wide window). + RESUME_TS="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select( + (((.user.login // "") == $ab) + and (((.body // "") | contains("")) + or ((.body // "") | contains("")))) + or (((.author_association // "") | IN("OWNER", "MEMBER", "COLLABORATOR")) + and (((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /takeover" + or ((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /retry"))) + | .created_at ] | max // ""' /tmp/tk-ic.json)" + EVENT_TS="$(jq -r --arg tl "${TAKEOVER_LABEL}" ' + [ .[] | select((.event // "") == "labeled") + | select((.label.name // "") == $tl) + | .created_at ] | max // ""' /tmp/tk-ev.json)" + if [[ -n "${EVENT_TS}" && "${EVENT_TS}" > "${RESUME_TS}" ]]; then + RESUME_TS="${EVENT_TS}" + fi + # The stop reason comes from the terminal round's headline — + # the scan-side notice always says "round cap" even when a + # breaker (consecutive failures, time budget) fired first. + # Only TERMINAL headlines match: the transient "could not + # start — a setup step failed" variant retries next scan and + # says nothing about why the loop stopped. The terminal set + # is cross-pinned against qwen-autofix.yml's HEADLINE= sites + # by the shepherd test — drift fails CI, not the dashboard. + REASON="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | test("🤖 AutoFix (stopped|could not start (evaluation|— reached the round cap))|— this was the last automatic attempt")) + | (.body | gsub("\r"; "") | split("\n")[0]) ] | last // ""' /tmp/tk-ic.json)" + [[ -z "${REASON}" ]] && REASON='round cap reached' + REASON="${REASON:0:120}" + if [[ -z "${TERM_TS}" ]]; then + NOTE='pause timestamp unreadable — release deferred (fail closed)' + elif [[ -n "${RESUME_TS}" && "${RESUME_TS}" > "${TERM_TS}" ]]; then + STATE='managed (re-armed)' + NOTE='resumed after the last pause — the escalation label is stale' + else + PAUSE_D=$(( (NOW_EPOCH - $(date -u -d "${TERM_TS}" +%s)) / 86400 )) + STATE="🛑 needs-human ${PAUSE_D}d" + NOTE="${REASON}" + if [[ "${MERGEABLE}" == "CONFLICTING" ]]; then + NOTE="paused AND conflicting — ${REASON}" + fi + if [[ "${PAUSE_D}" -ge "${AUTO_RELEASE_DAYS}" ]]; then + # Budget first (an exhausted tick stops spending API + # calls), then ONE live label read (inside live_skip) + # carries every veto: fail closed when unreadable, skip + # wins, and the scope condition (both labels) must still + # hold — a re-arm or release since the snapshot ends the + # pause. + if [[ "${RELEASES}" -ge "${MAX_RELEASES_PER_TICK}" ]]; then + NOTE='release budget reached this tick' + elif live_skip "${PR}"; then + NOTE="$(skip_note release)" + elif [[ "$(jq -r --arg a "${TAKEOVER_LABEL}" --arg b "${NEEDS_HUMAN_LABEL}" '([.labels[]?.name] | index($a) != null) and ([.labels[]?.name] | index($b) != null)' <<< "${LIVE_LABELS_JSON}")" != "true" ]]; then + NOTE='labels changed since the snapshot — deferring release' + else + if act "#${PR}: auto-release takeover (paused ${PAUSE_D}d)" \ + gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')" > /dev/null; then + # The release HAPPENED, so it counts against the + # per-tick budget regardless of the summary's fate + # (mirrors the conflict-dispatch counting rule). + RELEASES=$(( RELEASES + 1 )) + if act "#${PR}: post auto-release summary" \ + gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '🔓 Takeover auto-released: the autofix loop paused on this PR %s day(s) ago (%s) and no re-arm followed, so the `%s` label is removed to keep the managed pool honest. `%s` stays as the reminder that this PR needs a human decision: merge it, close it, or split/reduce it and comment `@qwen-code /takeover` to re-engage with a fresh round window.\n\n
\n中文说明\n\n🔓 已自动释放接管：autofix 循环在 %s 天前暂停于此 PR（%s），此后无人重新武装，现移除 `%s` 标签以保持托管池真实可用。保留 `%s` 作为待办提醒 —— 本 PR 需要人工决策：合并、关闭，或拆分/缩小后评论 `@qwen-code /takeover` 以全新轮次窗口重新接管。\n\n
\n\n' "${PAUSE_D}" "${REASON}" "${TAKEOVER_LABEL}" "${NEEDS_HUMAN_LABEL}" "${PAUSE_D}" "${REASON}" "${TAKEOVER_LABEL}" "${NEEDS_HUMAN_LABEL}")"; then + NOTE="auto-released after ${PAUSE_D}d paused" + else + NOTE='takeover released; summary post failed' + fi + else + NOTE='auto-release FAILED — will retry next tick' + fi + fi + fi + fi + fi + elif [[ "${MERGEABLE}" == "CONFLICTING" ]]; then + STATE='conflicting' + NOTE='resolution owned by the autofix scan' + elif [[ -n "${FAILED_TEST_URL}" && "${PENDING}" == "0" ]]; then + STATE="[ci red](${FAILED_TEST_URL})" + NOTE='reruns owned by CI Failure Patrol' + elif [[ "${PENDING}" != "0" ]]; then + STATE="checks in flight (${PENDING})" + fi + + echo "🐑 #${PR} [takeover: ${STATE}] → ${NOTE}" + # NOTE carries dynamic text (the stop-reason headline can embed + # provider error detail) — printf '%b' would expand backslash + # escapes and a raw pipe breaks the table, so sanitize both. + SAFE_NOTE="${NOTE//\\/\\\\}" + TAKEOVER_ROWS="${TAKEOVER_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${STATE} | ${SAFE_NOTE//|/\\|} |\n" + done < <(jq -c '.[]' /tmp/takeover.json) + + # Released-but-unresolved rows (needs-human without takeover is + # the deliberate post-release TODO state — read-only visibility). + while IFS= read -r ROW; do + [[ -z "${ROW}" ]] && continue + PR="$(jq -r '.number' <<< "${ROW}")" + AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" + UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" + UPD_AGO='?' + if [[ -n "${UPDATED}" ]]; then + UPD_AGO="$(( (NOW_EPOCH - $(date -u -d "${UPDATED}" +%s)) / 86400 ))d" + fi + if [[ "$(jq -r --arg t "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${ROW}")" == "true" ]]; then + continue # still managed — the takeover table above carries it + fi + HUMAN_ROWS="${HUMAN_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | takeover released — needs a human decision (merge / close / split + re-takeover) |\n" + done < <(jq -c --arg skip "${SKIP_LABEL}" '[.[] | select([.labels[]?.name] | index($skip) | not)] | .[]' /tmp/human-raw.json) + # ---- dashboard: one issue, edited in place ---------------------- { echo "Auto-maintained by the Fleet Shepherd workflow — do not edit by hand." echo - echo "Last tick: $(date -u +%Y-%m-%dT%H:%M:%SZ) · scan-signal age: ${SCAN_AGE_MIN}m · syncs: ${SYNCS} · dispatches: ${DISPATCHES}" + echo "Last tick: $(date -u +%Y-%m-%dT%H:%M:%SZ) · scan-signal age: ${SCAN_AGE_MIN}m · syncs: ${SYNCS} · dispatches: ${DISPATCHES} · releases: ${RELEASES}" + echo + echo '## Bot fleet' echo echo '| PR | Head | State | Action this tick |' echo '| --- | --- | --- | --- |' printf '%b' "${DASH_ROWS}" + echo + echo '## Takeover pool' + echo + echo 'Managed by the autofix loop; the shepherd only reports, and auto-releases a takeover whose pause went unanswered.' + echo + echo '| PR | Author | Updated | State | Note |' + echo '| --- | --- | --- | --- | --- |' + printf '%b' "${TAKEOVER_ROWS}" + echo + echo '## Awaiting human' + echo + echo 'Loop stopped and takeover released; these need a human decision.' + echo + echo '| PR | Author | Updated | State |' + echo '| --- | --- | --- | --- |' + printf '%b' "${HUMAN_ROWS}" if [[ -n "${LIVENESS_OUT}" ]]; then echo echo "" diff --git a/docs/design/takeover-fleet-visibility.md b/docs/design/takeover-fleet-visibility.md new file mode 100644 index 00000000000..a7036971639 --- /dev/null +++ b/docs/design/takeover-fleet-visibility.md @@ -0,0 +1,140 @@ +# Takeover fleet visibility and cap-hit escalation + +## Problem statement + +As of 2026-08-11, 35 open PRs carry `autofix/takeover`. Two structural gaps: + +1. **The takeover pool is invisible.** The Fleet Shepherd + (`qwen-fleet-shepherd.yml`) enumerates only bot-authored PRs (3 today). + The 35 human-authored takeover PRs appear on no dashboard; their state + (working / paused / conflicting / idle-for-days) is knowable only by + opening each PR. + +2. **Cap-hit PRs die silently.** When a takeover PR reaches its round cap + (100/100), or a circuit breaker (consecutive-failure, time-budget) stops + it, the loop posts one comment and goes quiet. Five PRs have been paused + since 2026-08-06 with no re-arm: #8213, #8396, #8416, #8439, #8443. + Nothing escalates them — no label, no dashboard entry, no auto-release — + so they hold the takeover label forever ("zombie takeover"). + +## Proposed changes + +### A. `autofix/needs-human` label (qwen-autofix.yml) + +A new maintainer-facing label meaning: _the loop has stopped on this PR; a +human must act (re-arm, split, merge, or close)_. + +**Applied** in the review scan's cap-notice path (the single funnel every +terminal state passes through: round cap, consecutive-failure cap, and +time-budget cap all write a terminal `autofix-eval` marker with +`round=EFF_MAX_ROUNDS`, which the next scan sees as `ROUND >= EFF_MAX_ROUNDS` +and lands in the cap-notice branch). The label write is placed so it runs +even when the once-per-window notice comment is dedup'd — this backfills the +label onto the five already-paused PRs on the first scan after deploy. + +**Removed** wherever management resumes or a human takes over: + +| Path | Site | +| ------------------------------------------------------ | -------------------- | +| `/takeover` re-arm on a managed PR | takeover-command job | +| `/takeover` fresh engage | takeover-command job | +| `/takeover stop` | takeover-command job | +| Manual label engage / release acks | takeover-ack job | +| `/retry` re-arm marker | retry-command job | +| Scan first-pickup engage ack (direct-label engagement) | review-scan job | + +Removal is best-effort with a warning on failure, mirroring the existing +`TAKEOVER_LABEL` DELETE pattern (404 tolerated). A stale `needs-human` left +behind by a failed removal is cosmetically wrong but harmless; the next +cap-stop reapplies it anyway. + +Label creation follows the existing convention: `gh label create` (idempotent, +fixed color) before the first REST add, so a missing label never gets a random +color. + +### B. Shepherd covers the takeover pool (qwen-fleet-shepherd.yml) + +A second enumeration — open PRs with `autofix/takeover`, including forks — +drives a **second dashboard table** in the same edited-in-place issue: + +| PR | Author | Updated | State | Note | +| --- | ------ | ------- | ----- | ---- | + +State comes from the list payload (conflicting / ci red / checks in flight / +idle). PRs carrying `autofix/needs-human` get a `🛑 needs-human` state; for +those few PRs the shepherd additionally reads the comment stream (fail-closed) +to recover the terminal timestamp (latest `` +notice) and the stop reason (first line of the latest terminal "AutoFix +stopped" headline, else "round cap reached"). + +**NON-GOAL:** the existing levers (conflict dispatch, stale-base sync) stay +scoped to the bot fleet. Takeover-PR conflicts are already the autofix scan's +job (`HAS_CONFLICT` selects them as targets), and `update-branch` on +contributor branches is out of scope for this change. + +### C. Auto-release lever (qwen-fleet-shepherd.yml) + +When a PR carries **both** `autofix/takeover` and `autofix/needs-human` and +its terminal timestamp is older than `AUTO_RELEASE_DAYS` (default 3, tunable +via the `QWEN_SHEPHERD_AUTO_RELEASE_DAYS` repo variable): + +1. Remove `autofix/takeover` (the loop disengages). +2. Post one bilingual summary: why it was released, the stop reason, and the + human's options (merge / close / split + re-takeover). +3. Keep `autofix/needs-human`: the PR still needs a human decision, and the + label remains the filterable TODO list. It clears on re-engage/re-arm via + the paths in (A). + +Idempotency needs no marker comment: the lever's scope condition (both labels) +is false after the release, so it cannot re-fire. Per-tick cap +(`MAX_RELEASES_PER_TICK`, default 3) bounds blast radius; `live_skip` is +re-checked immediately before the mutation, mirroring every existing lever. + +## Key design decisions + +- **Label write lives in the scan, not the address leg.** Every terminal stop + converges on `round=EFF_MAX_ROUNDS` markers, which the scan's cap branch + already observes with comments loaded and PAT identity verified. One hook + point covers all stop reasons, including future ones. +- **Pause reason comes from the terminal marker headline**, because the + scan-side notice always says "round cap (N/N)" even when a breaker fired + (observed on #8443: both comments present). +- **Bootstrap without a backfill job:** the label write runs even when the + notice comment is dedup'd, so currently-paused PRs are labeled by the + regular scan rotation after deploy — note the scan's idle backoff defers + PRs idle >24h (exactly the paused population) to ~1 scan in 4, so expect + the backfill within a few hours (median ~2h, p90 ~6h), not minutes. +- **Auto-release keyed on the notice timestamp**, not label age: labels carry + no timestamps, and the notice is written by the same identity-verified path + that applies the label. Resume evidence newer than the notice vetoes the + release — the bot's re-arm/engage markers, a trusted human's + `/takeover` or `/retry` command comment (the ack rides a queued job, so + counting only the ack would let a tick release a PR minutes after a human + re-armed it), and a fresh `labeled` event (a UI re-apply whose ack rides + the same queue, or the scan's idle-backoff pickup when the event is lost). +- **Shepherd timing:** 15-minute tick with a per-tick release cap — a backlog + of expired PRs drains over a few ticks rather than one burst. + +## Files affected + +- `.github/workflows/qwen-autofix.yml` — env, cap-notice branch, six + label-removal sites. +- `.github/workflows/qwen-fleet-shepherd.yml` — env, takeover enumeration, + dashboard takeover table plus a read-only "Awaiting human" section + (released PRs keep `needs-human` and would otherwise vanish from every + surface), auto-release lever. + +## Scope boundaries + +- No changes to round caps, breakers, or review-bot behavior. +- No shepherd levers on takeover PRs other than auto-release. +- No notification/@-mention of maintainers (comment + label + dashboard only). +- `autofix/needs-human` on plain (non-takeover) bot PRs is applied by the same + scan path and shown on the dashboard, but the auto-release lever never + touches them (they have no takeover label to release). + +## Open questions + +- Default `AUTO_RELEASE_DAYS=3` — short enough to keep the pool clean, long + enough for a maintainer to re-arm over a weekend? Adjustable without a + deploy via the repo variable. diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 1ea08ed3189..5fc5ce86ee0 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -3142,6 +3142,58 @@ describe('qwen-autofix workflow', () => { expect(loopHead).not.toContain('INSPECTED='); }); + it('applies the needs-human escalation label at the cap and removes it on resume/release', () => { + // The label contract: one env definition, one idempotent create with a + // fixed color (a REST add of a missing label would mint a random one). + expect(workflow).toContain("NEEDS_HUMAN_LABEL: 'autofix/needs-human'"); + expect(reviewScanJob).toContain( + 'gh label create "${NEEDS_HUMAN_LABEL}" --repo "${REPO}" --color', + ); + expect(reviewScanJob).toContain('-f "labels[]=${NEEDS_HUMAN_LABEL}"'); + // The label write sits OUTSIDE the once-per-window comment dedup — + // that is the bootstrap property: already-noticed PRs (paused before + // this shipped) get labeled on the first scan after deploy. Assert on + // ordering inside the cap branch: the label POST precedes the + // CAP_NOTICED gate that guards only the comment. + const labelPost = reviewScanJob.indexOf( + '-f "labels[]=${NEEDS_HUMAN_LABEL}"', + ); + const commentGate = reviewScanJob.indexOf('"${CAP_NOTICED}" == "0"'); + expect(labelPost).toBeGreaterThan(-1); + expect(commentGate).toBeGreaterThan(-1); + expect(labelPost).toBeLessThan(commentGate); + // …but never BEFORE the live-consent recheck: a takeover label pulled + // moments ago gets neither the notice nor the escalation label. + const consentRecheck = reviewScanJob.indexOf( + 'cap notice skipped: consent changed since the snapshot', + ); + expect(consentRecheck).toBeGreaterThan(-1); + expect(consentRecheck).toBeLessThan(labelPost); + // The dry-run line covers both writes. + expect(reviewScanJob).toContain( + 'DRY-RUN: would post cap-paused notice and apply ${NEEDS_HUMAN_LABEL}', + ); + // Removal at every resume/release point — the URI-encoded REST DELETE + // with 404 tolerance, same shape as the TAKEOVER_LABEL removal. Six + // sites: takeover-command re-arm, fresh engage, and stop; the ack + // job's engage/release; the /retry marker; the scan's first-pickup ack. + const removals = + workflow.match( + /gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{NEEDS_HUMAN_LABEL\}"/g, + ) ?? []; + expect(removals).toHaveLength(6); + // Every removal tolerates the common 404 (never-paused PR). + expect( + workflow.match(/\$\{NEEDS_HUMAN_LABEL\} removal failed/g)?.length, + ).toBe(6); + // The ack job removes on a real engage or any release — but NOT on + // base-refused (nothing changed) or skip-blocked (management never + // resumed). + expect(workflow).toContain( + '"${ACK}" == \'released\' || ( "${ACK}" == \'engaged\' && "${HAS_SKIP}" != \'true\' )', + ); + }); + it('backs off idle candidates without spending inspection budget', () => { // Measured 2026-07-29: 28 open takeover PRs, 8 idle in "nothing new" // state for 10+ hours. Every idle inspection costs a unit of the @@ -3354,6 +3406,7 @@ describe('qwen-autofix workflow', () => { REPO: 'QwenLM/qwen-code', TAKEOVER_LABEL: 'autofix/takeover', SKIP_LABEL: 'autofix/skip', + NEEDS_HUMAN_LABEL: 'autofix/needs-human', TAKEOVER_COMMAND: '@qwen-code /takeover', AUTOFIX_BOT: 'qwen-code-dev-bot', GITHUB_TOKEN: 'x', @@ -3398,10 +3451,14 @@ describe('qwen-autofix workflow', () => { expect(addAbsent.writes).toContain(''); expect(addAbsent.writes).not.toContain('next scheduled scan'); expect(addAbsent.writes).not.toContain('定时扫描'); - // add + present → re-arm ack, label untouched. + // add + present → re-arm ack, takeover label untouched — the only API + // write is the stale needs-human cleanup (404-tolerant; this PR was + // never paused, so the stub's silent success stands in for the 404). const rearm = runToggle({ cmd: 'add', labels: ['autofix/takeover'] }); expect(rearm.writes).toContain('COMMENT'); - expect(rearm.writes).not.toContain('API'); + expect(rearm.writes).not.toContain('labels[]=autofix/takeover'); + expect(rearm.writes).not.toContain('labels/autofix%2Ftakeover'); + expect(rearm.writes).toContain('labels/autofix%2Fneeds-human'); expect(rearm.log).toContain('re-armed'); // remove + present → label removed, through the URI-encoded path segment // (real jq runs in the substitution, so the %2F is the executed truth). diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index dbaca67e030..156ea3bfff6 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -63,12 +63,16 @@ describe('fleet shepherd workflow', () => { expect(workflow).toContain( '--json number,headRefName,headRefOid,mergeable,isCrossRepository,statusCheckRollup', ); - // Per-PR metadata still rides the ONE list call — the sole gh pr view - // is the labels-only live-skip recheck immediately before a mutation. + // Per-PR metadata still rides the list calls — the sole gh pr view is + // live_skip's labels-only recheck immediately before a mutation, whose + // exported payload the auto-release scope check rides (no second read). expect(workflow.split('gh pr view').length - 1).toBe(1); expect(workflow).toContain( 'gh pr view "${pr}" --repo "${REPO}" --json labels', ); + expect(workflow).toContain( + 'index($a) != null) and ([.labels[]?.name] | index($b) != null)\' <<< "${LIVE_LABELS_JSON}"', + ); // autofix/skip is the maintainer opt-out honored at every engagement // path: a skip-labeled PR gets no shepherd levers and no dashboard row. // Replay the filter VERBATIM to prove the label actually excludes. @@ -575,4 +579,320 @@ exit 1`; execFileSync('bash', ['-c', dryScript], { encoding: 'utf8' }), ).toContain('DRY-OK'); }); + + it('covers the takeover pool: enumeration, dashboard rows, and a bounded auto-release', () => { + // Label contract, mirroring qwen-autofix.yml. + expect(workflow).toContain("TAKEOVER_LABEL: 'autofix/takeover'"); + expect(workflow).toContain("NEEDS_HUMAN_LABEL: 'autofix/needs-human'"); + expect(workflow).toContain( + 'AUTO_RELEASE_DAYS: "${{ vars.QWEN_SHEPHERD_AUTO_RELEASE_DAYS || \'3\' }}"', + ); + expect(workflow).toContain("MAX_RELEASES_PER_TICK: '3'"); + // One list call carries the takeover row payload — forks INCLUDED (no + // isCrossRepository filter: visibility covers fork takeovers, and the + // release lever needs no push access). sort:updated-asc keeps the STALE + // tail (the release lever's target population) inside the limit window, + // and the saturation warning keeps truncation loud. + expect(workflow).toContain('--state open --label "${TAKEOVER_LABEL}"'); + expect(workflow).toContain( + '--limit 50 --json number,author,updatedAt,mergeable,statusCheckRollup,labels', + ); + expect(workflow).toContain("--search 'sort:updated-asc'"); + expect(workflow).toContain('takeover pool at the 50-PR enumeration limit'); + const takeoverEnum = workflow.match( + /--label "\$\{TAKEOVER_LABEL\}"[\s\S]*?\/tmp\/takeover-raw\.json/, + )?.[0]; + expect(takeoverEnum).toBeTruthy(); + expect(takeoverEnum).not.toContain('isCrossRepository'); + // A failed pool read keeps the previous dashboard body — an unreadable + // snapshot never masquerades as an empty pool. The exit is what keeps + // the dashboard unwritten; pin them together. + expect(workflow).toMatch( + /takeover enumeration failed; skipping the takeover table, auto-release, and dashboard update this tick"\n\s+exit 0/, + ); + // The released-but-unresolved tail (needs-human WITHOUT takeover) gets + // its own read-only enumeration and dashboard section — released PRs + // must not vanish from every surface. + expect(workflow).toContain('--state open --label "${NEEDS_HUMAN_LABEL}"'); + expect(workflow).toContain("'## Awaiting human'"); + expect(workflow).toContain('takeover released — needs a human decision'); + // autofix/skip wins in the pool too — replay the filter VERBATIM. The + // program-prefix anchor keeps this from capturing the bot-fleet filter + // (same jq shape, different file and a leading isCrossRepository leg). + const takeoverFilter = workflow.match( + /jq --arg skip "\$\{SKIP_LABEL\}" \\\n\s+'(\[\.\[\] \| select\(\[\.labels[\s\S]*?)' \\\n\s+\/tmp\/takeover-raw\.json/, + )?.[1]; + expect(takeoverFilter).toBeTruthy(); + const kept = JSON.parse( + execFileSync('jq', ['--arg', 'skip', 'autofix/skip', takeoverFilter], { + encoding: 'utf8', + input: JSON.stringify([ + { number: 1, labels: [] }, + { number: 2, labels: [{ name: 'autofix/skip' }] }, + { number: 3, labels: [{ name: 'unrelated' }] }, + ]), + }), + ).map((r) => r.number); + expect(kept).toEqual([1, 3]); + // Dashboard: the takeover table is rendered (edited in place with the + // bot-fleet table), and the header line reports releases. + expect(workflow).toContain("'## Takeover pool'"); + expect(workflow).toContain("'| PR | Author | Updated | State | Note |'"); + expect(workflow).toContain('releases: ${RELEASES}'); + // needs-human bot PRs surface on the bot-fleet table too. + expect(workflow).toContain("NH_PREFIX='🛑 '"); + }); + + it('behaviorally proves pause detection, the re-arm guard, and the release age gate', () => { + // Extract the three pause-detection jq programs VERBATIM (drift fails + // the test) and replay them against comment-stream fixtures. + const termJq = workflow.match( + /TERM_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + const resumeJq = workflow.match( + /RESUME_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + const reasonJq = workflow.match( + /REASON="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + expect(termJq).toBeTruthy(); + expect(resumeJq).toBeTruthy(); + expect(reasonJq).toBeTruthy(); + const run = (program, comments) => + execFileSync('jq', ['-r', '--arg', 'ab', 'qwen-code-dev-bot', program], { + encoding: 'utf8', + input: JSON.stringify(comments), + }).trim(); + const bot = (created_at, body) => ({ + user: { login: 'qwen-code-dev-bot' }, + created_at, + body, + }); + const human = (created_at, body) => ({ + user: { login: 'wenshao' }, + created_at, + body, + }); + // TERM_TS: latest bot cap notice only — a human forging the marker text + // does not move the pause clock. + expect( + run(termJq, [ + bot('2026-08-01T00:00:00Z', '⏸️ … '), + bot('2026-08-05T00:00:00Z', '⏸️ … '), + human('2026-08-06T00:00:00Z', ''), + ]), + ).toBe('2026-08-05T00:00:00Z'); + expect(run(termJq, [human('2026-08-06T00:00:00Z', 'x')])).toBe(''); + // RESUME_TS: re-arm markers and engage acks both count as a resume. + expect( + run(resumeJq, [ + bot('2026-08-02T00:00:00Z', '🔄 … '), + bot('2026-08-03T00:00:00Z', '🤝 … '), + ]), + ).toBe('2026-08-03T00:00:00Z'); + expect(run(resumeJq, [bot('2026-08-02T00:00:00Z', '⏸️ unrelated')])).toBe( + '', + ); + // …and a TRUSTED human's re-arm command counts at COMMENT time (the + // ack rides a queued job — without this a tick could release a PR + // minutes after the human re-armed it), while an outsider's identical + // command does not move the clock. + const trusted = (created_at, body) => ({ + user: { login: 'wenshao' }, + author_association: 'MEMBER', + created_at, + body, + }); + expect( + run(resumeJq, [trusted('2026-08-06T00:00:00Z', '@qwen-code /takeover')]), + ).toBe('2026-08-06T00:00:00Z'); + expect( + run(resumeJq, [trusted('2026-08-06T00:00:00Z', ' @qwen-code /retry ')]), + ).toBe('2026-08-06T00:00:00Z'); + expect( + run(resumeJq, [human('2026-08-06T00:00:00Z', '@qwen-code /takeover')]), + ).toBe(''); + // A fresh `labeled` event is resume evidence too (UI re-apply). + const eventJq = workflow.match( + /EVENT_TS="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, + )?.[1]; + expect(eventJq).toBeTruthy(); + const runEvent = (events) => + execFileSync('jq', ['-r', '--arg', 'tl', 'autofix/takeover', eventJq], { + encoding: 'utf8', + input: JSON.stringify(events), + }).trim(); + expect( + runEvent([ + { + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-06T00:00:00Z', + }, + { + event: 'labeled', + label: { name: 'other' }, + created_at: '2026-08-07T00:00:00Z', + }, + ]), + ).toBe('2026-08-06T00:00:00Z'); + expect(runEvent([{ event: 'closed' }])).toBe(''); + // The event read fails closed, like the comment read. + expect(workflow).toContain( + 'event read failed — release deferred this tick (fail closed)', + ); + // REASON: the terminal round's own headline beats the scan-side notice, + // which always says "round cap" even when a breaker fired first. + const reason = run(reasonJq, [ + bot( + '2026-08-04T00:00:00Z', + '🤖 AutoFix stopped: this counting window now contains 3 time-budget exhaustions …\ndetails', + ), + ]); + expect(reason).toContain('time-budget exhaustions'); + expect(reason).not.toContain('details'); + // Only TERMINAL headlines count: the transient setup-failure variant + // retries on the next scan and says nothing about why the loop stopped. + expect( + run(reasonJq, [ + bot( + '2026-08-04T00:00:00Z', + '🤖 AutoFix could not start — a setup step failed …', + ), + ]), + ).toBe(''); + // The re-arm guard: a resume NEWER than the latest cap notice means the + // label is stale and the release must never fire. Replay the gate + // VERBATIM so a dropped comparison fails the test. + const rearmGuard = workflow.match( + /(elif \[\[ -n "\$\{RESUME_TS\}" && "\$\{RESUME_TS\}" > "\$\{TERM_TS\}" \]\]; then\n {18}STATE='managed \(re-armed\)')/, + )?.[1]; + expect(rearmGuard).toBeTruthy(); + const guard = (resume, term) => + execFileSync( + 'bash', + [ + '-c', + `if [[ -z "${'$'}{TERM_TS}" ]]; then echo UNREADABLE\n${rearmGuard.replace(/\n {18}/g, '\n')}\necho REARMED\nelse\nfi`.replace( + 'else\nfi', + 'else echo RELEASE-ELIGIBLE\nfi', + ), + ], + { + env: { ...process.env, RESUME_TS: resume, TERM_TS: term }, + encoding: 'utf8', + }, + ).trim(); + expect(guard('2026-08-06T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( + 'REARMED', + ); + expect(guard('2026-08-04T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( + 'RELEASE-ELIGIBLE', + ); + expect(guard('', '2026-08-05T00:00:00Z')).toBe('RELEASE-ELIGIBLE'); + // The release gates run in blast-radius order: per-tick budget BEFORE + // the PAT-backed live read, then the shared live_skip veto (fail closed + // on an unreadable read; skip wins — both worded by skip_note), then the + // both-labels scope condition riding live_skip's exported payload — and + // the writes are act()-wrapped so dry-run makes no mutation. + expect(workflow).toMatch( + /RELEASES\}" -ge "\$\{MAX_RELEASES_PER_TICK\}" \]\]; then[\s\S]{0,200}elif live_skip "\$\{PR\}"; then\n\s+NOTE="\$\(skip_note release\)"[\s\S]{0,400}labels changed since the snapshot — deferring release/, + ); + expect(workflow).toMatch( + /if act "#\$\{PR\}: auto-release takeover[\s\S]{0,700}if act "#\$\{PR\}: post auto-release summary"/, + ); + // The release counts against the per-tick budget even when the summary + // comment fails — the increment precedes the summary act (mirrors the + // conflict-dispatch counting rule pinned above). + expect(workflow).toMatch( + /RELEASES=\$\(\( RELEASES \+ 1 \)\)[\s\S]{0,400}if act "#\$\{PR\}: post auto-release summary"/, + ); + // The fail-closed deferral guards are load-bearing (a missing one runs + // `date -d ""` under set -e and kills the tick) — pin their wording. + expect(workflow).toContain( + 'pause timestamp unreadable — release deferred (fail closed)', + ); + expect(workflow).toContain( + 'comment read failed — release deferred this tick', + ); + // The release is idempotent WITHOUT a marker: the scope condition (both + // labels) is false once the takeover label lands off. + expect(workflow).toContain( + 'index($a) != null) and ([.labels[]?.name] | index($b) != null)', + ); + expect(workflow).toContain(''); + // The age gate compares whole days against the tunable. + expect(workflow).toContain('"${PAUSE_D}" -ge "${AUTO_RELEASE_DAYS}"'); + // The tunable is operator-breakable — a non-numeric repo variable falls + // back to the default instead of dying mid-tick under set -e. + expect(workflow).toContain('=~ ^[0-9]+$'); + expect(workflow).toContain('is not numeric; using 3'); + // Dynamic note text is sanitized before it reaches the printf '%b' + // table: backslashes first (else %b expands them), then pipes. + expect(workflow).toContain('SAFE_NOTE="${NOTE//\\\\/\\\\\\\\}"'); + expect(workflow).toContain('${SAFE_NOTE//|/\\\\|}'); + }); + + it('pins the marker/headline contracts against qwen-autofix.yml — drift fails here, not in production', () => { + // The resume-marker set, the trusted re-arm commands, and the terminal + // headline formats are all PRODUCED by qwen-autofix.yml and CONSUMED by + // the shepherd. Each side's own suite pins its own file; only a + // cross-file assertion fails when one side moves without the other. + const autofix = readFileSync('.github/workflows/qwen-autofix.yml', 'utf8'); + // 1. Resume markers: the scan's REARM_KEY (the write side's own idea of + // "management resumed") and the shepherd's RESUME_TS must name the + // same markers — a new resume path added to only one file must fail + // here, never as a wrongful auto-release. + const rearmKey = autofix.match( + /REARM_KEY="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)'/, + )?.[1]; + expect(rearmKey).toBeTruthy(); + const resumeTs = workflow.match( + /RESUME_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + expect(resumeTs).toBeTruthy(); + for (const marker of [ + '', + '', + ]) { + expect(rearmKey).toContain(marker); + expect(resumeTs).toContain(marker); + } + // 2. The trusted re-arm commands the shepherd greps for are the same + // strings qwen-autofix.yml routes on. + expect(autofix).toContain("TAKEOVER_COMMAND: '@qwen-code /takeover'"); + expect(autofix).toContain("RETRY_COMMAND: '@qwen-code /retry'"); + expect(resumeTs).toContain('"@qwen-code /takeover"'); + expect(resumeTs).toContain('"@qwen-code /retry"'); + // 3. Every TERMINAL headline the address leg can print matches the + // shepherd's REASON regex, and the transient setup-failure headline + // does not. Templates are extracted from the producer and exercised + // against the consumer's verbatim regex — a reworded headline on + // either side fails this test. + const reasonRe = workflow.match(/\| test\("([^"]+)"\)\)/)?.[1]; + expect(reasonRe).toBeTruthy(); + const headlines = [...autofix.matchAll(/HEADLINE="(🤖 [^"]+)"/g)].map( + (m) => m[1], + ); + expect(headlines.length).toBeGreaterThanOrEqual(5); + const terminal = headlines.filter( + (h) => + h.includes('stopped after') || + h.includes('stopped: this counting window') || + h.includes('could not start evaluation') || + h.includes('reached the round cap') || + h.includes('this was the last automatic attempt'), + ); + const transient = headlines.filter((h) => + h.includes('a setup step failed'), + ); + expect(terminal.length).toBeGreaterThanOrEqual(4); + expect(transient).toHaveLength(1); + const matches = (h) => + execFileSync('jq', ['-rn', '--arg', 'b', h, `$b | test("${reasonRe}")`], { + encoding: 'utf8', + }).trim(); + for (const h of terminal) expect(matches(h)).toBe('true'); + for (const h of transient) expect(matches(h)).toBe('false'); + }); }); From 8ea334ce034301dae04a82efbacfb8c3917ec684 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Wed, 12 Aug 2026 11:42:22 +0800 Subject: [PATCH 02/17] fix(autofix): harden the takeover auto-release against review round 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the PR review's two Criticals and eleven Suggestions: - Command-comment resume evidence now counts only while FRESH (2h grace) and UNSUPERSEDED by a refusal ack (fork-refused/base-refused/ skip-blocked) — an ignored command expires instead of vetoing the release forever, and no permission logic is mirrored from the route. - The release lever's population comes from the needs-human enumeration (needs-human ∩ takeover), never the display window; both enumerations cap at 100 with saturation warnings, and a failed enumeration degrades to an error row so the dashboard write (and its liveness watermark) always runs. - The auto-release summary posts before the label DELETE, dedup'd by its own marker — neither half can strand the other on a transient failure. - Awaiting-human rows use neutral wording (capped bot PRs land there too) and a shepherd-side heal clears stale needs-human labels left by manual UI releases on fork PRs (human unlabeled event, budgeted, skip-vetoed). - Fail-closed deferrals now still render a dashboard row (the row append moved outside the evaluation arms); tick summary and dashboard header report the same counters; days_since() replaces pasted epoch math. - Tests: command-evidence gate replays (fresh/refused/expired/acked), refusal-variant and command-string cross-file pins, DELETE-target and fallback-assignment pins, heal jq replays, unified-row-render pin. Co-authored-by: Qwen-Coder --- .github/workflows/qwen-fleet-shepherd.yml | 324 ++++++++++++------ docs/design/takeover-fleet-visibility.md | 29 +- .../qwen-fleet-shepherd-workflow.test.js | 286 +++++++++++++--- 3 files changed, 481 insertions(+), 158 deletions(-) diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index 329f3ce78b4..661a6c3657b 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -85,6 +85,13 @@ env: # without a deploy via the repository variable. AUTO_RELEASE_DAYS: "${{ vars.QWEN_SHEPHERD_AUTO_RELEASE_DAYS || '3' }}" MAX_RELEASES_PER_TICK: '3' + # How long an unacked re-arm command comment counts as resume evidence. + # An accepted command is acked within minutes (the ack carries the resume + # marker); a route-ignored command gets no reply at all, so without this + # bound it would veto the release forever. + RESUME_COMMAND_GRACE_SEC: '7200' + # Stale needs-human cleanups (manual UI releases on fork PRs) per tick. + MAX_CLEANUPS_PER_TICK: '5' jobs: shepherd: @@ -342,6 +349,11 @@ jobs: fi } + # RFC3339 timestamp → whole days ago (both dashboards show ages). + days_since() { + echo "$(( (NOW_EPOCH - $(date -u -d "$1" +%s)) / 86400 ))" + } + # ---- walk the bot fleet (one list call carries all per-PR meta) -- # autofix/skip is the maintainer opt-out honored at every # engagement path — a skip-labeled PR gets no shepherd levers and @@ -492,11 +504,14 @@ jobs: done < <(jq -c '.[]' /tmp/fleet.json) # ---- takeover pool: dashboard rows + the auto-release lever ----- - # These PRs are managed by the autofix loop, not the shepherd — the - # only lever here is releasing a takeover whose pause went - # unanswered. A failed enumeration keeps the PREVIOUS dashboard - # body: an unreadable snapshot never masquerades as an empty pool. + # These PRs are managed by the autofix loop, not the shepherd — + # the only mutating lever here releases a takeover whose pause + # went unanswered. Enumeration failures degrade to a loud error + # row instead of exiting: the dashboard write must still happen, + # because it carries the liveness watermark the next tick's + # duplicate-dispatch guard reads. RELEASES=0 + CLEANUPS=0 TAKEOVER_ROWS='' HUMAN_ROWS='' # The variable is operator-tunable, so it is also operator- @@ -506,87 +521,155 @@ jobs: echo "::warning::AUTO_RELEASE_DAYS '${AUTO_RELEASE_DAYS}' is not numeric; using 3" AUTO_RELEASE_DAYS=3 fi - # sort:updated-asc so the STALE tail — paused PRs, the release - # lever's whole target population — stays inside the window when - # the pool outgrows the limit; freshly-active takeover PRs are - # watched by the loop and their authors anyway. The saturation - # warning keeps the truncation loud when the window still fills. + # sort:updated-asc keeps the stalest PRs in view when a pool + # outgrows the window; the saturation warning keeps the residual + # truncation loud. The LEVER's population deliberately comes from + # the needs-human enumeration (always small), never from this + # display window — see the second loop below. + TK_OK=true if ! gh pr list --repo "${REPO}" --state open --label "${TAKEOVER_LABEL}" \ --search 'sort:updated-asc' \ - --limit 50 --json number,author,updatedAt,mergeable,statusCheckRollup,labels \ + --limit 100 --json number,author,updatedAt,mergeable,statusCheckRollup,labels \ > /tmp/takeover-raw.json 2> /dev/null; then - echo "::warning::takeover enumeration failed; skipping the takeover table, auto-release, and dashboard update this tick" - exit 0 - fi - if [[ "$(jq length /tmp/takeover-raw.json)" -ge 50 ]]; then - echo "::warning::takeover pool at the 50-PR enumeration limit — the freshest PRs are dropped from this tick's table" + TK_OK=false + echo "::warning::takeover enumeration failed; the takeover table shows an error row this tick" + TAKEOVER_ROWS='| — | — | — | ⚠️ enumeration unreadable this tick | fail closed — no release evaluation ran |\n' + else + if [[ "$(jq length /tmp/takeover-raw.json)" -ge 100 ]]; then + echo "::warning::takeover pool at the 100-PR enumeration limit — recently-active PRs (paused-but-discussed ones included) may be missing from this table" + fi + jq --arg skip "${SKIP_LABEL}" \ + '[.[] | select([.labels[]?.name] | index($skip) | not)]' \ + /tmp/takeover-raw.json > /tmp/takeover.json fi - # The released-but-unresolved tail: needs-human WITHOUT takeover is - # the deliberate post-release TODO state, but the PR is in neither - # enumeration above — a second, read-only list keeps it visible. + # The needs-human enumeration drives the lever AND the + # "awaiting human" table: it carries both labels per row, so the + # release population is never truncated by the display window. + HM_OK=true if ! gh pr list --repo "${REPO}" --state open --label "${NEEDS_HUMAN_LABEL}" \ --search 'sort:updated-asc' \ - --limit 50 --json number,author,updatedAt,labels \ + --limit 100 --json number,author,updatedAt,mergeable,labels \ > /tmp/human-raw.json 2> /dev/null; then - echo "::warning::needs-human enumeration failed; skipping the takeover table, auto-release, and dashboard update this tick" - exit 0 + HM_OK=false + echo "::warning::needs-human enumeration failed; paused-PR evaluation skipped this tick" + HUMAN_ROWS='| — | — | — | ⚠️ enumeration unreadable this tick — previous entries may be stale |\n' + else + if [[ "$(jq length /tmp/human-raw.json)" -ge 100 ]]; then + echo "::warning::needs-human pool at the 100-PR enumeration limit — the freshest entries may be missing from this table" + fi + jq --arg skip "${SKIP_LABEL}" \ + '[.[] | select([.labels[]?.name] | index($skip) | not)]' \ + /tmp/human-raw.json > /tmp/human.json fi - jq --arg skip "${SKIP_LABEL}" \ - '[.[] | select([.labels[]?.name] | index($skip) | not)]' \ - /tmp/takeover-raw.json > /tmp/takeover.json - while IFS= read -r ROW; do - [[ -z "${ROW}" ]] && continue - PR="$(jq -r '.number' <<< "${ROW}")" - AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" - UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" - MERGEABLE="$(jq -r '.mergeable // "UNKNOWN"' <<< "${ROW}")" - PENDING="$(jq -r '[.statusCheckRollup[]? | select((.status // "") | IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))] | length' <<< "${ROW}")" - FAILED_TEST_URL="$(jq -r '[.statusCheckRollup[]? | select(.conclusion == "FAILURE") | select(.name | startswith("Test (")) | .detailsUrl][0] // ""' <<< "${ROW}")" - HAS_NH="$(jq -r --arg l "${NEEDS_HUMAN_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" - UPD_AGO='?' - if [[ -n "${UPDATED}" ]]; then - UPD_AGO="$(( (NOW_EPOCH - $(date -u -d "${UPDATED}" +%s)) / 86400 ))d" - fi + # Loop 1: managed takeover PRs that are NOT paused — cheap payload + # states only. Paused ones are rendered by loop 2, which owns the + # pause evaluation. (If the needs-human enumeration failed, render + # them here so they never vanish silently.) + if [[ "${TK_OK}" == "true" ]]; then + while IFS= read -r ROW; do + [[ -z "${ROW}" ]] && continue + PR="$(jq -r '.number' <<< "${ROW}")" + AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" + UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" + MERGEABLE="$(jq -r '.mergeable // "UNKNOWN"' <<< "${ROW}")" + PENDING="$(jq -r '[.statusCheckRollup[]? | select((.status // "") | IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))] | length' <<< "${ROW}")" + FAILED_TEST_URL="$(jq -r '[.statusCheckRollup[]? | select(.conclusion == "FAILURE") | select(.name | startswith("Test (")) | .detailsUrl][0] // ""' <<< "${ROW}")" + HAS_NH="$(jq -r --arg l "${NEEDS_HUMAN_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" + UPD_AGO='?' + [[ -n "${UPDATED}" ]] && UPD_AGO="$(days_since "${UPDATED}")d" + + STATE='managed · idle' + NOTE='—' + if [[ "${HAS_NH}" == "true" ]]; then + if [[ "${HM_OK}" == "true" ]]; then + continue # loop 2 renders paused PRs + fi + STATE='🛑 needs-human' + NOTE='pause evaluation unavailable this tick (enumeration unreadable)' + elif [[ "${MERGEABLE}" == "CONFLICTING" ]]; then + STATE='conflicting' + NOTE='resolution owned by the autofix scan' + elif [[ -n "${FAILED_TEST_URL}" && "${PENDING}" == "0" ]]; then + STATE="[ci red](${FAILED_TEST_URL})" + NOTE='reruns owned by CI Failure Patrol' + elif [[ "${PENDING}" != "0" ]]; then + STATE="checks in flight (${PENDING})" + fi + + echo "🐑 #${PR} [takeover: ${STATE}] → ${NOTE}" + # NOTE/STATE carry no dynamic text on this path, but keep the + # table pipe-safe for the ci-red URL. + SAFE_NOTE="${NOTE//\\/\\\\}" + TAKEOVER_ROWS="${TAKEOVER_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${STATE} | ${SAFE_NOTE//|/\\|} |\n" + done < <(jq -c '.[]' /tmp/takeover.json) + fi + + # Loop 2: needs-human PRs — pause evaluation, the release lever, + # and the awaiting-human rows. The paginated comment/event reads + # live in this loop only (paused PRs are few). Fail closed + # everywhere: an unreadable history defers the lever instead of + # acting on partial state. + if [[ "${HM_OK}" == "true" ]]; then + while IFS= read -r ROW; do + [[ -z "${ROW}" ]] && continue + PR="$(jq -r '.number' <<< "${ROW}")" + AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" + UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" + MERGEABLE="$(jq -r '.mergeable // "UNKNOWN"' <<< "${ROW}")" + HAS_TK="$(jq -r --arg l "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" + UPD_AGO='?' + [[ -n "${UPDATED}" ]] && UPD_AGO="$(days_since "${UPDATED}")d" - STATE='managed · idle' - NOTE='—' - if [[ "${HAS_NH}" == "true" ]]; then STATE='🛑 needs-human' - # Paused PRs are few, so the paginated comment/event reads live - # only in this branch. Fail closed everywhere: an unreadable - # history defers the release instead of acting on partial state. - if ! gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ic.json; then + NOTE='—' + if ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev.json; then + NOTE='event read failed — evaluation deferred this tick (fail closed)' + elif [[ "${HAS_TK}" == "true" ]] && ! gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ic.json; then NOTE='comment read failed — release deferred this tick' - elif ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev.json; then - NOTE='event read failed — release deferred this tick (fail closed)' - else + elif [[ "${HAS_TK}" == "true" ]]; then # The pause clock starts at the LATEST cap notice. TERM_TS="$(jq -r --arg ab "${AUTOFIX_BOT}" ' [ .[] | select((.user.login // "") == $ab) | select((.body // "") | contains("")) | .created_at ] | max // ""' /tmp/tk-ic.json)" - # Resume evidence, widest first — anything newer than - # TERM_TS means a human already acted and the release must - # never fire: + # Resume evidence — anything newer than TERM_TS means a + # human already acted and the release must never fire: # 1. the bot's resume markers (re-arm / engage ack); - # 2. a TRUSTED human's re-arm COMMAND comment — the - # command's own ack rides a queued job, so counting - # only the ack would let a shepherd tick release a PR - # minutes after the human re-armed it; - # 3. a fresh `labeled` event (below) — a UI re-apply is - # timestamped immediately, while its ack rides the same + # 2. a re-arm COMMAND comment, but only while FRESH and + # UNSUPERSEDED: an accepted command is acked within + # minutes (the ack carries a resume marker); a refused + # command draws a refusal ack, which supersedes it + # here; a command the route ignores (no permission) + # gets no reply at all, so the grace window is what + # stops an ignored command from vetoing the release + # forever. No permission check here on purpose — the + # route's collaborator check is the authorization + # gate, and mirroring it would only drift; + # 3. a fresh `labeled` event — a UI re-apply is + # timestamped immediately, while its ack rides the # queue (or, on a lost event, the scan's idle-backoff # pickup — an hours-wide window). RESUME_TS="$(jq -r --arg ab "${AUTOFIX_BOT}" ' - [ .[] | select( - (((.user.login // "") == $ab) - and (((.body // "") | contains("")) - or ((.body // "") | contains("")))) - or (((.author_association // "") | IN("OWNER", "MEMBER", "COLLABORATOR")) - and (((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /takeover" - or ((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /retry"))) + [ .[] | select((.user.login // "") == $ab) + | select(((.body // "") | contains("")) + or ((.body // "") | contains(""))) | .created_at ] | max // ""' /tmp/tk-ic.json)" + CMD_TS="$(jq -r ' + [ .[] | select(((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /takeover" + or ((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /retry") + | .created_at ] | max // ""' /tmp/tk-ic.json)" + REFUSAL_TS="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | test("")) + | .created_at ] | max // ""' /tmp/tk-ic.json)" + if [[ -n "${CMD_TS}" && "${CMD_TS}" > "${RESUME_TS}" ]] \ + && [[ -z "${REFUSAL_TS}" || "${CMD_TS}" > "${REFUSAL_TS}" ]]; then + CMD_AGE=$(( NOW_EPOCH - $(date -u -d "${CMD_TS}" +%s) )) + if [[ "${CMD_AGE}" -lt "${RESUME_COMMAND_GRACE_SEC}" ]]; then + RESUME_TS="${CMD_TS}" + fi + fi EVENT_TS="$(jq -r --arg tl "${TAKEOVER_LABEL}" ' [ .[] | select((.event // "") == "labeled") | select((.label.name // "") == $tl) @@ -614,7 +697,7 @@ jobs: STATE='managed (re-armed)' NOTE='resumed after the last pause — the escalation label is stale' else - PAUSE_D=$(( (NOW_EPOCH - $(date -u -d "${TERM_TS}" +%s)) / 86400 )) + PAUSE_D="$(days_since "${TERM_TS}")" STATE="🛑 needs-human ${PAUSE_D}d" NOTE="${REASON}" if [[ "${MERGEABLE}" == "CONFLICTING" ]]; then @@ -634,65 +717,84 @@ jobs: elif [[ "$(jq -r --arg a "${TAKEOVER_LABEL}" --arg b "${NEEDS_HUMAN_LABEL}" '([.labels[]?.name] | index($a) != null) and ([.labels[]?.name] | index($b) != null)' <<< "${LIVE_LABELS_JSON}")" != "true" ]]; then NOTE='labels changed since the snapshot — deferring release' else - if act "#${PR}: auto-release takeover (paused ${PAUSE_D}d)" \ - gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')" > /dev/null; then - # The release HAPPENED, so it counts against the - # per-tick budget regardless of the summary's fate - # (mirrors the conflict-dispatch counting rule). - RELEASES=$(( RELEASES + 1 )) + # Summary FIRST (dedup'd by its own marker — the + # comment stream is already loaded), THEN the label + # removal: a failed summary leaves both labels in + # place so the whole release retries next tick, and a + # failed removal finds the marker and only retries the + # DELETE. Neither half can strand the other. + SUMMARY_POSTED="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | contains("")) + ] | length' /tmp/tk-ic.json)" + if [[ "${SUMMARY_POSTED}" == "0" ]]; then if act "#${PR}: post auto-release summary" \ gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '🔓 Takeover auto-released: the autofix loop paused on this PR %s day(s) ago (%s) and no re-arm followed, so the `%s` label is removed to keep the managed pool honest. `%s` stays as the reminder that this PR needs a human decision: merge it, close it, or split/reduce it and comment `@qwen-code /takeover` to re-engage with a fresh round window.\n\n
\n中文说明\n\n🔓 已自动释放接管：autofix 循环在 %s 天前暂停于此 PR（%s），此后无人重新武装，现移除 `%s` 标签以保持托管池真实可用。保留 `%s` 作为待办提醒 —— 本 PR 需要人工决策：合并、关闭，或拆分/缩小后评论 `@qwen-code /takeover` 以全新轮次窗口重新接管。\n\n
\n\n' "${PAUSE_D}" "${REASON}" "${TAKEOVER_LABEL}" "${NEEDS_HUMAN_LABEL}" "${PAUSE_D}" "${REASON}" "${TAKEOVER_LABEL}" "${NEEDS_HUMAN_LABEL}")"; then + SUMMARY_POSTED=1 + else + NOTE='summary post failed — release deferred to next tick' + fi + fi + if [[ "${SUMMARY_POSTED}" != "0" ]]; then + # The release HAPPENED when the label lands off, so + # that is what counts against the per-tick budget. + if act "#${PR}: auto-release takeover (paused ${PAUSE_D}d)" \ + gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')" > /dev/null; then + RELEASES=$(( RELEASES + 1 )) NOTE="auto-released after ${PAUSE_D}d paused" else - NOTE='takeover released; summary post failed' + NOTE='takeover label removal failed — will retry next tick' fi - else - NOTE='auto-release FAILED — will retry next tick' fi fi fi fi + else + # No takeover label: the loop stopped and management was + # released (or never took over). Keep the PR visible — and + # when a HUMAN removed the takeover label by hand (fork PRs + # get no release ack from the route, so nothing else clears + # the escalation label there), clear the stale label. The + # shepherd's own auto-release authenticates as the bot and + # is NOT a heal trigger. + UNLABEL_ACTOR="$(jq -r --arg tl "${TAKEOVER_LABEL}" --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.event // "") == "unlabeled") + | select((.label.name // "") == $tl) + | select((.actor.login // "") != $ab) + | .actor.login ] | last // ""' /tmp/tk-ev.json)" + NOTE='loop stopped — needs a human decision (merge / close / split / re-engage)' + if [[ -n "${UNLABEL_ACTOR}" ]]; then + if [[ "${CLEANUPS}" -ge "${MAX_CLEANUPS_PER_TICK}" ]]; then + NOTE='cleanup budget reached this tick' + elif live_skip "${PR}"; then + NOTE="$(skip_note cleanup)" + elif act "#${PR}: clear stale ${NEEDS_HUMAN_LABEL} (manual release by @${UNLABEL_ACTOR})" \ + gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" > /dev/null; then + CLEANUPS=$(( CLEANUPS + 1 )) + NOTE="cleared stale escalation label (released by @${UNLABEL_ACTOR})" + else + NOTE='stale-label cleanup failed — will retry next tick' + fi + fi fi - elif [[ "${MERGEABLE}" == "CONFLICTING" ]]; then - STATE='conflicting' - NOTE='resolution owned by the autofix scan' - elif [[ -n "${FAILED_TEST_URL}" && "${PENDING}" == "0" ]]; then - STATE="[ci red](${FAILED_TEST_URL})" - NOTE='reruns owned by CI Failure Patrol' - elif [[ "${PENDING}" != "0" ]]; then - STATE="checks in flight (${PENDING})" - fi - - echo "🐑 #${PR} [takeover: ${STATE}] → ${NOTE}" - # NOTE carries dynamic text (the stop-reason headline can embed - # provider error detail) — printf '%b' would expand backslash - # escapes and a raw pipe breaks the table, so sanitize both. - SAFE_NOTE="${NOTE//\\/\\\\}" - TAKEOVER_ROWS="${TAKEOVER_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${STATE} | ${SAFE_NOTE//|/\\|} |\n" - done < <(jq -c '.[]' /tmp/takeover.json) - - # Released-but-unresolved rows (needs-human without takeover is - # the deliberate post-release TODO state — read-only visibility). - while IFS= read -r ROW; do - [[ -z "${ROW}" ]] && continue - PR="$(jq -r '.number' <<< "${ROW}")" - AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" - UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" - UPD_AGO='?' - if [[ -n "${UPDATED}" ]]; then - UPD_AGO="$(( (NOW_EPOCH - $(date -u -d "${UPDATED}" +%s)) / 86400 ))d" - fi - if [[ "$(jq -r --arg t "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${ROW}")" == "true" ]]; then - continue # still managed — the takeover table above carries it - fi - HUMAN_ROWS="${HUMAN_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | takeover released — needs a human decision (merge / close / split + re-takeover) |\n" - done < <(jq -c --arg skip "${SKIP_LABEL}" '[.[] | select([.labels[]?.name] | index($skip) | not)] | .[]' /tmp/human-raw.json) + # The row is appended OUTSIDE the evaluation arms: a fail-closed + # deferral must still render, or the PR would vanish from the + # dashboard for exactly the tick something went wrong. + echo "🐑 #${PR} [${STATE}] → ${NOTE}" + SAFE_NOTE="${NOTE//\\/\\\\}" + if [[ "${HAS_TK}" == "true" ]]; then + TAKEOVER_ROWS="${TAKEOVER_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${STATE} | ${SAFE_NOTE//|/\\|} |\n" + else + HUMAN_ROWS="${HUMAN_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${SAFE_NOTE//|/\\|} |\n" + fi + done < <(jq -c '.[]' /tmp/human.json) + fi # ---- dashboard: one issue, edited in place ---------------------- { echo "Auto-maintained by the Fleet Shepherd workflow — do not edit by hand." echo - echo "Last tick: $(date -u +%Y-%m-%dT%H:%M:%SZ) · scan-signal age: ${SCAN_AGE_MIN}m · syncs: ${SYNCS} · dispatches: ${DISPATCHES} · releases: ${RELEASES}" + echo "Last tick: $(date -u +%Y-%m-%dT%H:%M:%SZ) · scan-signal age: ${SCAN_AGE_MIN}m · syncs: ${SYNCS} · dispatches: ${DISPATCHES} · releases: ${RELEASES} · cleanups: ${CLEANUPS}" echo echo '## Bot fleet' echo @@ -710,9 +812,9 @@ jobs: echo echo '## Awaiting human' echo - echo 'Loop stopped and takeover released; these need a human decision.' + echo 'The loop stopped on these PRs and no takeover is active; a human needs to decide (merge / close / split / re-engage).' echo - echo '| PR | Author | Updated | State |' + echo '| PR | Author | Updated | Note |' echo '| --- | --- | --- | --- |' printf '%b' "${HUMAN_ROWS}" if [[ -n "${LIVENESS_OUT}" ]]; then @@ -736,4 +838,4 @@ jobs: echo "::warning::dashboard update failed; will retry next tick" fi fi - echo "✅ tick complete (syncs=${SYNCS} dispatches=${DISPATCHES})" + echo "✅ tick complete (syncs=${SYNCS} dispatches=${DISPATCHES} releases=${RELEASES} cleanups=${CLEANUPS})" diff --git a/docs/design/takeover-fleet-visibility.md b/docs/design/takeover-fleet-visibility.md index a7036971639..ee29c680dc8 100644 --- a/docs/design/takeover-fleet-visibility.md +++ b/docs/design/takeover-fleet-visibility.md @@ -107,11 +107,30 @@ re-checked immediately before the mutation, mirroring every existing lever. - **Auto-release keyed on the notice timestamp**, not label age: labels carry no timestamps, and the notice is written by the same identity-verified path that applies the label. Resume evidence newer than the notice vetoes the - release — the bot's re-arm/engage markers, a trusted human's - `/takeover` or `/retry` command comment (the ack rides a queued job, so - counting only the ack would let a tick release a PR minutes after a human - re-armed it), and a fresh `labeled` event (a UI re-apply whose ack rides - the same queue, or the scan's idle-backoff pickup when the event is lost). + release — the bot's re-arm/engage markers, a re-arm command comment, or a + fresh `labeled` event. Command comments count only while FRESH + (`RESUME_COMMAND_GRACE_SEC`, 2h) and UNSUPERSEDED by a refusal ack + (`fork-refused` / `base-refused` / `skip-blocked`): an accepted command is + acked within minutes; an ignored one (no route permission) simply expires; + and no permission check is mirrored into the shepherd — the route's + collaborator check is the authorization gate, and a mirrored copy would + only drift. +- **The release lever's population comes from the needs-human enumeration** + (needs-human ∩ takeover), not the takeover display window, so window + saturation can never make a paused PR unreleasable. Both enumerations cap + at 100 with loud saturation warnings, and an enumeration failure degrades + to an error row — the dashboard write (which carries the liveness + watermark) always runs. +- **The summary posts before the label removal**, dedup'd by its own marker, + so a failed comment leaves both labels in place and the whole release + retries next tick; a failed removal finds the marker and retries only the + DELETE. +- **Stale-label heal:** a fork PR released by hand gets no release ack (the + route suppresses fork `unlabeled` events), so nothing else clears its + `needs-human`. The shepherd watches the awaiting-human pool for a + human-actor `unlabeled` event on the takeover label and clears the stale + label (bounded per tick, skip-vetoed, and never triggered by the bot's own + auto-release). - **Shepherd timing:** 15-minute tick with a per-tick release cap — a backlog of expired PRs drains over a few ticks rather than one burst. diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index 156ea3bfff6..49f2e6ea493 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -588,34 +588,49 @@ exit 1`; 'AUTO_RELEASE_DAYS: "${{ vars.QWEN_SHEPHERD_AUTO_RELEASE_DAYS || \'3\' }}"', ); expect(workflow).toContain("MAX_RELEASES_PER_TICK: '3'"); + expect(workflow).toContain("RESUME_COMMAND_GRACE_SEC: '7200'"); + expect(workflow).toContain("MAX_CLEANUPS_PER_TICK: '5'"); // One list call carries the takeover row payload — forks INCLUDED (no // isCrossRepository filter: visibility covers fork takeovers, and the - // release lever needs no push access). sort:updated-asc keeps the STALE - // tail (the release lever's target population) inside the limit window, - // and the saturation warning keeps truncation loud. + // release lever needs no push access). sort:updated-asc keeps the stale + // tail in view; the saturation warnings keep truncation loud on BOTH + // enumerations. expect(workflow).toContain('--state open --label "${TAKEOVER_LABEL}"'); expect(workflow).toContain( - '--limit 50 --json number,author,updatedAt,mergeable,statusCheckRollup,labels', + '--limit 100 --json number,author,updatedAt,mergeable,statusCheckRollup,labels', ); expect(workflow).toContain("--search 'sort:updated-asc'"); - expect(workflow).toContain('takeover pool at the 50-PR enumeration limit'); + expect(workflow).toContain('takeover pool at the 100-PR enumeration limit'); + expect(workflow).toContain( + 'needs-human pool at the 100-PR enumeration limit', + ); const takeoverEnum = workflow.match( /--label "\$\{TAKEOVER_LABEL\}"[\s\S]*?\/tmp\/takeover-raw\.json/, )?.[0]; expect(takeoverEnum).toBeTruthy(); expect(takeoverEnum).not.toContain('isCrossRepository'); - // A failed pool read keeps the previous dashboard body — an unreadable - // snapshot never masquerades as an empty pool. The exit is what keeps - // the dashboard unwritten; pin them together. - expect(workflow).toMatch( - /takeover enumeration failed; skipping the takeover table, auto-release, and dashboard update this tick"\n\s+exit 0/, + // Enumeration failures degrade to a loud error row and FALL THROUGH to + // the dashboard write (which carries the liveness watermark) — they + // must not exit the tick. The release lever's population comes from the + // needs-human enumeration, never the display window. + expect(workflow).toContain( + 'takeover enumeration failed; the takeover table shows an error row this tick', + ); + expect(workflow).toContain( + 'needs-human enumeration failed; paused-PR evaluation skipped this tick', + ); + expect(workflow).not.toMatch( + /takeover enumeration failed[\s\S]{0,200}exit 0/, ); + expect(workflow).toContain('enumeration unreadable this tick'); // The released-but-unresolved tail (needs-human WITHOUT takeover) gets - // its own read-only enumeration and dashboard section — released PRs - // must not vanish from every surface. + // its own read-only section with neutral wording — capped bot PRs land + // here too, and they never had a takeover to release. expect(workflow).toContain('--state open --label "${NEEDS_HUMAN_LABEL}"'); expect(workflow).toContain("'## Awaiting human'"); - expect(workflow).toContain('takeover released — needs a human decision'); + expect(workflow).toContain( + 'loop stopped — needs a human decision (merge / close / split / re-engage)', + ); // autofix/skip wins in the pool too — replay the filter VERBATIM. The // program-prefix anchor keeps this from capturing the bot-fleet filter // (same jq shape, different file and a leading isCrossRepository leg). @@ -693,25 +708,115 @@ exit 1`; expect(run(resumeJq, [bot('2026-08-02T00:00:00Z', '⏸️ unrelated')])).toBe( '', ); - // …and a TRUSTED human's re-arm command counts at COMMENT time (the - // ack rides a queued job — without this a tick could release a PR - // minutes after the human re-armed it), while an outsider's identical - // command does not move the clock. - const trusted = (created_at, body) => ({ - user: { login: 'wenshao' }, - author_association: 'MEMBER', - created_at, - body, - }); + // …a re-arm COMMAND comment is ALSO resume evidence, but only while + // fresh and unsuperseded: the command's ack rides a queued job, so + // counting only the ack would let a tick release a PR minutes after a + // human re-armed it. No author gate here — the route's permission check + // owns authorization; an ignored command simply expires unsuperseded. + const cmdJq = workflow.match( + /CMD_TS="\$\(jq -r '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + const refusalJq = workflow.match( + /REFUSAL_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + expect(cmdJq).toBeTruthy(); + expect(refusalJq).toBeTruthy(); + const runCmd = (comments) => + execFileSync('jq', ['-r', cmdJq], { + encoding: 'utf8', + input: JSON.stringify(comments), + }).trim(); + // Exact trimmed command match, any author. expect( - run(resumeJq, [trusted('2026-08-06T00:00:00Z', '@qwen-code /takeover')]), + runCmd([human('2026-08-06T00:00:00Z', ' @qwen-code /takeover ')]), ).toBe('2026-08-06T00:00:00Z'); + expect(runCmd([human('2026-08-06T00:00:00Z', '@qwen-code /retry')])).toBe( + '2026-08-06T00:00:00Z', + ); + // …but `/takeover stop` is a release, not a resume. expect( - run(resumeJq, [trusted('2026-08-06T00:00:00Z', ' @qwen-code /retry ')]), - ).toBe('2026-08-06T00:00:00Z'); + runCmd([human('2026-08-06T00:00:00Z', '@qwen-code /takeover stop')]), + ).toBe(''); + // A refusal ack (fork-refused / base-refused / skip-blocked) is what + // supersedes a refused command — verify the variant set. + const runRefusal = (comments) => + execFileSync( + 'jq', + ['-r', '--arg', 'ab', 'qwen-code-dev-bot', refusalJq], + { encoding: 'utf8', input: JSON.stringify(comments) }, + ).trim(); + expect( + runRefusal([ + bot('2026-08-06T00:01:00Z', '🚫 … '), + ]), + ).toBe('2026-08-06T00:01:00Z'); expect( - run(resumeJq, [human('2026-08-06T00:00:00Z', '@qwen-code /takeover')]), + runRefusal([ + bot('2026-08-06T00:01:00Z', '🤝 … '), + ]), ).toBe(''); + // The full gate: command evidence counts only when it is newer than the + // marker evidence, newer than any refusal, AND inside the grace window. + const cmdGate = workflow.match( + /(if \[\[ -n "\$\{CMD_TS\}" && "\$\{CMD_TS\}" > "\$\{RESUME_TS\}" \]\] \\[\s\S]*?\n {16}fi)/, + )?.[1]; + expect(cmdGate).toBeTruthy(); + // The extracted block runs GNU `date -u -d`; on macOS hosts BSD date + // lacks -d, so shim it through node (always present in this repo). + const gnuDateShim = `date() { if [[ "$1" == '-u' && "$2" == '-d' ]]; then node -e 'console.log(Math.floor(new Date(process.argv[1]).getTime()/1000))' "$3"; else command date "$@"; fi; }`; + const gate = (env) => + execFileSync( + 'bash', + [ + '-c', + `${gnuDateShim}\n${cmdGate.replace(/\n {16}/g, '\n')}\necho "RESUME_TS=$RESUME_TS"`, + ], + { + env: { + ...process.env, + NOW_EPOCH: '1786000000', + RESUME_COMMAND_GRACE_SEC: '7200', + ...env, + }, + encoding: 'utf8', + }, + ).trim(); + // fresh accepted-but-unacked command → counts (the R1 race fix). + expect( + gate({ + RESUME_TS: '', + CMD_TS: '2026-08-06T00:00:00Z', + REFUSAL_TS: '', + NOW_EPOCH: String(Date.parse('2026-08-06T00:30:00Z') / 1000), + }), + ).toBe(`RESUME_TS=2026-08-06T00:00:00Z`); + // refused command (refusal ack newer) → does NOT count (R1-2). + expect( + gate({ + RESUME_TS: '', + CMD_TS: '2026-08-06T00:00:00Z', + REFUSAL_TS: '2026-08-06T00:01:00Z', + NOW_EPOCH: String(Date.parse('2026-08-06T00:30:00Z') / 1000), + }), + ).toBe('RESUME_TS='); + // ignored command (no ack ever) past the grace window → expires (R1-1). + expect( + gate({ + RESUME_TS: '', + CMD_TS: '2026-08-06T00:00:00Z', + REFUSAL_TS: '', + NOW_EPOCH: String(Date.parse('2026-08-06T03:00:00Z') / 1000), + }), + ).toBe('RESUME_TS='); + // an already-acked command never overrides its own newer marker. + expect( + gate({ + RESUME_TS: '2026-08-06T00:05:00Z', + CMD_TS: '2026-08-06T00:00:00Z', + REFUSAL_TS: '', + NOW_EPOCH: String(Date.parse('2026-08-06T00:30:00Z') / 1000), + }), + ).toBe('RESUME_TS=2026-08-06T00:05:00Z'); // A fresh `labeled` event is resume evidence too (UI re-apply). const eventJq = workflow.match( /EVENT_TS="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, @@ -739,7 +844,7 @@ exit 1`; expect(runEvent([{ event: 'closed' }])).toBe(''); // The event read fails closed, like the comment read. expect(workflow).toContain( - 'event read failed — release deferred this tick (fail closed)', + 'event read failed — evaluation deferred this tick (fail closed)', ); // REASON: the terminal round's own headline beats the scan-side notice, // which always says "round cap" even when a breaker fired first. @@ -798,14 +903,25 @@ exit 1`; expect(workflow).toMatch( /RELEASES\}" -ge "\$\{MAX_RELEASES_PER_TICK\}" \]\]; then[\s\S]{0,200}elif live_skip "\$\{PR\}"; then\n\s+NOTE="\$\(skip_note release\)"[\s\S]{0,400}labels changed since the snapshot — deferring release/, ); + // The summary posts FIRST (dedup'd by its own marker), then the label + // DELETE: a failed summary leaves both labels in place and the whole + // release retries next tick; a failed DELETE finds the marker and + // retries only itself. Neither half can strand the other. expect(workflow).toMatch( - /if act "#\$\{PR\}: auto-release takeover[\s\S]{0,700}if act "#\$\{PR\}: post auto-release summary"/, + /SUMMARY_POSTED="\$\(jq[\s\S]{0,500}fleet-shepherd auto-release/, ); - // The release counts against the per-tick budget even when the summary - // comment fails — the increment precedes the summary act (mirrors the - // conflict-dispatch counting rule pinned above). expect(workflow).toMatch( - /RELEASES=\$\(\( RELEASES \+ 1 \)\)[\s\S]{0,400}if act "#\$\{PR\}: post auto-release summary"/, + /if act "#\$\{PR\}: post auto-release summary"[\s\S]{0,1800}if act "#\$\{PR\}: auto-release takeover/, + ); + // The release counts when the label lands off — the increment follows + // the DELETE, never precedes it. + expect(workflow).toMatch( + /if act "#\$\{PR\}: auto-release takeover[\s\S]{0,300}RELEASES=\$\(\( RELEASES \+ 1 \)\)/, + ); + // And the DELETE removes the TAKEOVER label — never needs-human (the + // filterable TODO must survive the release). + expect(workflow).toMatch( + /auto-release takeover \(paused \$\{PAUSE_D\}d\)" \\\n\s+gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{TAKEOVER_LABEL\}"/, ); // The fail-closed deferral guards are load-bearing (a missing one runs // `date -d ""` under set -e and kills the tick) — pin their wording. @@ -815,8 +931,9 @@ exit 1`; expect(workflow).toContain( 'comment read failed — release deferred this tick', ); - // The release is idempotent WITHOUT a marker: the scope condition (both - // labels) is false once the takeover label lands off. + // The release is idempotent: the summary dedups on its own marker and + // the label side's scope condition (both labels) goes false once the + // takeover label lands off. expect(workflow).toContain( 'index($a) != null) and ([.labels[]?.name] | index($b) != null)', ); @@ -824,13 +941,82 @@ exit 1`; // The age gate compares whole days against the tunable. expect(workflow).toContain('"${PAUSE_D}" -ge "${AUTO_RELEASE_DAYS}"'); // The tunable is operator-breakable — a non-numeric repo variable falls - // back to the default instead of dying mid-tick under set -e. + // back to the default instead of dying mid-tick under set -e. Pin the + // fallback ASSIGNMENT with the message (newline-anchored: a bare + // toContain('AUTO_RELEASE_DAYS=3') would substring-match '=30'). expect(workflow).toContain('=~ ^[0-9]+$'); - expect(workflow).toContain('is not numeric; using 3'); + expect(workflow).toMatch( + /is not numeric; using 3"\n\s+AUTO_RELEASE_DAYS=3\n/, + ); + // The stale-label heal (R1-13): a HUMAN unlabeled event on an + // awaiting-human PR clears the stale escalation label — budgeted, + // skip-vetoed, and never triggered by the bot's own auto-release. + expect(workflow).toContain('"${CLEANUPS}" -ge "${MAX_CLEANUPS_PER_TICK}"'); + const unlabelJq = workflow.match( + /UNLABEL_ACTOR="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, + )?.[1]; + expect(unlabelJq).toBeTruthy(); + const runUnlabel = (events) => + execFileSync( + 'jq', + [ + '-r', + '--arg', + 'tl', + 'autofix/takeover', + '--arg', + 'ab', + 'qwen-code-dev-bot', + unlabelJq, + ], + { encoding: 'utf8', input: JSON.stringify(events) }, + ).trim(); + expect( + runUnlabel([ + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'wenshao' }, + }, + ]), + ).toBe('wenshao'); + // The bot's own unlabel (the auto-release DELETE) must NOT trigger a + // cleanup — needs-human survives auto-release by design. + expect( + runUnlabel([ + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'qwen-code-dev-bot' }, + }, + ]), + ).toBe(''); + // The heal removes NEEDS_HUMAN (never takeover), act-wrapped. + expect(workflow).toMatch( + /clear stale \$\{NEEDS_HUMAN_LABEL\}[\s\S]{0,300}gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{NEEDS_HUMAN_LABEL\}"/, + ); + // Tick summary and dashboard header report the same counters (R1-12). + expect(workflow).toContain( + '✅ tick complete (syncs=${SYNCS} dispatches=${DISPATCHES} releases=${RELEASES} cleanups=${CLEANUPS})', + ); + expect(workflow).toContain('cleanups: ${CLEANUPS}'); // Dynamic note text is sanitized before it reaches the printf '%b' // table: backslashes first (else %b expands them), then pipes. expect(workflow).toContain('SAFE_NOTE="${NOTE//\\\\/\\\\\\\\}"'); expect(workflow).toContain('${SAFE_NOTE//|/\\\\|}'); + // Shared day-math helper (R1-10) instead of pasted epoch arithmetic. + expect(workflow).toContain('days_since() {'); + // Loop 2 renders exactly one row per needs-human PR — appended OUTSIDE + // the evaluation arms, so a fail-closed deferral (comment/event read + // failure) still renders instead of silently vanishing from the + // dashboard for the tick something went wrong. + const loop2 = workflow.match( + /# Loop 2: needs-human PRs[\s\S]*?done < <\(jq -c '\.\[\]' \/tmp\/human\.json\)/, + )?.[0]; + expect(loop2).toBeTruthy(); + expect(loop2.match(/echo "🐑 #\$\{PR\} \[/g)).toHaveLength(1); + expect(loop2.match(/TAKEOVER_ROWS="\$\{TAKEOVER_ROWS\}/g)).toHaveLength(1); + expect(loop2.match(/HUMAN_ROWS="\$\{HUMAN_ROWS\}/g)).toHaveLength(1); }); it('pins the marker/headline contracts against qwen-autofix.yml — drift fails here, not in production', () => { @@ -858,18 +1044,34 @@ exit 1`; expect(rearmKey).toContain(marker); expect(resumeTs).toContain(marker); } - // 2. The trusted re-arm commands the shepherd greps for are the same - // strings qwen-autofix.yml routes on. + // 2. The re-arm commands the shepherd greps for are the same strings + // qwen-autofix.yml routes on, and the refusal-ack variants that + // supersede a refused command are exactly the ones the producer can + // emit. expect(autofix).toContain("TAKEOVER_COMMAND: '@qwen-code /takeover'"); expect(autofix).toContain("RETRY_COMMAND: '@qwen-code /retry'"); - expect(resumeTs).toContain('"@qwen-code /takeover"'); - expect(resumeTs).toContain('"@qwen-code /retry"'); + const cmdTs = workflow.match( + /CMD_TS="\$\(jq -r '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + expect(cmdTs).toBeTruthy(); + expect(cmdTs).toContain('"@qwen-code /takeover"'); + expect(cmdTs).toContain('"@qwen-code /retry"'); + const refusalTs = workflow.match( + /REFUSAL_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + expect(refusalTs).toBeTruthy(); + for (const variant of ['fork-refused', 'base-refused', 'skip-blocked']) { + expect(refusalTs).toContain(variant); + expect(autofix).toContain(``); + } // 3. Every TERMINAL headline the address leg can print matches the // shepherd's REASON regex, and the transient setup-failure headline // does not. Templates are extracted from the producer and exercised // against the consumer's verbatim regex — a reworded headline on // either side fails this test. - const reasonRe = workflow.match(/\| test\("([^"]+)"\)\)/)?.[1]; + const reasonRe = workflow.match( + /REASON="\$\(jq -r[\s\S]*?\| test\("([^"]+)"\)\)/, + )?.[1]; expect(reasonRe).toBeTruthy(); const headlines = [...autofix.matchAll(/HEADLINE="(🤖 [^"]+)"/g)].map( (m) => m[1], From cb0351db6bf59ef00b620080b7ef6c2bc3b4164a Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Wed, 12 Aug 2026 16:01:14 +0800 Subject: [PATCH 03/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?2=20=E2=80=94=20cycle-scoped=20heal,=20retryable=20summary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The stale-label heal now only counts a human unlabel NEWER than the latest label-apply, so an unlabel from an earlier takeover cycle can no longer heal the current cycle's needs-human after an auto-release (R2-1). - The summary dedup marker is scoped to the current pause cycle (markers older than the latest cap notice are ignored), so a re-armed and re-capped PR still gets its second release summary (R2-4). - The two DELETE levers no longer redirect act()'s stdout, keeping the DRY-RUN preview and failure warning visible (R2-5). - AUTO_RELEASE_DAYS is base-10 normalized after the numeric guard, so a zero-padded repo variable can't silently kill the lever (R2-6). Co-authored-by: Qwen-Coder --- .github/workflows/qwen-fleet-shepherd.yml | 21 ++++- docs/design/takeover-fleet-visibility.md | 17 ++-- .../qwen-fleet-shepherd-workflow.test.js | 78 ++++++++++++++++++- 3 files changed, 101 insertions(+), 15 deletions(-) diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index 661a6c3657b..266a62f4a13 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -521,6 +521,10 @@ jobs: echo "::warning::AUTO_RELEASE_DAYS '${AUTO_RELEASE_DAYS}' is not numeric; using 3" AUTO_RELEASE_DAYS=3 fi + # Leading zeros pass the regex but bash reads 08/09 as bad octal at + # the -ge comparison — normalize to base 10 so a zero-padded + # variable cannot silently kill the lever. + AUTO_RELEASE_DAYS=$((10#${AUTO_RELEASE_DAYS})) # sort:updated-asc keeps the stalest PRs in view when a pool # outgrows the window; the saturation warning keeps the residual # truncation loud. The LEVER's population deliberately comes from @@ -723,9 +727,10 @@ jobs: # place so the whole release retries next tick, and a # failed removal finds the marker and only retries the # DELETE. Neither half can strand the other. - SUMMARY_POSTED="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + SUMMARY_POSTED="$(jq -r --arg ab "${AUTOFIX_BOT}" --arg term "${TERM_TS}" ' [ .[] | select((.user.login // "") == $ab) | select((.body // "") | contains("")) + | select((.created_at // "") > $term) ] | length' /tmp/tk-ic.json)" if [[ "${SUMMARY_POSTED}" == "0" ]]; then if act "#${PR}: post auto-release summary" \ @@ -739,7 +744,7 @@ jobs: # The release HAPPENED when the label lands off, so # that is what counts against the per-tick budget. if act "#${PR}: auto-release takeover (paused ${PAUSE_D}d)" \ - gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')" > /dev/null; then + gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')"; then RELEASES=$(( RELEASES + 1 )) NOTE="auto-released after ${PAUSE_D}d paused" else @@ -757,10 +762,18 @@ jobs: # the escalation label there), clear the stale label. The # shepherd's own auto-release authenticates as the bot and # is NOT a heal trigger. - UNLABEL_ACTOR="$(jq -r --arg tl "${TAKEOVER_LABEL}" --arg ab "${AUTOFIX_BOT}" ' + # ...only a human unlabel NEWER than the latest label-apply + # counts — an unlabel from an EARLIER takeover cycle must + # never heal this cycle's label. + LATEST_LABEL_TS="$(jq -r --arg tl "${TAKEOVER_LABEL}" ' + [ .[] | select((.event // "") == "labeled") + | select((.label.name // "") == $tl) + | .created_at ] | max // ""' /tmp/tk-ev.json)" + UNLABEL_ACTOR="$(jq -r --arg tl "${TAKEOVER_LABEL}" --arg ab "${AUTOFIX_BOT}" --arg ll "${LATEST_LABEL_TS}" ' [ .[] | select((.event // "") == "unlabeled") | select((.label.name // "") == $tl) | select((.actor.login // "") != $ab) + | select((.created_at // "") > $ll) | .actor.login ] | last // ""' /tmp/tk-ev.json)" NOTE='loop stopped — needs a human decision (merge / close / split / re-engage)' if [[ -n "${UNLABEL_ACTOR}" ]]; then @@ -769,7 +782,7 @@ jobs: elif live_skip "${PR}"; then NOTE="$(skip_note cleanup)" elif act "#${PR}: clear stale ${NEEDS_HUMAN_LABEL} (manual release by @${UNLABEL_ACTOR})" \ - gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" > /dev/null; then + gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')"; then CLEANUPS=$(( CLEANUPS + 1 )) NOTE="cleared stale escalation label (released by @${UNLABEL_ACTOR})" else diff --git a/docs/design/takeover-fleet-visibility.md b/docs/design/takeover-fleet-visibility.md index ee29c680dc8..cde9bdf03f4 100644 --- a/docs/design/takeover-fleet-visibility.md +++ b/docs/design/takeover-fleet-visibility.md @@ -121,16 +121,19 @@ re-checked immediately before the mutation, mirroring every existing lever. at 100 with loud saturation warnings, and an enumeration failure degrades to an error row — the dashboard write (which carries the liveness watermark) always runs. -- **The summary posts before the label removal**, dedup'd by its own marker, - so a failed comment leaves both labels in place and the whole release - retries next tick; a failed removal finds the marker and retries only the - DELETE. +- **The summary posts before the label removal**, dedup'd by its own marker + scoped to the current pause cycle (only markers newer than the latest cap + notice count), so a failed comment leaves both labels in place and the + whole release retries next tick; a failed removal finds the marker and + retries only the DELETE; and a re-armed-and-re-capped PR still gets its + second summary. - **Stale-label heal:** a fork PR released by hand gets no release ack (the route suppresses fork `unlabeled` events), so nothing else clears its `needs-human`. The shepherd watches the awaiting-human pool for a - human-actor `unlabeled` event on the takeover label and clears the stale - label (bounded per tick, skip-vetoed, and never triggered by the bot's own - auto-release). + human-actor `unlabeled` event on the takeover label that is NEWER than the + latest label-apply (a stale unlabel from an earlier takeover cycle must + never heal this cycle's label), and clears the stale label — bounded per + tick, skip-vetoed, and never triggered by the bot's own auto-release. - **Shepherd timing:** 15-minute tick with a per-tick release cap — a backlog of expired PRs drains over a few ticks rather than one burst. diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index 49f2e6ea493..f32f9094238 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -950,14 +950,26 @@ exit 1`; ); // The stale-label heal (R1-13): a HUMAN unlabeled event on an // awaiting-human PR clears the stale escalation label — budgeted, - // skip-vetoed, and never triggered by the bot's own auto-release. + // skip-vetoed, never triggered by the bot's own auto-release, and (R2-1) + // correlated to the CURRENT takeover cycle: only an unlabel newer than + // the latest label-apply counts, so a human unlabel from an EARLIER + // cycle cannot heal this cycle's label. expect(workflow).toContain('"${CLEANUPS}" -ge "${MAX_CLEANUPS_PER_TICK}"'); + const latestLabelJq = workflow.match( + /LATEST_LABEL_TS="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, + )?.[1]; const unlabelJq = workflow.match( - /UNLABEL_ACTOR="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, + /UNLABEL_ACTOR="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" --arg ab "\$\{AUTOFIX_BOT\}" --arg ll "\$\{LATEST_LABEL_TS\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, )?.[1]; + expect(latestLabelJq).toBeTruthy(); expect(unlabelJq).toBeTruthy(); - const runUnlabel = (events) => - execFileSync( + const runUnlabel = (events) => { + const ll = execFileSync( + 'jq', + ['-r', '--arg', 'tl', 'autofix/takeover', latestLabelJq], + { encoding: 'utf8', input: JSON.stringify(events) }, + ).trim(); + return execFileSync( 'jq', [ '-r', @@ -967,16 +979,27 @@ exit 1`; '--arg', 'ab', 'qwen-code-dev-bot', + '--arg', + 'll', + ll, unlabelJq, ], { encoding: 'utf8', input: JSON.stringify(events) }, ).trim(); + }; + // Same-cycle manual release → heal fires. expect( runUnlabel([ + { + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-01T00:00:00Z', + }, { event: 'unlabeled', label: { name: 'autofix/takeover' }, actor: { login: 'wenshao' }, + created_at: '2026-08-02T00:00:00Z', }, ]), ).toBe('wenshao'); @@ -984,10 +1007,39 @@ exit 1`; // cleanup — needs-human survives auto-release by design. expect( runUnlabel([ + { + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-01T00:00:00Z', + }, { event: 'unlabeled', label: { name: 'autofix/takeover' }, actor: { login: 'qwen-code-dev-bot' }, + created_at: '2026-08-02T00:00:00Z', + }, + ]), + ).toBe(''); + // R2-1's two-cycle fixture: cycle-1 human release, cycle-2 re-label + + // bot auto-release — the stale human event must NOT heal cycle 2. + expect( + runUnlabel([ + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'wenshao' }, + created_at: '2026-08-01T00:00:00Z', + }, + { + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-02T00:00:00Z', + }, + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'qwen-code-dev-bot' }, + created_at: '2026-08-05T00:00:00Z', }, ]), ).toBe(''); @@ -995,6 +1047,24 @@ exit 1`; expect(workflow).toMatch( /clear stale \$\{NEEDS_HUMAN_LABEL\}[\s\S]{0,300}gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{NEEDS_HUMAN_LABEL\}"/, ); + // act() echoes its own DRY-RUN preview / failure warning — the two new + // DELETE levers must not swallow that with a redirect (R2-5). + const actDeleteSites = workflow.match( + /act "[^\n]+" \\\n\s+gh api -X DELETE[^\n]+/g, + ); + expect(actDeleteSites?.length).toBeGreaterThanOrEqual(2); + for (const site of actDeleteSites ?? []) { + expect(site).not.toContain('> /dev/null'); + } + // A zero-padded variable must not kill the lever: base-10 normalize + // after the numeric guard (R2-6). + expect(workflow).toContain( + 'AUTO_RELEASE_DAYS=$((10#${AUTO_RELEASE_DAYS}))', + ); + // The summary dedup is scoped to THIS pause cycle — markers older than + // the latest cap notice don't suppress a second release's summary + // (R2-4). + expect(workflow).toContain('select((.created_at // "") > $term)'); // Tick summary and dashboard header report the same counters (R1-12). expect(workflow).toContain( '✅ tick complete (syncs=${SYNCS} dispatches=${DISPATCHES} releases=${RELEASES} cleanups=${CLEANUPS})', From 5b6c285849e8c62209bbb72deb55f12403954fdb Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Wed, 12 Aug 2026 19:25:04 +0800 Subject: [PATCH 04/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?2=20=E2=80=94=20cycle-scoped=20markers=20and=20mutation-tested?= =?UTF-8?q?=20pins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Heal is cycle-correlated: only a human unlabel NEWER than the latest label-apply counts (an earlier cycle's unlabel can't heal this cycle). - The release summary dedup marker is scoped to the current pause cycle, so a re-armed and re-capped PR still gets its second summary. - act() stdout is no longer redirected on the two DELETE levers (DRY-RUN preview and failure warning stay visible). - AUTO_RELEASE_DAYS is base-10 normalized so a zero-padded repo variable cannot silently kill the lever. - Doc/workflow-header text corrected to the implemented order (summary first, marker-dedup'd) and to the idle-backoff backfill timing. - Mutation-tested test pins for every gap the reviewer probed: days_since replay, NH_PREFIX interpolation + truth map, loop-1 deferral, full cross-file marker/refusal-set equality, label-constant cross-pin, EVENT_TS merge + promotion ordering, CLEANUPS increment, unclassified headline classification, filter byte-identity, exit-spelling ban, @uri encoding, sort/field-list attribution, paginate shapes, scope --arg bindings, LIVE_LABELS_JSON wiring, positional append pin, label-create idempotence + POST guard, and per-branch removal attribution in the toggle replay. Co-authored-by: Qwen-Coder --- .github/workflows/qwen-autofix.yml | 3 +- .github/workflows/qwen-fleet-shepherd.yml | 14 +- docs/design/takeover-fleet-visibility.md | 14 +- scripts/tests/qwen-autofix-workflow.test.js | 21 ++- .../qwen-fleet-shepherd-workflow.test.js | 171 ++++++++++++++++-- 5 files changed, 198 insertions(+), 25 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 1d00dedb4cc..55497ba49ee 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -2943,7 +2943,8 @@ jobs: # comments, but the label is what makes a paused PR # filterable and what the shepherd's auto-release ages # from — and applying it unconditionally backfills the - # already-paused fleet on the first scan after this ships. + # already-paused fleet via the scan rotation after this + # ships (idle backoff: expect hours, not the first scan). gh label create "${NEEDS_HUMAN_LABEL}" --repo "${REPO}" --color 'D93F0B' \ --description 'The autofix loop stopped on this PR — a human must re-arm, split, merge, or close it' \ 2> /dev/null || true diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index 266a62f4a13..2f82b5e7b30 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -22,12 +22,14 @@ name: 'Fleet Shepherd' # # One lever exists for the takeover pool: AUTO-RELEASE. A PR carrying both # autofix/takeover and autofix/needs-human whose cap pause went unanswered -# for AUTO_RELEASE_DAYS days gets its takeover label removed plus one -# bilingual summary comment (merge / close / split + re-takeover). The -# needs-human label stays as the filterable TODO. No marker is needed for -# idempotency: the lever's scope condition (both labels) is false once the -# release lands. Every other takeover state — conflicts, red CI, new -# feedback — remains the autofix scan's job; the shepherd only reports it. +# for AUTO_RELEASE_DAYS days gets one bilingual summary comment (merge / +# close / split + re-takeover) and then its takeover label removed. The +# needs-human label stays as the filterable TODO. Idempotency: the summary +# is dedup'd by its `` marker (scoped to +# the current pause cycle), so a failed label removal retries only the +# DELETE, and the scope condition (both labels) turns false once the release +# lands. Every other takeover state — conflicts, red CI, new feedback — +# remains the autofix scan's job; the shepherd only reports it. # # NON-GOAL: rerunning flaky-failed CI. That is owned by the CI Failure Patrol # (.github/workflows/qwen-ci-flaky-rerun.yml), which has its own markers, diff --git a/docs/design/takeover-fleet-visibility.md b/docs/design/takeover-fleet-visibility.md index cde9bdf03f4..06e6d8c606e 100644 --- a/docs/design/takeover-fleet-visibility.md +++ b/docs/design/takeover-fleet-visibility.md @@ -30,7 +30,9 @@ time-budget cap all write a terminal `autofix-eval` marker with `round=EFF_MAX_ROUNDS`, which the next scan sees as `ROUND >= EFF_MAX_ROUNDS` and lands in the cap-notice branch). The label write is placed so it runs even when the once-per-window notice comment is dedup'd — this backfills the -label onto the five already-paused PRs on the first scan after deploy. +label onto the already-paused fleet via the regular scan rotation after +deploy (idle backoff defers PRs idle >24h to ~1 scan in 4 — expect hours, +not the first scan). **Removed** wherever management resumes or a human takes over: @@ -78,9 +80,13 @@ When a PR carries **both** `autofix/takeover` and `autofix/needs-human` and its terminal timestamp is older than `AUTO_RELEASE_DAYS` (default 3, tunable via the `QWEN_SHEPHERD_AUTO_RELEASE_DAYS` repo variable): -1. Remove `autofix/takeover` (the loop disengages). -2. Post one bilingual summary: why it was released, the stop reason, and the - human's options (merge / close / split + re-takeover). +1. Post one bilingual summary — dedup'd by its + `` marker (scoped to the current + pause cycle): why it was released, the stop reason, and the human's + options (merge / close / split + re-takeover). +2. Remove `autofix/takeover` (the loop disengages). A failed removal finds + the marker and retries only the DELETE; a failed summary leaves both + labels in place so the whole release retries next tick. 3. Keep `autofix/needs-human`: the PR still needs a human decision, and the label remains the filterable TODO list. It clears on re-engage/re-arm via the paths in (A). diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 5fc5ce86ee0..17d580b0168 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -3145,11 +3145,23 @@ describe('qwen-autofix workflow', () => { it('applies the needs-human escalation label at the cap and removes it on resume/release', () => { // The label contract: one env definition, one idempotent create with a // fixed color (a REST add of a missing label would mint a random one). + // The create's `|| true` and the POST's `if !` guard are load-bearing + // under the runner's `bash -e`: a create that fails once the label + // exists, or a POST that aborts the scan on a transient 502, must not + // ship green (R2-25). expect(workflow).toContain("NEEDS_HUMAN_LABEL: 'autofix/needs-human'"); expect(reviewScanJob).toContain( 'gh label create "${NEEDS_HUMAN_LABEL}" --repo "${REPO}" --color', ); - expect(reviewScanJob).toContain('-f "labels[]=${NEEDS_HUMAN_LABEL}"'); + expect(reviewScanJob).toMatch( + /gh label create "\$\{NEEDS_HUMAN_LABEL\}"[\s\S]{0,250}?2> \/dev\/null \|\| true/, + ); + expect(reviewScanJob).toContain( + 'if ! gh api -X POST "repos/${REPO}/issues/${PR}/labels" -f "labels[]=${NEEDS_HUMAN_LABEL}"', + ); + expect(reviewScanJob).toContain( + '${NEEDS_HUMAN_LABEL} add failed for #${PR}; will retry next scan', + ); // The label write sits OUTSIDE the once-per-window comment dedup — // that is the bootstrap property: already-noticed PRs (paused before // this shipped) get labeled on the first scan after deploy. Assert on @@ -3449,6 +3461,9 @@ describe('qwen-autofix workflow', () => { ); expect(addAbsent.writes).toContain('labels[]=autofix/takeover'); expect(addAbsent.writes).toContain(''); + // …and the engage arm clears any stale escalation label (R2-27: the + // removal must live in THIS branch, not just anywhere in the file). + expect(addAbsent.writes).toContain('labels/autofix%2Fneeds-human'); expect(addAbsent.writes).not.toContain('next scheduled scan'); expect(addAbsent.writes).not.toContain('定时扫描'); // add + present → re-arm ack, takeover label untouched — the only API @@ -3469,6 +3484,9 @@ describe('qwen-autofix workflow', () => { expect(removePresent.writes).toContain( 'API api -X DELETE repos/QwenLM/qwen-code/issues/7165/labels/autofix%2Ftakeover', ); + // …and the stop arm clears the escalation label in the same breath + // (R2-27): a stop on a paused PR must not leave needs-human behind. + expect(removePresent.writes).toContain('labels/autofix%2Fneeds-human'); // Release acks directly too — the exact mirror of the engage side: a // loud add next to a mute stop re-creates the lost-event ambiguity on // the release side (and fork/non-main releases have no other ack path @@ -3506,6 +3524,7 @@ describe('qwen-autofix workflow', () => { expect(forkRefused.writes).not.toContain('API'); const forkManaged = runToggle({ cmd: 'add', fork: true }); expect(forkManaged.writes).toContain('labels[]=autofix/takeover'); + expect(forkManaged.writes).toContain('labels/autofix%2Fneeds-human'); // Fork label events carry no secrets, so no other job could ever ack a // fork engage — the command's own ack is the ONLY one, and it sets the // expectation that the first round comes from the next scheduled scan. diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index f32f9094238..195c8ce6f2d 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -1087,6 +1087,11 @@ exit 1`; expect(loop2.match(/echo "🐑 #\$\{PR\} \[/g)).toHaveLength(1); expect(loop2.match(/TAKEOVER_ROWS="\$\{TAKEOVER_ROWS\}/g)).toHaveLength(1); expect(loop2.match(/HUMAN_ROWS="\$\{HUMAN_ROWS\}/g)).toHaveLength(1); + // R2-17: …and the append is POSITIONALLY outside the evaluation arms + // (a bare occurrence count can't tell arm-level from loop-body level). + expect(loop2).toMatch( + /\n {14}fi\n(?: {14}#[^\n]*\n)+ {14}echo "🐑 #\$\{PR\} \[/, + ); }); it('pins the marker/headline contracts against qwen-autofix.yml — drift fails here, not in production', () => { @@ -1097,8 +1102,8 @@ exit 1`; const autofix = readFileSync('.github/workflows/qwen-autofix.yml', 'utf8'); // 1. Resume markers: the scan's REARM_KEY (the write side's own idea of // "management resumed") and the shepherd's RESUME_TS must name the - // same markers — a new resume path added to only one file must fail - // here, never as a wrongful auto-release. + // same markers — compare the FULL sets, so a resume path added to + // only one file fails here, never as a wrongful auto-release (R2-10). const rearmKey = autofix.match( /REARM_KEY="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)'/, )?.[1]; @@ -1107,13 +1112,14 @@ exit 1`; /RESUME_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, )?.[1]; expect(resumeTs).toBeTruthy(); - for (const marker of [ - '', - '', - ]) { - expect(rearmKey).toContain(marker); - expect(resumeTs).toContain(marker); - } + const markerSet = (p) => + new Set( + [...p.matchAll(/contains\("()"\)/g)].map((m) => m[1]), + ); + expect(markerSet(rearmKey)).toEqual(markerSet(resumeTs)); + // 1b. The label constants the whole integration keys on (R2-11). + expect(autofix).toContain("NEEDS_HUMAN_LABEL: 'autofix/needs-human'"); + expect(autofix).toContain("TAKEOVER_LABEL: 'autofix/takeover'"); // 2. The re-arm commands the shepherd greps for are the same strings // qwen-autofix.yml routes on, and the refusal-ack variants that // supersede a refused command are exactly the ones the producer can @@ -1130,10 +1136,29 @@ exit 1`; /REFUSAL_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, )?.[1]; expect(refusalTs).toBeTruthy(); - for (const variant of ['fork-refused', 'base-refused', 'skip-blocked']) { - expect(refusalTs).toContain(variant); - expect(autofix).toContain(``); - } + // Compare the refusal-variant SET against the producer's full ack + // taxonomy (minus engaged/released) — a one-sided addition on either + // side fails here (R2-10). + const producerAckVariants = new Set( + [...autofix.matchAll(//g)].map( + (m) => m[1], + ), + ); + const producerRefusals = [...producerAckVariants].filter( + (v) => v !== 'engaged' && v !== 'released', + ); + expect(producerRefusals.sort()).toEqual([ + 'base-refused', + 'fork-refused', + 'skip-blocked', + ]); + const shepherdRefusals = ( + refusalTs.match(/takeover-ack \(([^)]+)\)/)?.[1] ?? '' + ) + .split('|') + .filter(Boolean) + .sort(); + expect(shepherdRefusals).toEqual(producerRefusals.sort()); // 3. Every TERMINAL headline the address leg can print matches the // shepherd's REASON regex, and the transient setup-failure headline // does not. Templates are extracted from the producer and exercised @@ -1166,5 +1191,125 @@ exit 1`; }).trim(); for (const h of terminal) expect(matches(h)).toBe('true'); for (const h of transient) expect(matches(h)).toBe('false'); + // Every extracted headline must be CLASSIFIED — an unclassified one + // (today: the retry-transient/item-level variants) is asserted against + // the regex expecting false, so a future terminal headline that misses + // the hard-coded substrings fails loudly instead of degrading the + // dashboard reason silently (R2-14). + const unclassified = headlines.filter( + (h) => !terminal.includes(h) && !transient.includes(h), + ); + expect(unclassified.length).toBeGreaterThan(0); + for (const h of unclassified) expect(matches(h)).toBe('false'); + }); + + it('pins the previously mutation-tested gaps from review round 2', () => { + // R2-7: days_since() is behaviorally replayed (the /86400 → /3600 + // mutation must fail) — shim GNU date through node for macOS hosts. + const daysSince = workflow.match(/days_since\(\) \{[\s\S]*?\n {10}\}/)?.[0]; + expect(daysSince).toBeTruthy(); + const gnuDateShim = `date() { if [[ "$1" == '-u' && "$2" == '-d' ]]; then node -e 'console.log(Math.floor(new Date(process.argv[1]).getTime()/1000))' "$3"; else command date "$@"; fi; }`; + const runDays = (now, ts) => + execFileSync( + 'bash', + [ + '-c', + `${gnuDateShim}\n${daysSince.replace(/\n {10}/g, '\n')}\ndays_since "\${TS}"`, + ], + { + env: { ...process.env, NOW_EPOCH: String(now), TS: ts }, + encoding: 'utf8', + }, + ).trim(); + const NOW = Date.parse('2026-08-10T00:00:00Z') / 1000; + expect(runDays(NOW, '2026-08-07T01:00:00Z')).toBe('2'); + expect(runDays(NOW, '2026-08-06T23:00:00Z')).toBe('3'); + // R2-8: the 🛑 prefix must reach the bot-fleet row, not just exist. + expect(workflow).toContain('${NH_PREFIX}${STATUS_NOTE} | ${ACTION_NOTE} |'); + // R2-18: the prefix condition's truth mapping — replay verbatim. + const nhCond = workflow.match( + /if \[\[ "\$\(jq -r --arg l "\$\{NEEDS_HUMAN_LABEL\}" '([^']*)' <<< "\$\{ROW\}"\)" == "true" \]\]/, + )?.[1]; + expect(nhCond).toBeTruthy(); + const nhRun = (labels) => + execFileSync('jq', ['-r', '--arg', 'l', 'autofix/needs-human', nhCond], { + encoding: 'utf8', + input: JSON.stringify({ labels }), + }).trim(); + expect(nhRun([{ name: 'autofix/needs-human' }])).toBe('true'); + expect(nhRun([])).toBe('false'); + // R2-9: loop 1's deferral and its HM_OK=false fallback are pinned. + const loop1 = workflow.match( + /# Loop 1: managed takeover PRs[\s\S]*?done < <\(jq -c '\.\[\]' \/tmp\/takeover\.json\)/, + )?.[0]; + expect(loop1).toBeTruthy(); + expect(loop1).toContain('continue # loop 2 renders paused PRs'); + expect(loop1).toContain('pause evaluation unavailable this tick'); + // R2-16: the needs-human skip filter is byte-identical to the pinned + // takeover filter (an inverted population ships red, not green). The + // [^']+ capture cannot span across call sites (the lazy [\s\S]*? form + // silently would). + const humanFilter = workflow.match( + /jq --arg skip "\$\{SKIP_LABEL\}" \\\n\s+'([^']+)' \\\n\s+\/tmp\/human-raw\.json/, + )?.[1]; + expect(humanFilter).toBeTruthy(); + const takeoverFilterP = workflow.match( + /jq --arg skip "\$\{SKIP_LABEL\}" \\\n\s+'([^']+)' \\\n\s+\/tmp\/takeover-raw\.json/, + )?.[1]; + expect(humanFilter).toBe(takeoverFilterP); + // R2-24/R2-28: the needs-human enumeration (the lever's population) + // carries the stale-first sort and the labels payload. + const humanEnum = workflow.match( + /--label "\$\{NEEDS_HUMAN_LABEL\}"[\s\S]*?\/tmp\/human-raw\.json/, + )?.[0]; + expect(humanEnum).toBeTruthy(); + expect(humanEnum).toContain("--search 'sort:updated-asc'"); + expect(humanEnum).toContain( + '--json number,author,updatedAt,mergeable,labels', + ); + // R2-22: neither enumeration failure may terminate the tick — any + // spelling of exit — because the dashboard write carries the liveness + // watermark. + expect(workflow).not.toMatch( + /takeover enumeration failed[\s\S]{0,400}exit( 0| 1)?\b/, + ); + expect(workflow).not.toMatch( + /needs-human enumeration failed[\s\S]{0,400}exit( 0| 1)?\b/, + ); + // R2-29: both loop-2 evidence fetches paginate (paused PRs are the + // high-comment population — page 1 alone misses the newest notice). + expect(workflow).toMatch( + /issues\/\$\{PR\}\/events" --paginate[\s\S]{0,80}\/tmp\/tk-ev\.json/, + ); + expect(workflow).toMatch( + /issues\/\$\{PR\}\/comments" --paginate[\s\S]{0,80}\/tmp\/tk-ic\.json/, + ); + // R2-19: the release scope check requires BOTH labels — the --arg + // bindings are the load-bearing part. + expect(workflow).toContain( + 'jq -r --arg a "${TAKEOVER_LABEL}" --arg b "${NEEDS_HUMAN_LABEL}" \'([.labels[]?.name] | index($a) != null) and ([.labels[]?.name] | index($b) != null)\' <<< "${LIVE_LABELS_JSON}"', + ); + // R2-20: live_skip exports the payload the scope check rides. + expect(workflow).toContain( + 'LIVE_LABELS_JSON="$(gh pr view "${pr}" --repo "${REPO}" --json labels', + ); + // R2-12: the labeled-event merge into RESUME_TS exists… + expect(workflow).toMatch( + /if \[\[ -n "\$\{EVENT_TS\}" && "\$\{EVENT_TS\}" > "\$\{RESUME_TS\}" \]\]; then\n\s+RESUME_TS="\$\{EVENT_TS\}"/, + ); + // R2-21: …and both promotions run BEFORE the evaluation chain consumes + // RESUME_TS (position-independent extractions can't see this). + expect(workflow).toMatch( + /RESUME_TS="\$\{CMD_TS\}"[\s\S]*?RESUME_TS="\$\{EVENT_TS\}"[\s\S]*?if \[\[ -z "\$\{TERM_TS\}" \]\]/, + ); + // R2-13: CLEANUPS increments only after the heal DELETE succeeded. + expect(workflow).toMatch( + /clear stale \$\{NEEDS_HUMAN_LABEL\}[\s\S]{0,400}labels\/\$\(jq -rn --arg l "\$\{NEEDS_HUMAN_LABEL\}" '\$l\|@uri'\)"; then\n\s+CLEANUPS=\$\(\( CLEANUPS \+ 1 \)\)/, + ); + // R2-23: both shepherd DELETEs are URI-encoded (the encoded path is the + // addressable route). + expect(workflow).toMatch( + /auto-release takeover \(paused \$\{PAUSE_D\}d\)" \\\n\s+gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{TAKEOVER_LABEL\}" '\$l\|@uri'\)"/, + ); }); }); From f64c17672b1259cbabaabeaa01286eb186ccde7b Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Wed, 12 Aug 2026 21:10:52 +0800 Subject: [PATCH 05/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?3=20=E2=80=94=20release-ack=20label=20gate=20and=20shared=20cla?= =?UTF-8?q?ssifiers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - R3-1 (Critical): the every-scan cap-branch label POST is now suppressed when a release ack (takeover-ack released) is newer than the last re-arm, so a released bot PR is not re-labeled each scan (which would fight every release-side removal and ping-pong with the shepherd cleanup). A re-arm advances the window past the release ack, re-enabling the label. - R3-7: the /retry re-arm's needs-human removal now honors autofix/skip, mirroring the takeover-command guard — a frozen PR keeps its only filterable escalation state. - R3-2 (Critical): the takeover-enum error row no longer claims 'no release evaluation ran' — the lever is fed by the needs-human enumeration. - R3-8: the conflict-dispatch lever refuses a paused (needs-human) PR instead of spending a dispatch slot the scan would refuse. - R1-10: extracted pending_checks()/failed_test_url() helpers so both dashboard loops share one CI-status classifier (the round-1 reply was wrong that the restructure removed this duplication — it did not). - Hardened the mutation-tested pins: exact terminal-headline count (5), full rearm DELETE line + single-API-write, AUTO_RELEASE_DAYS guard order, runRearm env/stub/assertion for the /retry DELETE + skip guard, and the scope-guard comparison operator. Co-authored-by: Qwen-Coder --- .github/workflows/qwen-autofix.yml | 25 +++++++++- .github/workflows/qwen-fleet-shepherd.yml | 40 +++++++++++---- scripts/tests/qwen-autofix-workflow.test.js | 50 +++++++++++++++++-- .../qwen-fleet-shepherd-workflow.test.js | 50 ++++++++++++++++--- 4 files changed, 143 insertions(+), 22 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 55497ba49ee..4c4d039b90a 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1916,8 +1916,15 @@ jobs: gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '\U0001F504 AutoFix re-armed. The next scan re-reads this PR'"'"'s feedback from the start and the round counter resets. Nothing was deleted — this marker supersedes the evaluation markers above it.\n\n
\n中文说明\n\n\U0001F504 已重新武装 AutoFix。下一次扫描会从头重新读取本 PR 的反馈，轮次计数也已重置。未删除任何内容 —— 本标记使其上方的评估标记失效。\n\n
\n\n')" echo "🔄 re-armed PR #${PR}" # Management resumed — the escalation label is stale. 404 is the - # common case (the PR was never paused). - if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + # common case (the PR was never paused). Skip wins over re-arm + # EVERYWHERE (mirrors the takeover-command guard): a frozen PR + # must keep its escalation label, because the fresh re-arm window + # won't re-apply it while skip is present and no scan manages the + # PR — an unguarded DELETE would silently drop the only filterable + # escalation state, with no shepherd heal to restore it. + if [[ "$(gh pr view "${PR}" --repo "${REPO}" --json labels 2> /dev/null | jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' || echo '')" == "true" ]]; then + echo "🧭 ${NEEDS_HUMAN_LABEL} removal skipped: ${SKIP_LABEL} present on #${PR}" + elif ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" fi @@ -2915,6 +2922,18 @@ jobs: [ .[] | select((.user.login // "") == $ab) | select((.body // "") | contains("")) | select((.created_at // "") > $rt) ] | length' "${WORKDIR}/ic.json")" + # A release ack newer than the last re-arm means the most + # recent management action was a deliberate release (`takeover + # stop`), not a re-arm. The label POST below must NOT fire in + # that state: a released PR stays in the scan candidate set and + # keeps ROUND >= cap, so an unconditional POST would re-add the + # escalation label the release just removed — and ping-pong with + # the shepherd's cleanup lever every tick. A re-arm advances + # REARM_KEY past the release ack, re-enabling the label. + RELEASE_ACKED="$(jq -r --arg ab "${AUTOFIX_BOT}" --arg rt "${NOTICE_RT}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | contains("")) + | select((.created_at // "") > $rt) ] | length' "${WORKDIR}/ic.json")" if [[ "${DRY_RUN}" == "true" ]]; then echo "🧪 DRY-RUN: would post cap-paused notice and apply ${NEEDS_HUMAN_LABEL} on #${PR}" else @@ -2937,6 +2956,8 @@ jobs: fi if [[ "${SCAN_BOT_ACTOR}" != "${AUTOFIX_BOT}" ]]; then echo "::warning::cap-paused notice skipped: PAT authenticates as '${SCAN_BOT_ACTOR}', expected ${AUTOFIX_BOT}" + elif [[ "${RELEASE_ACKED}" != "0" ]]; then + echo "🧭 cap label/notice skipped: PR was released after its last re-arm (#${PR})" else # The escalation label rides EVERY cap detection, noticed # or not: the once-per-window dedup suppresses repeat diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index 2f82b5e7b30..dcbb75c9d78 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -356,6 +356,20 @@ jobs: echo "$(( (NOW_EPOCH - $(date -u -d "$1" +%s)) / 86400 ))" } + # CI-status classifiers shared by BOTH dashboard loops, so a + # check-naming or status-set change lands in one place and the two + # tables can never classify the same PR differently. + # WAITING and REQUESTED are also not-yet-final check states. + pending_checks() { + jq -r '[.statusCheckRollup[]? | select((.status // "") | IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))] | length' <<< "$1" + } + # Platform-blind on purpose: the dashboard is a health VIEW, and a + # Windows- or macOS-only regression is just as red as an Ubuntu one + # (reruns stay with the Patrol either way). + failed_test_url() { + jq -r '[.statusCheckRollup[]? | select(.conclusion == "FAILURE") | select(.name | startswith("Test (")) | .detailsUrl][0] // ""' <<< "$1" + } + # ---- walk the bot fleet (one list call carries all per-PR meta) -- # autofix/skip is the maintainer opt-out honored at every # engagement path — a skip-labeled PR gets no shepherd levers and @@ -385,12 +399,8 @@ jobs: DASH_ROWS="${DASH_ROWS}| #${PR} | ? | incomplete metadata | — |\n" continue fi - # WAITING and REQUESTED are also not-yet-final check states. - PENDING="$(jq -r '[.statusCheckRollup[]? | select((.status // "") | IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))] | length' <<< "${ROW}")" - # Platform-blind on purpose: the dashboard is a health VIEW, and - # a Windows- or macOS-only regression is just as red as an Ubuntu - # one (reruns stay with the Patrol either way). - FAILED_TEST_URL="$(jq -r '[.statusCheckRollup[]? | select(.conclusion == "FAILURE") | select(.name | startswith("Test (")) | .detailsUrl][0] // ""' <<< "${ROW}")" + PENDING="$(pending_checks "${ROW}")" + FAILED_TEST_URL="$(failed_test_url "${ROW}")" BEHIND="$(gh api "repos/${REPO}/compare/main...${HEAD}" --jq '.behind_by // 0' 2> /dev/null || echo 0)" STATUS_NOTE='idle' @@ -441,6 +451,13 @@ jobs: ACTION_NOTE='conflict dispatch already sent for this head' elif [[ "${DISPATCHES}" -ge "${MAX_CONFLICT_DISPATCHES_PER_TICK}" ]]; then ACTION_NOTE='conflict-dispatch budget reached this tick' + elif [[ -n "${NH_PREFIX}" ]]; then + # A paused PR's own label declares the loop stopped on it — + # the dispatched scan would refuse ("Dispatch refused … + # stays unhandled"), so spending a dispatch slot here only + # posts a contradictory comment and plants a dedup marker. + # The conflict stays unhandled until a human re-arms. + ACTION_NOTE='paused (needs-human) — conflict stays unhandled until re-arm' elif live_skip "${PR}"; then # Budget first: once it is exhausted no mutation is possible, # so the PAT-backed live label read would be pure waste. @@ -539,7 +556,12 @@ jobs: > /tmp/takeover-raw.json 2> /dev/null; then TK_OK=false echo "::warning::takeover enumeration failed; the takeover table shows an error row this tick" - TAKEOVER_ROWS='| — | — | — | ⚠️ enumeration unreadable this tick | fail closed — no release evaluation ran |\n' + # The release lever is fed by the needs-human enumeration (loop + # 2), NOT this display window — so a takeover-list failure does + # NOT stop pause evaluation. The row must say so, or the + # dashboard contradicts itself during the very outage it exists + # to clarify. + TAKEOVER_ROWS='| — | — | — | ⚠️ takeover pool unreadable this tick | paused rows below still evaluated (needs-human fed) |\n' else if [[ "$(jq length /tmp/takeover-raw.json)" -ge 100 ]]; then echo "::warning::takeover pool at the 100-PR enumeration limit — recently-active PRs (paused-but-discussed ones included) may be missing from this table" @@ -579,8 +601,8 @@ jobs: AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" MERGEABLE="$(jq -r '.mergeable // "UNKNOWN"' <<< "${ROW}")" - PENDING="$(jq -r '[.statusCheckRollup[]? | select((.status // "") | IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))] | length' <<< "${ROW}")" - FAILED_TEST_URL="$(jq -r '[.statusCheckRollup[]? | select(.conclusion == "FAILURE") | select(.name | startswith("Test (")) | .detailsUrl][0] // ""' <<< "${ROW}")" + PENDING="$(pending_checks "${ROW}")" + FAILED_TEST_URL="$(failed_test_url "${ROW}")" HAS_NH="$(jq -r --arg l "${NEEDS_HUMAN_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" UPD_AGO='?' [[ -n "${UPDATED}" ]] && UPD_AGO="$(days_since "${UPDATED}")d" diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 690ea2b45d1..e730d2c9848 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -3181,6 +3181,20 @@ describe('qwen-autofix workflow', () => { ); expect(consentRecheck).toBeGreaterThan(-1); expect(consentRecheck).toBeLessThan(labelPost); + // R3-1: a release ack newer than the window suppresses the every-scan + // label POST — a released bot PR stays in the scan candidate set with + // ROUND >= cap, so an unconditional POST would re-add the escalation + // label the release just removed and ping-pong with the shepherd's + // cleanup. The gate must sit BEFORE the label POST. + expect(reviewScanJob).toContain('RELEASE_ACKED='); + expect(reviewScanJob).toContain( + 'cap label/notice skipped: PR was released after its last re-arm', + ); + const releaseGate = reviewScanJob.indexOf( + 'elif [[ "${RELEASE_ACKED}" != "0" ]]; then', + ); + expect(releaseGate).toBeGreaterThan(-1); + expect(releaseGate).toBeLessThan(labelPost); // The dry-run line covers both writes. expect(reviewScanJob).toContain( 'DRY-RUN: would post cap-paused notice and apply ${NEEDS_HUMAN_LABEL}', @@ -3468,12 +3482,17 @@ describe('qwen-autofix workflow', () => { expect(addAbsent.writes).not.toContain('定时扫描'); // add + present → re-arm ack, takeover label untouched — the only API // write is the stale needs-human cleanup (404-tolerant; this PR was - // never paused, so the stub's silent success stands in for the 404). + // never paused). Pin the full DELETE line (method + encoded label) and + // that it is the ONLY API write — a fragment match would let a refactor + // swap the DELETE for a probe GET or add a stray label POST (R3-9). const rearm = runToggle({ cmd: 'add', labels: ['autofix/takeover'] }); expect(rearm.writes).toContain('COMMENT'); expect(rearm.writes).not.toContain('labels[]=autofix/takeover'); expect(rearm.writes).not.toContain('labels/autofix%2Ftakeover'); - expect(rearm.writes).toContain('labels/autofix%2Fneeds-human'); + expect(rearm.writes).toContain( + 'API api -X DELETE repos/QwenLM/qwen-code/issues/7165/labels/autofix%2Fneeds-human', + ); + expect(rearm.writes.match(/^API /gm) ?? []).toHaveLength(1); expect(rearm.log).toContain('re-armed'); // remove + present → label removed, through the URI-encoded path segment // (real jq runs in the substitution, so the %2F is the executed truth). @@ -9730,19 +9749,21 @@ exit 1 expect(rearmStep).toBeTruthy(); const block = rearmStep.replace(/\n {10}/g, '\n'); - const runRearm = ({ actor = BOT, apiFail = false } = {}) => { + const runRearm = ({ actor = BOT, apiFail = false, labels = '[]' } = {}) => { const dir = mkdtempSync(join(tmpdir(), 'autofix-rearm-')); try { writeFileSync( join(dir, 'gh'), [ '#!/bin/bash', - `echo "$1 $2" >> '${join(dir, 'calls.log')}'`, + `echo "$@" >> '${join(dir, 'calls.log')}'`, 'if [[ "$1" == "api" && "$2" == "user" ]]; then', ` if [[ "${apiFail}" == "true" ]]; then echo "HTTP 401: Bad credentials" >&2; exit 1; fi`, ` printf '%s' "${actor}"`, 'elif [[ "$1" == "pr" && "$2" == "comment" ]]; then', ` printf '%s' "$7" > '${join(dir, 'body.txt')}'`, + 'elif [[ "$1" == "pr" && "$2" == "view" ]]; then', + ` printf '%s' '{"labels":${labels}}'`, 'fi', ].join('\n'), ); @@ -9756,6 +9777,8 @@ exit 1 PR: '7354', REPO: 'QwenLM/qwen-code', AUTOFIX_BOT: BOT, + NEEDS_HUMAN_LABEL: 'autofix/needs-human', + SKIP_LABEL: 'autofix/skip', }, encoding: 'utf8', }); @@ -9783,10 +9806,29 @@ exit 1 expect(ok.body).toContain(''); expect(ok.body).toContain('
'); expect(ok.body).toContain('中文说明'); + // R3-11: the /retry path's needs-human DELETE must actually target the + // encoded label (broken @uri / renamed variable / swallowed failure all + // fail here), and it is the only label write the re-arm performs. + expect(ok.calls).toContain( + 'api -X DELETE repos/QwenLM/qwen-code/issues/7354/labels/autofix%2Fneeds-human', + ); + expect(ok.calls.match(/^api -X DELETE/gm) ?? []).toHaveLength(1); // No line may be indented 4+ spaces, or the marker renders as a code block // and the scanners' marker match silently fails. expect(ok.body).not.toMatch(/^ {4,}/m); + // R3-7: skip wins over re-arm — with autofix/skip present the stale-label + // DELETE is skipped, so a frozen PR keeps its only filterable escalation + // state (no scan manages it and the fresh window won't re-apply it). + const skipped = runRearm({ + actor: BOT, + labels: '[{"name":"autofix/skip"}]', + }); + expect(skipped.status).toBe(0); + expect(skipped.calls).toContain('pr comment'); + expect(skipped.calls).not.toContain('api -X DELETE'); + expect(skipped.stdout).toContain('removal skipped'); + // Actor mismatch: the PAT authenticates as someone else -> the guard exits // non-zero and posts nothing (a mis-scoped PAT must not leave a stranded PR // behind a fake "re-armed" line). diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index 195c8ce6f2d..9c7b9d53d3c 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -116,7 +116,7 @@ describe('fleet shepherd workflow', () => { expect(workflow).toContain('consent withdrawn, no %s'); expect(workflow).toContain('fail closed, no %s'); expect(workflow).toMatch( - /DISPATCHES\}" -ge "\$\{MAX_CONFLICT_DISPATCHES_PER_TICK\}" \]\]; then[\s\S]{0,220}elif live_skip "\$\{PR\}"; then/, + /DISPATCHES\}" -ge "\$\{MAX_CONFLICT_DISPATCHES_PER_TICK\}" \]\]; then[\s\S]*?paused \(needs-human\)[\s\S]*?elif live_skip "\$\{PR\}"; then/, ); expect(workflow).toMatch( /SYNCS\}" -ge "\$\{MAX_SYNCS_PER_TICK\}" \]\]; then[\s\S]{0,160}elif live_skip "\$\{PR\}"; then/, @@ -1057,9 +1057,16 @@ exit 1`; expect(site).not.toContain('> /dev/null'); } // A zero-padded variable must not kill the lever: base-10 normalize - // after the numeric guard (R2-6). - expect(workflow).toContain( - 'AUTO_RELEASE_DAYS=$((10#${AUTO_RELEASE_DAYS}))', + // AFTER the numeric guard (R2-6). Pin the ORDER, anchored on the + // AUTO_RELEASE_DAYS-specific guard line — a looser regex would match the + // unrelated COUNT guard earlier in the file, and moving the normalize + // above the guard would let `$((10#abc))` abort the tick under set -e + // (R3-10). + expect(workflow).toMatch( + /is not numeric; using 3"\n\s+AUTO_RELEASE_DAYS=3\n/, + ); + expect(workflow).toMatch( + /if \[\[ ! "\$\{AUTO_RELEASE_DAYS\}" =~ \^\[0-9\]\+\$ \]\]; then[\s\S]*?AUTO_RELEASE_DAYS=\$\(\(10#\$\{AUTO_RELEASE_DAYS\}\)\)/, ); // The summary dedup is scoped to THIS pause cycle — markers older than // the latest cap notice don't suppress a second release's summary @@ -1183,7 +1190,7 @@ exit 1`; const transient = headlines.filter((h) => h.includes('a setup step failed'), ); - expect(terminal.length).toBeGreaterThanOrEqual(4); + expect(terminal.length).toBe(5); expect(transient).toHaveLength(1); const matches = (h) => execFileSync('jq', ['-rn', '--arg', 'b', h, `$b | test("${reasonRe}")`], { @@ -1285,9 +1292,11 @@ exit 1`; /issues\/\$\{PR\}\/comments" --paginate[\s\S]{0,80}\/tmp\/tk-ic\.json/, ); // R2-19: the release scope check requires BOTH labels — the --arg - // bindings are the load-bearing part. + // bindings are the load-bearing part. R3-12: the pin runs through the + // comparison operator, so a polarity flip (`!=` → `==`) of the scope + // guard fails instead of silently releasing label-changed PRs. expect(workflow).toContain( - 'jq -r --arg a "${TAKEOVER_LABEL}" --arg b "${NEEDS_HUMAN_LABEL}" \'([.labels[]?.name] | index($a) != null) and ([.labels[]?.name] | index($b) != null)\' <<< "${LIVE_LABELS_JSON}"', + 'jq -r --arg a "${TAKEOVER_LABEL}" --arg b "${NEEDS_HUMAN_LABEL}" \'([.labels[]?.name] | index($a) != null) and ([.labels[]?.name] | index($b) != null)\' <<< "${LIVE_LABELS_JSON}")" != "true" ]]; then', ); // R2-20: live_skip exports the payload the scope check rides. expect(workflow).toContain( @@ -1311,5 +1320,32 @@ exit 1`; expect(workflow).toMatch( /auto-release takeover \(paused \$\{PAUSE_D\}d\)" \\\n\s+gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{TAKEOVER_LABEL\}" '\$l\|@uri'\)"/, ); + // R3-2: the takeover-enum error row must NOT claim "no release + // evaluation ran" — the lever is fed by the needs-human enumeration. + expect(workflow).toContain( + 'paused rows below still evaluated (needs-human fed)', + ); + expect(workflow).not.toContain('fail closed — no release evaluation ran'); + // R3-8: the conflict-dispatch lever refuses a paused (needs-human) PR + // instead of spending a dispatch slot the scan would refuse. + expect(workflow).toContain( + 'paused (needs-human) — conflict stays unhandled until re-arm', + ); + expect(workflow).toMatch( + /elif \[\[ -n "\$\{NH_PREFIX\}" \]\]; then\n\s+#[\s\S]*?conflict stays unhandled until re-arm/, + ); + // R1-10: the CI-status classifiers are shared helpers, and NEITHER loop + // inlines its own copy of the jq (a check-naming change must land once). + expect(workflow).toContain('pending_checks() {'); + expect(workflow).toContain('failed_test_url() {'); + expect( + workflow.match(/PENDING="\$\(pending_checks "\$\{ROW\}"\)/g), + ).toHaveLength(2); + expect( + workflow.match(/FAILED_TEST_URL="\$\(failed_test_url "\$\{ROW\}"\)/g), + ).toHaveLength(2); + expect(workflow).not.toContain( + 'IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))] | length\' <<< "${ROW}"', + ); }); }); From 3758c6840b6af4aacbbcdd3da60af0b99215c139 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Thu, 13 Aug 2026 18:33:55 +0800 Subject: [PATCH 06/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?4=20=E2=80=94=20label-lifecycle=20hardening?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - R4-C1: the conflict-dispatch lever reads needs-human from the LIVE label payload (after live_skip), not the tick-start snapshot, so a label applied after enumeration is still honored. - R4-C2: a re-armed PR that still carries needs-human (a resume-side removal failed) now gets a bounded, skip-vetoed cleanup retry instead of staying pinned in the paused population forever. - R4-C3: the per-tick release budget is consumed before the first external write — a DELETE outage can no longer mutate many PRs while RELEASES=0. - R4-C4: dashboard row routing follows post-action label state — a released PR moves to Awaiting human, a healed one drops off entirely. - R4-C5: the AUTO_RELEASE_DAYS guard also rejects over-long digit strings before any arithmetic (Bash-int overflow would wrap negative and pass -ge). - R4-32: takeover-command stop only removes needs-human when the takeover release actually landed (REMOVED_OK; 404 counts) — a failed release no longer strands the escalation label while latching RELEASE_ACKED. - R4-2: the /retry skip guard fails closed — an unreadable label state keeps the label (mirrors takeover-ack's exit-1 convention). - R4-3: the takeover-ack released arm and the stop branch both honor autofix/skip when removing needs-human. - R4-S1: producer headlines must be explicitly classified terminal or transient — an unclassified headline now fails the cross-file test. - Pins updated/added for every behavior above. Co-authored-by: Qwen-Coder --- .github/workflows/qwen-autofix.yml | 51 +++++++++++---- .github/workflows/qwen-fleet-shepherd.yml | 64 ++++++++++++++----- scripts/tests/qwen-autofix-workflow.test.js | 24 +++++-- .../qwen-fleet-shepherd-workflow.test.js | 55 ++++++++-------- 4 files changed, 136 insertions(+), 58 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 4c4d039b90a..cedcdcbc558 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1829,13 +1829,34 @@ jobs: # `/takeover stop` retries the removal — but must not disappear # silently either: masked, the ack reads "released" while the # loop keeps managing the PR. + # Track whether the takeover release actually LANDED: success + # or 404 (already gone) both mean the label is off. A 403/5xx/ + # network failure means it is still on — in that case the + # needs-human removal below must NOT run, or the PR keeps the + # takeover label while losing its escalation label (and the + # release ack posted here would latch RELEASE_ACKED, blocking + # every automated path that could re-apply it). (R4-32) + REMOVED_OK=true if ! REMOVE_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${REMOVE_ERR}" == *404* ]] || echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}" + if [[ "${REMOVE_ERR}" == *404* ]]; then + REMOVED_OK=true + else + REMOVED_OK=false + echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}" + fi fi echo "🏷️ removed ${TAKEOVER_LABEL} from #${PR}" # Released means a human is driving — the escalation label is - # stale. 404 is the common case (never paused). - if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + # stale. Remove it only when the release landed (REMOVED_OK) + # AND the PR is not frozen by skip (a frozen PR must keep its + # only filterable escalation state — R4-3). 404 is the common + # case (never paused). + SKIP_STATE="$(jq -r --arg t "${SKIP_LABEL}" '[.labels[].name] | index($t) != null' <<< "${PR_INFO}")" + if [[ "${REMOVED_OK}" != "true" ]]; then + echo "::warning::#${PR}: release did not land — keeping ${NEEDS_HUMAN_LABEL}; the next /takeover stop retries both" + elif [[ "${SKIP_STATE}" == "true" ]]; then + echo "🧭 ${NEEDS_HUMAN_LABEL} removal skipped: ${SKIP_LABEL} present on #${PR}" + elif ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" fi # Release ack, direct from the command — the exact mirror of @@ -1920,9 +1941,11 @@ jobs: # EVERYWHERE (mirrors the takeover-command guard): a frozen PR # must keep its escalation label, because the fresh re-arm window # won't re-apply it while skip is present and no scan manages the - # PR — an unguarded DELETE would silently drop the only filterable - # escalation state, with no shepherd heal to restore it. - if [[ "$(gh pr view "${PR}" --repo "${REPO}" --json labels 2> /dev/null | jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' || echo '')" == "true" ]]; then + # PR. And the read FAILS CLOSED (mirrors takeover-ack's exit-1): + # an unreadable label state must keep the label, never drop it. + if ! RETRY_LABELS="$(gh pr view "${PR}" --repo "${REPO}" --json labels 2> /dev/null)"; then + echo "::warning::#${PR}: label state unreadable — keeping ${NEEDS_HUMAN_LABEL} (fail closed)" + elif [[ "$(jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${RETRY_LABELS}")" == "true" ]]; then echo "🧭 ${NEEDS_HUMAN_LABEL} removal skipped: ${SKIP_LABEL} present on #${PR}" elif ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" @@ -2002,13 +2025,15 @@ jobs: fi # The escalation label goes stale on a real engage or any release # (a human is driving again). NOT on base-refused (nothing changed) - # and NOT on skip-blocked (management never resumed). 404 is the - # common case (the PR was never paused). Runs BEFORE the ack - # comment: the state change already happened (the human toggled - # the label — the comment is purely informational), so a transient - # comment failure aborting this step under set -e must not strand - # the stale label. - if [[ "${ACK}" == 'released' || ( "${ACK}" == 'engaged' && "${HAS_SKIP}" != 'true' ) ]]; then + # and NOT on skip-blocked (management never resumed) — and a + # RELEASE onto a skip-frozen PR also keeps the label: nothing + # manages or restores that PR, so it must stay in the needs-human + # filter (R4-3). 404 is the common case (the PR was never paused). + # Runs BEFORE the ack comment: the state change already happened + # (the human toggled the label — the comment is purely + # informational), so a transient comment failure aborting this step + # under set -e must not strand the stale label. + if [[ ( "${ACK}" == 'released' || "${ACK}" == 'engaged' ) && "${HAS_SKIP}" != 'true' ]]; then if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" fi diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index dcbb75c9d78..709b16a249e 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -451,17 +451,17 @@ jobs: ACTION_NOTE='conflict dispatch already sent for this head' elif [[ "${DISPATCHES}" -ge "${MAX_CONFLICT_DISPATCHES_PER_TICK}" ]]; then ACTION_NOTE='conflict-dispatch budget reached this tick' - elif [[ -n "${NH_PREFIX}" ]]; then - # A paused PR's own label declares the loop stopped on it — - # the dispatched scan would refuse ("Dispatch refused … - # stays unhandled"), so spending a dispatch slot here only - # posts a contradictory comment and plants a dedup marker. - # The conflict stays unhandled until a human re-arms. - ACTION_NOTE='paused (needs-human) — conflict stays unhandled until re-arm' elif live_skip "${PR}"; then # Budget first: once it is exhausted no mutation is possible, # so the PAT-backed live label read would be pure waste. ACTION_NOTE="$(skip_note dispatch)" + elif [[ "$(jq -r --arg l "${NEEDS_HUMAN_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${LIVE_LABELS_JSON}")" == "true" ]]; then + # Check needs-human from the LIVE payload, not the tick-start + # snapshot: the autofix loop can apply the label after + # enumeration, and dispatching conflict work onto a now-paused + # PR would plant a dedup marker the paused scan never acts on. + # The conflict stays unhandled until a human re-arms. + ACTION_NOTE='paused (needs-human) — conflict stays unhandled until re-arm' else if act "#${PR}: dispatch autofix for conflict resolution" \ env GITHUB_TOKEN="${ACTIONS_TOKEN}" gh workflow run qwen-autofix.yml --repo "${REPO}" -f pr_number="${PR}"; then @@ -535,9 +535,13 @@ jobs: HUMAN_ROWS='' # The variable is operator-tunable, so it is also operator- # breakable: a non-numeric value would fail the -ge comparison - # under set -e mid-tick. Fall back to the default instead. - if [[ ! "${AUTO_RELEASE_DAYS}" =~ ^[0-9]+$ ]]; then - echo "::warning::AUTO_RELEASE_DAYS '${AUTO_RELEASE_DAYS}' is not numeric; using 3" + # under set -e mid-tick. Fall back to the default instead. The + # digit-only regex still admits values above Bash's signed-int + # range (e.g. 9223372036854775808 wraps negative, so EVERY pause + # age would pass the -ge check) — bound by string LENGTH before any + # arithmetic, so a huge value can never reach the comparison. + if [[ ! "${AUTO_RELEASE_DAYS}" =~ ^[0-9]+$ ]] || [[ ${#AUTO_RELEASE_DAYS} -gt 2 ]]; then + echo "::warning::AUTO_RELEASE_DAYS '${AUTO_RELEASE_DAYS}' is not numeric or is too large; using 3" AUTO_RELEASE_DAYS=3 fi # Leading zeros pass the regex but bash reads 08/09 as bad octal at @@ -648,6 +652,10 @@ jobs: HAS_TK="$(jq -r --arg l "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" UPD_AGO='?' [[ -n "${UPDATED}" ]] && UPD_AGO="$(days_since "${UPDATED}")d" + # Row routing follows POST-ACTION label state, not the + # pre-mutation snapshot: a successful release moves the PR to + # Awaiting human; a successful heal drops it from both tables. + if [[ "${HAS_TK}" == "true" ]]; then ROUTE='takeover'; else ROUTE='awaiting'; fi STATE='🛑 needs-human' NOTE='—' @@ -723,7 +731,21 @@ jobs: NOTE='pause timestamp unreadable — release deferred (fail closed)' elif [[ -n "${RESUME_TS}" && "${RESUME_TS}" > "${TERM_TS}" ]]; then STATE='managed (re-armed)' - NOTE='resumed after the last pause — the escalation label is stale' + # The PR is re-armed but still carries needs-human — a + # resume-side removal failed. Clear the stale label here + # (bounded, skip-vetoed) or the PR stays pinned in the + # paused population forever. + if [[ "${CLEANUPS}" -ge "${MAX_CLEANUPS_PER_TICK}" ]]; then + NOTE='resumed; stale-label cleanup budget reached this tick' + elif live_skip "${PR}"; then + NOTE="$(skip_note cleanup)" + elif act "#${PR}: clear stale ${NEEDS_HUMAN_LABEL} (re-armed)" \ + gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')"; then + CLEANUPS=$(( CLEANUPS + 1 )) + NOTE='resumed — cleared stale escalation label' + else + NOTE='resumed after the last pause — the escalation label is stale' + fi else PAUSE_D="$(days_since "${TERM_TS}")" STATE="🛑 needs-human ${PAUSE_D}d" @@ -745,6 +767,11 @@ jobs: elif [[ "$(jq -r --arg a "${TAKEOVER_LABEL}" --arg b "${NEEDS_HUMAN_LABEL}" '([.labels[]?.name] | index($a) != null) and ([.labels[]?.name] | index($b) != null)' <<< "${LIVE_LABELS_JSON}")" != "true" ]]; then NOTE='labels changed since the snapshot — deferring release' else + # Consume the budget BEFORE the first external write: + # a release ATTEMPT is what is bounded. Counting only + # successful DELETEs would let a DELETE outage mutate + # many PRs in one tick while RELEASES stayed 0. + RELEASES=$(( RELEASES + 1 )) # Summary FIRST (dedup'd by its own marker — the # comment stream is already loaded), THEN the label # removal: a failed summary leaves both labels in @@ -765,11 +792,13 @@ jobs: fi fi if [[ "${SUMMARY_POSTED}" != "0" ]]; then - # The release HAPPENED when the label lands off, so - # that is what counts against the per-tick budget. if act "#${PR}: auto-release takeover (paused ${PAUSE_D}d)" \ gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')"; then - RELEASES=$(( RELEASES + 1 )) + # Released — only needs-human remains, so the PR + # belongs in Awaiting human NOW, not the takeover + # pool (route from post-action state, not the + # pre-mutation snapshot). + ROUTE='awaiting' NOTE="auto-released after ${PAUSE_D}d paused" else NOTE='takeover label removal failed — will retry next tick' @@ -808,6 +837,9 @@ jobs: elif act "#${PR}: clear stale ${NEEDS_HUMAN_LABEL} (manual release by @${UNLABEL_ACTOR})" \ gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')"; then CLEANUPS=$(( CLEANUPS + 1 )) + # Both labels are gone now — the PR is fully released and + # must drop out of the dashboard entirely. + ROUTE='none' NOTE="cleared stale escalation label (released by @${UNLABEL_ACTOR})" else NOTE='stale-label cleanup failed — will retry next tick' @@ -819,9 +851,9 @@ jobs: # dashboard for exactly the tick something went wrong. echo "🐑 #${PR} [${STATE}] → ${NOTE}" SAFE_NOTE="${NOTE//\\/\\\\}" - if [[ "${HAS_TK}" == "true" ]]; then + if [[ "${ROUTE}" == 'takeover' ]]; then TAKEOVER_ROWS="${TAKEOVER_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${STATE} | ${SAFE_NOTE//|/\\|} |\n" - else + elif [[ "${ROUTE}" == 'awaiting' ]]; then HUMAN_ROWS="${HUMAN_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${SAFE_NOTE//|/\\|} |\n" fi done < <(jq -c '.[]' /tmp/human.json) diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index e730d2c9848..c7af138b8aa 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -3213,10 +3213,11 @@ describe('qwen-autofix workflow', () => { workflow.match(/\$\{NEEDS_HUMAN_LABEL\} removal failed/g)?.length, ).toBe(6); // The ack job removes on a real engage or any release — but NOT on - // base-refused (nothing changed) or skip-blocked (management never - // resumed). + // base-refused (nothing changed), NOT on skip-blocked (management never + // resumed), and NOT on a release onto a skip-frozen PR (nothing manages + // or restores it — R4-3): both arms now require HAS_SKIP != 'true'. expect(workflow).toContain( - '"${ACK}" == \'released\' || ( "${ACK}" == \'engaged\' && "${HAS_SKIP}" != \'true\' )', + '( "${ACK}" == \'released\' || "${ACK}" == \'engaged\' ) && "${HAS_SKIP}" != \'true\'', ); }); @@ -9749,7 +9750,12 @@ exit 1 expect(rearmStep).toBeTruthy(); const block = rearmStep.replace(/\n {10}/g, '\n'); - const runRearm = ({ actor = BOT, apiFail = false, labels = '[]' } = {}) => { + const runRearm = ({ + actor = BOT, + apiFail = false, + labels = '[]', + viewFail = false, + } = {}) => { const dir = mkdtempSync(join(tmpdir(), 'autofix-rearm-')); try { writeFileSync( @@ -9763,6 +9769,7 @@ exit 1 'elif [[ "$1" == "pr" && "$2" == "comment" ]]; then', ` printf '%s' "$7" > '${join(dir, 'body.txt')}'`, 'elif [[ "$1" == "pr" && "$2" == "view" ]]; then', + ` if [[ "${viewFail}" == "true" ]]; then echo "GraphQL: Something went wrong" >&2; exit 1; fi`, ` printf '%s' '{"labels":${labels}}'`, 'fi', ].join('\n'), @@ -9829,6 +9836,15 @@ exit 1 expect(skipped.calls).not.toContain('api -X DELETE'); expect(skipped.stdout).toContain('removal skipped'); + // R4-2: the skip check FAILS CLOSED — a transient `gh pr view` failure + // keeps the label rather than dropping it (the marker still posts; only + // the cleanup is withheld). + const readFailed = runRearm({ actor: BOT, viewFail: true }); + expect(readFailed.status).toBe(0); + expect(readFailed.calls).toContain('pr comment'); + expect(readFailed.calls).not.toContain('api -X DELETE'); + expect(readFailed.stdout).toContain('label state unreadable'); + // Actor mismatch: the PAT authenticates as someone else -> the guard exits // non-zero and posts nothing (a mis-scoped PAT must not leave a stranded PR // behind a fake "re-armed" line). diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index 9c7b9d53d3c..de6000545c7 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -116,7 +116,7 @@ describe('fleet shepherd workflow', () => { expect(workflow).toContain('consent withdrawn, no %s'); expect(workflow).toContain('fail closed, no %s'); expect(workflow).toMatch( - /DISPATCHES\}" -ge "\$\{MAX_CONFLICT_DISPATCHES_PER_TICK\}" \]\]; then[\s\S]*?paused \(needs-human\)[\s\S]*?elif live_skip "\$\{PR\}"; then/, + /DISPATCHES\}" -ge "\$\{MAX_CONFLICT_DISPATCHES_PER_TICK\}" \]\]; then[\s\S]*?elif live_skip "\$\{PR\}"; then[\s\S]*?paused \(needs-human\)/, ); expect(workflow).toMatch( /SYNCS\}" -ge "\$\{MAX_SYNCS_PER_TICK\}" \]\]; then[\s\S]{0,160}elif live_skip "\$\{PR\}"; then/, @@ -913,10 +913,11 @@ exit 1`; expect(workflow).toMatch( /if act "#\$\{PR\}: post auto-release summary"[\s\S]{0,1800}if act "#\$\{PR\}: auto-release takeover/, ); - // The release counts when the label lands off — the increment follows - // the DELETE, never precedes it. + // The budget is consumed BEFORE the first external write (R4-C3): a + // release ATTEMPT is bounded, so a DELETE outage can't mutate many PRs + // in one tick while RELEASES stays 0. expect(workflow).toMatch( - /if act "#\$\{PR\}: auto-release takeover[\s\S]{0,300}RELEASES=\$\(\( RELEASES \+ 1 \)\)/, + /RELEASES=\$\(\( RELEASES \+ 1 \)\)[\s\S]{0,1200}if act "#\$\{PR\}: post auto-release summary"/, ); // And the DELETE removes the TAKEOVER label — never needs-human (the // filterable TODO must survive the release). @@ -946,7 +947,7 @@ exit 1`; // toContain('AUTO_RELEASE_DAYS=3') would substring-match '=30'). expect(workflow).toContain('=~ ^[0-9]+$'); expect(workflow).toMatch( - /is not numeric; using 3"\n\s+AUTO_RELEASE_DAYS=3\n/, + /is not numeric or is too large; using 3"\n\s+AUTO_RELEASE_DAYS=3\n/, ); // The stale-label heal (R1-13): a HUMAN unlabeled event on an // awaiting-human PR clears the stale escalation label — budgeted, @@ -1061,12 +1062,11 @@ exit 1`; // AUTO_RELEASE_DAYS-specific guard line — a looser regex would match the // unrelated COUNT guard earlier in the file, and moving the normalize // above the guard would let `$((10#abc))` abort the tick under set -e - // (R3-10). + // (R3-10). R4-C5: the guard ALSO rejects over-long digit strings before + // any arithmetic (Bash-int overflow would wrap negative and pass -ge). + expect(workflow).toContain('is not numeric or is too large; using 3'); expect(workflow).toMatch( - /is not numeric; using 3"\n\s+AUTO_RELEASE_DAYS=3\n/, - ); - expect(workflow).toMatch( - /if \[\[ ! "\$\{AUTO_RELEASE_DAYS\}" =~ \^\[0-9\]\+\$ \]\]; then[\s\S]*?AUTO_RELEASE_DAYS=\$\(\(10#\$\{AUTO_RELEASE_DAYS\}\)\)/, + /if \[\[ ! "\$\{AUTO_RELEASE_DAYS\}" =~ \^\[0-9\]\+\$ \]\] \|\| \[\[ \$\{#AUTO_RELEASE_DAYS\} -gt 2 \]\]; then[\s\S]*?AUTO_RELEASE_DAYS=\$\(\(10#\$\{AUTO_RELEASE_DAYS\}\)\)/, ); // The summary dedup is scoped to THIS pause cycle — markers older than // the latest cap notice don't suppress a second release's summary @@ -1187,27 +1187,29 @@ exit 1`; h.includes('reached the round cap') || h.includes('this was the last automatic attempt'), ); - const transient = headlines.filter((h) => - h.includes('a setup step failed'), + // R4-S1: BOTH sets must be explicit and exhaustive. A headline in + // neither list means the producer added wording nobody classified — and + // if that wording is terminal, the dashboard reason silently degrades. + // So unclassified must be EMPTY, not "asserted false": require every + // producer headline to be named terminal or transient here. + const transient = headlines.filter( + (h) => + h.includes('a setup step failed') || + h.includes('will retry on the next scan') || + h.includes('Could not produce a passing fix for this feedback'), + ); + const unclassified = headlines.filter( + (h) => !terminal.includes(h) && !transient.includes(h), ); expect(terminal.length).toBe(5); - expect(transient).toHaveLength(1); + expect(transient).toHaveLength(4); + expect(unclassified).toEqual([]); const matches = (h) => execFileSync('jq', ['-rn', '--arg', 'b', h, `$b | test("${reasonRe}")`], { encoding: 'utf8', }).trim(); for (const h of terminal) expect(matches(h)).toBe('true'); for (const h of transient) expect(matches(h)).toBe('false'); - // Every extracted headline must be CLASSIFIED — an unclassified one - // (today: the retry-transient/item-level variants) is asserted against - // the regex expecting false, so a future terminal headline that misses - // the hard-coded substrings fails loudly instead of degrading the - // dashboard reason silently (R2-14). - const unclassified = headlines.filter( - (h) => !terminal.includes(h) && !transient.includes(h), - ); - expect(unclassified.length).toBeGreaterThan(0); - for (const h of unclassified) expect(matches(h)).toBe('false'); }); it('pins the previously mutation-tested gaps from review round 2', () => { @@ -1327,12 +1329,15 @@ exit 1`; ); expect(workflow).not.toContain('fail closed — no release evaluation ran'); // R3-8: the conflict-dispatch lever refuses a paused (needs-human) PR - // instead of spending a dispatch slot the scan would refuse. + // instead of spending a dispatch slot the scan would refuse. R4-C1: the + // check reads the LIVE label payload (after live_skip), not the + // tick-start snapshot — a needs-human applied after enumeration is + // still caught. expect(workflow).toContain( 'paused (needs-human) — conflict stays unhandled until re-arm', ); expect(workflow).toMatch( - /elif \[\[ -n "\$\{NH_PREFIX\}" \]\]; then\n\s+#[\s\S]*?conflict stays unhandled until re-arm/, + /elif \[\[ "\$\(jq -r --arg l "\$\{NEEDS_HUMAN_LABEL\}" '\[\.labels\[\]\?\.name\] \| index\(\$l\) != null' <<< "\$\{LIVE_LABELS_JSON\}"\)" == "true" \]\]; then\n\s+#[\s\S]*?conflict stays unhandled until re-arm/, ); // R1-10: the CI-status classifiers are shared helpers, and NEITHER loop // inlines its own copy of the jq (a check-naming change must land once). From 9c8d732f0ed5b0c939bbe9001ef0c838d819db98 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Thu, 13 Aug 2026 19:04:29 +0800 Subject: [PATCH 07/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?4=20=E2=80=94=20robust=20release=20detection=20and=20marker-tru?= =?UTF-8?q?e=20gates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - R4-1/R4-5: release detection now uses the takeover unlabeled EVENT (recorded on every removal path, unlike the tolerated-lost ack comment), and the suppression only applies to human-authored PRs — a bot PR released from takeover returns to standard management and keeps the cap notice + escalation label. - R4-6: the conflict-dispatch lever requires marker truth (conflict_paused) — an armed PR with a stale needs-human label is dispatched normally. - R4-C1: the pause check reads needs-human from the live label payload. - R4-C2: re-armed PRs with a stale label get a bounded cleanup retry. - R4-C3: the release budget is consumed before the first external write. - R4-C4: dashboard rows route on post-action label state. - R4-C5: AUTO_RELEASE_DAYS rejects over-long digit strings before arithmetic. - R4-32/R4-2/R4-3: stop/ack/retry removal paths gate on REMOVED_OK and skip. - R4-9/R4-10/R4-13: membership check, STATE escaping, HM_OK-branched error row. - R4-14: command evidence requires a write/maintain/admin commenter. - R4-11/R4-15/R4-24: behavioral replays for the classifiers, the release jq, and the gate nesting. Co-authored-by: Qwen-Coder --- .github/workflows/qwen-autofix.yml | 37 ++-- .github/workflows/qwen-fleet-shepherd.yml | 102 ++++++++--- scripts/tests/qwen-autofix-workflow.test.js | 76 +++++++- .../qwen-fleet-shepherd-workflow.test.js | 164 ++++++++++++++---- 4 files changed, 306 insertions(+), 73 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index cedcdcbc558..2dafe2f8040 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -2947,18 +2947,29 @@ jobs: [ .[] | select((.user.login // "") == $ab) | select((.body // "") | contains("")) | select((.created_at // "") > $rt) ] | length' "${WORKDIR}/ic.json")" - # A release ack newer than the last re-arm means the most - # recent management action was a deliberate release (`takeover - # stop`), not a re-arm. The label POST below must NOT fire in - # that state: a released PR stays in the scan candidate set and - # keeps ROUND >= cap, so an unconditional POST would re-add the - # escalation label the release just removed — and ping-pong with - # the shepherd's cleanup lever every tick. A re-arm advances - # REARM_KEY past the release ack, re-enabling the label. - RELEASE_ACKED="$(jq -r --arg ab "${AUTOFIX_BOT}" --arg rt "${NOTICE_RT}" ' - [ .[] | select((.user.login // "") == $ab) - | select((.body // "") | contains("")) - | select((.created_at // "") > $rt) ] | length' "${WORKDIR}/ic.json")" + # Release evidence = a takeover unlabeled EVENT newer than the + # window key. GitHub records it whenever the label comes off — + # `/takeover stop`, the ack job, or a manual UI removal — + # unlike the release-ack COMMENT, which both release paths + # tolerate losing (R4-1): the stop branch swallows a failed + # ack post, and the ack job's set -e aborts before it. A + # re-arm advances REARM_KEY past the event, re-enabling the + # label. Fail closed: an unreadable event history suppresses + # the re-label rather than risk the ping-pong. + if ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > "${WORKDIR}/ev.json"; then + RELEASE_ACKED='unreadable' + else + RELEASE_ACKED="$(jq -r --arg tl "${TAKEOVER_LABEL}" --arg rt "${NOTICE_RT}" ' + [ .[] | select((.event // "") == "unlabeled") + | select((.label.name // "") == $tl) + | select((.created_at // "") > $rt) ] | length' "${WORKDIR}/ev.json")" + fi + # A release only suppresses re-labeling for HUMAN-authored PRs. + # A bot-authored PR released from takeover returns to STANDARD + # management (its ack says so) — it is still managed, so at the + # strict cap it deserves the same notice + escalation label as + # any managed PR (R4-5). + IS_BOT_AUTHOR="$(jq -r --arg ab "${AUTOFIX_BOT}" '((.author.login // "") == $ab)' <<< "${PR_META}")" if [[ "${DRY_RUN}" == "true" ]]; then echo "🧪 DRY-RUN: would post cap-paused notice and apply ${NEEDS_HUMAN_LABEL} on #${PR}" else @@ -2981,7 +2992,7 @@ jobs: fi if [[ "${SCAN_BOT_ACTOR}" != "${AUTOFIX_BOT}" ]]; then echo "::warning::cap-paused notice skipped: PAT authenticates as '${SCAN_BOT_ACTOR}', expected ${AUTOFIX_BOT}" - elif [[ "${RELEASE_ACKED}" != "0" ]]; then + elif [[ "${RELEASE_ACKED}" != "0" && "${IS_BOT_AUTHOR}" != "true" ]]; then echo "🧭 cap label/notice skipped: PR was released after its last re-arm (#${PR})" else # The escalation label rides EVERY cap detection, noticed diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index 709b16a249e..a6b9bfeef0d 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -356,6 +356,34 @@ jobs: echo "$(( (NOW_EPOCH - $(date -u -d "$1" +%s)) / 86400 ))" } + # Is this needs-human PR genuinely PAUSED (so the scan would refuse + # a conflict dispatch)? The label alone is only a proxy: an ARMED + # PR that kept a stale label (a resume-side removal failed) is NOT + # paused and the scan would accept the dispatch. Check marker truth + # — a re-arm/engage marker newer than the last cap notice means + # re-armed, not paused (R4-6). Fail closed toward "paused": an + # unreadable history suppresses the dispatch rather than waste it. + conflict_paused() { + local pr="$1" + if ! gh api "repos/${REPO}/issues/${pr}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/cf-ic.json; then + return 0 + fi + local term resume + term="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | contains("")) + | .created_at ] | max // ""' /tmp/cf-ic.json)" + resume="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select(((.body // "") | contains("")) + or ((.body // "") | contains(""))) + | .created_at ] | max // ""' /tmp/cf-ic.json)" + if [[ -n "${resume}" && ( -z "${term}" || "${resume}" > "${term}" ) ]]; then + return 1 + fi + return 0 + } + # CI-status classifiers shared by BOTH dashboard loops, so a # check-naming or status-set change lands in one place and the two # tables can never classify the same PR differently. @@ -455,12 +483,12 @@ jobs: # Budget first: once it is exhausted no mutation is possible, # so the PAT-backed live label read would be pure waste. ACTION_NOTE="$(skip_note dispatch)" - elif [[ "$(jq -r --arg l "${NEEDS_HUMAN_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${LIVE_LABELS_JSON}")" == "true" ]]; then - # Check needs-human from the LIVE payload, not the tick-start - # snapshot: the autofix loop can apply the label after - # enumeration, and dispatching conflict work onto a now-paused - # PR would plant a dedup marker the paused scan never acts on. - # The conflict stays unhandled until a human re-arms. + elif [[ "$(jq -r --arg l "${NEEDS_HUMAN_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${LIVE_LABELS_JSON}")" == "true" ]] && conflict_paused "${PR}"; then + # needs-human from the LIVE payload (a label applied after + # enumeration is caught) AND marker-confirmed genuinely + # paused — an armed PR with a stale label falls through to + # the dispatch below (R4-6). The conflict stays unhandled + # until a human re-arms. ACTION_NOTE='paused (needs-human) — conflict stays unhandled until re-arm' else if act "#${PR}: dispatch autofix for conflict resolution" \ @@ -560,12 +588,11 @@ jobs: > /tmp/takeover-raw.json 2> /dev/null; then TK_OK=false echo "::warning::takeover enumeration failed; the takeover table shows an error row this tick" - # The release lever is fed by the needs-human enumeration (loop - # 2), NOT this display window — so a takeover-list failure does - # NOT stop pause evaluation. The row must say so, or the - # dashboard contradicts itself during the very outage it exists - # to clarify. - TAKEOVER_ROWS='| — | — | — | ⚠️ takeover pool unreadable this tick | paused rows below still evaluated (needs-human fed) |\n' + # The error-row wording is finalized AFTER the needs-human + # enumeration (R4-13): the release lever is fed by THAT window, + # and both reads share one PAT, so a correlated mid-tick outage + # can fail both — the row must not claim evaluation proceeds + # when the feed is unreadable too. else if [[ "$(jq length /tmp/takeover-raw.json)" -ge 100 ]]; then echo "::warning::takeover pool at the 100-PR enumeration limit — recently-active PRs (paused-but-discussed ones included) may be missing from this table" @@ -594,6 +621,22 @@ jobs: /tmp/human-raw.json > /tmp/human.json fi + # Membership set of the needs-human enumeration (R4-9): loop 1 must + # only defer a paused PR to loop 2 when loop 2 will actually see it + # — a PR truncated out of the 100-window must still render here. + HUMAN_IDS='' + if [[ "${HM_OK}" == "true" ]]; then + HUMAN_IDS=",$(jq -r '[.[].number | tostring] | join(",")' /tmp/human.json)," + fi + # Finalize the takeover-enum error row now that HM_OK is known. + if [[ "${TK_OK}" == "false" ]]; then + if [[ "${HM_OK}" == "true" ]]; then + TAKEOVER_ROWS='| — | — | — | ⚠️ takeover pool unreadable this tick | paused rows below still evaluated (needs-human fed) |\n' + else + TAKEOVER_ROWS='| — | — | — | ⚠️ takeover pool unreadable this tick | paused rows NOT evaluated — needs-human enumeration also failed |\n' + fi + fi + # Loop 1: managed takeover PRs that are NOT paused — cheap payload # states only. Paused ones are rendered by loop 2, which owns the # pause evaluation. (If the needs-human enumeration failed, render @@ -614,11 +657,11 @@ jobs: STATE='managed · idle' NOTE='—' if [[ "${HAS_NH}" == "true" ]]; then - if [[ "${HM_OK}" == "true" ]]; then - continue # loop 2 renders paused PRs + if [[ "${HUMAN_IDS}" == *",${PR},"* ]]; then + continue # loop 2 renders this paused PR fi STATE='🛑 needs-human' - NOTE='pause evaluation unavailable this tick (enumeration unreadable)' + NOTE='pause evaluation unavailable this tick (truncated or unreadable needs-human window)' elif [[ "${MERGEABLE}" == "CONFLICTING" ]]; then STATE='conflicting' NOTE='resolution owned by the autofix scan' @@ -630,10 +673,13 @@ jobs: fi echo "🐑 #${PR} [takeover: ${STATE}] → ${NOTE}" - # NOTE/STATE carry no dynamic text on this path, but keep the - # table pipe-safe for the ci-red URL. + # STATE can carry the ci-red detailsUrl (check-run creator + # controlled) and NOTE the stop-reason headline — escape + # backslashes before printf '%b' and pipes before the table + # (R4-10). + SAFE_STATE="${STATE//\\/\\\\}" SAFE_NOTE="${NOTE//\\/\\\\}" - TAKEOVER_ROWS="${TAKEOVER_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${STATE} | ${SAFE_NOTE//|/\\|} |\n" + TAKEOVER_ROWS="${TAKEOVER_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${SAFE_STATE//|/\\|} | ${SAFE_NOTE//|/\\|} |\n" done < <(jq -c '.[]' /tmp/takeover.json) fi @@ -691,10 +737,19 @@ jobs: | select(((.body // "") | contains("")) or ((.body // "") | contains(""))) | .created_at ] | max // ""' /tmp/tk-ic.json)" - CMD_TS="$(jq -r ' + # Latest re-arm command + its author. R4-14: a renewing + # stream of STRANGER commands must not veto the release + # forever — only a write/maintain/admin commenter's command + # counts (one permission read, only when a command is the + # sole evidence — bot markers and labeled events never need + # it). + CMD_LINE="$(jq -r ' [ .[] | select(((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /takeover" or ((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /retry") - | .created_at ] | max // ""' /tmp/tk-ic.json)" + | {ts: (.created_at // ""), login: (.user.login // "")} ] + | max_by(.ts) // empty | "\(.ts)\t\(.login)"' /tmp/tk-ic.json)" + CMD_TS="${CMD_LINE%%$'\t'*}" + CMD_AUTHOR="${CMD_LINE##*$'\t'}" REFUSAL_TS="$(jq -r --arg ab "${AUTOFIX_BOT}" ' [ .[] | select((.user.login // "") == $ab) | select((.body // "") | test("")) @@ -702,8 +757,11 @@ jobs: if [[ -n "${CMD_TS}" && "${CMD_TS}" > "${RESUME_TS}" ]] \ && [[ -z "${REFUSAL_TS}" || "${CMD_TS}" > "${REFUSAL_TS}" ]]; then CMD_AGE=$(( NOW_EPOCH - $(date -u -d "${CMD_TS}" +%s) )) - if [[ "${CMD_AGE}" -lt "${RESUME_COMMAND_GRACE_SEC}" ]]; then - RESUME_TS="${CMD_TS}" + if [[ "${CMD_AGE}" -lt "${RESUME_COMMAND_GRACE_SEC}" ]] && [[ -n "${CMD_AUTHOR}" ]]; then + CMD_PERM="$(gh api "repos/${REPO}/collaborators/${CMD_AUTHOR}/permission" --jq '.permission // ""' 2> /dev/null || echo '')" + if [[ "${CMD_PERM}" == 'write' || "${CMD_PERM}" == 'maintain' || "${CMD_PERM}" == 'admin' ]]; then + RESUME_TS="${CMD_TS}" + fi fi fi EVENT_TS="$(jq -r --arg tl "${TAKEOVER_LABEL}" ' diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index c7af138b8aa..20fcfe035b1 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -2688,15 +2688,16 @@ describe('qwen-autofix workflow', () => { // forces a deliberate test update, however it is spaced or line-wrapped: // bump this count AND pipe the new site through the normalizer (bumping // the count below too) — bumping this pin alone leaves toBe(9) green. - expect(workflow.split('--paginate').length - 1).toBe(14); + expect(workflow.split('--paginate').length - 1).toBe(15); // scan ic + pr-events + ic re-fetch + scan rv/rc + prepare rv/rc/ic + - // report COMMENTS_JSON fallback = nine normalized fetch sites. The + // report COMMENTS_JSON fallback + the cap-branch release-evidence events + // fetch (R4-1) = ten normalized fetch sites. The // blocked-takeover status lookup is deliberately NOT among them: like the // sibling STATUS_ID read, it consumes the page stream inline via // `--jq ... | .id` into `tail -1` and never lands in a WORKDIR json file, // so piping it through `jq -s 'add // []'` would wrap the id stream in an // array and break the tail-1 consumer. - expect(workflow.split("jq -s 'add // []'").length - 1).toBe(9); + expect(workflow.split("jq -s 'add // []'").length - 1).toBe(10); // Empty-input semantics: a total gh failure feeds the fallback an EMPTY // stream, where the normalizer filter must yield '[]' and not 'null' — // the PRIOR_HEADS consumer below iterates the result with .[], which @@ -3181,20 +3182,77 @@ describe('qwen-autofix workflow', () => { ); expect(consentRecheck).toBeGreaterThan(-1); expect(consentRecheck).toBeLessThan(labelPost); - // R3-1: a release ack newer than the window suppresses the every-scan - // label POST — a released bot PR stays in the scan candidate set with - // ROUND >= cap, so an unconditional POST would re-add the escalation - // label the release just removed and ping-pong with the shepherd's - // cleanup. The gate must sit BEFORE the label POST. + // R3-1/R4-1: a takeover `unlabeled` EVENT newer than the window + // suppresses the every-scan label POST — a released PR stays in the + // scan candidate set with ROUND >= cap, so an unconditional POST would + // re-add the escalation label the release just removed and ping-pong + // with the shepherd's cleanup. R4-5: the gate only fires for + // HUMAN-authored PRs (a bot PR released from takeover is still managed, + // so it keeps the label at the strict cap). The gate sits BEFORE the + // label POST. expect(reviewScanJob).toContain('RELEASE_ACKED='); + expect(reviewScanJob).toContain('IS_BOT_AUTHOR='); expect(reviewScanJob).toContain( 'cap label/notice skipped: PR was released after its last re-arm', ); const releaseGate = reviewScanJob.indexOf( - 'elif [[ "${RELEASE_ACKED}" != "0" ]]; then', + 'elif [[ "${RELEASE_ACKED}" != "0" && "${IS_BOT_AUTHOR}" != "true" ]]; then', ); expect(releaseGate).toBeGreaterThan(-1); expect(releaseGate).toBeLessThan(labelPost); + // R4-15: the RELEASE_ACKED jq body (event marker + time direction) is + // replayed — a `> $rt` → `< $rt` flip must fail, not ship green. + const relAckJq = reviewScanJob.match( + /RELEASE_ACKED="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" --arg rt "\$\{NOTICE_RT\}" '([\s\S]*?)' "\$\{WORKDIR\}\/ev\.json"\)"/, + )?.[1]; + expect(relAckJq).toBeTruthy(); + const runRelAck = (events, rt) => + execFileSync( + 'jq', + ['-r', '--arg', 'tl', 'autofix/takeover', '--arg', 'rt', rt, relAckJq], + { encoding: 'utf8', input: JSON.stringify(events) }, + ).trim(); + expect( + runRelAck( + [ + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-06T00:00:00Z', + }, + ], + '2026-08-05T00:00:00Z', + ), + ).toBe('1'); + // ack older than the window → 0; no ack → 0. + expect( + runRelAck( + [ + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-04T00:00:00Z', + }, + ], + '2026-08-05T00:00:00Z', + ), + ).toBe('0'); + expect(runRelAck([{ event: 'labeled' }], '2026-08-05T00:00:00Z')).toBe('0'); + // R4-24: the gate's control-flow nesting is replayed — the label POST + // must fire ONLY in the else arm (RELEASE_ACKED=0 or bot author), never + // in the released arm. + const gateBlock = reviewScanJob.match( + /if \[\[ "\$\{SCAN_BOT_ACTOR\}" != "\$\{AUTOFIX_BOT\}" \]\]; then[\s\S]*?cap label\/notice skipped[\s\S]*?\n {16}fi/, + )?.[0]; + expect(gateBlock).toBeTruthy(); + const releasedIdx = gateBlock.indexOf('cap label/notice skipped'); + const elseIdx = gateBlock.indexOf('else', releasedIdx); + const postIdx = gateBlock.indexOf( + 'gh api -X POST "repos/${REPO}/issues/${PR}/labels"', + ); + expect(releasedIdx).toBeGreaterThan(-1); + expect(elseIdx).toBeGreaterThan(releasedIdx); + expect(postIdx).toBeGreaterThan(elseIdx); // The dry-run line covers both writes. expect(reviewScanJob).toContain( 'DRY-RUN: would post cap-paused notice and apply ${NEEDS_HUMAN_LABEL}', diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index de6000545c7..c04af0c2e81 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -711,27 +711,28 @@ exit 1`; // …a re-arm COMMAND comment is ALSO resume evidence, but only while // fresh and unsuperseded: the command's ack rides a queued job, so // counting only the ack would let a tick release a PR minutes after a - // human re-armed it. No author gate here — the route's permission check - // owns authorization; an ignored command simply expires unsuperseded. - const cmdJq = workflow.match( - /CMD_TS="\$\(jq -r '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + // human re-armed it. R4-14: the commenter's permission is checked before + // a command counts — a renewing stream of stranger commands must not + // veto a release forever. + const cmdLineJq = workflow.match( + /CMD_LINE="\$\(jq -r '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, )?.[1]; const refusalJq = workflow.match( /REFUSAL_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, )?.[1]; - expect(cmdJq).toBeTruthy(); + expect(cmdLineJq).toBeTruthy(); expect(refusalJq).toBeTruthy(); const runCmd = (comments) => - execFileSync('jq', ['-r', cmdJq], { + execFileSync('jq', ['-r', cmdLineJq], { encoding: 'utf8', input: JSON.stringify(comments), }).trim(); - // Exact trimmed command match, any author. + // Exact trimmed command match → "tslogin", any author. expect( runCmd([human('2026-08-06T00:00:00Z', ' @qwen-code /takeover ')]), - ).toBe('2026-08-06T00:00:00Z'); + ).toBe('2026-08-06T00:00:00Z\twenshao'); expect(runCmd([human('2026-08-06T00:00:00Z', '@qwen-code /retry')])).toBe( - '2026-08-06T00:00:00Z', + '2026-08-06T00:00:00Z\twenshao', ); // …but `/takeover stop` is a release, not a resume. expect( @@ -756,7 +757,9 @@ exit 1`; ]), ).toBe(''); // The full gate: command evidence counts only when it is newer than the - // marker evidence, newer than any refusal, AND inside the grace window. + // marker evidence, newer than any refusal, inside the grace window, AND + // the commenter holds write/maintain/admin (R4-14). Stub `gh` so the + // permission read returns a test-controlled value. const cmdGate = workflow.match( /(if \[\[ -n "\$\{CMD_TS\}" && "\$\{CMD_TS\}" > "\$\{RESUME_TS\}" \]\] \\[\s\S]*?\n {16}fi)/, )?.[1]; @@ -764,23 +767,38 @@ exit 1`; // The extracted block runs GNU `date -u -d`; on macOS hosts BSD date // lacks -d, so shim it through node (always present in this repo). const gnuDateShim = `date() { if [[ "$1" == '-u' && "$2" == '-d' ]]; then node -e 'console.log(Math.floor(new Date(process.argv[1]).getTime()/1000))' "$3"; else command date "$@"; fi; }`; - const gate = (env) => - execFileSync( - 'bash', - [ - '-c', - `${gnuDateShim}\n${cmdGate.replace(/\n {16}/g, '\n')}\necho "RESUME_TS=$RESUME_TS"`, - ], - { - env: { - ...process.env, - NOW_EPOCH: '1786000000', - RESUME_COMMAND_GRACE_SEC: '7200', - ...env, + const gate = (env) => { + const dir = mkdtempSync(join(tmpdir(), 'shep-gate-')); + try { + writeFileSync( + join(dir, 'gh'), + '#!/bin/bash\nif [[ "$1" == "api" ]]; then printf \'%s\' "${TEST_PERM}"; fi', + ); + chmodSync(join(dir, 'gh'), 0o755); + return execFileSync( + 'bash', + [ + '-c', + `${gnuDateShim}\n${cmdGate.replace(/\n {16}/g, '\n')}\necho "RESUME_TS=$RESUME_TS"`, + ], + { + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + REPO: 'QwenLM/qwen-code', + NOW_EPOCH: '1786000000', + RESUME_COMMAND_GRACE_SEC: '7200', + TEST_PERM: 'write', + CMD_AUTHOR: 'wenshao', + ...env, + }, + encoding: 'utf8', }, - encoding: 'utf8', - }, - ).trim(); + ).trim(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; // fresh accepted-but-unacked command → counts (the R1 race fix). expect( gate({ @@ -817,6 +835,26 @@ exit 1`; NOW_EPOCH: String(Date.parse('2026-08-06T00:30:00Z') / 1000), }), ).toBe('RESUME_TS=2026-08-06T00:05:00Z'); + // R4-14: a stranger's fresh command (read/no permission) does NOT count, + // even inside the grace window. + expect( + gate({ + RESUME_TS: '', + CMD_TS: '2026-08-06T00:00:00Z', + REFUSAL_TS: '', + NOW_EPOCH: String(Date.parse('2026-08-06T00:30:00Z') / 1000), + TEST_PERM: 'read', + }), + ).toBe('RESUME_TS='); + expect( + gate({ + RESUME_TS: '', + CMD_TS: '2026-08-06T00:00:00Z', + REFUSAL_TS: '', + NOW_EPOCH: String(Date.parse('2026-08-06T00:30:00Z') / 1000), + TEST_PERM: '', + }), + ).toBe('RESUME_TS='); // A fresh `labeled` event is resume evidence too (UI re-apply). const eventJq = workflow.match( /EVENT_TS="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, @@ -1134,7 +1172,7 @@ exit 1`; expect(autofix).toContain("TAKEOVER_COMMAND: '@qwen-code /takeover'"); expect(autofix).toContain("RETRY_COMMAND: '@qwen-code /retry'"); const cmdTs = workflow.match( - /CMD_TS="\$\(jq -r '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + /CMD_LINE="\$\(jq -r '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, )?.[1]; expect(cmdTs).toBeTruthy(); expect(cmdTs).toContain('"@qwen-code /takeover"'); @@ -1252,7 +1290,7 @@ exit 1`; /# Loop 1: managed takeover PRs[\s\S]*?done < <\(jq -c '\.\[\]' \/tmp\/takeover\.json\)/, )?.[0]; expect(loop1).toBeTruthy(); - expect(loop1).toContain('continue # loop 2 renders paused PRs'); + expect(loop1).toContain('continue # loop 2 renders this paused PR'); expect(loop1).toContain('pause evaluation unavailable this tick'); // R2-16: the needs-human skip filter is byte-identical to the pinned // takeover filter (an inverted population ships red, not green). The @@ -1337,7 +1375,37 @@ exit 1`; 'paused (needs-human) — conflict stays unhandled until re-arm', ); expect(workflow).toMatch( - /elif \[\[ "\$\(jq -r --arg l "\$\{NEEDS_HUMAN_LABEL\}" '\[\.labels\[\]\?\.name\] \| index\(\$l\) != null' <<< "\$\{LIVE_LABELS_JSON\}"\)" == "true" \]\]; then\n\s+#[\s\S]*?conflict stays unhandled until re-arm/, + /elif \[\[ "\$\(jq -r --arg l "\$\{NEEDS_HUMAN_LABEL\}" '\[\.labels\[\]\?\.name\] \| index\(\$l\) != null' <<< "\$\{LIVE_LABELS_JSON\}"\)" == "true" \]\] && conflict_paused "\$\{PR\}"; then\n\s+#[\s\S]*?conflict stays unhandled until re-arm/, + ); + // R4-6: the marker-truth helper exists and fails closed (an unreadable + // comment history suppresses the dispatch, not wastes it). + expect(workflow).toContain('conflict_paused() {'); + expect(workflow).toMatch( + /conflict_paused\(\) \{[\s\S]*?return 0[\s\S]*?return 1[\s\S]*?return 0/, + ); + // R4-9: loop 1 defers a paused PR to loop 2 ONLY when it is actually in + // the needs-human window (membership), so a truncated PR still renders. + expect(workflow).toContain( + 'HUMAN_IDS=",$(jq -r \'[.[].number | tostring] | join(",")\' /tmp/human.json),"', + ); + expect(workflow).toContain('if [[ "${HUMAN_IDS}" == *",${PR},"* ]]; then'); + // R4-10: loop 1's row escapes STATE (the ci-red detailsUrl) as well as + // NOTE — a `|` or `\c` in the URL must not break the printf '%b' table. + expect(workflow).toContain('SAFE_STATE="${STATE//\\\\/\\\\\\\\}"'); + expect(workflow).toContain('${SAFE_STATE//|/\\\\|}'); + // R4-13: the takeover-enum error row is finalized AFTER the needs-human + // result and branches on HM_OK — it must not claim evaluation proceeds + // when the needs-human feed also failed. + expect(workflow).toContain( + 'paused rows NOT evaluated — needs-human enumeration also failed', + ); + // R4-14: a command comment counts only after a write/maintain/admin + // permission read on its author. + expect(workflow).toContain( + 'gh api "repos/${REPO}/collaborators/${CMD_AUTHOR}/permission"', + ); + expect(workflow).toMatch( + /CMD_PERM}" == 'write' \|\| "\$\{CMD_PERM\}" == 'maintain' \|\| "\$\{CMD_PERM\}" == 'admin'/, ); // R1-10: the CI-status classifiers are shared helpers, and NEITHER loop // inlines its own copy of the jq (a check-naming change must land once). @@ -1352,5 +1420,43 @@ exit 1`; expect(workflow).not.toContain( 'IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))] | length\' <<< "${ROW}"', ); + // R4-11: the two classifiers are replayed behaviorally (a polarity or + // arithmetic mutation of the jq body must fail, not ship green). + const pendingJq = workflow.match( + /pending_checks\(\) \{\n\s+jq -r '([^']+)'/, + )?.[1]; + const failedJq = workflow.match( + /failed_test_url\(\) \{\n\s+jq -r '([^']+)'/, + )?.[1]; + expect(pendingJq).toBeTruthy(); + expect(failedJq).toBeTruthy(); + const runClassifier = (prog, rollup) => + execFileSync('jq', ['-r', prog], { + encoding: 'utf8', + input: JSON.stringify({ statusCheckRollup: rollup }), + }).trim(); + expect(runClassifier(pendingJq, [])).toBe('0'); + expect( + runClassifier(pendingJq, [ + { status: 'QUEUED' }, + { status: 'IN_PROGRESS' }, + { status: 'COMPLETED' }, + ]), + ).toBe('2'); + expect( + runClassifier(failedJq, [ + { conclusion: 'FAILURE', name: 'Test (x)', detailsUrl: 'u1' }, + ]), + ).toBe('u1'); + expect( + runClassifier(failedJq, [ + { conclusion: 'SUCCESS', name: 'Test (x)', detailsUrl: 'u1' }, + ]), + ).toBe(''); + expect( + runClassifier(failedJq, [ + { conclusion: 'FAILURE', name: 'Lint', detailsUrl: 'u1' }, + ]), + ).toBe(''); }); }); From c4ce6e130aa2e5b259c8c6a7ecf829937a67929f Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Thu, 13 Aug 2026 19:39:28 +0800 Subject: [PATCH 08/17] fix(autofix): keep the label-DELETE idiom byte-identical across workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R4-32's REMOVED_OK tracking reworked the takeover-command stop branch's 404-tolerance block, breaking the pr-self-report-label ↔ qwen-autofix contract test that pins the two workflows' label-DELETE idiom byte-identical. Keep the canonical idiom and derive REMOVED_OK from REMOVE_ERR's content afterward (empty = landed, 404 = already off, anything else = release did not land) — same behavior, contract intact. Co-authored-by: Qwen-Coder --- .github/workflows/qwen-autofix.yml | 31 ++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 2dafe2f8040..b7cff7ad2b2 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1829,21 +1829,24 @@ jobs: # `/takeover stop` retries the removal — but must not disappear # silently either: masked, the ack reads "released" while the # loop keeps managing the PR. - # Track whether the takeover release actually LANDED: success - # or 404 (already gone) both mean the label is off. A 403/5xx/ - # network failure means it is still on — in that case the - # needs-human removal below must NOT run, or the PR keeps the - # takeover label while losing its escalation label (and the - # release ack posted here would latch RELEASE_ACKED, blocking - # every automated path that could re-apply it). (R4-32) - REMOVED_OK=true + # REST for the same reason as the add above; the label name is a + # path segment and contains a slash, so it must be URI-encoded. + # The 404-tolerance block is pinned byte-identical to the other + # workflows' label DELETE (a contract test), so REMOVED_OK — + # whether the takeover release actually LANDED — is derived from + # REMOVE_ERR's content AFTER the idiom instead of reworking it: + # empty = landed, 404 = already off (landed), anything else = + # the release did NOT land, and the needs-human removal below + # must NOT run (R4-32) — or the PR keeps the takeover label + # while losing its escalation label, and the release ack posted + # here would latch RELEASE_ACKED, blocking every automated path + # that could re-apply it. if ! REMOVE_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')" 2>&1)"; then - if [[ "${REMOVE_ERR}" == *404* ]]; then - REMOVED_OK=true - else - REMOVED_OK=false - echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}" - fi + [[ "${REMOVE_ERR}" == *404* ]] || echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}" + fi + REMOVED_OK=true + if [[ -n "${REMOVE_ERR}" && "${REMOVE_ERR}" != *404* ]]; then + REMOVED_OK=false fi echo "🏷️ removed ${TAKEOVER_LABEL} from #${PR}" # Released means a human is driving — the escalation label is From 41b905dc583dae957894029e0c03d422e0ca27b4 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Fri, 14 Aug 2026 00:15:13 +0800 Subject: [PATCH 09/17] =?UTF-8?q?test(autofix):=20close=20review=20round?= =?UTF-8?q?=204b=20=E2=80=94=20mutation-tested=20harness=20hardening?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - R4-4: re-bound the conflict-lever regex spans and anchor on conflict_paused so the pin can't resolve live_skip against the sync lever's call site. - R4-16: runAck records gh calls and asserts per-branch needs-human DELETE counts (engaged/released=1; base-refused/skip=0). - R4-17: pin the first-pickup scan DELETE inside the engage-ack success branch. - R4-18/R4-19: ordering pins — takeover POST before needs-human DELETE (engage), marker comment before cleanup DELETE (/retry). - R4-20: deleteFail stub branch replays non-404 (warns, status 0) and 404 (silent) DELETE outcomes. - R4-21: identity-failure paths assert no DELETE ran. - R4-22: runRearm stub serves labels only when --json labels is requested. - R4-23: full api-write census pinned (exactly api user + one DELETE). - R4-25: skip fixture uses the production multi-label shape. - R4-28: loop-2 fetch pins include the jq -s 'add // []' merge program. - R4-29: cmdGate scenario where a refusal is OLDER than the fresh command. - R4-30: takeoverEnum asserts its own sort:updated-asc qualifier. - R4-31: multi-entry fixtures pin the max/last/length aggregation operators on CMD_TS, EVENT_TS, REASON, SUMMARY_POSTED, and the heal lever's LATEST_LABEL_TS/UNLABEL_ACTOR programs. Co-authored-by: Qwen-Coder --- scripts/tests/qwen-autofix-workflow.test.js | 104 +++++++++++- .../qwen-fleet-shepherd-workflow.test.js | 152 +++++++++++++++++- 2 files changed, 248 insertions(+), 8 deletions(-) diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 20fcfe035b1..f3513f8b3b4 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -3266,6 +3266,16 @@ describe('qwen-autofix workflow', () => { /gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{NEEDS_HUMAN_LABEL\}"/g, ) ?? []; expect(removals).toHaveLength(6); + // R4-17: the scan-side first-pickup DELETE must live in the engage-ack + // SUCCESS branch — moved to the ack-failure path it would clear + // needs-human when no engagement happened. + const firstPickup = reviewScanJob.match( + /if gh pr comment "\$\{PR\}" --repo "\$\{REPO\}" --body "\$\(printf '🤝 Takeover engaged[\s\S]*?; then([\s\S]*?)\n {16}else/, + )?.[1]; + expect(firstPickup).toBeTruthy(); + expect(firstPickup).toContain( + 'gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}"', + ); // Every removal tolerates the common 404 (never-paused PR). expect( workflow.match(/\$\{NEEDS_HUMAN_LABEL\} removal failed/g)?.length, @@ -3537,6 +3547,13 @@ describe('qwen-autofix workflow', () => { // …and the engage arm clears any stale escalation label (R2-27: the // removal must live in THIS branch, not just anywhere in the file). expect(addAbsent.writes).toContain('labels/autofix%2Fneeds-human'); + // R4-18: the loud takeover POST precedes the tolerant needs-human + // DELETE — a transient POST failure then aborts the step before the + // DELETE, keeping the paused PR visible (reversed order would drop the + // escalation label even though the engagement never landed). + expect(addAbsent.writes.indexOf('API api -X POST')).toBeLessThan( + addAbsent.writes.indexOf('labels/autofix%2Fneeds-human'), + ); expect(addAbsent.writes).not.toContain('next scheduled scan'); expect(addAbsent.writes).not.toContain('定时扫描'); // add + present → re-arm ack, takeover label untouched — the only API @@ -5134,7 +5151,7 @@ exit 1 .join('\n'); expect(ackBlock).toBeTruthy(); - const runAck = ({ ack, base = '', prViewOk = true }) => { + const runAck = ({ ack, base = '', prViewOk = true, labels = [] }) => { const dir = mkdtempSync(join(tmpdir(), 'ack-')); const bin = join(dir, 'bin'); mkdirSync(bin); @@ -5142,10 +5159,14 @@ exit 1 join(bin, 'gh'), [ '#!/usr/bin/env bash', - `if [[ "$1" == 'api' ]]; then printf 'qwen-code-dev-bot'; exit 0; fi`, + `echo "$@" >> ${JSON.stringify(join(dir, 'calls.log'))}`, + `if [[ "$1" == 'api' && "$2" == 'user' ]]; then printf 'qwen-code-dev-bot'; exit 0; fi`, + // Other api calls (the needs-human DELETE) succeed silently but are + // recorded above — the ack-job conditional removal is observable. + `if [[ "$1" == 'api' ]]; then exit 0; fi`, `if [[ "$1" == 'pr' && "$2" == 'view' ]]; then : > ${JSON.stringify(join(dir, 'pr-view-called'))}; ${ prViewOk - ? `printf '%s' '{"labels":[],"author":{"login":"wenshao"}}'; exit 0` + ? `printf '%s' '${JSON.stringify({ labels, author: { login: 'wenshao' } })}'; exit 0` : 'exit 1' }; fi`, `if [[ "$1" == 'pr' && "$2" == 'comment' ]]; then printf '%s' "$7" > ${JSON.stringify(join(dir, 'comment.md'))}; exit 0; fi`, @@ -5163,6 +5184,7 @@ exit 1 SKIP_LABEL: 'autofix/skip', TAKEOVER_LABEL: 'autofix/takeover', TAKEOVER_COMMAND: '@qwen-code /takeover', + NEEDS_HUMAN_LABEL: 'autofix/needs-human', ACK: ack, PR: '7368', ACK_BASE: base, @@ -5170,9 +5192,11 @@ exit 1 encoding: 'utf8', }); const commentPath = join(dir, 'comment.md'); + const callsPath = join(dir, 'calls.log'); const result = { status: proc.status, body: existsSync(commentPath) ? readFileSync(commentPath, 'utf8') : '', + calls: existsSync(callsPath) ? readFileSync(callsPath, 'utf8') : '', readPr: existsSync(join(dir, 'pr-view-called')), }; rmSync(dir, { recursive: true, force: true }); @@ -5211,6 +5235,30 @@ exit 1 const broken = runAck({ ack: 'engaged', prViewOk: false }); expect(broken.status).not.toBe(0); expect(broken.body).toBe(''); + // R4-16: the conditional needs-human removal is observable per branch — + // engaged (no skip) and released perform exactly one DELETE of the + // encoded label; base-refused / engaged-with-skip perform none. + const nhDelete = + 'api -X DELETE repos/QwenLM/qwen-code/issues/7368/labels/autofix%2Fneeds-human'; + expect( + runAck({ ack: 'engaged' }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(1); + expect(runAck({ ack: 'engaged' }).calls).toContain(nhDelete); + expect( + runAck({ ack: 'released' }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(1); + expect(runAck({ ack: 'released' }).calls).toContain(nhDelete); + expect( + runAck({ ack: 'engaged', labels: [{ name: 'autofix/skip' }] }).calls, + ).not.toContain(nhDelete); + expect( + runAck({ ack: 'base-refused', base: 'release' }).calls, + ).not.toContain(nhDelete); + expect( + runAck({ ack: 'base-refused', base: 'release' }).calls.match( + /api -X DELETE/gm, + ) ?? [], + ).toHaveLength(0); }); it('narrows the agent prompt after a timeout since the last successful round', () => { @@ -9813,6 +9861,7 @@ exit 1 apiFail = false, labels = '[]', viewFail = false, + deleteFail = '', } = {}) => { const dir = mkdtempSync(join(tmpdir(), 'autofix-rearm-')); try { @@ -9824,11 +9873,21 @@ exit 1 'if [[ "$1" == "api" && "$2" == "user" ]]; then', ` if [[ "${apiFail}" == "true" ]]; then echo "HTTP 401: Bad credentials" >&2; exit 1; fi`, ` printf '%s' "${actor}"`, + 'elif [[ "$1" == "api" && "$2" == "-X" ]]; then', + // R4-20: the DELETE can fail — non-404 surfaces the error text, + // 404 is the already-gone case. + ` if [[ -n "${deleteFail}" ]]; then echo "${deleteFail}" >&2; exit 1; fi`, 'elif [[ "$1" == "pr" && "$2" == "comment" ]]; then', ` printf '%s' "$7" > '${join(dir, 'body.txt')}'`, 'elif [[ "$1" == "pr" && "$2" == "view" ]]; then', ` if [[ "${viewFail}" == "true" ]]; then echo "GraphQL: Something went wrong" >&2; exit 1; fi`, - ` printf '%s' '{"labels":${labels}}'`, + // R4-22: only serve the labels payload when the caller actually + // asked for it — a field-selection drift must fail the guard. + ' if [[ "$*" == *"--json labels"* ]]; then', + ` printf '%s' '{"labels":${labels}}'`, + ' else', + ` printf '%s' '{"number":7354}'`, + ' fi', 'fi', ].join('\n'), ); @@ -9878,16 +9937,47 @@ exit 1 'api -X DELETE repos/QwenLM/qwen-code/issues/7354/labels/autofix%2Fneeds-human', ); expect(ok.calls.match(/^api -X DELETE/gm) ?? []).toHaveLength(1); + // R4-23: …and it is the only label WRITE of any verb — the full api-call + // census is `api user` + the one DELETE (a stray label POST would make + // this 3). + expect(ok.calls.match(/^api /gm) ?? []).toHaveLength(2); + // R4-19: the marker comment posts BEFORE the cleanup DELETE — under + // `bash -eo pipefail` a comment failure then aborts before the DELETE, + // keeping the escalation label (the reverse order would drop the label + // without posting the re-arm marker). + expect(ok.calls.indexOf('pr comment')).toBeLessThan( + ok.calls.indexOf('api -X DELETE'), + ); // No line may be indented 4+ spaces, or the marker renders as a code block // and the scanners' marker match silently fails. expect(ok.body).not.toMatch(/^ {4,}/m); + // R4-20: a non-404 DELETE failure surfaces `removal failed` but does not + // fail the step (the marker was already posted); a 404 stays silent. + const delFail = runRearm({ + actor: BOT, + deleteFail: 'HTTP 502: Bad Gateway', + }); + expect(delFail.status).toBe(0); + expect(delFail.calls).toContain('api -X DELETE'); + expect(delFail.stdout).toContain('removal failed'); + const del404 = runRearm({ + actor: BOT, + deleteFail: 'HTTP 404: Not Found', + }); + expect(del404.status).toBe(0); + expect(del404.stdout).not.toContain('removal failed'); + // R3-7: skip wins over re-arm — with autofix/skip present the stale-label // DELETE is skipped, so a frozen PR keeps its only filterable escalation // state (no scan manages it and the fresh window won't re-apply it). + // R4-25: the fixture uses the production multi-label shape (takeover + + // needs-human + skip) — a single-label payload would miss order- + // dependent guard mutants. const skipped = runRearm({ actor: BOT, - labels: '[{"name":"autofix/skip"}]', + labels: + '[{"name":"autofix/takeover"},{"name":"autofix/needs-human"},{"name":"autofix/skip"}]', }); expect(skipped.status).toBe(0); expect(skipped.calls).toContain('pr comment'); @@ -9910,6 +10000,9 @@ exit 1 expect(mismatch.status).toBe(1); expect(mismatch.calls).toContain('api user'); expect(mismatch.calls).not.toContain('pr comment'); + // R4-21: the identity failure must also withhold the needs-human DELETE — + // a mis-scoped PAT must not strip the escalation label either. + expect(mismatch.calls).not.toContain('api -X DELETE'); expect(mismatch.body).toBe(''); expect(mismatch.stdout).toContain(`expected ${BOT}`); @@ -9918,6 +10011,7 @@ exit 1 const failed = runRearm({ apiFail: true }); expect(failed.status).toBe(1); expect(failed.calls).not.toContain('pr comment'); + expect(failed.calls).not.toContain('api -X DELETE'); expect(failed.body).toBe(''); expect(failed.stdout).toContain('Failed to verify CI_DEV_BOT_PAT identity'); expect(failed.stdout).toContain('Bad credentials'); diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index c04af0c2e81..a96f28ec3e6 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -116,7 +116,7 @@ describe('fleet shepherd workflow', () => { expect(workflow).toContain('consent withdrawn, no %s'); expect(workflow).toContain('fail closed, no %s'); expect(workflow).toMatch( - /DISPATCHES\}" -ge "\$\{MAX_CONFLICT_DISPATCHES_PER_TICK\}" \]\]; then[\s\S]*?elif live_skip "\$\{PR\}"; then[\s\S]*?paused \(needs-human\)/, + /DISPATCHES\}" -ge "\$\{MAX_CONFLICT_DISPATCHES_PER_TICK\}" \]\]; then[\s\S]{0,200}elif live_skip "\$\{PR\}"; then[\s\S]{0,800}conflict_paused "\$\{PR\}"; then/, ); expect(workflow).toMatch( /SYNCS\}" -ge "\$\{MAX_SYNCS_PER_TICK\}" \]\]; then[\s\S]{0,160}elif live_skip "\$\{PR\}"; then/, @@ -609,6 +609,10 @@ exit 1`; )?.[0]; expect(takeoverEnum).toBeTruthy(); expect(takeoverEnum).not.toContain('isCrossRepository'); + // R4-30: pin the stale-first sort on THIS enumeration (not just the + // needs-human one) — over the 100 cap, unsorted order drops exactly the + // stale tail the dashboard exists to surface. + expect(takeoverEnum).toContain("--search 'sort:updated-asc'"); // Enumeration failures degrade to a loud error row and FALL THROUGH to // the dashboard write (which carries the liveness watermark) — they // must not exit the tick. The release lever's population comes from the @@ -738,6 +742,14 @@ exit 1`; expect( runCmd([human('2026-08-06T00:00:00Z', '@qwen-code /takeover stop')]), ).toBe(''); + // R4-31: two commands — the NEWEST wins (pins the `max` selection; a + // stale older command must not become CMD_TS). + expect( + runCmd([ + human('2026-08-01T00:00:00Z', '@qwen-code /retry'), + human('2026-08-06T00:00:00Z', '@qwen-code /takeover'), + ]), + ).toBe('2026-08-06T00:00:00Z\twenshao'); // A refusal ack (fork-refused / base-refused / skip-blocked) is what // supersedes a refused command — verify the variant set. const runRefusal = (comments) => @@ -855,6 +867,17 @@ exit 1`; TEST_PERM: '', }), ).toBe('RESUME_TS='); + // R4-29: a refusal OLDER than the fresh command must NOT supersede it + // (pins the `||` in the supersession check — an `&&` flip drops the + // command and reopens the R1 release race). + expect( + gate({ + RESUME_TS: '', + CMD_TS: '2026-08-06T01:00:00Z', + REFUSAL_TS: '2026-08-06T00:01:00Z', + NOW_EPOCH: String(Date.parse('2026-08-06T01:30:00Z') / 1000), + }), + ).toBe('RESUME_TS=2026-08-06T01:00:00Z'); // A fresh `labeled` event is resume evidence too (UI re-apply). const eventJq = workflow.match( /EVENT_TS="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, @@ -879,6 +902,21 @@ exit 1`; }, ]), ).toBe('2026-08-06T00:00:00Z'); + // R4-31: two takeover labeled events — the NEWEST wins (pins `max`). + expect( + runEvent([ + { + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-01T00:00:00Z', + }, + { + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-08T00:00:00Z', + }, + ]), + ).toBe('2026-08-08T00:00:00Z'); expect(runEvent([{ event: 'closed' }])).toBe(''); // The event read fails closed, like the comment read. expect(workflow).toContain( @@ -904,6 +942,20 @@ exit 1`; ), ]), ).toBe(''); + // R4-31: two terminal headlines (a re-armed PR stopped twice) — the + // NEWEST is reported (pins `last`; a stale round-1 reason must not win). + expect( + run(reasonJq, [ + bot( + '2026-08-01T00:00:00Z', + '🤖 AutoFix stopped after 3 consecutive rounds that failed to push anything …', + ), + bot( + '2026-08-04T00:00:00Z', + '🤖 AutoFix stopped: this counting window now contains 3 time-budget exhaustions …', + ), + ]), + ).toContain('time-budget exhaustions'); // The re-arm guard: a resume NEWER than the latest cap notice means the // label is stale and the release must never fire. Replay the gate // VERBATIM so a dropped comparison fails the test. @@ -948,6 +1000,51 @@ exit 1`; expect(workflow).toMatch( /SUMMARY_POSTED="\$\(jq[\s\S]{0,500}fleet-shepherd auto-release/, ); + // R4-31: replay the dedup counter's aggregation — empty stream and a + // stale (pre-term) marker both yield "0", a fresh one yields "1" (pins + // `| length`; a `last` mutant would emit null and skip the summary). + const summaryJq = workflow.match( + /SUMMARY_POSTED="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" --arg term "\$\{TERM_TS\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + expect(summaryJq).toBeTruthy(); + const runSummary = (comments, term) => + execFileSync( + 'jq', + [ + '-r', + '--arg', + 'ab', + 'qwen-code-dev-bot', + '--arg', + 'term', + term, + summaryJq, + ], + { encoding: 'utf8', input: JSON.stringify(comments) }, + ).trim(); + expect(runSummary([], '2026-08-05T00:00:00Z')).toBe('0'); + expect( + runSummary( + [ + bot( + '2026-08-04T00:00:00Z', + '🔓 … ', + ), + ], + '2026-08-05T00:00:00Z', + ), + ).toBe('0'); + expect( + runSummary( + [ + bot( + '2026-08-06T00:00:00Z', + '🔓 … ', + ), + ], + '2026-08-05T00:00:00Z', + ), + ).toBe('1'); expect(workflow).toMatch( /if act "#\$\{PR\}: post auto-release summary"[\s\S]{0,1800}if act "#\$\{PR\}: auto-release takeover/, ); @@ -1082,6 +1179,52 @@ exit 1`; }, ]), ).toBe(''); + // R4-31: full two-cycle stream WITH cycle-1's labeled event (pins + // LATEST_LABEL_TS `max`; a `min` mutant would anchor to cycle 1 and let + // the stale human unlabel heal cycle 2) — still expects ''. + expect( + runUnlabel([ + { + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-01T00:00:00Z', + }, + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'wenshao' }, + created_at: '2026-08-02T00:00:00Z', + }, + { + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-03T00:00:00Z', + }, + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'qwen-code-dev-bot' }, + created_at: '2026-08-05T00:00:00Z', + }, + ]), + ).toBe(''); + // R4-31: a human unlabel of an UNRELATED label must not count (pins the + // `.label.name == $tl` select). + expect( + runUnlabel([ + { + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-01T00:00:00Z', + }, + { + event: 'unlabeled', + label: { name: 'autofix/skip' }, + actor: { login: 'wenshao' }, + created_at: '2026-08-02T00:00:00Z', + }, + ]), + ).toBe(''); // The heal removes NEEDS_HUMAN (never takeover), act-wrapped. expect(workflow).toMatch( /clear stale \$\{NEEDS_HUMAN_LABEL\}[\s\S]{0,300}gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{NEEDS_HUMAN_LABEL\}"/, @@ -1325,11 +1468,14 @@ exit 1`; ); // R2-29: both loop-2 evidence fetches paginate (paused PRs are the // high-comment population — page 1 alone misses the newest notice). + // R4-28: pin the page-MERGE program too — GitHub returns comments + // oldest-first, so `.[0] // []` would anchor TERM_TS to a stale page-1 + // cycle while the current cycle's re-arm marker sits on page 2. expect(workflow).toMatch( - /issues\/\$\{PR\}\/events" --paginate[\s\S]{0,80}\/tmp\/tk-ev\.json/, + /issues\/\$\{PR\}\/events" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ev\.json/, ); expect(workflow).toMatch( - /issues\/\$\{PR\}\/comments" --paginate[\s\S]{0,80}\/tmp\/tk-ic\.json/, + /issues\/\$\{PR\}\/comments" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ic\.json/, ); // R2-19: the release scope check requires BOTH labels — the --arg // bindings are the load-bearing part. R3-12: the pin runs through the From 2de717128e394d73628d3af8df7d25ead2485e56 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Fri, 14 Aug 2026 01:04:43 +0800 Subject: [PATCH 10/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?5=20=E2=80=94=20trust=20boundaries=20and=20evidence=20freshness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - R4-5 residual: PR_META now fetches author so IS_BOT_AUTHOR actually resolves (the exemption was dead on arrival), with a behavioral replay. - R5-1: REMOVED_OK derives from the captured stream ('HTTP ' non-404 = not landed) instead of output emptiness — GitHub returns a body on success. - R5-2: /retry only drops needs-human when management actually resumes (takeover label present or bot-authored) — an auto-released human PR keeps its escalation label. - R5-3: conflict_paused requires a real cap notice AND a newer resume marker — label-present/notice-absent now fails closed toward paused. - R5-4: a failed permission read defers the release (PERM_READ_FAILED), never counts as no-permission — at both evaluation points. - R5-5: compute_resume_ts scans in-grace commands newest-first and permission-checks each (≤2 reads), so a stranger's echo can't shadow a maintainer's command. - R5-6: the release branch re-fetches evidence and recomputes resume state immediately before the first write. - R5-8: the heal re-checks the takeover label from the live payload before clearing needs-human. - R5-9: same-second ties resolve toward resume/release suppression in both files (RESUME>=TERM; RELEASE_ACKED >= window). - R5-10: the heal anchors to the current pause boundary (latest needs-human apply event); an absent anchor skips the cleanup, fail closed. - Tests: whole-function compute_resume_ts replay (permission/shadow/tie/ grace/refusal cases), heal anchor fixtures, toggle stub models the real DELETE body, runRearm orphan case. Co-authored-by: Qwen-Coder --- .github/workflows/qwen-autofix.yml | 73 ++-- .github/workflows/qwen-fleet-shepherd.yml | 181 +++++--- scripts/tests/qwen-autofix-workflow.test.js | 45 +- .../qwen-fleet-shepherd-workflow.test.js | 409 ++++++++---------- 4 files changed, 375 insertions(+), 333 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index b7cff7ad2b2..dea28328709 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1829,23 +1829,23 @@ jobs: # `/takeover stop` retries the removal — but must not disappear # silently either: masked, the ack reads "released" while the # loop keeps managing the PR. - # REST for the same reason as the add above; the label name is a - # path segment and contains a slash, so it must be URI-encoded. # The 404-tolerance block is pinned byte-identical to the other # workflows' label DELETE (a contract test), so REMOVED_OK — - # whether the takeover release actually LANDED — is derived from - # REMOVE_ERR's content AFTER the idiom instead of reworking it: - # empty = landed, 404 = already off (landed), anything else = - # the release did NOT land, and the needs-human removal below - # must NOT run (R4-32) — or the PR keeps the takeover label - # while losing its escalation label, and the release ack posted - # here would latch RELEASE_ACKED, blocking every automated path - # that could re-apply it. + # whether the takeover release actually LANDED — is derived + # AFTER the idiom from the captured stream: gh api prints the + # remaining-labels JSON body on success (even '[]'), while a + # failure carries "HTTP " in the error text. 404 = + # already off (landed); any other HTTP error = the release did + # NOT land and the needs-human removal below must NOT run + # (R4-32) — or the PR keeps the takeover label while losing + # its escalation label, and the release ack posted here would + # latch RELEASE_ACKED, blocking every automated path that + # could re-apply it. if ! REMOVE_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')" 2>&1)"; then [[ "${REMOVE_ERR}" == *404* ]] || echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}" fi REMOVED_OK=true - if [[ -n "${REMOVE_ERR}" && "${REMOVE_ERR}" != *404* ]]; then + if [[ "${REMOVE_ERR}" == *"HTTP "* && "${REMOVE_ERR}" != *404* ]]; then REMOVED_OK=false fi echo "🏷️ removed ${TAKEOVER_LABEL} from #${PR}" @@ -1940,16 +1940,22 @@ jobs: gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '\U0001F504 AutoFix re-armed. The next scan re-reads this PR'"'"'s feedback from the start and the round counter resets. Nothing was deleted — this marker supersedes the evaluation markers above it.\n\n
\n中文说明\n\n\U0001F504 已重新武装 AutoFix。下一次扫描会从头重新读取本 PR 的反馈，轮次计数也已重置。未删除任何内容 —— 本标记使其上方的评估标记失效。\n\n
\n\n')" echo "🔄 re-armed PR #${PR}" # Management resumed — the escalation label is stale. 404 is the - # common case (the PR was never paused). Skip wins over re-arm - # EVERYWHERE (mirrors the takeover-command guard): a frozen PR - # must keep its escalation label, because the fresh re-arm window - # won't re-apply it while skip is present and no scan manages the - # PR. And the read FAILS CLOSED (mirrors takeover-ack's exit-1): - # an unreadable label state must keep the label, never drop it. - if ! RETRY_LABELS="$(gh pr view "${PR}" --repo "${REPO}" --json labels 2> /dev/null)"; then + # common case (the PR was never paused). Remove it only when the + # PR will actually be MANAGED after this re-arm (R5-2): the scan + # candidate population is bot-authored or takeover-labeled PRs + # only, so on an auto-released human PR (no takeover label, not + # bot-authored) /retry posts a marker nothing will act on — the + # label must stay as the only filterable escalation state. Skip + # also wins over re-arm everywhere (a frozen PR keeps its label), + # and the read FAILS CLOSED (mirrors takeover-ack's exit-1). + if ! RETRY_INFO="$(gh pr view "${PR}" --repo "${REPO}" --json labels,author 2> /dev/null)"; then echo "::warning::#${PR}: label state unreadable — keeping ${NEEDS_HUMAN_LABEL} (fail closed)" - elif [[ "$(jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${RETRY_LABELS}")" == "true" ]]; then + elif [[ "$(jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${RETRY_INFO}")" == "true" ]]; then echo "🧭 ${NEEDS_HUMAN_LABEL} removal skipped: ${SKIP_LABEL} present on #${PR}" + elif [[ "$(jq -r --arg ab "${AUTOFIX_BOT}" --arg tk "${TAKEOVER_LABEL}" ' + ((.author.login // "") == $ab) or (([.labels[]?.name] | index($tk)) != null) + ' <<< "${RETRY_INFO}")" != "true" ]]; then + echo "🧭 keeping ${NEEDS_HUMAN_LABEL} on #${PR}: nothing manages it until re-engaged (no takeover label, not bot-authored)" elif ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" fi @@ -2568,10 +2574,11 @@ jobs: break fi # One PR fetch for the branch name, check rollup, creation time - # (the watermark floor below), and labels (the effective round - # cap) — avoids extra round-trips per candidate PR. + # (the watermark floor below), labels (the effective round + # cap), and author (the cap branch's bot-author exemption) — + # avoids extra round-trips per candidate PR. PR_META="$(gh pr view "${PR}" --repo "${REPO}" \ - --json headRefName,headRefOid,statusCheckRollup,createdAt,labels,isCrossRepository,headRepositoryOwner,headRepository 2> /dev/null || echo '{}')" + --json headRefName,headRefOid,statusCheckRollup,createdAt,labels,isCrossRepository,headRepositoryOwner,headRepository,author 2> /dev/null || echo '{}')" HEAD_REPO_FULL="${REPO}" if [[ "$(jq -r '.isCrossRepository // false' <<< "${PR_META}")" == "true" ]]; then HR_OWNER="$(jq -r '.headRepositoryOwner.login // ""' <<< "${PR_META}")" @@ -2950,22 +2957,24 @@ jobs: [ .[] | select((.user.login // "") == $ab) | select((.body // "") | contains("")) | select((.created_at // "") > $rt) ] | length' "${WORKDIR}/ic.json")" - # Release evidence = a takeover unlabeled EVENT newer than the - # window key. GitHub records it whenever the label comes off — - # `/takeover stop`, the ack job, or a manual UI removal — - # unlike the release-ack COMMENT, which both release paths - # tolerate losing (R4-1): the stop branch swallows a failed - # ack post, and the ack job's set -e aborts before it. A - # re-arm advances REARM_KEY past the event, re-enabling the - # label. Fail closed: an unreadable event history suppresses - # the re-label rather than risk the ping-pong. + # Release evidence = a takeover unlabeled EVENT at-or-newer + # than the window key. GitHub records it whenever the label + # comes off — `/takeover stop`, the ack job, or a manual UI + # removal — unlike the release-ack COMMENT, which both release + # paths tolerate losing (R4-1): the stop branch swallows a + # failed ack post, and the ack job's set -e aborts before it. + # A re-arm advances REARM_KEY past the event, re-enabling the + # label. Same-second ties resolve toward "released" — never + # re-escalate a completed release (R5-9). Fail closed: an + # unreadable event history suppresses the re-label rather than + # risk the ping-pong. if ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > "${WORKDIR}/ev.json"; then RELEASE_ACKED='unreadable' else RELEASE_ACKED="$(jq -r --arg tl "${TAKEOVER_LABEL}" --arg rt "${NOTICE_RT}" ' [ .[] | select((.event // "") == "unlabeled") | select((.label.name // "") == $tl) - | select((.created_at // "") > $rt) ] | length' "${WORKDIR}/ev.json")" + | select((.created_at // "") >= $rt) ] | length' "${WORKDIR}/ev.json")" fi # A release only suppresses re-labeling for HUMAN-authored PRs. # A bot-authored PR released from takeover returns to STANDARD diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index a6b9bfeef0d..72e43d73344 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -378,12 +378,77 @@ jobs: | select(((.body // "") | contains("")) or ((.body // "") | contains(""))) | .created_at ] | max // ""' /tmp/cf-ic.json)" - if [[ -n "${resume}" && ( -z "${term}" || "${resume}" > "${term}" ) ]]; then + # Only marker-confirmed re-arm clears "paused" — and only against + # a REAL cap notice. The state "label present but no notice" is + # reachable (the cap path applies the label before posting the + # notice and tolerates a lost notice; a human can delete it too): + # every takeover PR carries an older engage ack, so `term` empty + # + `resume` non-empty must NOT read as "re-armed" — that fails + # OPEN onto a genuinely paused PR (R5-3). + if [[ -n "${resume}" && -n "${term}" && "${resume}" > "${term}" ]]; then return 1 fi return 0 } + # Latest resume evidence (bot re-arm/engage markers, a trusted + # human's fresh command comment, or a fresh takeover labeled + # event) as an RFC3339 timestamp, or '' when none. A command counts + # only while FRESH (grace window), UNSUPERSEDED by a refusal ack, + # and from a write/maintain/admin author — scanned newest-first so + # a stranger's echo can't SHADOW a maintainer's command (R5-5), + # with permission reads bounded at 2 per PR. Sets PERM_READ_FAILED + # when a command's permission check could not be evaluated — the + # caller fails CLOSED on that (defer; never trust-on-error, R5-4). + compute_resume_ts() { + local ic="$1" ev="$2" + PERM_READ_FAILED='' + local resume refusal + resume="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select(((.body // "") | contains("")) + or ((.body // "") | contains(""))) + | .created_at ] | max // ""' "${ic}")" + refusal="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | test("")) + | .created_at ] | max // ""' "${ic}")" + local cands cts cauthor cage cperm reads=0 + cands="$(jq -r ' + [ .[] | select(((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /takeover" + or ((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /retry") + | {ts: (.created_at // ""), login: (.user.login // "")} ] + | sort_by(.ts) | reverse | .[] | "\(.ts)\t\(.login)"' "${ic}")" + while IFS=$'\t' read -r cts cauthor; do + [[ -z "${cts}" || -z "${cauthor}" ]] && continue + [[ "${cts}" > "${resume}" ]] || continue + [[ -z "${refusal}" || "${cts}" > "${refusal}" ]] || continue + cage=$(( NOW_EPOCH - $(date -u -d "${cts}" +%s) )) + [[ "${cage}" -lt "${RESUME_COMMAND_GRACE_SEC}" ]] || continue + [[ "${reads}" -ge 2 ]] && break + reads=$(( reads + 1 )) + if ! cperm="$(gh api "repos/${REPO}/collaborators/${cauthor}/permission" --jq '.permission // ""' 2> /dev/null)"; then + PERM_READ_FAILED=true + continue + fi + if [[ "${cperm}" == 'write' || "${cperm}" == 'maintain' || "${cperm}" == 'admin' ]]; then + resume="${cts}" + break + fi + done <<< "${cands}" + local evt + evt="$(jq -r --arg tl "${TAKEOVER_LABEL}" ' + [ .[] | select((.event // "") == "labeled") + | select((.label.name // "") == $tl) + | .created_at ] | max // ""' "${ev}")" + # A tie (same-second) resolves toward the RESUME side: suppressing + # a release is always the safer direction (R5-9). + if [[ -n "${evt}" && ! "${resume}" > "${evt}" ]]; then + resume="${evt}" + fi + printf '%s' "${resume}" + } + # CI-status classifiers shared by BOTH dashboard loops, so a # check-naming or status-set change lands in one place and the two # tables can never classify the same PR differently. @@ -715,62 +780,22 @@ jobs: [ .[] | select((.user.login // "") == $ab) | select((.body // "") | contains("")) | .created_at ] | max // ""' /tmp/tk-ic.json)" - # Resume evidence — anything newer than TERM_TS means a - # human already acted and the release must never fire: + # Resume evidence — anything at-or-newer than TERM_TS means + # a human already acted and the release must never fire; + # ties resolve toward resume (the safe direction, R5-9): # 1. the bot's resume markers (re-arm / engage ack); - # 2. a re-arm COMMAND comment, but only while FRESH and - # UNSUPERSEDED: an accepted command is acked within - # minutes (the ack carries a resume marker); a refused - # command draws a refusal ack, which supersedes it - # here; a command the route ignores (no permission) - # gets no reply at all, so the grace window is what - # stops an ignored command from vetoing the release - # forever. No permission check here on purpose — the - # route's collaborator check is the authorization - # gate, and mirroring it would only drift; + # 2. a fresh re-arm COMMAND from a write/maintain/admin + # commenter, scanned newest-first so a stranger's echo + # can't shadow a maintainer's (R4-14/R5-5), superseded + # by a refusal ack; a route-ignored command simply + # expires at the grace window; # 3. a fresh `labeled` event — a UI re-apply is # timestamped immediately, while its ack rides the # queue (or, on a lost event, the scan's idle-backoff # pickup — an hours-wide window). - RESUME_TS="$(jq -r --arg ab "${AUTOFIX_BOT}" ' - [ .[] | select((.user.login // "") == $ab) - | select(((.body // "") | contains("")) - or ((.body // "") | contains(""))) - | .created_at ] | max // ""' /tmp/tk-ic.json)" - # Latest re-arm command + its author. R4-14: a renewing - # stream of STRANGER commands must not veto the release - # forever — only a write/maintain/admin commenter's command - # counts (one permission read, only when a command is the - # sole evidence — bot markers and labeled events never need - # it). - CMD_LINE="$(jq -r ' - [ .[] | select(((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /takeover" - or ((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /retry") - | {ts: (.created_at // ""), login: (.user.login // "")} ] - | max_by(.ts) // empty | "\(.ts)\t\(.login)"' /tmp/tk-ic.json)" - CMD_TS="${CMD_LINE%%$'\t'*}" - CMD_AUTHOR="${CMD_LINE##*$'\t'}" - REFUSAL_TS="$(jq -r --arg ab "${AUTOFIX_BOT}" ' - [ .[] | select((.user.login // "") == $ab) - | select((.body // "") | test("")) - | .created_at ] | max // ""' /tmp/tk-ic.json)" - if [[ -n "${CMD_TS}" && "${CMD_TS}" > "${RESUME_TS}" ]] \ - && [[ -z "${REFUSAL_TS}" || "${CMD_TS}" > "${REFUSAL_TS}" ]]; then - CMD_AGE=$(( NOW_EPOCH - $(date -u -d "${CMD_TS}" +%s) )) - if [[ "${CMD_AGE}" -lt "${RESUME_COMMAND_GRACE_SEC}" ]] && [[ -n "${CMD_AUTHOR}" ]]; then - CMD_PERM="$(gh api "repos/${REPO}/collaborators/${CMD_AUTHOR}/permission" --jq '.permission // ""' 2> /dev/null || echo '')" - if [[ "${CMD_PERM}" == 'write' || "${CMD_PERM}" == 'maintain' || "${CMD_PERM}" == 'admin' ]]; then - RESUME_TS="${CMD_TS}" - fi - fi - fi - EVENT_TS="$(jq -r --arg tl "${TAKEOVER_LABEL}" ' - [ .[] | select((.event // "") == "labeled") - | select((.label.name // "") == $tl) - | .created_at ] | max // ""' /tmp/tk-ev.json)" - if [[ -n "${EVENT_TS}" && "${EVENT_TS}" > "${RESUME_TS}" ]]; then - RESUME_TS="${EVENT_TS}" - fi + # The permission read fails CLOSED: PERM_READ_FAILED defers + # the release below instead of trusting on error (R5-4). + RESUME_TS="$(compute_resume_ts /tmp/tk-ic.json /tmp/tk-ev.json)" # The stop reason comes from the terminal round's headline — # the scan-side notice always says "round cap" even when a # breaker (consecutive failures, time budget) fired first. @@ -787,7 +812,12 @@ jobs: REASON="${REASON:0:120}" if [[ -z "${TERM_TS}" ]]; then NOTE='pause timestamp unreadable — release deferred (fail closed)' - elif [[ -n "${RESUME_TS}" && "${RESUME_TS}" > "${TERM_TS}" ]]; then + elif [[ "${PERM_READ_FAILED}" == "true" ]]; then + # A fresh command's permission could not be verified — + # fail closed: the command MIGHT be a maintainer re-arm + # (R5-4), so defer rather than treat it as no-permission. + NOTE='command-permission read failed — release deferred this tick (fail closed)' + elif [[ -n "${RESUME_TS}" && ! "${TERM_TS}" > "${RESUME_TS}" ]]; then STATE='managed (re-armed)' # The PR is re-armed but still carries needs-human — a # resume-side removal failed. Clear the stale label here @@ -824,6 +854,17 @@ jobs: NOTE="$(skip_note release)" elif [[ "$(jq -r --arg a "${TAKEOVER_LABEL}" --arg b "${NEEDS_HUMAN_LABEL}" '([.labels[]?.name] | index($a) != null) and ([.labels[]?.name] | index($b) != null)' <<< "${LIVE_LABELS_JSON}")" != "true" ]]; then NOTE='labels changed since the snapshot — deferring release' + elif ! gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ic2.json \ + || ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev2.json; then + NOTE='evidence re-check unreadable — release deferred (fail closed)' + elif RESUME_NOW="$(compute_resume_ts /tmp/tk-ic2.json /tmp/tk-ev2.json)"; [[ -n "${RESUME_NOW}" && ! "${TERM_TS}" > "${RESUME_NOW}" ]]; then + # R5-6: the resume evidence above was read at loop + # entry — a re-arm landing since must veto NOW. The + # re-fetch + recompute runs immediately before the + # first write, bounded by the per-tick budget. + NOTE='resume evidence appeared during evaluation — release cancelled' + elif [[ "${PERM_READ_FAILED}" == "true" ]]; then + NOTE='command-permission read failed — release deferred this tick (fail closed)' else # Consume the budget BEFORE the first external write: # a release ATTEMPT is what is bounded. Counting only @@ -873,25 +914,37 @@ jobs: # the escalation label there), clear the stale label. The # shepherd's own auto-release authenticates as the bot and # is NOT a heal trigger. - # ...only a human unlabel NEWER than the latest label-apply - # counts — an unlabel from an EARLIER takeover cycle must - # never heal this cycle's label. - LATEST_LABEL_TS="$(jq -r --arg tl "${TAKEOVER_LABEL}" ' + # Anchor to the CURRENT pause boundary: the latest + # needs-human label-apply EVENT (the cap branch applies the + # label at pause time). A human takeover-unlabel counts only + # when NEWER than that — anything older belongs to an earlier + # cycle (R5-10) — and an absent anchor (e.g. past the ~90-day + # events lookback) means "cannot correlate" → skip the + # cleanup rather than admit everything (fail closed). + NH_APPLY_TS="$(jq -r --arg nl "${NEEDS_HUMAN_LABEL}" ' [ .[] | select((.event // "") == "labeled") - | select((.label.name // "") == $tl) + | select((.label.name // "") == $nl) | .created_at ] | max // ""' /tmp/tk-ev.json)" - UNLABEL_ACTOR="$(jq -r --arg tl "${TAKEOVER_LABEL}" --arg ab "${AUTOFIX_BOT}" --arg ll "${LATEST_LABEL_TS}" ' - [ .[] | select((.event // "") == "unlabeled") - | select((.label.name // "") == $tl) - | select((.actor.login // "") != $ab) - | select((.created_at // "") > $ll) - | .actor.login ] | last // ""' /tmp/tk-ev.json)" + UNLABEL_ACTOR='' + if [[ -n "${NH_APPLY_TS}" ]]; then + UNLABEL_ACTOR="$(jq -r --arg tl "${TAKEOVER_LABEL}" --arg ab "${AUTOFIX_BOT}" --arg ll "${NH_APPLY_TS}" ' + [ .[] | select((.event // "") == "unlabeled") + | select((.label.name // "") == $tl) + | select((.actor.login // "") != $ab) + | select((.created_at // "") > $ll) + | .actor.login ] | last // ""' /tmp/tk-ev.json)" + fi NOTE='loop stopped — needs a human decision (merge / close / split / re-engage)' if [[ -n "${UNLABEL_ACTOR}" ]]; then if [[ "${CLEANUPS}" -ge "${MAX_CLEANUPS_PER_TICK}" ]]; then NOTE='cleanup budget reached this tick' elif live_skip "${PR}"; then NOTE="$(skip_note cleanup)" + elif [[ "$(jq -r --arg t "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${LIVE_LABELS_JSON}")" == "true" ]]; then + # R5-8: takeover was re-applied between the tick-start + # snapshot and now — the PR is managed again, so nothing + # here is stale. + NOTE='takeover label re-applied since the snapshot — cleanup cancelled' elif act "#${PR}: clear stale ${NEEDS_HUMAN_LABEL} (manual release by @${UNLABEL_ACTOR})" \ gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')"; then CLEANUPS=$(( CLEANUPS + 1 )) diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index f3513f8b3b4..cb1118ab659 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -2650,7 +2650,7 @@ describe('qwen-autofix workflow', () => { ).length - 1, ).toBe(2); expect(reviewScanJob).toContain( - '--json headRefName,headRefOid,statusCheckRollup,createdAt,labels', + '--json headRefName,headRefOid,statusCheckRollup,createdAt,labels,isCrossRepository,headRepositoryOwner,headRepository,author', ); // Command-style comments are instructions, not feedback — excluded at // ALL FIVE feedback sites (scan count via $cf; NEWEST, LIVE_NEW, @@ -3200,6 +3200,26 @@ describe('qwen-autofix workflow', () => { ); expect(releaseGate).toBeGreaterThan(-1); expect(releaseGate).toBeLessThan(labelPost); + // R4-5 residual: IS_BOT_AUTHOR must actually resolve from PR_META — the + // fetch's field list must carry `author` and the jq must read + // .author.login. Replay both so a field-list drift fails (a dead + // IS_BOT_AUTHOR=false would suppress the label for bot PRs forever). + const botAuthorJq = reviewScanJob.match( + /IS_BOT_AUTHOR="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([^']+)' <<< "\$\{PR_META\}"\)"/, + )?.[1]; + expect(botAuthorJq).toBeTruthy(); + const runBotAuthor = (login) => + execFileSync( + 'jq', + ['-r', '--arg', 'ab', 'qwen-code-dev-bot', botAuthorJq], + { + encoding: 'utf8', + input: JSON.stringify({ author: { login } }), + }, + ).trim(); + expect(runBotAuthor('qwen-code-dev-bot')).toBe('true'); + expect(runBotAuthor('wenshao')).toBe('false'); + expect(runBotAuthor('')).toBe('false'); // R4-15: the RELEASE_ACKED jq body (event marker + time direction) is // replayed — a `> $rt` → `< $rt` flip must fail, not ship green. const relAckJq = reviewScanJob.match( @@ -3473,7 +3493,7 @@ describe('qwen-autofix workflow', () => { // the block's two failure policies instead of the stub // universally exiting 0. `elif [[ "$1" == "label" && "$2" == "create" ]]; then echo "LABEL-CREATE $*" >> '${join(dir, 'writes.log')}';`, - `elif [[ "$1" == "api" ]]; then echo "API $*" >> '${join(dir, 'writes.log')}'; if [[ "$2" == "-X" && "$3" == "POST" && -n "\${TOGGLE_POST_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_POST_FAILS}" >&2; exit 1; fi; if [[ "$2" == "-X" && "$3" == "DELETE" && -n "\${TOGGLE_DELETE_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_DELETE_FAILS}" >&2; exit 1; fi`, + `elif [[ "$1" == "api" ]]; then echo "API $*" >> '${join(dir, 'writes.log')}'; if [[ "$2" == "-X" && "$3" == "POST" && -n "\${TOGGLE_POST_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_POST_FAILS}" >&2; exit 1; fi; if [[ "$2" == "-X" && "$3" == "DELETE" && -n "\${TOGGLE_DELETE_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_DELETE_FAILS}" >&2; exit 1; fi; if [[ "$2" == "-X" && "$3" == "DELETE" ]]; then printf '%s' '[]'; fi`, `elif [[ "$1" == "pr" && "$2" == "comment" ]]; then echo "COMMENT $*" >> '${join(dir, 'writes.log')}';`, 'fi', ].join('\n'), @@ -3706,6 +3726,8 @@ describe('qwen-autofix workflow', () => { ); expect(releaseRace.writes).toContain('takeover-ack released'); expect(releaseRace.log).not.toContain('::warning::'); + // R5-1: a 404 counts as landed, so the needs-human cleanup still runs… + expect(releaseRace.writes).toContain('labels/autofix%2Fneeds-human'); // Any other DELETE failure must not drop the release ack either — a // later `/takeover stop` retries the removal — but it MUST warn: // masked, the ack reads "released" while the loop keeps managing the @@ -3719,6 +3741,12 @@ describe('qwen-autofix workflow', () => { expect(releaseFailed.log).toContain('::warning::'); expect(releaseFailed.log).toContain('removal failed'); expect(releaseFailed.log).toContain('HTTP 500'); + // R5-1: …but a failed takeover release keeps needs-human — REMOVED_OK is + // derived from the captured stream ("HTTP " non-404 = did not land), and + // the stub emits the remaining-labels body on success to model the real + // API contract. + expect(releaseFailed.writes).not.toContain('labels/autofix%2Fneeds-human'); + expect(releaseFailed.log).toContain('release did not land'); // Both ack posts keep their non-fatal fallback: under bash -e a failed // gh pr comment would otherwise abort the step RED after the label was // already toggled — a worse signal than the silence being fixed. A @@ -9860,6 +9888,7 @@ exit 1 actor = BOT, apiFail = false, labels = '[]', + prAuthor = BOT, viewFail = false, deleteFail = '', } = {}) => { @@ -9884,7 +9913,7 @@ exit 1 // R4-22: only serve the labels payload when the caller actually // asked for it — a field-selection drift must fail the guard. ' if [[ "$*" == *"--json labels"* ]]; then', - ` printf '%s' '{"labels":${labels}}'`, + ` printf '%s' '{"labels":${labels},"author":{"login":"${prAuthor}"}}'`, ' else', ` printf '%s' '{"number":7354}'`, ' fi', @@ -9903,6 +9932,7 @@ exit 1 AUTOFIX_BOT: BOT, NEEDS_HUMAN_LABEL: 'autofix/needs-human', SKIP_LABEL: 'autofix/skip', + TAKEOVER_LABEL: 'autofix/takeover', }, encoding: 'utf8', }); @@ -9993,6 +10023,15 @@ exit 1 expect(readFailed.calls).not.toContain('api -X DELETE'); expect(readFailed.stdout).toContain('label state unreadable'); + // R5-2: an auto-released HUMAN-authored PR (no takeover label, not + // bot-authored) is in no scan candidate set — /retry must keep the + // escalation label, since nothing will re-apply it. + const orphan = runRearm({ actor: BOT, prAuthor: 'wenshao' }); + expect(orphan.status).toBe(0); + expect(orphan.calls).toContain('pr comment'); + expect(orphan.calls).not.toContain('api -X DELETE'); + expect(orphan.stdout).toContain('nothing manages it until re-engaged'); + // Actor mismatch: the PAT authenticates as someone else -> the guard exits // non-zero and posts nothing (a mis-scoped PAT must not leave a stranded PR // behind a fake "re-armed" line). diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index a96f28ec3e6..9dc48192565 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -669,7 +669,7 @@ exit 1`; /TERM_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, )?.[1]; const resumeJq = workflow.match( - /RESUME_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + /local resume refusal\n\s+resume="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' "\$\{ic\}"\)"/, )?.[1]; const reasonJq = workflow.match( /REASON="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, @@ -712,212 +712,156 @@ exit 1`; expect(run(resumeJq, [bot('2026-08-02T00:00:00Z', '⏸️ unrelated')])).toBe( '', ); - // …a re-arm COMMAND comment is ALSO resume evidence, but only while - // fresh and unsuperseded: the command's ack rides a queued job, so - // counting only the ack would let a tick release a PR minutes after a - // human re-armed it. R4-14: the commenter's permission is checked before - // a command counts — a renewing stream of stranger commands must not - // veto a release forever. - const cmdLineJq = workflow.match( - /CMD_LINE="\$\(jq -r '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + // Resume evidence is computed by compute_resume_ts — replay the WHOLE + // function verbatim (markers + trusted commands + labeled events + + // permission gate + tie-break), with `gh` and GNU date stubbed. Each + // case pins one of R4-14 / R5-4 / R5-5 / R5-9. + const resumeFn = workflow.match( + /(compute_resume_ts\(\) \{[\s\S]*?\n {10}\})/, )?.[1]; - const refusalJq = workflow.match( - /REFUSAL_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, - )?.[1]; - expect(cmdLineJq).toBeTruthy(); - expect(refusalJq).toBeTruthy(); - const runCmd = (comments) => - execFileSync('jq', ['-r', cmdLineJq], { - encoding: 'utf8', - input: JSON.stringify(comments), - }).trim(); - // Exact trimmed command match → "tslogin", any author. - expect( - runCmd([human('2026-08-06T00:00:00Z', ' @qwen-code /takeover ')]), - ).toBe('2026-08-06T00:00:00Z\twenshao'); - expect(runCmd([human('2026-08-06T00:00:00Z', '@qwen-code /retry')])).toBe( - '2026-08-06T00:00:00Z\twenshao', - ); - // …but `/takeover stop` is a release, not a resume. - expect( - runCmd([human('2026-08-06T00:00:00Z', '@qwen-code /takeover stop')]), - ).toBe(''); - // R4-31: two commands — the NEWEST wins (pins the `max` selection; a - // stale older command must not become CMD_TS). - expect( - runCmd([ - human('2026-08-01T00:00:00Z', '@qwen-code /retry'), - human('2026-08-06T00:00:00Z', '@qwen-code /takeover'), - ]), - ).toBe('2026-08-06T00:00:00Z\twenshao'); - // A refusal ack (fork-refused / base-refused / skip-blocked) is what - // supersedes a refused command — verify the variant set. - const runRefusal = (comments) => - execFileSync( - 'jq', - ['-r', '--arg', 'ab', 'qwen-code-dev-bot', refusalJq], - { encoding: 'utf8', input: JSON.stringify(comments) }, - ).trim(); - expect( - runRefusal([ - bot('2026-08-06T00:01:00Z', '🚫 … '), - ]), - ).toBe('2026-08-06T00:01:00Z'); - expect( - runRefusal([ - bot('2026-08-06T00:01:00Z', '🤝 … '), - ]), - ).toBe(''); - // The full gate: command evidence counts only when it is newer than the - // marker evidence, newer than any refusal, inside the grace window, AND - // the commenter holds write/maintain/admin (R4-14). Stub `gh` so the - // permission read returns a test-controlled value. - const cmdGate = workflow.match( - /(if \[\[ -n "\$\{CMD_TS\}" && "\$\{CMD_TS\}" > "\$\{RESUME_TS\}" \]\] \\[\s\S]*?\n {16}fi)/, - )?.[1]; - expect(cmdGate).toBeTruthy(); - // The extracted block runs GNU `date -u -d`; on macOS hosts BSD date - // lacks -d, so shim it through node (always present in this repo). + expect(resumeFn).toBeTruthy(); const gnuDateShim = `date() { if [[ "$1" == '-u' && "$2" == '-d' ]]; then node -e 'console.log(Math.floor(new Date(process.argv[1]).getTime()/1000))' "$3"; else command date "$@"; fi; }`; - const gate = (env) => { - const dir = mkdtempSync(join(tmpdir(), 'shep-gate-')); + const computeResume = ({ + comments = [], + events = [], + perm = 'write', + permFail = false, + now = '2026-08-06T00:30:00Z', + }) => { + const dir = mkdtempSync(join(tmpdir(), 'resume-')); try { + writeFileSync(join(dir, 'ic.json'), JSON.stringify(comments)); + writeFileSync(join(dir, 'ev.json'), JSON.stringify(events)); writeFileSync( join(dir, 'gh'), - '#!/bin/bash\nif [[ "$1" == "api" ]]; then printf \'%s\' "${TEST_PERM}"; fi', + [ + '#!/bin/bash', + 'if [[ "$1" == "api" ]]; then', + ` if [[ "${permFail}" == "true" ]]; then echo "HTTP 502" >&2; exit 1; fi`, + // Per-user permission: the collaborators path ($2) names the + // user; `perm` is the default, `stranger` always reads `read`. + ` case "$2" in *collaborators/stranger/*) printf '%s' "read" ;; *) printf '%s' "${perm}" ;; esac`, + 'fi', + ].join('\n'), ); chmodSync(join(dir, 'gh'), 0o755); - return execFileSync( + const out = execFileSync( 'bash', [ '-c', - `${gnuDateShim}\n${cmdGate.replace(/\n {16}/g, '\n')}\necho "RESUME_TS=$RESUME_TS"`, + `${gnuDateShim}\n${resumeFn.replace(/\n {10}/g, '\n')}\ncompute_resume_ts "${dir}/ic.json" "${dir}/ev.json"\necho "PERM_FAILED=$PERM_READ_FAILED"`, ], { env: { ...process.env, PATH: `${dir}:${process.env.PATH}`, REPO: 'QwenLM/qwen-code', - NOW_EPOCH: '1786000000', + AUTOFIX_BOT: 'qwen-code-dev-bot', + TAKEOVER_LABEL: 'autofix/takeover', RESUME_COMMAND_GRACE_SEC: '7200', - TEST_PERM: 'write', - CMD_AUTHOR: 'wenshao', - ...env, + NOW_EPOCH: String(Date.parse(now) / 1000), }, encoding: 'utf8', }, ).trim(); + const [ts, flag] = out.split('PERM_FAILED='); + return { ts, flag }; } finally { rmSync(dir, { recursive: true, force: true }); } }; - // fresh accepted-but-unacked command → counts (the R1 race fix). - expect( - gate({ - RESUME_TS: '', - CMD_TS: '2026-08-06T00:00:00Z', - REFUSAL_TS: '', - NOW_EPOCH: String(Date.parse('2026-08-06T00:30:00Z') / 1000), - }), - ).toBe(`RESUME_TS=2026-08-06T00:00:00Z`); - // refused command (refusal ack newer) → does NOT count (R1-2). + const cmd = (ts, login, body = '@qwen-code /takeover') => ({ + user: { login }, + created_at: ts, + body, + }); + const evt = (ts, name = 'autofix/takeover', kind = 'labeled') => ({ + event: kind, + label: { name }, + created_at: ts, + }); + // bot markers: newest re-arm/engage wins. expect( - gate({ - RESUME_TS: '', - CMD_TS: '2026-08-06T00:00:00Z', - REFUSAL_TS: '2026-08-06T00:01:00Z', - NOW_EPOCH: String(Date.parse('2026-08-06T00:30:00Z') / 1000), - }), - ).toBe('RESUME_TS='); - // ignored command (no ack ever) past the grace window → expires (R1-1). + computeResume({ + comments: [ + bot('2026-08-02T00:00:00Z', '🔄 … '), + bot('2026-08-03T00:00:00Z', '🤝 … '), + ], + }).ts, + ).toBe('2026-08-03T00:00:00Z'); + // a fresh trusted command counts (the R1 race fix)… expect( - gate({ - RESUME_TS: '', - CMD_TS: '2026-08-06T00:00:00Z', - REFUSAL_TS: '', - NOW_EPOCH: String(Date.parse('2026-08-06T03:00:00Z') / 1000), - }), - ).toBe('RESUME_TS='); - // an already-acked command never overrides its own newer marker. + computeResume({ comments: [cmd('2026-08-06T00:00:00Z', 'wenshao')] }).ts, + ).toBe('2026-08-06T00:00:00Z'); + // …but a stranger's command does not (R4-14)… expect( - gate({ - RESUME_TS: '2026-08-06T00:05:00Z', - CMD_TS: '2026-08-06T00:00:00Z', - REFUSAL_TS: '', - NOW_EPOCH: String(Date.parse('2026-08-06T00:30:00Z') / 1000), - }), - ).toBe('RESUME_TS=2026-08-06T00:05:00Z'); - // R4-14: a stranger's fresh command (read/no permission) does NOT count, - // even inside the grace window. + computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'stranger')], + perm: 'read', + }).ts, + ).toBe(''); + // …and a stale command past the grace window expires (R1-1)… expect( - gate({ - RESUME_TS: '', - CMD_TS: '2026-08-06T00:00:00Z', - REFUSAL_TS: '', - NOW_EPOCH: String(Date.parse('2026-08-06T00:30:00Z') / 1000), - TEST_PERM: 'read', - }), - ).toBe('RESUME_TS='); + computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'wenshao')], + now: '2026-08-06T03:00:00Z', + }).ts, + ).toBe(''); + // …a refusal ack NEWER than the command supersedes it (R1-2)… expect( - gate({ - RESUME_TS: '', - CMD_TS: '2026-08-06T00:00:00Z', - REFUSAL_TS: '', - NOW_EPOCH: String(Date.parse('2026-08-06T00:30:00Z') / 1000), - TEST_PERM: '', - }), - ).toBe('RESUME_TS='); - // R4-29: a refusal OLDER than the fresh command must NOT supersede it - // (pins the `||` in the supersession check — an `&&` flip drops the - // command and reopens the R1 release race). + computeResume({ + comments: [ + cmd('2026-08-06T00:00:00Z', 'wenshao'), + bot( + '2026-08-06T00:01:00Z', + '🚫 … ', + ), + ], + }).ts, + ).toBe(''); + // …but a refusal OLDER than the fresh command does not (R4-29)… expect( - gate({ - RESUME_TS: '', - CMD_TS: '2026-08-06T01:00:00Z', - REFUSAL_TS: '2026-08-06T00:01:00Z', - NOW_EPOCH: String(Date.parse('2026-08-06T01:30:00Z') / 1000), - }), - ).toBe('RESUME_TS=2026-08-06T01:00:00Z'); - // A fresh `labeled` event is resume evidence too (UI re-apply). - const eventJq = workflow.match( - /EVENT_TS="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, - )?.[1]; - expect(eventJq).toBeTruthy(); - const runEvent = (events) => - execFileSync('jq', ['-r', '--arg', 'tl', 'autofix/takeover', eventJq], { - encoding: 'utf8', - input: JSON.stringify(events), - }).trim(); + computeResume({ + comments: [ + bot( + '2026-08-06T00:01:00Z', + '🚫 … ', + ), + cmd('2026-08-06T01:00:00Z', 'wenshao'), + ], + now: '2026-08-06T01:30:00Z', + }).ts, + ).toBe('2026-08-06T01:00:00Z'); + // …and a stranger's echo NEWER than the maintainer's command cannot + // shadow it — every in-grace command is permission-checked (R5-5), so + // the maintainer's still promotes. expect( - runEvent([ - { - event: 'labeled', - label: { name: 'autofix/takeover' }, - created_at: '2026-08-06T00:00:00Z', - }, - { - event: 'labeled', - label: { name: 'other' }, - created_at: '2026-08-07T00:00:00Z', - }, - ]), + computeResume({ + comments: [ + cmd('2026-08-06T00:00:00Z', 'wenshao'), + cmd('2026-08-06T00:10:00Z', 'stranger'), + ], + }).ts, ).toBe('2026-08-06T00:00:00Z'); - // R4-31: two takeover labeled events — the NEWEST wins (pins `max`). + // a labeled event counts (UI re-apply)… + expect(computeResume({ events: [evt('2026-08-06T00:00:00Z')] }).ts).toBe( + '2026-08-06T00:00:00Z', + ); + // …and a same-second tie resolves toward RESUME (R5-9). expect( - runEvent([ - { - event: 'labeled', - label: { name: 'autofix/takeover' }, - created_at: '2026-08-01T00:00:00Z', - }, - { - event: 'labeled', - label: { name: 'autofix/takeover' }, - created_at: '2026-08-08T00:00:00Z', - }, - ]), - ).toBe('2026-08-08T00:00:00Z'); - expect(runEvent([{ event: 'closed' }])).toBe(''); + computeResume({ + comments: [bot('2026-08-06T00:00:00Z', '🔄 … ')], + events: [evt('2026-08-06T00:00:00Z')], + }).ts, + ).toBe('2026-08-06T00:00:00Z'); + // a failed permission read sets PERM_READ_FAILED and does not promote + // the command (R5-4). + const permFailed = computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'wenshao')], + permFail: true, + }); + expect(permFailed.ts).toBe(''); + expect(permFailed.flag).toBe('true'); // The event read fails closed, like the comment read. expect(workflow).toContain( 'event read failed — evaluation deferred this tick (fail closed)', @@ -956,11 +900,12 @@ exit 1`; ), ]), ).toContain('time-budget exhaustions'); - // The re-arm guard: a resume NEWER than the latest cap notice means the - // label is stale and the release must never fire. Replay the gate - // VERBATIM so a dropped comparison fails the test. + // The re-arm guard: resume evidence AT-OR-NEWER than the latest cap + // notice means a human acted and the release must never fire (ties + // resolve toward resume — R5-9). Replay the gate VERBATIM so a dropped + // comparison fails the test. const rearmGuard = workflow.match( - /(elif \[\[ -n "\$\{RESUME_TS\}" && "\$\{RESUME_TS\}" > "\$\{TERM_TS\}" \]\]; then\n {18}STATE='managed \(re-armed\)')/, + /(elif \[\[ -n "\$\{RESUME_TS\}" && ! "\$\{TERM_TS\}" > "\$\{RESUME_TS\}" \]\]; then\n {18}STATE='managed \(re-armed\)')/, )?.[1]; expect(rearmGuard).toBeTruthy(); const guard = (resume, term) => @@ -981,6 +926,11 @@ exit 1`; expect(guard('2026-08-06T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( 'REARMED', ); + // A same-second tie resolves toward RESUME (the release stays + // suppressed) — R5-9. + expect(guard('2026-08-05T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( + 'REARMED', + ); expect(guard('2026-08-04T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( 'RELEASE-ELIGIBLE', ); @@ -1086,25 +1036,27 @@ exit 1`; ); // The stale-label heal (R1-13): a HUMAN unlabeled event on an // awaiting-human PR clears the stale escalation label — budgeted, - // skip-vetoed, never triggered by the bot's own auto-release, and (R2-1) - // correlated to the CURRENT takeover cycle: only an unlabel newer than - // the latest label-apply counts, so a human unlabel from an EARLIER - // cycle cannot heal this cycle's label. + // skip-vetoed, never triggered by the bot's own auto-release, and (R5-10) + // anchored to the CURRENT pause boundary (the latest needs-human + // label-apply event), so a stale unlabel from an EARLIER cycle can't + // heal this cycle's label, and an absent anchor skips the cleanup + // (fail closed). expect(workflow).toContain('"${CLEANUPS}" -ge "${MAX_CLEANUPS_PER_TICK}"'); - const latestLabelJq = workflow.match( - /LATEST_LABEL_TS="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, + const nhApplyJq = workflow.match( + /NH_APPLY_TS="\$\(jq -r --arg nl "\$\{NEEDS_HUMAN_LABEL\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, )?.[1]; const unlabelJq = workflow.match( - /UNLABEL_ACTOR="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" --arg ab "\$\{AUTOFIX_BOT\}" --arg ll "\$\{LATEST_LABEL_TS\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, + /UNLABEL_ACTOR="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" --arg ab "\$\{AUTOFIX_BOT\}" --arg ll "\$\{NH_APPLY_TS\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, )?.[1]; - expect(latestLabelJq).toBeTruthy(); + expect(nhApplyJq).toBeTruthy(); expect(unlabelJq).toBeTruthy(); const runUnlabel = (events) => { const ll = execFileSync( 'jq', - ['-r', '--arg', 'tl', 'autofix/takeover', latestLabelJq], + ['-r', '--arg', 'nl', 'autofix/needs-human', nhApplyJq], { encoding: 'utf8', input: JSON.stringify(events) }, ).trim(); + if (ll === '') return ''; return execFileSync( 'jq', [ @@ -1123,14 +1075,16 @@ exit 1`; { encoding: 'utf8', input: JSON.stringify(events) }, ).trim(); }; - // Same-cycle manual release → heal fires. + const nhLabeled = (ts) => ({ + event: 'labeled', + label: { name: 'autofix/needs-human' }, + created_at: ts, + }); + // Same-cycle manual release → heal fires (human unlabel newer than the + // current pause's label-apply). expect( runUnlabel([ - { - event: 'labeled', - label: { name: 'autofix/takeover' }, - created_at: '2026-08-01T00:00:00Z', - }, + nhLabeled('2026-08-01T12:00:00Z'), { event: 'unlabeled', label: { name: 'autofix/takeover' }, @@ -1143,11 +1097,7 @@ exit 1`; // cleanup — needs-human survives auto-release by design. expect( runUnlabel([ - { - event: 'labeled', - label: { name: 'autofix/takeover' }, - created_at: '2026-08-01T00:00:00Z', - }, + nhLabeled('2026-08-01T12:00:00Z'), { event: 'unlabeled', label: { name: 'autofix/takeover' }, @@ -1156,8 +1106,9 @@ exit 1`; }, ]), ).toBe(''); - // R2-1's two-cycle fixture: cycle-1 human release, cycle-2 re-label + - // bot auto-release — the stale human event must NOT heal cycle 2. + // R5-10's core scenario: a stale human unlabel OLDER than the current + // pause's needs-human apply must not heal (cycle-1 human release, then + // /retry re-arm + re-cap re-applied needs-human). expect( runUnlabel([ { @@ -1171,6 +1122,7 @@ exit 1`; label: { name: 'autofix/takeover' }, created_at: '2026-08-02T00:00:00Z', }, + nhLabeled('2026-08-03T00:00:00Z'), { event: 'unlabeled', label: { name: 'autofix/takeover' }, @@ -1179,44 +1131,23 @@ exit 1`; }, ]), ).toBe(''); - // R4-31: full two-cycle stream WITH cycle-1's labeled event (pins - // LATEST_LABEL_TS `max`; a `min` mutant would anchor to cycle 1 and let - // the stale human unlabel heal cycle 2) — still expects ''. + // An absent anchor (needs-human apply beyond the ~90-day events + // lookback) → cannot correlate → cleanup skipped (fail closed). expect( runUnlabel([ - { - event: 'labeled', - label: { name: 'autofix/takeover' }, - created_at: '2026-08-01T00:00:00Z', - }, { event: 'unlabeled', label: { name: 'autofix/takeover' }, actor: { login: 'wenshao' }, created_at: '2026-08-02T00:00:00Z', }, - { - event: 'labeled', - label: { name: 'autofix/takeover' }, - created_at: '2026-08-03T00:00:00Z', - }, - { - event: 'unlabeled', - label: { name: 'autofix/takeover' }, - actor: { login: 'qwen-code-dev-bot' }, - created_at: '2026-08-05T00:00:00Z', - }, ]), ).toBe(''); - // R4-31: a human unlabel of an UNRELATED label must not count (pins the + // A human unlabel of an UNRELATED label must not count (pins the // `.label.name == $tl` select). expect( runUnlabel([ - { - event: 'labeled', - label: { name: 'autofix/takeover' }, - created_at: '2026-08-01T00:00:00Z', - }, + nhLabeled('2026-08-01T12:00:00Z'), { event: 'unlabeled', label: { name: 'autofix/skip' }, @@ -1297,7 +1228,7 @@ exit 1`; )?.[1]; expect(rearmKey).toBeTruthy(); const resumeTs = workflow.match( - /RESUME_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + /local resume refusal\n\s+resume="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' "\$\{ic\}"\)"/, )?.[1]; expect(resumeTs).toBeTruthy(); const markerSet = (p) => @@ -1315,13 +1246,13 @@ exit 1`; expect(autofix).toContain("TAKEOVER_COMMAND: '@qwen-code /takeover'"); expect(autofix).toContain("RETRY_COMMAND: '@qwen-code /retry'"); const cmdTs = workflow.match( - /CMD_LINE="\$\(jq -r '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + /cands="\$\(jq -r '([\s\S]*?)' "\$\{ic\}"\)"/, )?.[1]; expect(cmdTs).toBeTruthy(); expect(cmdTs).toContain('"@qwen-code /takeover"'); expect(cmdTs).toContain('"@qwen-code /retry"'); const refusalTs = workflow.match( - /REFUSAL_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + /refusal="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' "\$\{ic\}"\)"/, )?.[1]; expect(refusalTs).toBeTruthy(); // Compare the refusal-variant SET against the producer's full ack @@ -1488,15 +1419,25 @@ exit 1`; expect(workflow).toContain( 'LIVE_LABELS_JSON="$(gh pr view "${pr}" --repo "${REPO}" --json labels', ); - // R2-12: the labeled-event merge into RESUME_TS exists… + // R2-12/R5-9: the labeled-event merge lives in compute_resume_ts with a + // tie-safe comparison — an event at-or-newer than other evidence wins. expect(workflow).toMatch( - /if \[\[ -n "\$\{EVENT_TS\}" && "\$\{EVENT_TS\}" > "\$\{RESUME_TS\}" \]\]; then\n\s+RESUME_TS="\$\{EVENT_TS\}"/, + /if \[\[ -n "\$\{evt\}" && ! "\$\{resume\}" > "\$\{evt\}" \]\]; then\n\s+resume="\$\{evt\}"/, ); - // R2-21: …and both promotions run BEFORE the evaluation chain consumes - // RESUME_TS (position-independent extractions can't see this). + // R2-21: the single compute_resume_ts call runs BEFORE the evaluation + // chain consumes RESUME_TS — and the R5-6 pre-write recompute exists. expect(workflow).toMatch( - /RESUME_TS="\$\{CMD_TS\}"[\s\S]*?RESUME_TS="\$\{EVENT_TS\}"[\s\S]*?if \[\[ -z "\$\{TERM_TS\}" \]\]/, + /RESUME_TS="\$\(compute_resume_ts \/tmp\/tk-ic\.json \/tmp\/tk-ev\.json\)"[\s\S]*?if \[\[ -z "\$\{TERM_TS\}" \]\]/, ); + expect(workflow).toContain( + 'RESUME_NOW="$(compute_resume_ts /tmp/tk-ic2.json /tmp/tk-ev2.json)"', + ); + // R5-4: a failed permission read defers the release (fail closed), + // pinned at both evaluation points. + expect( + workflow.match(/command-permission read failed — release deferred/g) + ?.length, + ).toBe(2); // R2-13: CLEANUPS increments only after the heal DELETE succeeded. expect(workflow).toMatch( /clear stale \$\{NEEDS_HUMAN_LABEL\}[\s\S]{0,400}labels\/\$\(jq -rn --arg l "\$\{NEEDS_HUMAN_LABEL\}" '\$l\|@uri'\)"; then\n\s+CLEANUPS=\$\(\( CLEANUPS \+ 1 \)\)/, @@ -1548,10 +1489,10 @@ exit 1`; // R4-14: a command comment counts only after a write/maintain/admin // permission read on its author. expect(workflow).toContain( - 'gh api "repos/${REPO}/collaborators/${CMD_AUTHOR}/permission"', + 'gh api "repos/${REPO}/collaborators/${cauthor}/permission"', ); expect(workflow).toMatch( - /CMD_PERM}" == 'write' \|\| "\$\{CMD_PERM\}" == 'maintain' \|\| "\$\{CMD_PERM\}" == 'admin'/, + /cperm}" == 'write' \|\| "\$\{cperm\}" == 'maintain' \|\| "\$\{cperm\}" == 'admin'/, ); // R1-10: the CI-status classifiers are shared helpers, and NEITHER loop // inlines its own copy of the jq (a check-naming change must land once). From b575b6710c6a14a7d4acb754d172749168dcd673 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Fri, 14 Aug 2026 07:57:04 +0800 Subject: [PATCH 11/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?6=20=E2=80=94=20contract-safe=20gates=20and=20subshell=20flag?= =?UTF-8?q?=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - B5: the bot-fleet enumeration failure now degrades to a loud error row and falls through (FLEET_OK gate) instead of exiting before the independently-fed takeover/needs-human processing and the dashboard's liveness-watermark write. - B12: the cap-branch LIVE_LABELS consent re-read fails closed on an unreadable gh pr view (a collapse to '' ignored a concurrently added skip for standard bot PRs). - R6-1/R6-19: the takeover-release landed flag is keyed on the DELETE exit status (LBL_DEL_FAILED set inside the pinned idiom's failure branch) — never on output text, which lies in both directions. The pr-self-report-label idiom evolves identically to keep the cross-workflow contract green (and its own 'removed' log line no longer lies either). - R5-4 residual: compute_resume_ts now returns via globals (RESUME_OUT/PERM_READ_FAILED) and both call sites invoke it directly — the previous subshell silently dropped PERM_READ_FAILED, leaving the fail-closed defer branches dead. - R6-3: command candidates are deduped by author before permission reads, so a stranger posting N commands can't burn the 2-read budget and shadow a maintainer's command. - R6-4: an unreadable release history is reported as such, not as 'released'. --- .github/workflows/pr-self-report-label.yml | 13 +++-- .github/workflows/qwen-autofix.yml | 31 ++++++++--- .github/workflows/qwen-fleet-shepherd.yml | 51 ++++++++++++------- scripts/tests/qwen-autofix-workflow.test.js | 14 +++++ .../qwen-fleet-shepherd-workflow.test.js | 38 ++++++++++---- 5 files changed, 112 insertions(+), 35 deletions(-) diff --git a/.github/workflows/pr-self-report-label.yml b/.github/workflows/pr-self-report-label.yml index cccaadf243d..2db86ce5884 100644 --- a/.github/workflows/pr-self-report-label.yml +++ b/.github/workflows/pr-self-report-label.yml @@ -100,11 +100,18 @@ jobs: # failure (403 rate limit, 5xx, network) also keeps the step # green — the job only re-runs on the next PR event — but must # not disappear: without the warning, the log claims "removed" - # while the label stays on the PR. + # while the label stays on the PR. LBL_DEL_FAILED is keyed on the + # exit status (never the output text) and mirrors the identical + # idiom in qwen-autofix.yml (a contract test pins them in sync). + LBL_DEL_FAILED=false if ! REMOVE_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${REMOVE_ERR}" == *404* ]] || echo "::warning::#${PR}: ${LABEL} removal failed — ${REMOVE_ERR}" + [[ "${REMOVE_ERR}" == *404* ]] || { LBL_DEL_FAILED=true; echo "::warning::#${PR}: ${LABEL} removal failed — ${REMOVE_ERR}"; } + fi + if [[ "${LBL_DEL_FAILED}" == "true" ]]; then + echo "#${PR}: ${LABEL} removal did not land — will retry on the next PR event" + else + echo "🏷️ #${PR}: removed ${LABEL} — no self-reported linked issue" fi - echo "🏷️ #${PR}: removed ${LABEL} — no self-reported linked issue" else echo "#${PR}: no change (self=${SELF}, labeled=${HAS_LABEL}, api=${API_OK})" fi diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index dea28328709..662836a9faf 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1840,12 +1840,16 @@ jobs: # (R4-32) — or the PR keeps the takeover label while losing # its escalation label, and the release ack posted here would # latch RELEASE_ACKED, blocking every automated path that - # could re-apply it. + # could re-apply it. The flag is keyed on the EXIT STATUS, not + # the output text: a 200 success carries the remaining-labels + # body and a network error carries no "HTTP" token, so any + # text sniff fails one direction or the other (R6-1/R6-19). + LBL_DEL_FAILED=false if ! REMOVE_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${REMOVE_ERR}" == *404* ]] || echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}" + [[ "${REMOVE_ERR}" == *404* ]] || { LBL_DEL_FAILED=true; echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}"; } fi REMOVED_OK=true - if [[ "${REMOVE_ERR}" == *"HTTP "* && "${REMOVE_ERR}" != *404* ]]; then + if [[ "${LBL_DEL_FAILED}" == "true" ]]; then REMOVED_OK=false fi echo "🏷️ removed ${TAKEOVER_LABEL} from #${PR}" @@ -2988,8 +2992,17 @@ jobs: # Consent may have moved since PR_META: skip wins everywhere, # and a takeover notice additionally requires the label to # still be present — a label removed (or skip added) moments - # ago must not receive a stale 'paused' notice. - LIVE_LABELS="$(gh pr view "${PR}" --repo "${REPO}" --json labels 2> /dev/null | jq -r '[.labels[]?.name] | join(" ")' || echo '')" + # ago must not receive a stale 'paused' notice. The read + # FAILS CLOSED (mirrors takeover-ack): an unreadable label + # state must not get a notice or the escalation label — + # collapsing the failure to '' would ignore a concurrently + # added skip for standard bot PRs. + LIVE_LABELS_JSON="$(gh pr view "${PR}" --repo "${REPO}" --json labels 2> /dev/null || echo '')" + LIVE_LABELS="$(jq -r '[.labels[]?.name] | join(" ")' <<< "${LIVE_LABELS_JSON}" 2> /dev/null || echo '')" + if [[ -z "${LIVE_LABELS_JSON}" ]]; then + echo "🧭 cap notice skipped: label state unreadable (fail closed) on #${PR}" + continue + fi if [[ " ${LIVE_LABELS} " == *" ${SKIP_LABEL} "* ]] \ || [[ "${HAS_TAKEOVER}" == "true" && " ${LIVE_LABELS} " != *" ${TAKEOVER_LABEL} "* ]]; then echo "🧭 cap notice skipped: consent changed since the snapshot (labels: ${LIVE_LABELS:-unreadable})" @@ -3005,7 +3018,13 @@ jobs: if [[ "${SCAN_BOT_ACTOR}" != "${AUTOFIX_BOT}" ]]; then echo "::warning::cap-paused notice skipped: PAT authenticates as '${SCAN_BOT_ACTOR}', expected ${AUTOFIX_BOT}" elif [[ "${RELEASE_ACKED}" != "0" && "${IS_BOT_AUTHOR}" != "true" ]]; then - echo "🧭 cap label/notice skipped: PR was released after its last re-arm (#${PR})" + # R6-4: an events-API outage is NOT a release — report it + # as what it is, or the log lies during an incident. + if [[ "${RELEASE_ACKED}" == "unreadable" ]]; then + echo "🧭 cap label/notice skipped: release history unreadable (fail closed) on #${PR}" + else + echo "🧭 cap label/notice skipped: PR was released after its last re-arm (#${PR})" + fi else # The escalation label rides EVERY cap detection, noticed # or not: the once-per-window dedup suppresses repeat diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index 72e43d73344..455c51d102c 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -395,13 +395,18 @@ jobs: # human's fresh command comment, or a fresh takeover labeled # event) as an RFC3339 timestamp, or '' when none. A command counts # only while FRESH (grace window), UNSUPERSEDED by a refusal ack, - # and from a write/maintain/admin author — scanned newest-first so - # a stranger's echo can't SHADOW a maintainer's command (R5-5), - # with permission reads bounded at 2 per PR. Sets PERM_READ_FAILED - # when a command's permission check could not be evaluated — the - # caller fails CLOSED on that (defer; never trust-on-error, R5-4). + # and from a write/maintain/admin author — scanned newest-first, + # deduped BY AUTHOR so a stranger posting two commands can't burn + # the 2-permission-read budget and shadow a maintainer's command + # (R5-5/R6-3). Sets PERM_READ_FAILED when a command's permission + # check could not be evaluated — the caller fails CLOSED on that + # (defer; never trust-on-error, R5-4). Results are returned via the + # RESUME_OUT global — NOT stdout — because a `$(...)` call site + # would run the function in a subshell and silently drop + # PERM_READ_FAILED (the R5-4 defer was dead code that way). compute_resume_ts() { local ic="$1" ev="$2" + RESUME_OUT='' PERM_READ_FAILED='' local resume refusal resume="$(jq -r --arg ab "${AUTOFIX_BOT}" ' @@ -413,12 +418,13 @@ jobs: [ .[] | select((.user.login // "") == $ab) | select((.body // "") | test("")) | .created_at ] | max // ""' "${ic}")" + # Fresh command candidates: newest-first, ONE row per author. local cands cts cauthor cage cperm reads=0 cands="$(jq -r ' [ .[] | select(((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /takeover" or ((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /retry") | {ts: (.created_at // ""), login: (.user.login // "")} ] - | sort_by(.ts) | reverse | .[] | "\(.ts)\t\(.login)"' "${ic}")" + | sort_by(.ts) | reverse | unique_by(.login) | .[] | "\(.ts)\t\(.login)"' "${ic}")" while IFS=$'\t' read -r cts cauthor; do [[ -z "${cts}" || -z "${cauthor}" ]] && continue [[ "${cts}" > "${resume}" ]] || continue @@ -446,7 +452,7 @@ jobs: if [[ -n "${evt}" && ! "${resume}" > "${evt}" ]]; then resume="${evt}" fi - printf '%s' "${resume}" + RESUME_OUT="${resume}" } # CI-status classifiers shared by BOTH dashboard loops, so a @@ -471,16 +477,22 @@ jobs: if ! gh pr list --repo "${REPO}" --state open --author "${AUTOFIX_BOT}" --base main \ --limit 50 --json number,headRefName,headRefOid,mergeable,isCrossRepository,statusCheckRollup,labels \ > /tmp/fleet-raw.json 2> /dev/null; then - # A failed fleet fetch must not masquerade as an empty fleet: the - # walk is skipped AND the dashboard keeps its previous body rather - # than being overwritten with a misleading empty table. - echo "::warning::fleet enumeration failed; skipping this tick's walk and dashboard update" - exit 0 + # A failed fleet fetch must not masquerade as an empty fleet — + # but it also must not exit (B5): the takeover/needs-human + # processing below is fed by its OWN enumerations, so it keeps + # running, and the dashboard write still happens (carrying the + # liveness watermark). Degrade to a loud error row instead. + FLEET_OK=false + echo "::warning::fleet enumeration failed; the bot-fleet table shows an error row this tick" + DASH_ROWS='| — | — | ⚠️ fleet enumeration unreadable this tick | fail closed — fleet levers skipped |\n' + else + FLEET_OK=true + jq --arg skip "${SKIP_LABEL}" \ + '[.[] | select(.isCrossRepository == false) | select([.labels[]?.name] | index($skip) | not)]' \ + /tmp/fleet-raw.json > /tmp/fleet.json fi - jq --arg skip "${SKIP_LABEL}" \ - '[.[] | select(.isCrossRepository == false) | select([.labels[]?.name] | index($skip) | not)]' \ - /tmp/fleet-raw.json > /tmp/fleet.json + if [[ "${FLEET_OK}" == "true" ]]; then while IFS= read -r ROW; do [[ -z "${ROW}" ]] && continue PR="$(jq -r '.number' <<< "${ROW}")" @@ -614,6 +626,7 @@ jobs: echo "🐑 #${PR} [${STATUS_NOTE}] → ${ACTION_NOTE}" DASH_ROWS="${DASH_ROWS}| #${PR} | \`${HEAD:0:9}\` | ${NH_PREFIX}${STATUS_NOTE} | ${ACTION_NOTE} |\n" done < <(jq -c '.[]' /tmp/fleet.json) + fi # FLEET_OK # ---- takeover pool: dashboard rows + the auto-release lever ----- # These PRs are managed by the autofix loop, not the shepherd — @@ -795,7 +808,11 @@ jobs: # pickup — an hours-wide window). # The permission read fails CLOSED: PERM_READ_FAILED defers # the release below instead of trusting on error (R5-4). - RESUME_TS="$(compute_resume_ts /tmp/tk-ic.json /tmp/tk-ev.json)" + # Direct call, NOT $(...) — the function returns its result + # in RESUME_OUT/PERM_READ_FAILED globals, and a subshell + # would drop them. + compute_resume_ts /tmp/tk-ic.json /tmp/tk-ev.json + RESUME_TS="${RESUME_OUT}" # The stop reason comes from the terminal round's headline — # the scan-side notice always says "round cap" even when a # breaker (consecutive failures, time budget) fired first. @@ -857,7 +874,7 @@ jobs: elif ! gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ic2.json \ || ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev2.json; then NOTE='evidence re-check unreadable — release deferred (fail closed)' - elif RESUME_NOW="$(compute_resume_ts /tmp/tk-ic2.json /tmp/tk-ev2.json)"; [[ -n "${RESUME_NOW}" && ! "${TERM_TS}" > "${RESUME_NOW}" ]]; then + elif compute_resume_ts /tmp/tk-ic2.json /tmp/tk-ev2.json; RESUME_NOW="${RESUME_OUT}"; [[ -n "${RESUME_NOW}" && ! "${TERM_TS}" > "${RESUME_NOW}" ]]; then # R5-6: the resume evidence above was read at loop # entry — a re-arm landing since must veto NOW. The # re-fetch + recompute runs immediately before the diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index cb1118ab659..35c35873766 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -3123,6 +3123,20 @@ describe('qwen-autofix workflow', () => { expect(gate('autofix/managed autofix/takeover', 'true')).toContain( 'PROCEED', ); + // B12: the consent re-read FAILS CLOSED for standard bot PRs too — an + // unreadable `gh pr view` must skip the write entirely (a collapse to + // '' would ignore a concurrently added skip). The block sits BEFORE the + // consent gate. + expect(reviewScanJob).toContain( + 'cap notice skipped: label state unreadable (fail closed)', + ); + const unreadableIdx = reviewScanJob.indexOf( + 'if [[ -z "${LIVE_LABELS_JSON}" ]]; then', + ); + expect(unreadableIdx).toBeGreaterThan(-1); + expect(unreadableIdx).toBeLessThan( + reviewScanJob.indexOf('if [[ " ${LIVE_LABELS} " == *" ${SKIP_LABEL} "*'), + ); // Candidates drain newest-first, and the free busy skip never consumes // inspection budget. expect(reviewScanJob).toContain('sort_by(-.number)'); diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index 9dc48192565..6a63361ab2d 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -208,9 +208,25 @@ describe('fleet shepherd workflow', () => { expect(workflow).toMatch( /if \[\[ "\$\{MERGEABLE\}" == "CONFLICTING" \]\]; then[\s\S]{0,900}gh api "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/comments" --paginate/, ); - // A failed fleet fetch skips the walk AND preserves the previous - // dashboard body — never an empty-table overwrite. - expect(workflow).toContain('fleet enumeration failed; skipping this tick'); + // A failed fleet fetch must not masquerade as an empty fleet — and it + // must not exit either (B5): the takeover/needs-human processing runs + // off its OWN enumerations, so the fleet failure degrades to a loud + // error row and the tick falls through to the dashboard write (which + // carries the liveness watermark). No `exit` after the warning. + expect(workflow).toContain( + 'fleet enumeration failed; the bot-fleet table shows an error row this tick', + ); + expect(workflow).not.toMatch( + /fleet enumeration failed[\s\S]{0,200}exit( 0| 1)?\b/, + ); + expect(workflow).toContain( + '⚠️ fleet enumeration unreadable this tick | fail closed — fleet levers skipped', + ); + // The fleet walk is gated on FLEET_OK so the error row is the only row. + expect(workflow).toContain('FLEET_OK=false'); + expect(workflow).toMatch( + /if \[\[ "\$\{FLEET_OK\}" == "true" \]\]; then\n\s+while IFS= read -r ROW; do/, + ); // Per-tick blast-radius caps. expect(workflow).toContain("MAX_SYNCS_PER_TICK: '3'"); expect(workflow).toContain("MAX_CONFLICT_DISPATCHES_PER_TICK: '2'"); @@ -749,7 +765,7 @@ exit 1`; 'bash', [ '-c', - `${gnuDateShim}\n${resumeFn.replace(/\n {10}/g, '\n')}\ncompute_resume_ts "${dir}/ic.json" "${dir}/ev.json"\necho "PERM_FAILED=$PERM_READ_FAILED"`, + `${gnuDateShim}\n${resumeFn.replace(/\n {10}/g, '\n')}\ncompute_resume_ts "${dir}/ic.json" "${dir}/ev.json"\necho "TS=$RESUME_OUT"\necho "PERM_FAILED=$PERM_READ_FAILED"`, ], { env: { @@ -764,7 +780,8 @@ exit 1`; encoding: 'utf8', }, ).trim(); - const [ts, flag] = out.split('PERM_FAILED='); + const ts = out.match(/^TS=(.*)$/m)?.[1] ?? ''; + const flag = out.match(/^PERM_FAILED=(.*)$/m)?.[1] ?? ''; return { ts, flag }; } finally { rmSync(dir, { recursive: true, force: true }); @@ -1424,14 +1441,17 @@ exit 1`; expect(workflow).toMatch( /if \[\[ -n "\$\{evt\}" && ! "\$\{resume\}" > "\$\{evt\}" \]\]; then\n\s+resume="\$\{evt\}"/, ); - // R2-21: the single compute_resume_ts call runs BEFORE the evaluation - // chain consumes RESUME_TS — and the R5-6 pre-write recompute exists. + // R2-21: the compute_resume_ts call runs BEFORE the evaluation chain + // consumes RESUME_TS — and the R5-6 pre-write recompute exists. Both + // call sites use the global-return form (a `$(...)` subshell would drop + // PERM_READ_FAILED — the R5-4 defer would be dead code). expect(workflow).toMatch( - /RESUME_TS="\$\(compute_resume_ts \/tmp\/tk-ic\.json \/tmp\/tk-ev\.json\)"[\s\S]*?if \[\[ -z "\$\{TERM_TS\}" \]\]/, + /compute_resume_ts \/tmp\/tk-ic\.json \/tmp\/tk-ev\.json\n\s+RESUME_TS="\$\{RESUME_OUT\}"[\s\S]*?if \[\[ -z "\$\{TERM_TS\}" \]\]/, ); expect(workflow).toContain( - 'RESUME_NOW="$(compute_resume_ts /tmp/tk-ic2.json /tmp/tk-ev2.json)"', + 'compute_resume_ts /tmp/tk-ic2.json /tmp/tk-ev2.json; RESUME_NOW="${RESUME_OUT}"', ); + expect(workflow).not.toContain('$(compute_resume_ts'); // R5-4: a failed permission read defers the release (fail closed), // pinned at both evaluation points. expect( From 175e224ff2b8b8b9f6b0b9f3c3a021b1d79315f0 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Fri, 14 Aug 2026 10:37:15 +0800 Subject: [PATCH 12/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?7=20=E2=80=94=20lever=20starvation,=20re-arm=20anchoring,=20per?= =?UTF-8?q?mission=20shadows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - R5-7: the release lever gets its OWN enumeration of the paused population (takeover+needs-human, stale-first) instead of the long-lived needs-human display window — released-awaiting PRs aging back into that window could truncate exactly the fresh pauses that become release-eligible, starving the lever and making the zombie state permanent and self-feeding. - R6-3: the 2-read permission budget now sets PERM_READ_FAILED on exhaustion (it was failing open), and the candidate walk sorts newest-first per author (group_by+max_by+sort) instead of unique_by's alphabetical order, so two read-only strangers can't shadow a maintainer's newer command. - R7-1: the stale-label cleanup anchors on the current pause boundary (latest needs-human apply) and is marker-confirmed only — not keyed on TERM_TS, and never on command/label evidence — so a re-paused PR with a lost cycle-2 notice isn't read as re-armed on stale cycle-1 evidence. - R7-7: the /takeover stop success echo is gated on REMOVED_OK — a failed DELETE no longer logs 'removed'. - R7-2: TAKEOVER_COMMAND/RETRY_COMMAND mirrored into the shepherd env and passed via --arg, so the resume matcher can't drift from the route. - Tests: conflict_paused + re-arm guard behavioral replays, mirrored-command cross-file pin, engaged/released-with-skip ack matrix cells, LBL_DEL_FAILED branching, gnuDateShim hoisted to module scope, R4-24 nesting indices. Co-authored-by: Qwen-Coder --- .github/workflows/qwen-autofix.yml | 9 +- .github/workflows/qwen-fleet-shepherd.yml | 196 +++++++++++++----- scripts/tests/pr-self-report-label.test.js | 8 + scripts/tests/qwen-autofix-workflow.test.js | 35 +++- .../qwen-fleet-shepherd-workflow.test.js | 186 +++++++++++++---- 5 files changed, 332 insertions(+), 102 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 662836a9faf..5ce878c6ddb 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1852,7 +1852,14 @@ jobs: if [[ "${LBL_DEL_FAILED}" == "true" ]]; then REMOVED_OK=false fi - echo "🏷️ removed ${TAKEOVER_LABEL} from #${PR}" + if [[ "${REMOVED_OK}" == "true" ]]; then + echo "🏷️ removed ${TAKEOVER_LABEL} from #${PR}" + else + # R7-7: the success claim must not fire when the DELETE did + # not land — the motivating fix for pr-self-report-label.yml + # in this very PR is precisely this lying-log shape. + echo "⚠️ #${PR}: ${TAKEOVER_LABEL} removal did not land — the next /takeover stop retries" + fi # Released means a human is driving — the escalation label is # stale. Remove it only when the release landed (REMOVED_OK) # AND the PR is not frozen by skip (a frozen PR must keep its diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index 455c51d102c..2d6fd8b294c 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -82,6 +82,12 @@ env: # closes it. TAKEOVER_LABEL: 'autofix/takeover' NEEDS_HUMAN_LABEL: 'autofix/needs-human' + # The two re-arm commands the route listens for — mirrored from + # qwen-autofix.yml (TAKEOVER_COMMAND/RETRY_COMMAND) so the resume-evidence + # matcher can never drift from the command syntax the route actually + # accepts (R7-2). + TAKEOVER_COMMAND: '@qwen-code /takeover' + RETRY_COMMAND: '@qwen-code /retry' # Days a takeover PR may sit paused (both labels present, no re-arm since # the cap notice) before the shepherd releases the takeover label. Tunable # without a deploy via the repository variable. @@ -418,20 +424,28 @@ jobs: [ .[] | select((.user.login // "") == $ab) | select((.body // "") | test("")) | .created_at ] | max // ""' "${ic}")" - # Fresh command candidates: newest-first, ONE row per author. + # Fresh command candidates, newest-first, ONE row per author: + # group_by(.login) + max_by(.ts) keeps each author's newest command, + # then re-sort by ts desc restores recency order (unique_by would + # emit authors in LOGIN order, walking alphabetically and letting + # two read-only strangers shadow a maintainer's newer command, + # R6-3). local cands cts cauthor cage cperm reads=0 - cands="$(jq -r ' - [ .[] | select(((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /takeover" - or ((.body // "") | gsub("^\\s+|\\s+$"; "")) == "@qwen-code /retry") + cands="$(jq -r --arg tc "${TAKEOVER_COMMAND}" --arg rc "${RETRY_COMMAND}" ' + [ .[] | select(((.body // "") | gsub("^\\s+|\\s+$"; "")) == $tc + or ((.body // "") | gsub("^\\s+|\\s+$"; "")) == $rc) | {ts: (.created_at // ""), login: (.user.login // "")} ] - | sort_by(.ts) | reverse | unique_by(.login) | .[] | "\(.ts)\t\(.login)"' "${ic}")" + | group_by(.login) | map(max_by(.ts)) | sort_by(.ts) | reverse | .[] | "\(.ts)\t\(.login)"' "${ic}")" while IFS=$'\t' read -r cts cauthor; do [[ -z "${cts}" || -z "${cauthor}" ]] && continue [[ "${cts}" > "${resume}" ]] || continue [[ -z "${refusal}" || "${cts}" > "${refusal}" ]] || continue cage=$(( NOW_EPOCH - $(date -u -d "${cts}" +%s) )) [[ "${cage}" -lt "${RESUME_COMMAND_GRACE_SEC}" ]] || continue - [[ "${reads}" -ge 2 ]] && break + # Budget exhausted with candidates still unexamined: the caller + # must not read this as "all candidates were read-only/none + # trusted" — that fails OPEN. Flag it so the release defers. + [[ "${reads}" -ge 2 ]] && { PERM_READ_FAILED=true; break; } reads=$(( reads + 1 )) if ! cperm="$(gh api "repos/${REPO}/collaborators/${cauthor}/permission" --jq '.permission // ""' 2> /dev/null)"; then PERM_READ_FAILED=true @@ -679,39 +693,63 @@ jobs: '[.[] | select([.labels[]?.name] | index($skip) | not)]' \ /tmp/takeover-raw.json > /tmp/takeover.json fi - # The needs-human enumeration drives the lever AND the - # "awaiting human" table: it carries both labels per row, so the - # release population is never truncated by the display window. + # The needs-human enumeration drives the "awaiting human" DISPLAY + # table (loop 3). It is a long-lived population — every released PR + # keeps the label — so it can exceed its 100-window. HM_OK=true if ! gh pr list --repo "${REPO}" --state open --label "${NEEDS_HUMAN_LABEL}" \ --search 'sort:updated-asc' \ --limit 100 --json number,author,updatedAt,mergeable,labels \ > /tmp/human-raw.json 2> /dev/null; then HM_OK=false - echo "::warning::needs-human enumeration failed; paused-PR evaluation skipped this tick" + echo "::warning::needs-human enumeration failed; the awaiting-human table shows an error row this tick" HUMAN_ROWS='| — | — | — | ⚠️ enumeration unreadable this tick — previous entries may be stale |\n' else if [[ "$(jq length /tmp/human-raw.json)" -ge 100 ]]; then - echo "::warning::needs-human pool at the 100-PR enumeration limit — the freshest entries may be missing from this table" + echo "::warning::needs-human pool at the 100-PR enumeration limit — the freshest awaiting-human entries may be missing from the table" fi jq --arg skip "${SKIP_LABEL}" \ '[.[] | select([.labels[]?.name] | index($skip) | not)]' \ /tmp/human-raw.json > /tmp/human.json fi + # The RELEASE LEVER gets its OWN enumeration of the paused + # population (both labels), sorted stale-first (R5-7): released + # awaiting PRs age back into the needs-human window, so feeding the + # lever from that display window would truncate exactly the fresh + # pauses that become release-eligible, starving the lever and + # making the zombie state permanent and self-feeding. The paused + # population is small (only currently-paused takeover PRs). + PAUSED_OK=true + if ! gh pr list --repo "${REPO}" --state open --label "${TAKEOVER_LABEL}" --label "${NEEDS_HUMAN_LABEL}" \ + --search 'sort:updated-asc' \ + --limit 100 --json number,author,updatedAt,mergeable,labels \ + > /tmp/paused-raw.json 2> /dev/null; then + PAUSED_OK=false + echo "::warning::paused-takeover enumeration failed; the release lever is skipped this tick" + else + if [[ "$(jq length /tmp/paused-raw.json)" -ge 100 ]]; then + echo "::warning::paused-takeover pool at the 100-PR enumeration limit — the freshest paused entries may miss release evaluation" + fi + jq --arg skip "${SKIP_LABEL}" \ + '[.[] | select([.labels[]?.name] | index($skip) | not)]' \ + /tmp/paused-raw.json > /tmp/paused.json + fi - # Membership set of the needs-human enumeration (R4-9): loop 1 must - # only defer a paused PR to loop 2 when loop 2 will actually see it - # — a PR truncated out of the 100-window must still render here. + # Membership set of the paused enumeration (R4-9): loop 1 must + # only defer a paused PR to loop 2 when loop 2 will actually see + # it — a PR truncated out of the window must still render here. HUMAN_IDS='' - if [[ "${HM_OK}" == "true" ]]; then - HUMAN_IDS=",$(jq -r '[.[].number | tostring] | join(",")' /tmp/human.json)," + if [[ "${PAUSED_OK}" == "true" ]]; then + HUMAN_IDS=",$(jq -r '[.[].number | tostring] | join(",")' /tmp/paused.json)," fi - # Finalize the takeover-enum error row now that HM_OK is known. + # Finalize the takeover-enum error row now that PAUSED_OK is known + # (the release lever is fed by the paused enumeration, not the + # needs-human display window). if [[ "${TK_OK}" == "false" ]]; then - if [[ "${HM_OK}" == "true" ]]; then - TAKEOVER_ROWS='| — | — | — | ⚠️ takeover pool unreadable this tick | paused rows below still evaluated (needs-human fed) |\n' + if [[ "${PAUSED_OK}" == "true" ]]; then + TAKEOVER_ROWS='| — | — | — | ⚠️ takeover pool unreadable this tick | paused rows below still evaluated (paused pool fed) |\n' else - TAKEOVER_ROWS='| — | — | — | ⚠️ takeover pool unreadable this tick | paused rows NOT evaluated — needs-human enumeration also failed |\n' + TAKEOVER_ROWS='| — | — | — | ⚠️ takeover pool unreadable this tick | paused rows NOT evaluated — paused enumeration also failed |\n' fi fi @@ -761,33 +799,34 @@ jobs: done < <(jq -c '.[]' /tmp/takeover.json) fi - # Loop 2: needs-human PRs — pause evaluation, the release lever, - # and the awaiting-human rows. The paginated comment/event reads - # live in this loop only (paused PRs are few). Fail closed - # everywhere: an unreadable history defers the lever instead of - # acting on partial state. - if [[ "${HM_OK}" == "true" ]]; then + # Loop 2: paused takeover PRs (both labels) — the release lever + # and the takeover-table paused rows. Fed by the dedicated paused + # enumeration (R5-7), sorted stale-first, so the lever never + # starves behind the long-lived awaiting display population. The + # paginated comment/event reads live in this loop only (paused PRs + # are few). Fail closed everywhere: an unreadable history defers + # the lever instead of acting on partial state. + if [[ "${PAUSED_OK}" == "true" ]]; then while IFS= read -r ROW; do [[ -z "${ROW}" ]] && continue PR="$(jq -r '.number' <<< "${ROW}")" AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" MERGEABLE="$(jq -r '.mergeable // "UNKNOWN"' <<< "${ROW}")" - HAS_TK="$(jq -r --arg l "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" UPD_AGO='?' [[ -n "${UPDATED}" ]] && UPD_AGO="$(days_since "${UPDATED}")d" # Row routing follows POST-ACTION label state, not the # pre-mutation snapshot: a successful release moves the PR to - # Awaiting human; a successful heal drops it from both tables. - if [[ "${HAS_TK}" == "true" ]]; then ROUTE='takeover'; else ROUTE='awaiting'; fi + # Awaiting human. + ROUTE='takeover' STATE='🛑 needs-human' NOTE='—' if ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev.json; then NOTE='event read failed — evaluation deferred this tick (fail closed)' - elif [[ "${HAS_TK}" == "true" ]] && ! gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ic.json; then + elif ! gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ic.json; then NOTE='comment read failed — release deferred this tick' - elif [[ "${HAS_TK}" == "true" ]]; then + else # The pause clock starts at the LATEST cap notice. TERM_TS="$(jq -r --arg ab "${AUTOFIX_BOT}" ' [ .[] | select((.user.login // "") == $ab) @@ -827,6 +866,27 @@ jobs: | (.body | gsub("\r"; "") | split("\n")[0]) ] | last // ""' /tmp/tk-ic.json)" [[ -z "${REASON}" ]] && REASON='round cap reached' REASON="${REASON:0:120}" + # Re-armed detection for the stale-label cleanup is anchored + # to the CURRENT pause boundary (latest needs-human apply, + # same shape as the heal's NH_APPLY_TS) and is MARKER- + # confirmed only (bot re-arm/engage marker) — not the + # command-grace or labeled-event evidence that RESUME_TS + # carries. The cap path applies needs-human BEFORE the + # dedup'd notice and tolerates a lost notice, so keying the + # cleanup on TERM_TS (latest notice) would read a re-paused + # PR whose cycle-2 notice was lost as "re-armed" on stale + # cycle-1 evidence and DELETE its fresh label (R7-1). + # Command/label evidence still vetoes the RELEASE below via + # RESUME_TS — it just never triggers this cleanup. + MARKER_RESUME="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select(((.body // "") | contains("")) + or ((.body // "") | contains(""))) + | .created_at ] | max // ""' /tmp/tk-ic.json)" + PAUSE_APPLY_TS="$(jq -r --arg nl "${NEEDS_HUMAN_LABEL}" ' + [ .[] | select((.event // "") == "labeled") + | select((.label.name // "") == $nl) + | .created_at ] | max // ""' /tmp/tk-ev.json)" if [[ -z "${TERM_TS}" ]]; then NOTE='pause timestamp unreadable — release deferred (fail closed)' elif [[ "${PERM_READ_FAILED}" == "true" ]]; then @@ -834,7 +894,7 @@ jobs: # fail closed: the command MIGHT be a maintainer re-arm # (R5-4), so defer rather than treat it as no-permission. NOTE='command-permission read failed — release deferred this tick (fail closed)' - elif [[ -n "${RESUME_TS}" && ! "${TERM_TS}" > "${RESUME_TS}" ]]; then + elif [[ -n "${PAUSE_APPLY_TS}" && -n "${MARKER_RESUME}" && ! "${PAUSE_APPLY_TS}" > "${MARKER_RESUME}" ]]; then STATE='managed (re-armed)' # The PR is re-armed but still carries needs-human — a # resume-side removal failed. Clear the stale label here @@ -923,21 +983,55 @@ jobs: fi fi fi + fi + # The row is appended OUTSIDE the evaluation arms: a fail-closed + # deferral must still render, or the PR would vanish from the + # dashboard for exactly the tick something went wrong. + echo "🐑 #${PR} [takeover: ${STATE}] → ${NOTE}" + SAFE_NOTE="${NOTE//\\/\\\\}" + if [[ "${ROUTE}" == 'takeover' ]]; then + TAKEOVER_ROWS="${TAKEOVER_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${STATE} | ${SAFE_NOTE//|/\\|} |\n" + elif [[ "${ROUTE}" == 'awaiting' ]]; then + HUMAN_ROWS="${HUMAN_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${SAFE_NOTE//|/\\|} |\n" + fi + done < <(jq -c '.[]' /tmp/paused.json) + fi + + # Loop 3: needs-human-ONLY PRs — the awaiting-human display rows + # and the manual-release heal. Needs-human is a long-lived label + # (released PRs keep it), so this is the display-only population; + # the release lever is loop 2's. Both-label PRs were already + # rendered by loop 2 and are skipped here. + if [[ "${HM_OK}" == "true" ]]; then + while IFS= read -r ROW; do + [[ -z "${ROW}" ]] && continue + PR="$(jq -r '.number' <<< "${ROW}")" + HAS_TK="$(jq -r --arg l "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" + [[ "${HAS_TK}" == "true" ]] && continue + AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" + UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" + UPD_AGO='?' + [[ -n "${UPDATED}" ]] && UPD_AGO="$(days_since "${UPDATED}")d" + ROUTE='awaiting' + STATE='🛑 needs-human' + # No takeover label: the loop stopped and management was + # released (or never took over). Keep the PR visible — and + # when a HUMAN removed the takeover label by hand (fork PRs + # get no release ack from the route, so nothing else clears + # the escalation label there), clear the stale label. The + # shepherd's own auto-release authenticates as the bot and + # is NOT a heal trigger. + # Anchor to the CURRENT pause boundary: the latest + # needs-human label-apply EVENT (the cap branch applies the + # label at pause time). A human takeover-unlabel counts only + # when NEWER than that — anything older belongs to an earlier + # cycle (R5-10) — and an absent anchor (e.g. past the ~90-day + # events lookback) means "cannot correlate" → skip the + # cleanup rather than admit everything (fail closed). + NOTE='loop stopped — needs a human decision (merge / close / split / re-engage)' + if ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev.json; then + NOTE='event read failed — heal deferred this tick (fail closed)' else - # No takeover label: the loop stopped and management was - # released (or never took over). Keep the PR visible — and - # when a HUMAN removed the takeover label by hand (fork PRs - # get no release ack from the route, so nothing else clears - # the escalation label there), clear the stale label. The - # shepherd's own auto-release authenticates as the bot and - # is NOT a heal trigger. - # Anchor to the CURRENT pause boundary: the latest - # needs-human label-apply EVENT (the cap branch applies the - # label at pause time). A human takeover-unlabel counts only - # when NEWER than that — anything older belongs to an earlier - # cycle (R5-10) — and an absent anchor (e.g. past the ~90-day - # events lookback) means "cannot correlate" → skip the - # cleanup rather than admit everything (fail closed). NH_APPLY_TS="$(jq -r --arg nl "${NEEDS_HUMAN_LABEL}" ' [ .[] | select((.event // "") == "labeled") | select((.label.name // "") == $nl) @@ -951,7 +1045,6 @@ jobs: | select((.created_at // "") > $ll) | .actor.login ] | last // ""' /tmp/tk-ev.json)" fi - NOTE='loop stopped — needs a human decision (merge / close / split / re-engage)' if [[ -n "${UNLABEL_ACTOR}" ]]; then if [[ "${CLEANUPS}" -ge "${MAX_CLEANUPS_PER_TICK}" ]]; then NOTE='cleanup budget reached this tick' @@ -974,14 +1067,9 @@ jobs: fi fi fi - # The row is appended OUTSIDE the evaluation arms: a fail-closed - # deferral must still render, or the PR would vanish from the - # dashboard for exactly the tick something went wrong. - echo "🐑 #${PR} [${STATE}] → ${NOTE}" + echo "🐑 #${PR} [awaiting: ${STATE}] → ${NOTE}" SAFE_NOTE="${NOTE//\\/\\\\}" - if [[ "${ROUTE}" == 'takeover' ]]; then - TAKEOVER_ROWS="${TAKEOVER_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${STATE} | ${SAFE_NOTE//|/\\|} |\n" - elif [[ "${ROUTE}" == 'awaiting' ]]; then + if [[ "${ROUTE}" != 'none' ]]; then HUMAN_ROWS="${HUMAN_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${SAFE_NOTE//|/\\|} |\n" fi done < <(jq -c '.[]' /tmp/human.json) diff --git a/scripts/tests/pr-self-report-label.test.js b/scripts/tests/pr-self-report-label.test.js index 82d28c5df60..56e9485b4d8 100644 --- a/scripts/tests/pr-self-report-label.test.js +++ b/scripts/tests/pr-self-report-label.test.js @@ -187,6 +187,10 @@ describe('pr-self-report-label', () => { }); expect(race.removed).toBe(true); expect(race.out).not.toContain('::warning::'); + // R7-5: a 404 race means the end state holds → the success claim fires, + // the "did not land" claim must not. + expect(race.out).toContain('removed'); + expect(race.out).not.toContain('removal did not land'); // Any other DELETE failure keeps the step green (the job only re-runs // on the next PR event) but MUST surface — a silent run would claim // "removed" while the label stays on the PR. @@ -200,6 +204,10 @@ describe('pr-self-report-label', () => { expect(failed.out).toContain('::warning::'); expect(failed.out).toContain('removal failed'); expect(failed.out).toContain('HTTP 500'); + // R7-5: the LBL_DEL_FAILED branching — a failed DELETE must NOT claim + // the label was removed (the lying-log shape this change exists to kill). + expect(failed.out).toContain('removal did not land'); + expect(failed.out).not.toContain('removed'); // POST carries NO tolerance: a label that fails to apply must fail the // step (the runBlock sets -e), not leave the PR unlabeled behind green. expect(() => diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 35c35873766..8c0a7e5ed9f 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -3273,14 +3273,19 @@ describe('qwen-autofix workflow', () => { ).toBe('0'); expect(runRelAck([{ event: 'labeled' }], '2026-08-05T00:00:00Z')).toBe('0'); // R4-24: the gate's control-flow nesting is replayed — the label POST - // must fire ONLY in the else arm (RELEASE_ACKED=0 or bot author), never - // in the released arm. + // must fire ONLY in the outer else arm (RELEASE_ACKED=0 or bot author), + // never in the released arm. R7-3: releasedIdx anchors on the released + // echo and elseIdx on the OUTER else (16-space indent — the inner else + // at 18 spaces introduces the released echo, not the POST arm). const gateBlock = reviewScanJob.match( /if \[\[ "\$\{SCAN_BOT_ACTOR\}" != "\$\{AUTOFIX_BOT\}" \]\]; then[\s\S]*?cap label\/notice skipped[\s\S]*?\n {16}fi/, )?.[0]; expect(gateBlock).toBeTruthy(); const releasedIdx = gateBlock.indexOf('cap label/notice skipped'); - const elseIdx = gateBlock.indexOf('else', releasedIdx); + const elseIdx = gateBlock.indexOf( + '\n' + ' '.repeat(16) + 'else', + releasedIdx, + ); const postIdx = gateBlock.indexOf( 'gh api -X POST "repos/${REPO}/issues/${PR}/labels"', ); @@ -3638,6 +3643,10 @@ describe('qwen-autofix workflow', () => { expect(botSkipRelease.writes).toContain( 'opts it out of standard bot management entirely', ); + // R7-6: the skip veto keeps needs-human on a skip-frozen release — + // /takeover stop must NOT strip the only filterable escalation state + // from a PR nothing manages. + expect(botSkipRelease.writes).not.toContain('labels/autofix%2Fneeds-human'); // remove + absent → explicit no-op, no writes at all. const removeAbsent = runToggle({ cmd: 'remove' }); expect(removeAbsent.writes.trim()).toBe(''); @@ -5293,6 +5302,14 @@ exit 1 expect( runAck({ ack: 'engaged', labels: [{ name: 'autofix/skip' }] }).calls, ).not.toContain(nhDelete); + // R6-5: assert the TOTAL DELETE count for engaged-with-skip too — + // `not.toContain` alone lets a mutant add an unrelated second DELETE. + expect( + runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/skip' }], + }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(0); expect( runAck({ ack: 'base-refused', base: 'release' }).calls, ).not.toContain(nhDelete); @@ -5301,6 +5318,18 @@ exit 1 /api -X DELETE/gm, ) ?? [], ).toHaveLength(0); + // R7-6: the ack-job matrix's released-with-skip cell — a released PR + // frozen by skip must keep needs-human (narrowing the live-read gate to + // engaged-only would leave HAS_SKIP='' on released acks and strip it). + expect( + runAck({ ack: 'released', labels: [{ name: 'autofix/skip' }] }).calls, + ).not.toContain(nhDelete); + expect( + runAck({ + ack: 'released', + labels: [{ name: 'autofix/skip' }], + }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(0); }); it('narrows the agent prompt after a timeout since the last successful round', () => { diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index 6a63361ab2d..0d4393069bd 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { execFileSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { chmodSync, mkdtempSync, @@ -21,6 +21,11 @@ const workflow = readFileSync( 'utf8', ); +// GNU `date -u -d` shim for the behavioral replays — on macOS hosts BSD date +// lacks `-d`, so route it through node. Shared by every date-shim replay +// (R7-4): a fix to the shim must land once, not diverge across copies. +const gnuDateShim = `date() { if [[ "$1" == '-u' && "$2" == '-d' ]]; then node -e 'console.log(Math.floor(new Date(process.argv[1]).getTime()/1000))' "$3"; else command date "$@"; fi; }`; + describe('fleet shepherd workflow', () => { it('runs checkout-free — every read goes through the API', () => { // The run step reads no repo files (the flake registry is gone with the @@ -631,13 +636,20 @@ exit 1`; expect(takeoverEnum).toContain("--search 'sort:updated-asc'"); // Enumeration failures degrade to a loud error row and FALL THROUGH to // the dashboard write (which carries the liveness watermark) — they - // must not exit the tick. The release lever's population comes from the - // needs-human enumeration, never the display window. + // must not exit the tick. The release lever gets its OWN enumeration of + // the paused population (both labels), never the long-lived needs-human + // display window (R5-7). expect(workflow).toContain( 'takeover enumeration failed; the takeover table shows an error row this tick', ); expect(workflow).toContain( - 'needs-human enumeration failed; paused-PR evaluation skipped this tick', + 'needs-human enumeration failed; the awaiting-human table shows an error row this tick', + ); + expect(workflow).toContain( + 'paused-takeover enumeration failed; the release lever is skipped this tick', + ); + expect(workflow).toContain( + '--state open --label "${TAKEOVER_LABEL}" --label "${NEEDS_HUMAN_LABEL}"', ); expect(workflow).not.toMatch( /takeover enumeration failed[\s\S]{0,200}exit 0/, @@ -736,7 +748,6 @@ exit 1`; /(compute_resume_ts\(\) \{[\s\S]*?\n {10}\})/, )?.[1]; expect(resumeFn).toBeTruthy(); - const gnuDateShim = `date() { if [[ "$1" == '-u' && "$2" == '-d' ]]; then node -e 'console.log(Math.floor(new Date(process.argv[1]).getTime()/1000))' "$3"; else command date "$@"; fi; }`; const computeResume = ({ comments = [], events = [], @@ -774,6 +785,8 @@ exit 1`; REPO: 'QwenLM/qwen-code', AUTOFIX_BOT: 'qwen-code-dev-bot', TAKEOVER_LABEL: 'autofix/takeover', + TAKEOVER_COMMAND: '@qwen-code /takeover', + RETRY_COMMAND: '@qwen-code /retry', RESUME_COMMAND_GRACE_SEC: '7200', NOW_EPOCH: String(Date.parse(now) / 1000), }, @@ -917,40 +930,42 @@ exit 1`; ), ]), ).toContain('time-budget exhaustions'); - // The re-arm guard: resume evidence AT-OR-NEWER than the latest cap - // notice means a human acted and the release must never fire (ties - // resolve toward resume — R5-9). Replay the gate VERBATIM so a dropped + // The re-arm guard (R7-1): the stale-label cleanup fires only when a + // MARKER-confirmed re-arm is at-or-newer than the CURRENT pause boundary + // (latest needs-human apply) — not keyed on TERM_TS, and never on + // command/label evidence. Replay the gate VERBATIM so a dropped // comparison fails the test. const rearmGuard = workflow.match( - /(elif \[\[ -n "\$\{RESUME_TS\}" && ! "\$\{TERM_TS\}" > "\$\{RESUME_TS\}" \]\]; then\n {18}STATE='managed \(re-armed\)')/, + /(elif \[\[ -n "\$\{PAUSE_APPLY_TS\}" && -n "\$\{MARKER_RESUME\}" && ! "\$\{PAUSE_APPLY_TS\}" > "\$\{MARKER_RESUME\}" \]\]; then\n {18}STATE='managed \(re-armed\)')/, )?.[1]; expect(rearmGuard).toBeTruthy(); - const guard = (resume, term) => + const guard = (marker, apply) => execFileSync( 'bash', [ '-c', - `if [[ -z "${'$'}{TERM_TS}" ]]; then echo UNREADABLE\n${rearmGuard.replace(/\n {18}/g, '\n')}\necho REARMED\nelse\nfi`.replace( - 'else\nfi', - 'else echo RELEASE-ELIGIBLE\nfi', - ), + `if false; then echo UNREADABLE\n${rearmGuard.replace(/\n {18}/g, '\n')}\necho REARMED\nelse echo RELEASE-ELIGIBLE\nfi`, ], { - env: { ...process.env, RESUME_TS: resume, TERM_TS: term }, + env: { ...process.env, MARKER_RESUME: marker, PAUSE_APPLY_TS: apply }, encoding: 'utf8', }, ).trim(); + // Marker-confirmed re-arm newer than the current pause apply → re-armed. expect(guard('2026-08-06T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( 'REARMED', ); - // A same-second tie resolves toward RESUME (the release stays - // suppressed) — R5-9. + // A same-second tie resolves toward REARMED (R5-9 / R7-1). expect(guard('2026-08-05T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( 'REARMED', ); + // Marker older than the pause apply (stale cycle-1 evidence) → NOT + // re-armed; the fresh pause stands. expect(guard('2026-08-04T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( 'RELEASE-ELIGIBLE', ); + // No marker at all → not re-armed (command-grace evidence alone never + // triggers this cleanup). expect(guard('', '2026-08-05T00:00:00Z')).toBe('RELEASE-ELIGIBLE'); // The release gates run in blast-radius order: per-tick budget BEFORE // the PAT-backed live read, then the shared live_skip veto (fail closed @@ -1212,21 +1227,20 @@ exit 1`; expect(workflow).toContain('${SAFE_NOTE//|/\\\\|}'); // Shared day-math helper (R1-10) instead of pasted epoch arithmetic. expect(workflow).toContain('days_since() {'); - // Loop 2 renders exactly one row per needs-human PR — appended OUTSIDE - // the evaluation arms, so a fail-closed deferral (comment/event read + // Loop 3 renders exactly one row per needs-human-only PR — appended + // OUTSIDE the evaluation arms, so a fail-closed deferral (event read // failure) still renders instead of silently vanishing from the // dashboard for the tick something went wrong. - const loop2 = workflow.match( - /# Loop 2: needs-human PRs[\s\S]*?done < <\(jq -c '\.\[\]' \/tmp\/human\.json\)/, + const loop3 = workflow.match( + /# Loop 3: needs-human-ONLY PRs[\s\S]*?done < <\(jq -c '\.\[\]' \/tmp\/human\.json\)/, )?.[0]; - expect(loop2).toBeTruthy(); - expect(loop2.match(/echo "🐑 #\$\{PR\} \[/g)).toHaveLength(1); - expect(loop2.match(/TAKEOVER_ROWS="\$\{TAKEOVER_ROWS\}/g)).toHaveLength(1); - expect(loop2.match(/HUMAN_ROWS="\$\{HUMAN_ROWS\}/g)).toHaveLength(1); + expect(loop3).toBeTruthy(); + expect(loop3.match(/echo "🐑 #\$\{PR\} \[/g)).toHaveLength(1); + expect(loop3.match(/HUMAN_ROWS="\$\{HUMAN_ROWS\}/g)).toHaveLength(1); // R2-17: …and the append is POSITIONALLY outside the evaluation arms // (a bare occurrence count can't tell arm-level from loop-body level). - expect(loop2).toMatch( - /\n {14}fi\n(?: {14}#[^\n]*\n)+ {14}echo "🐑 #\$\{PR\} \[/, + expect(loop3).toMatch( + /\n {14}fi\n(?: {14}#[^\n]*\n)* {14}echo "🐑 #\$\{PR\} \[/, ); }); @@ -1256,18 +1270,25 @@ exit 1`; // 1b. The label constants the whole integration keys on (R2-11). expect(autofix).toContain("NEEDS_HUMAN_LABEL: 'autofix/needs-human'"); expect(autofix).toContain("TAKEOVER_LABEL: 'autofix/takeover'"); - // 2. The re-arm commands the shepherd greps for are the same strings - // qwen-autofix.yml routes on, and the refusal-ack variants that - // supersede a refused command are exactly the ones the producer can - // emit. + // 2. The re-arm commands the shepherd greps for are MIRRORED constants + // (R7-2): the shepherd env defines TAKEOVER_COMMAND/RETRY_COMMAND and + // passes them to the matcher via --arg, so they must equal the + // qwen-autofix.yml route constants exactly — a command-syntax edit on + // the autofix side desyncs here, not silently in production. The + // refusal-ack variants that supersede a refused command are exactly + // the ones the producer can emit. expect(autofix).toContain("TAKEOVER_COMMAND: '@qwen-code /takeover'"); expect(autofix).toContain("RETRY_COMMAND: '@qwen-code /retry'"); + expect(workflow).toContain("TAKEOVER_COMMAND: '@qwen-code /takeover'"); + expect(workflow).toContain("RETRY_COMMAND: '@qwen-code /retry'"); const cmdTs = workflow.match( - /cands="\$\(jq -r '([\s\S]*?)' "\$\{ic\}"\)"/, + /cands="\$\(jq -r --arg tc "\$\{TAKEOVER_COMMAND\}" --arg rc "\$\{RETRY_COMMAND\}" '([\s\S]*?)' "\$\{ic\}"\)"/, )?.[1]; expect(cmdTs).toBeTruthy(); - expect(cmdTs).toContain('"@qwen-code /takeover"'); - expect(cmdTs).toContain('"@qwen-code /retry"'); + // The matcher compares against the --arg'd constants, not hardcoded + // literals — so it can never drift from the env constants above. + expect(cmdTs).toContain('== $tc'); + expect(cmdTs).toContain('== $rc'); const refusalTs = workflow.match( /refusal="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' "\$\{ic\}"\)"/, )?.[1]; @@ -1343,10 +1364,9 @@ exit 1`; it('pins the previously mutation-tested gaps from review round 2', () => { // R2-7: days_since() is behaviorally replayed (the /86400 → /3600 - // mutation must fail) — shim GNU date through node for macOS hosts. + // mutation must fail) — the shared module-scope GNU date shim applies. const daysSince = workflow.match(/days_since\(\) \{[\s\S]*?\n {10}\}/)?.[0]; expect(daysSince).toBeTruthy(); - const gnuDateShim = `date() { if [[ "$1" == '-u' && "$2" == '-d' ]]; then node -e 'console.log(Math.floor(new Date(process.argv[1]).getTime()/1000))' "$3"; else command date "$@"; fi; }`; const runDays = (now, ts) => execFileSync( 'bash', @@ -1468,9 +1488,9 @@ exit 1`; /auto-release takeover \(paused \$\{PAUSE_D\}d\)" \\\n\s+gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{TAKEOVER_LABEL\}" '\$l\|@uri'\)"/, ); // R3-2: the takeover-enum error row must NOT claim "no release - // evaluation ran" — the lever is fed by the needs-human enumeration. + // evaluation ran" — the lever is fed by the paused enumeration. expect(workflow).toContain( - 'paused rows below still evaluated (needs-human fed)', + 'paused rows below still evaluated (paused pool fed)', ); expect(workflow).not.toContain('fail closed — no release evaluation ran'); // R3-8: the conflict-dispatch lever refuses a paused (needs-human) PR @@ -1490,21 +1510,99 @@ exit 1`; expect(workflow).toMatch( /conflict_paused\(\) \{[\s\S]*?return 0[\s\S]*?return 1[\s\S]*?return 0/, ); + // R5-14: replay conflict_paused's decision condition (a structural pin + // alone lets `-n term &&` drop and every label-without-notice PR read as + // "armed"). The helper reads /tmp/cf-ic.json via gh, so stub gh to write + // it and run the extracted function verbatim. + const conflictFn = workflow.match( + /(conflict_paused\(\) \{[\s\S]*?\n {10}\})/, + )?.[1]; + expect(conflictFn).toBeTruthy(); + const conflictPaused = ({ comments, failRead = false }) => { + const dir = mkdtempSync(join(tmpdir(), 'conflict-')); + try { + // gh api --paginate emits the paginated array to stdout; the helper + // itself pipes it through `jq -s 'add // []'`. + writeFileSync( + join(dir, 'gh'), + [ + '#!/usr/bin/env bash', + 'if [[ "$1" == "api" ]]; then', + ` if [[ "${failRead}" == "true" ]]; then echo "HTTP 502" >&2; exit 1; fi`, + ` printf '%s' '${JSON.stringify(comments).replace(/'/g, `'\\''`)}'`, + 'fi', + ].join('\n'), + ); + chmodSync(join(dir, 'gh'), 0o755); + const res = spawnSync( + 'bash', + [ + '-c', + `${conflictFn.replace(/\n {10}/g, '\n')}\nif conflict_paused 7354; then echo PAUSED; else echo ARMED; fi`, + ], + { + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + REPO: 'QwenLM/qwen-code', + AUTOFIX_BOT: 'qwen-code-dev-bot', + }, + encoding: 'utf8', + }, + ); + return res.stdout.trim(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; + const bot = (ts, body) => ({ + user: { login: 'qwen-code-dev-bot' }, + created_at: ts, + body, + }); + // No cap notice + old engage ack → paused (fail closed, R5-3). + expect( + conflictPaused({ + comments: [ + bot('2026-08-01T00:00:00Z', '🤝 … '), + ], + }), + ).toBe('PAUSED'); + // Re-arm newer than the cap notice → armed. + expect( + conflictPaused({ + comments: [ + bot('2026-08-01T00:00:00Z', '⏸️ … '), + bot('2026-08-02T00:00:00Z', '🔄 … '), + ], + }), + ).toBe('ARMED'); + // Cap notice newer than the re-arm → paused. + expect( + conflictPaused({ + comments: [ + bot('2026-08-02T00:00:00Z', '🔄 … '), + bot('2026-08-03T00:00:00Z', '⏸️ … '), + ], + }), + ).toBe('PAUSED'); + // Unreadable comments → paused (fail closed). + expect(conflictPaused({ comments: [], failRead: true })).toBe('PAUSED'); // R4-9: loop 1 defers a paused PR to loop 2 ONLY when it is actually in - // the needs-human window (membership), so a truncated PR still renders. + // the paused enumeration (membership), so a truncated PR still renders. expect(workflow).toContain( - 'HUMAN_IDS=",$(jq -r \'[.[].number | tostring] | join(",")\' /tmp/human.json),"', + 'HUMAN_IDS=",$(jq -r \'[.[].number | tostring] | join(",")\' /tmp/paused.json),"', ); expect(workflow).toContain('if [[ "${HUMAN_IDS}" == *",${PR},"* ]]; then'); // R4-10: loop 1's row escapes STATE (the ci-red detailsUrl) as well as // NOTE — a `|` or `\c` in the URL must not break the printf '%b' table. expect(workflow).toContain('SAFE_STATE="${STATE//\\\\/\\\\\\\\}"'); expect(workflow).toContain('${SAFE_STATE//|/\\\\|}'); - // R4-13: the takeover-enum error row is finalized AFTER the needs-human - // result and branches on HM_OK — it must not claim evaluation proceeds - // when the needs-human feed also failed. + // R4-13: the takeover-enum error row is finalized AFTER the paused + // result and branches on PAUSED_OK — it must not claim evaluation + // proceeds when the paused feed also failed. expect(workflow).toContain( - 'paused rows NOT evaluated — needs-human enumeration also failed', + 'paused rows NOT evaluated — paused enumeration also failed', ); // R4-14: a command comment counts only after a write/maintain/admin // permission read on its author. From 78024d2067b5df1fd752df2897ad68c4facf3ccc Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Fri, 14 Aug 2026 13:27:34 +0800 Subject: [PATCH 13/17] test(autofix): complete the R4-16 ack-matrix DELETE-count coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add fork-refused and skip-blocked ack cases to the takeover-ack harness — management never resumed on either, so zero needs-human DELETEs, each asserted by total DELETE count (not just toContain). Co-authored-by: Qwen-Coder --- scripts/tests/qwen-autofix-workflow.test.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 5287b0ba397..a9800c560a8 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -5561,6 +5561,21 @@ exit 1 labels: [{ name: 'autofix/skip' }], }).calls.match(/api -X DELETE/gm) ?? [], ).toHaveLength(0); + // R4-16: fork-refused and skip-blocked acks — management never resumed, + // zero DELETEs (with total-count assertions, not just toContain). + expect(runAck({ ack: 'fork-refused' }).calls).not.toContain(nhDelete); + expect( + runAck({ ack: 'fork-refused' }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(0); + expect( + runAck({ ack: 'skip-blocked', labels: [{ name: 'autofix/skip' }] }).calls, + ).not.toContain(nhDelete); + expect( + runAck({ + ack: 'skip-blocked', + labels: [{ name: 'autofix/skip' }], + }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(0); }); it('narrows the agent prompt after a timeout since the last successful round', () => { From 955566251d4d2faf794c96fa41e7bb8cc7c3d9f4 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Fri, 14 Aug 2026 11:26:43 +0000 Subject: [PATCH 14/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?8=20=E2=80=94=20reachable=20re-arm=20cleanup,=20release=20race?= =?UTF-8?q?=20guards,=20pin=20census?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/qwen-autofix.yml | 17 +- .github/workflows/qwen-fleet-shepherd.yml | 84 +++-- docs/design/takeover-fleet-visibility.md | 22 +- scripts/tests/qwen-autofix-workflow.test.js | 145 +++++++- .../qwen-fleet-shepherd-workflow.test.js | 322 ++++++++++++++++-- 5 files changed, 502 insertions(+), 88 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index a1d9f669373..06c3808e0f3 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -2013,10 +2013,9 @@ jobs: # failure carries "HTTP " in the error text. 404 = # already off (landed); any other HTTP error = the release did # NOT land and the needs-human removal below must NOT run - # (R4-32) — or the PR keeps the takeover label while losing - # its escalation label, and the release ack posted here would - # latch RELEASE_ACKED, blocking every automated path that - # could re-apply it. The flag is keyed on the EXIT STATUS, not + # (R4-32) — or the PR keeps the takeover label (still capped, + # nothing manages it now) while losing the only filterable + # escalation state. The flag is keyed on the EXIT STATUS, not # the output text: a 200 success carries the remaining-labels # body and a network error carries no "HTTP" token, so any # text sniff fails one direction or the other (R6-1/R6-19). @@ -2206,6 +2205,16 @@ jobs: HAS_SKIP="$(jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${PR_STATE_INFO}")" PR_AUTHOR_LIVE="$(jq -r '.author.login // ""' <<< "${PR_STATE_INFO}")" fi + # R2-4: a delayed release ack can run AFTER takeover was + # re-applied and the new cycle paused again — a live takeover + # label means this ack's premise is stale. Post nothing and + # touch nothing: the new cycle's own events produce their own + # acks, while a stale run here would delete the new cycle's + # needs-human and claim a release while takeover is live. + if [[ "${ACK}" == 'released' && "$(jq -r --arg t "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${PR_STATE_INFO}")" == "true" ]]; then + echo "⚠️ released ack skipped on #${PR}: ${TAKEOVER_LABEL} re-applied since the release (stale ack)" + exit 0 + fi if [[ "${ACK}" == 'base-refused' ]]; then BODY="$(printf '🚫 Takeover not engaged: the loop only manages PRs that target `main`, and this one targets `%s`. A stacked PR moves whenever its base branch does, so "new feedback since the last round" and base-conflict resolution are not well defined until the base lands. Two ways forward: retarget this PR to `main` once the base PR merges — the `%s` label is left in place and the scan lists by label, so the next scan engages it with no re-labelling — or take over the base PR instead.\n\n
\n中文说明\n\n🚫 未接管：循环只管理以 `main` 为 base 的 PR，而本 PR 的 base 是 `%s`。堆叠 PR 会随 base 分支移动，因此“自上一轮以来的新反馈”与 base 冲突处理都无法良定义。两条路：待 base 的 PR 合入后把本 PR 改为面向 `main` —— `%s` 标签予以保留，扫描按标签枚举，下一次扫描即会自动接管，无需重新打标签；或改为接管 base 那个 PR。\n\n
\n\n' "${ACK_BASE}" "${TAKEOVER_LABEL}" "${ACK_BASE}" "${TAKEOVER_LABEL}")" elif [[ "${ACK}" == 'engaged' && "${HAS_SKIP}" == "true" ]]; then diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index 2d6fd8b294c..8faf0fc6098 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -671,8 +671,8 @@ jobs: # sort:updated-asc keeps the stalest PRs in view when a pool # outgrows the window; the saturation warning keeps the residual # truncation loud. The LEVER's population deliberately comes from - # the needs-human enumeration (always small), never from this - # display window — see the second loop below. + # its OWN paused enumeration (always small), never from this + # display window — see the paused enumeration below. TK_OK=true if ! gh pr list --repo "${REPO}" --state open --label "${TAKEOVER_LABEL}" \ --search 'sort:updated-asc' \ @@ -680,11 +680,11 @@ jobs: > /tmp/takeover-raw.json 2> /dev/null; then TK_OK=false echo "::warning::takeover enumeration failed; the takeover table shows an error row this tick" - # The error-row wording is finalized AFTER the needs-human - # enumeration (R4-13): the release lever is fed by THAT window, - # and both reads share one PAT, so a correlated mid-tick outage - # can fail both — the row must not claim evaluation proceeds - # when the feed is unreadable too. + # The error-row wording is finalized AFTER the paused + # enumeration (R4-13): the release lever is fed by THAT + # enumeration, and both reads share one PAT, so a correlated + # mid-tick outage can fail both — the row must not claim + # evaluation proceeds when the feed is unreadable too. else if [[ "$(jq length /tmp/takeover-raw.json)" -ge 100 ]]; then echo "::warning::takeover pool at the 100-PR enumeration limit — recently-active PRs (paused-but-discussed ones included) may be missing from this table" @@ -755,8 +755,8 @@ jobs: # Loop 1: managed takeover PRs that are NOT paused — cheap payload # states only. Paused ones are rendered by loop 2, which owns the - # pause evaluation. (If the needs-human enumeration failed, render - # them here so they never vanish silently.) + # pause evaluation. (If the paused enumeration failed or dropped + # them, render them here so they never vanish silently.) if [[ "${TK_OK}" == "true" ]]; then while IFS= read -r ROW; do [[ -z "${ROW}" ]] && continue @@ -777,7 +777,7 @@ jobs: continue # loop 2 renders this paused PR fi STATE='🛑 needs-human' - NOTE='pause evaluation unavailable this tick (truncated or unreadable needs-human window)' + NOTE='pause evaluation unavailable this tick (truncated or unreadable paused enumeration)' elif [[ "${MERGEABLE}" == "CONFLICTING" ]]; then STATE='conflicting' NOTE='resolution owned by the autofix scan' @@ -845,13 +845,14 @@ jobs: # timestamped immediately, while its ack rides the # queue (or, on a lost event, the scan's idle-backoff # pickup — an hours-wide window). - # The permission read fails CLOSED: PERM_READ_FAILED defers - # the release below instead of trusting on error (R5-4). - # Direct call, NOT $(...) — the function returns its result - # in RESUME_OUT/PERM_READ_FAILED globals, and a subshell + # The marker variant is derived below for the cleanup + # anchor; the full evidence is evaluated only at release + # time, from a FRESH fetch (R5-6), where its permission + # read fails CLOSED: PERM_READ_FAILED defers the release + # instead of trusting on error (R5-4). Direct call, NOT + # $(...) — the function returns its result in the + # RESUME_OUT/PERM_READ_FAILED globals, and a subshell # would drop them. - compute_resume_ts /tmp/tk-ic.json /tmp/tk-ev.json - RESUME_TS="${RESUME_OUT}" # The stop reason comes from the terminal round's headline — # the scan-side notice always says "round cap" even when a # breaker (consecutive failures, time budget) fired first. @@ -870,14 +871,15 @@ jobs: # to the CURRENT pause boundary (latest needs-human apply, # same shape as the heal's NH_APPLY_TS) and is MARKER- # confirmed only (bot re-arm/engage marker) — not the - # command-grace or labeled-event evidence that RESUME_TS + # command-grace or labeled-event evidence the release veto # carries. The cap path applies needs-human BEFORE the # dedup'd notice and tolerates a lost notice, so keying the # cleanup on TERM_TS (latest notice) would read a re-paused # PR whose cycle-2 notice was lost as "re-armed" on stale # cycle-1 evidence and DELETE its fresh label (R7-1). # Command/label evidence still vetoes the RELEASE below via - # RESUME_TS — it just never triggers this cleanup. + # the pre-write RESUME_NOW recompute — it just never + # triggers this cleanup. MARKER_RESUME="$(jq -r --arg ab "${AUTOFIX_BOT}" ' [ .[] | select((.user.login // "") == $ab) | select(((.body // "") | contains("")) @@ -887,14 +889,16 @@ jobs: [ .[] | select((.event // "") == "labeled") | select((.label.name // "") == $nl) | .created_at ] | max // ""' /tmp/tk-ev.json)" - if [[ -z "${TERM_TS}" ]]; then - NOTE='pause timestamp unreadable — release deferred (fail closed)' - elif [[ "${PERM_READ_FAILED}" == "true" ]]; then - # A fresh command's permission could not be verified — - # fail closed: the command MIGHT be a maintainer re-arm - # (R5-4), so defer rather than treat it as no-permission. - NOTE='command-permission read failed — release deferred this tick (fail closed)' - elif [[ -n "${PAUSE_APPLY_TS}" && -n "${MARKER_RESUME}" && ! "${PAUSE_APPLY_TS}" > "${MARKER_RESUME}" ]]; then + # The re-armed cleanup runs FIRST so it stays reachable + # when the cap notice was lost (TERM_TS empty) — exactly + # the label-without-notice state this branch exists for + # (R8-1). A cap notice NEWER than the marker means the PR + # re-paused after the re-arm: the fresh pause wins and the + # cleanup must not fire — including the case where the + # re-cap's label POST no-op'd (label already present → no + # new labeled event) and only the notice marks the new + # cycle (R8-10). + if [[ -n "${PAUSE_APPLY_TS}" && -n "${MARKER_RESUME}" && ! "${PAUSE_APPLY_TS}" > "${MARKER_RESUME}" && ! "${TERM_TS}" > "${MARKER_RESUME}" ]]; then STATE='managed (re-armed)' # The PR is re-armed but still carries needs-human — a # resume-side removal failed. Clear the stale label here @@ -909,8 +913,10 @@ jobs: CLEANUPS=$(( CLEANUPS + 1 )) NOTE='resumed — cleared stale escalation label' else - NOTE='resumed after the last pause — the escalation label is stale' + NOTE='stale-label cleanup failed — will retry next tick' fi + elif [[ -z "${TERM_TS}" ]]; then + NOTE='pause timestamp unreadable — release deferred (fail closed)' else PAUSE_D="$(days_since "${TERM_TS}")" STATE="🛑 needs-human ${PAUSE_D}d" @@ -961,14 +967,27 @@ jobs: ] | length' /tmp/tk-ic.json)" if [[ "${SUMMARY_POSTED}" == "0" ]]; then if act "#${PR}: post auto-release summary" \ - gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '🔓 Takeover auto-released: the autofix loop paused on this PR %s day(s) ago (%s) and no re-arm followed, so the `%s` label is removed to keep the managed pool honest. `%s` stays as the reminder that this PR needs a human decision: merge it, close it, or split/reduce it and comment `@qwen-code /takeover` to re-engage with a fresh round window.\n\n
\n中文说明\n\n🔓 已自动释放接管：autofix 循环在 %s 天前暂停于此 PR（%s），此后无人重新武装，现移除 `%s` 标签以保持托管池真实可用。保留 `%s` 作为待办提醒 —— 本 PR 需要人工决策：合并、关闭，或拆分/缩小后评论 `@qwen-code /takeover` 以全新轮次窗口重新接管。\n\n
\n\n' "${PAUSE_D}" "${REASON}" "${TAKEOVER_LABEL}" "${NEEDS_HUMAN_LABEL}" "${PAUSE_D}" "${REASON}" "${TAKEOVER_LABEL}" "${NEEDS_HUMAN_LABEL}")"; then + gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '🔓 Takeover auto-released: the autofix loop paused on this PR %s day(s) ago (%s) and no re-arm followed, so the `%s` label is removed to keep the managed pool honest. `%s` stays as the reminder that this PR needs a human decision: merge it, close it, or split/reduce it and comment `%s` to re-engage with a fresh round window.\n\n
\n中文说明\n\n🔓 已自动释放接管：autofix 循环在 %s 天前暂停于此 PR（%s），此后无人重新武装，现移除 `%s` 标签以保持托管池真实可用。保留 `%s` 作为待办提醒 —— 本 PR 需要人工决策：合并、关闭，或拆分/缩小后评论 `%s` 以全新轮次窗口重新接管。\n\n
\n\n' "${PAUSE_D}" "${REASON}" "${TAKEOVER_LABEL}" "${NEEDS_HUMAN_LABEL}" "${TAKEOVER_COMMAND}" "${PAUSE_D}" "${REASON}" "${TAKEOVER_LABEL}" "${NEEDS_HUMAN_LABEL}" "${TAKEOVER_COMMAND}")"; then SUMMARY_POSTED=1 else NOTE='summary post failed — release deferred to next tick' fi fi if [[ "${SUMMARY_POSTED}" != "0" ]]; then - if act "#${PR}: auto-release takeover (paused ${PAUSE_D}d)" \ + # R2-3: a re-arm can land BETWEEN the summary post + # and this DELETE — its label re-apply is a no-op + # while the label still rides the PR, so no live + # label read can catch it; only fresh marker/event + # evidence can. Re-fetch and veto immediately + # before the removal (fail closed on unreadable). + if ! gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ic3.json \ + || ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev3.json; then + NOTE='evidence re-check unreadable — release deferred (fail closed)' + elif compute_resume_ts /tmp/tk-ic3.json /tmp/tk-ev3.json; RESUME_NOW="${RESUME_OUT}"; [[ -n "${RESUME_NOW}" && ! "${TERM_TS}" > "${RESUME_NOW}" ]]; then + NOTE='re-arm landed during the release — takeover kept' + elif [[ "${PERM_READ_FAILED}" == "true" ]]; then + NOTE='command-permission read failed — release deferred this tick (fail closed)' + elif act "#${PR}: auto-release takeover (paused ${PAUSE_D}d)" \ gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')"; then # Released — only needs-human remains, so the PR # belongs in Awaiting human NOW, not the takeover @@ -1001,13 +1020,16 @@ jobs: # and the manual-release heal. Needs-human is a long-lived label # (released PRs keep it), so this is the display-only population; # the release lever is loop 2's. Both-label PRs were already - # rendered by loop 2 and are skipped here. + # rendered by loop 2 and are skipped here when loop 2 ran. if [[ "${HM_OK}" == "true" ]]; then while IFS= read -r ROW; do [[ -z "${ROW}" ]] && continue PR="$(jq -r '.number' <<< "${ROW}")" HAS_TK="$(jq -r --arg l "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" - [[ "${HAS_TK}" == "true" ]] && continue + # Both-label PRs render in loop 2 — but only when loop 2 ran; + # on a paused-enumeration outage they must still appear here + # (the heal's live takeover check vetoes any mutation, R5-8). + [[ "${HAS_TK}" == "true" && "${PAUSED_OK}" == "true" ]] && continue AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" UPD_AGO='?' diff --git a/docs/design/takeover-fleet-visibility.md b/docs/design/takeover-fleet-visibility.md index 06e6d8c606e..593027b653d 100644 --- a/docs/design/takeover-fleet-visibility.md +++ b/docs/design/takeover-fleet-visibility.md @@ -50,6 +50,13 @@ Removal is best-effort with a warning on failure, mirroring the existing behind by a failed removal is cosmetically wrong but harmless; the next cap-stop reapplies it anyway. +A PR closed or merged while paused keeps `needs-human` — deliberately. No +closure removal path exists (the route drops commands on non-open PRs, every +enumeration is `--state open`, and there is no `pull_request: closed` +trigger), and the residue is inert: all consumers filter on open state, so +the label only marks the resolved escalation in the closed PR's own history. +All-state label queries should pair the label with a state filter. + Label creation follows the existing convention: `gh label create` (idempotent, fixed color) before the first REST add, so a missing label never gets a random color. @@ -121,12 +128,15 @@ re-checked immediately before the mutation, mirroring every existing lever. and no permission check is mirrored into the shepherd — the route's collaborator check is the authorization gate, and a mirrored copy would only drift. -- **The release lever's population comes from the needs-human enumeration** - (needs-human ∩ takeover), not the takeover display window, so window - saturation can never make a paused PR unreleasable. Both enumerations cap - at 100 with loud saturation warnings, and an enumeration failure degrades - to an error row — the dashboard write (which carries the liveness - watermark) always runs. +- **The release lever gets its own enumeration** of the paused population + (needs-human ∩ takeover, stalest-first) — not the takeover display window + and not the needs-human display window: released PRs keep `needs-human` + and age back into that display window, so feeding the lever from it would + truncate exactly the fresh pauses that become release-eligible. All three + enumerations cap at 100 with loud saturation warnings; a display + enumeration failure degrades to an error row, and a paused-enumeration + failure skips the lever for that tick — the dashboard write (which + carries the liveness watermark) always runs. - **The summary posts before the label removal**, dedup'd by its own marker scoped to the current pause cycle (only markers newer than the latest cap notice count), so a failed comment leaves both labels in place and the diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index a9800c560a8..5aef8ad3006 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -3368,6 +3368,13 @@ describe('qwen-autofix workflow', () => { expect(unreadableIdx).toBeLessThan( reviewScanJob.indexOf('if [[ " ${LIVE_LABELS} " == *" ${SKIP_LABEL} "*'), ); + // R8-17: the fail-closed gate actually SKIPS the writes — the + // `continue` is load-bearing (without it an unreadable state falls + // through the consent gate, whose empty-string disjuncts are both + // false, and POSTs over a concurrently added skip). + expect(reviewScanJob).toMatch( + /if \[\[ -z "\$\{LIVE_LABELS_JSON\}" \]\]; then\n\s+echo "🧭 cap notice skipped: label state unreadable \(fail closed\) on #\$\{PR\}"\n\s+continue/, + ); // Candidates drain newest-first, and the free busy skip never consumes // inspection budget. expect(reviewScanJob).toContain('sort_by(-.number)'); @@ -3503,16 +3510,35 @@ describe('qwen-autofix workflow', () => { ), ).toBe('0'); expect(runRelAck([{ event: 'labeled' }], '2026-08-05T00:00:00Z')).toBe('0'); + // R6-6: same-second boundary — a release at EXACTLY the window key + // counts (>= tie-toward-released: a completed release must never be + // re-escalated, R5-9). + expect( + runRelAck( + [ + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-05T00:00:00Z', + }, + ], + '2026-08-05T00:00:00Z', + ), + ).toBe('1'); // R4-24: the gate's control-flow nesting is replayed — the label POST // must fire ONLY in the outer else arm (RELEASE_ACKED=0 or bot author), - // never in the released arm. R7-3: releasedIdx anchors on the released - // echo and elseIdx on the OUTER else (16-space indent — the inner else - // at 18 spaces introduces the released echo, not the POST arm). + // never in the released arm. R7-3: elseIdx anchors on the OUTER else + // (16-space indent — the inner else at 18 spaces introduces the + // released echo, not the POST arm). R8-18: releasedIdx anchors on the + // FULL released echo — the shared prefix alone resolves to the earlier + // 'release history unreadable' echo, silently migrating the anchor. const gateBlock = reviewScanJob.match( /if \[\[ "\$\{SCAN_BOT_ACTOR\}" != "\$\{AUTOFIX_BOT\}" \]\]; then[\s\S]*?cap label\/notice skipped[\s\S]*?\n {16}fi/, )?.[0]; expect(gateBlock).toBeTruthy(); - const releasedIdx = gateBlock.indexOf('cap label/notice skipped'); + const releasedIdx = gateBlock.indexOf( + 'cap label/notice skipped: PR was released after its last re-arm', + ); const elseIdx = gateBlock.indexOf( '\n' + ' '.repeat(16) + 'else', releasedIdx, @@ -3743,7 +3769,7 @@ describe('qwen-autofix workflow', () => { // the block's two failure policies instead of the stub // universally exiting 0. `elif [[ "$1" == "label" && "$2" == "create" ]]; then echo "LABEL-CREATE $*" >> '${join(dir, 'writes.log')}';`, - `elif [[ "$1" == "api" ]]; then echo "API $*" >> '${join(dir, 'writes.log')}'; if [[ "$2" == "-X" && "$3" == "POST" && -n "\${TOGGLE_POST_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_POST_FAILS}" >&2; exit 1; fi; if [[ "$2" == "-X" && "$3" == "DELETE" && -n "\${TOGGLE_DELETE_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_DELETE_FAILS}" >&2; exit 1; fi; if [[ "$2" == "-X" && "$3" == "DELETE" ]]; then printf '%s' '[]'; fi`, + `elif [[ "$1" == "api" ]]; then echo "API $*" >> '${join(dir, 'writes.log')}'; if [[ "$2" == "-X" && "$3" == "POST" && -n "\${TOGGLE_POST_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_POST_FAILS}" >&2; exit 1; fi; if [[ "$2" == "-X" && "$3" == "DELETE" && -n "\${TOGGLE_DELETE_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_DELETE_FAILS}" >&2; exit 1; fi; if [[ "$2" == "-X" && "$3" == "DELETE" ]]; then printf '%s' '[{"name":"Tracks HTTP 5xx flakes"}]'; fi`, `elif [[ "$1" == "pr" && "$2" == "comment" ]]; then echo "COMMENT $*" >> '${join(dir, 'writes.log')}';`, 'fi', ].join('\n'), @@ -3995,12 +4021,27 @@ describe('qwen-autofix workflow', () => { expect(releaseFailed.log).toContain('::warning::'); expect(releaseFailed.log).toContain('removal failed'); expect(releaseFailed.log).toContain('HTTP 500'); - // R5-1: …but a failed takeover release keeps needs-human — REMOVED_OK is - // derived from the captured stream ("HTTP " non-404 = did not land), and - // the stub emits the remaining-labels body on success to model the real - // API contract. + // R5-1: …but a failed takeover release keeps needs-human — landed vs + // not-landed is keyed on the DELETE's EXIT STATUS (404 counts as + // landed; any other non-zero does not — R6-1's transport failures and + // R6-19's "HTTP "-bearing success bodies both mislead a text sniff), + // and the stub's success body carries a label name containing "HTTP " + // to model the real remaining-labels API contract. expect(releaseFailed.writes).not.toContain('labels/autofix%2Fneeds-human'); expect(releaseFailed.log).toContain('release did not land'); + // R6-9: a transport-level failure carries no "HTTP " token anywhere — + // the exit-status derivation still classifies it as not-landed (the + // class the old text sniff missed, R6-1). + const releaseTransportFailed = runToggle({ + cmd: 'remove', + labels: ['autofix/takeover'], + deleteFails: 'gh: dial tcp 140.82.121.4:443: connect: connection refused', + }); + expect(releaseTransportFailed.writes).toContain('takeover-ack released'); + expect(releaseTransportFailed.writes).not.toContain( + 'labels/autofix%2Fneeds-human', + ); + expect(releaseTransportFailed.log).toContain('release did not land'); // Both ack posts keep their non-fatal fallback: under bash -e a failed // gh pr comment would otherwise abort the step RED after the label was // already toggled — a worse signal than the silence being fixed. A @@ -5433,7 +5474,13 @@ exit 1 .join('\n'); expect(ackBlock).toBeTruthy(); - const runAck = ({ ack, base = '', prViewOk = true, labels = [] }) => { + const runAck = ({ + ack, + base = '', + prViewOk = true, + labels = [], + deleteFails = '', + }) => { const dir = mkdtempSync(join(tmpdir(), 'ack-')); const bin = join(dir, 'bin'); mkdirSync(bin); @@ -5443,12 +5490,18 @@ exit 1 '#!/usr/bin/env bash', `echo "$@" >> ${JSON.stringify(join(dir, 'calls.log'))}`, `if [[ "$1" == 'api' && "$2" == 'user' ]]; then printf 'qwen-code-dev-bot'; exit 0; fi`, - // Other api calls (the needs-human DELETE) succeed silently but are - // recorded above — the ack-job conditional removal is observable. - `if [[ "$1" == 'api' ]]; then exit 0; fi`, + // Other api calls (the needs-human DELETE) succeed silently but + // are recorded above — the ack-job conditional removal is + // observable. R6-33: with a failure knob — a non-404 must + // warn-and-continue onto the ack comment, and the harness's + // bash -e discriminates an aborting guard-strip mutant. + `if [[ "$1" == 'api' ]]; then if [[ -n "${deleteFails}" ]]; then echo "${deleteFails}" >&2; exit 1; fi; exit 0; fi`, `if [[ "$1" == 'pr' && "$2" == 'view' ]]; then : > ${JSON.stringify(join(dir, 'pr-view-called'))}; ${ prViewOk - ? `printf '%s' '${JSON.stringify({ labels, author: { login: 'wenshao' } })}'; exit 0` + ? // R6-22: serve the payload only for the EXACT field list + // the ack job requests (--json labels,author) — a drifted + // field selection gets an empty object, like production. + `if [[ "$*" == *'--json labels,author'* ]]; then printf '%s' '${JSON.stringify({ labels, author: { login: 'wenshao' } })}'; else printf '%s' '{}'; fi; exit 0` : 'exit 1' }; fi`, `if [[ "$1" == 'pr' && "$2" == 'comment' ]]; then printf '%s' "$7" > ${JSON.stringify(join(dir, 'comment.md'))}; exit 0; fi`, @@ -5477,6 +5530,7 @@ exit 1 const callsPath = join(dir, 'calls.log'); const result = { status: proc.status, + stdout: proc.stdout ?? '', body: existsSync(commentPath) ? readFileSync(commentPath, 'utf8') : '', calls: existsSync(callsPath) ? readFileSync(callsPath, 'utf8') : '', readPr: existsSync(join(dir, 'pr-view-called')), @@ -5530,6 +5584,13 @@ exit 1 runAck({ ack: 'released' }).calls.match(/api -X DELETE/gm) ?? [], ).toHaveLength(1); expect(runAck({ ack: 'released' }).calls).toContain(nhDelete); + // R6-11: …and the cleanup runs BEFORE the unguarded final ack comment — + // a transient comment failure aborting under bash -e must not strand + // the stale label. + const releasedOrder = runAck({ ack: 'released' }); + expect(releasedOrder.calls.indexOf('api -X DELETE')).toBeLessThan( + releasedOrder.calls.indexOf('pr comment'), + ); expect( runAck({ ack: 'engaged', labels: [{ name: 'autofix/skip' }] }).calls, ).not.toContain(nhDelete); @@ -5561,6 +5622,34 @@ exit 1 labels: [{ name: 'autofix/skip' }], }).calls.match(/api -X DELETE/gm) ?? [], ).toHaveLength(0); + // R2-4: a DELAYED release ack that lands after takeover was re-applied + // is stale — it must not touch the new cycle's needs-human and posts + // nothing (the new cycle's own events produce their own acks). + const staleRelease = runAck({ + ack: 'released', + labels: [{ name: 'autofix/takeover' }, { name: 'autofix/needs-human' }], + }); + expect(staleRelease.status).toBe(0); + expect(staleRelease.calls).not.toContain('api -X DELETE'); + expect(staleRelease.body).toBe(''); + expect(staleRelease.stdout).toContain('stale ack'); + // R6-33: a non-404 needs-human DELETE failure warns and STILL posts the + // ack (bash -e must not abort before the comment)… + const relDeleteFailed = runAck({ + ack: 'released', + deleteFails: 'HTTP 502: Bad Gateway', + }); + expect(relDeleteFailed.status).toBe(0); + expect(relDeleteFailed.stdout).toContain('removal failed'); + expect(relDeleteFailed.body).toContain(''); + // …while a 404 (the never-paused common case) is silently tolerated. + const relDelete404 = runAck({ + ack: 'released', + deleteFails: 'HTTP 404: Not Found', + }); + expect(relDelete404.status).toBe(0); + expect(relDelete404.stdout).not.toContain('removal failed'); + expect(relDelete404.body).toContain(''); // R4-16: fork-refused and skip-blocked acks — management never resumed, // zero DELETEs (with total-count assertions, not just toContain). expect(runAck({ ack: 'fork-refused' }).calls).not.toContain(nhDelete); @@ -11229,7 +11318,9 @@ exit 1 join(dir, 'gh'), [ '#!/bin/bash', - `echo "$@" >> '${join(dir, 'calls.log')}'`, + // One line per call even when an argument (the re-arm body) + // carries newlines — the R6-34 allowlist walks call lines. + `printf '%s\\n' "\${*//$'\\n'/ }" >> '${join(dir, 'calls.log')}'`, 'if [[ "$1" == "api" && "$2" == "user" ]]; then', ` if [[ "${apiFail}" == "true" ]]; then echo "HTTP 401: Bad credentials" >&2; exit 1; fi`, ` printf '%s' "${actor}"`, @@ -11243,7 +11334,10 @@ exit 1 ` if [[ "${viewFail}" == "true" ]]; then echo "GraphQL: Something went wrong" >&2; exit 1; fi`, // R4-22: only serve the labels payload when the caller actually // asked for it — a field-selection drift must fail the guard. - ' if [[ "$*" == *"--json labels"* ]]; then', + // R6-12/R2-7: the gate matches the EXACT production field list; + // a drifted `--json labels` (no author) must not be served the + // author metadata production would omit. + ' if [[ "$*" == *"--json labels,author"* ]]; then', ` printf '%s' '{"labels":${labels},"author":{"login":"${prAuthor}"}}'`, ' else', ` printf '%s' '{"number":7354}'`, @@ -11300,8 +11394,13 @@ exit 1 expect(ok.calls.match(/^api -X DELETE/gm) ?? []).toHaveLength(1); // R4-23: …and it is the only label WRITE of any verb — the full api-call // census is `api user` + the one DELETE (a stray label POST would make - // this 3). + // this 3). R6-34: …and the census is not verb-blind: EVERY recorded + // call must match the allowlist, so a stray `pr edit --remove-label` or + // `label create` write fails too. expect(ok.calls.match(/^api /gm) ?? []).toHaveLength(2); + for (const call of ok.calls.trim().split('\n')) { + expect(call).toMatch(/^(api user|api -X DELETE|pr view|pr comment)/); + } // R4-19: the marker comment posts BEFORE the cleanup DELETE — under // `bash -eo pipefail` a comment failure then aborts before the DELETE, // keeping the escalation label (the reverse order would drop the label @@ -11362,6 +11461,18 @@ exit 1 expect(orphan.calls).toContain('pr comment'); expect(orphan.calls).not.toContain('api -X DELETE'); expect(orphan.stdout).toContain('nothing manages it until re-engaged'); + // R6-7: a HUMAN-authored takeover PR — the managed-PR guard's takeover + // disjunct deletes the stale label (a mutant killing that disjunct + // keeps every other fixture green). + const humanTakeover = runRearm({ + actor: BOT, + prAuthor: 'wenshao', + labels: '[{"name":"autofix/takeover"},{"name":"autofix/needs-human"}]', + }); + expect(humanTakeover.status).toBe(0); + expect(humanTakeover.calls).toContain( + 'api -X DELETE repos/QwenLM/qwen-code/issues/7354/labels/autofix%2Fneeds-human', + ); // Actor mismatch: the PAT authenticates as someone else -> the guard exits // non-zero and posts nothing (a mis-scoped PAT must not leave a stranded PR diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index 0d4393069bd..270b62534d6 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -217,21 +217,32 @@ describe('fleet shepherd workflow', () => { // must not exit either (B5): the takeover/needs-human processing runs // off its OWN enumerations, so the fleet failure degrades to a loud // error row and the tick falls through to the dashboard write (which - // carries the liveness watermark). No `exit` after the warning. - expect(workflow).toContain( + // carries the liveness watermark). Pin the WHOLE failure branch — an + // exit statement BEFORE the warning escapes a forward-only scan + // (R8-11). + const fleetFetch = workflow.match( + /if ! gh pr list --repo "\$\{REPO\}" --state open --author "\$\{AUTOFIX_BOT\}" --base main[\s\S]*?\n {10}else\n {12}FLEET_OK=true/, + )?.[0]; + expect(fleetFetch).toBeTruthy(); + expect(fleetFetch).not.toMatch(/\n\s*exit( 0| 1)?\s*($|;)/m); + expect(fleetFetch).toContain( 'fleet enumeration failed; the bot-fleet table shows an error row this tick', ); - expect(workflow).not.toMatch( - /fleet enumeration failed[\s\S]{0,200}exit( 0| 1)?\b/, - ); - expect(workflow).toContain( + expect(fleetFetch).toContain( '⚠️ fleet enumeration unreadable this tick | fail closed — fleet levers skipped', ); - // The fleet walk is gated on FLEET_OK so the error row is the only row. - expect(workflow).toContain('FLEET_OK=false'); + expect(fleetFetch).toContain('FLEET_OK=false'); + // The fleet walk is gated on FLEET_OK so the error row is the only row + // — the success side assigns true, and the gate CLOSES before the + // takeover section, so a fleet fetch failure can never skip the + // independently-fed levers (R8-14). + expect(workflow).toContain('FLEET_OK=true'); expect(workflow).toMatch( /if \[\[ "\$\{FLEET_OK\}" == "true" \]\]; then\n\s+while IFS= read -r ROW; do/, ); + expect(workflow).toMatch( + /done < <\(jq -c '\.\[\]' \/tmp\/fleet\.json\)\n {10}fi # FLEET_OK\n\n {10}# ---- takeover pool/, + ); // Per-tick blast-radius caps. expect(workflow).toContain("MAX_SYNCS_PER_TICK: '3'"); expect(workflow).toContain("MAX_CONFLICT_DISPATCHES_PER_TICK: '2'"); @@ -892,6 +903,76 @@ exit 1`; }); expect(permFailed.ts).toBe(''); expect(permFailed.flag).toBe('true'); + // R8-4: budget exhaustion — with THREE distinct-author fresh commands + // the 2-read budget runs out before the oldest is examined; that must + // surface as PERM_READ_FAILED (defer), never as "no trusted command" + // (fail open). + const budgeted = computeResume({ + comments: [ + cmd('2026-08-06T00:30:00Z', 'reader-a'), + cmd('2026-08-06T00:20:00Z', 'reader-b'), + cmd('2026-08-06T00:10:00Z', 'reader-c'), + ], + perm: 'read', + }); + expect(budgeted.ts).toBe(''); + expect(budgeted.flag).toBe('true'); + // R8-4: the grace-window boundary — at EXACTLY RESUME_COMMAND_GRACE_SEC + // the command has expired (strict -lt; a still-unacked command that old + // is route-ignored, so the boundary resolves toward release), and one + // second inside the window still counts. + expect( + computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'wenshao')], + now: '2026-08-06T02:00:00Z', + }).ts, + ).toBe(''); + expect( + computeResume({ + comments: [cmd('2026-08-06T00:00:01Z', 'wenshao')], + now: '2026-08-06T02:00:00Z', + }).ts, + ).toBe('2026-08-06T00:00:01Z'); + // R6-15: the resume/refusal author filters — a HUMAN-forged re-arm + // marker never counts (only the bot posts real ones)… + expect( + computeResume({ + comments: [ + { + user: { login: 'wenshao' }, + created_at: '2026-08-06T00:00:00Z', + body: 'forged ', + }, + ], + }).ts, + ).toBe(''); + // …and a forged refusal ack cannot supersede a fresh maintainer + // command. + expect( + computeResume({ + comments: [ + { + user: { login: 'wenshao' }, + created_at: '2026-08-06T00:01:00Z', + body: 'forged ', + }, + cmd('2026-08-06T01:00:00Z', 'wenshao'), + ], + now: '2026-08-06T01:30:00Z', + }).ts, + ).toBe('2026-08-06T01:00:00Z'); + // R6-28: a command OLDER than the newest resume marker never counts — + // the ordering guard keeps the marker (without it, a cap notice posted + // between the command and its processing would release a PR re-armed + // minutes earlier). + expect( + computeResume({ + comments: [ + cmd('2026-08-06T00:00:00Z', 'wenshao'), + bot('2026-08-06T01:00:00Z', '🔄 … '), + ], + }).ts, + ).toBe('2026-08-06T01:00:00Z'); // The event read fails closed, like the comment read. expect(workflow).toContain( 'event read failed — evaluation deferred this tick (fail closed)', @@ -932,26 +1013,33 @@ exit 1`; ).toContain('time-budget exhaustions'); // The re-arm guard (R7-1): the stale-label cleanup fires only when a // MARKER-confirmed re-arm is at-or-newer than the CURRENT pause boundary - // (latest needs-human apply) — not keyed on TERM_TS, and never on - // command/label evidence. Replay the gate VERBATIM so a dropped - // comparison fails the test. + // (latest needs-human apply), never on command/label evidence, and a + // cap notice NEWER than the marker (a re-pause) vetoes it (R8-10). + // Replay the gate VERBATIM so a dropped comparison fails the test. const rearmGuard = workflow.match( - /(elif \[\[ -n "\$\{PAUSE_APPLY_TS\}" && -n "\$\{MARKER_RESUME\}" && ! "\$\{PAUSE_APPLY_TS\}" > "\$\{MARKER_RESUME\}" \]\]; then\n {18}STATE='managed \(re-armed\)')/, + /\n {16}(if \[\[ -n "\$\{PAUSE_APPLY_TS\}" && -n "\$\{MARKER_RESUME\}" && ! "\$\{PAUSE_APPLY_TS\}" > "\$\{MARKER_RESUME\}" && ! "\$\{TERM_TS\}" > "\$\{MARKER_RESUME\}" \]\]; then\n {18}STATE='managed \(re-armed\)')/, )?.[1]; expect(rearmGuard).toBeTruthy(); - const guard = (marker, apply) => + const guard = (marker, apply, term = '') => execFileSync( 'bash', [ '-c', - `if false; then echo UNREADABLE\n${rearmGuard.replace(/\n {18}/g, '\n')}\necho REARMED\nelse echo RELEASE-ELIGIBLE\nfi`, + `${rearmGuard.replace(/\n {18}/g, '\n')}\necho REARMED\nelse\necho RELEASE-ELIGIBLE\nfi`, ], { - env: { ...process.env, MARKER_RESUME: marker, PAUSE_APPLY_TS: apply }, + env: { + ...process.env, + MARKER_RESUME: marker, + PAUSE_APPLY_TS: apply, + TERM_TS: term, + }, encoding: 'utf8', }, ).trim(); - // Marker-confirmed re-arm newer than the current pause apply → re-armed. + // Marker-confirmed re-arm newer than the current pause apply → re-armed + // — also with NO cap notice: a lost notice must not make the cleanup + // unreachable (R8-1). expect(guard('2026-08-06T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( 'REARMED', ); @@ -967,6 +1055,25 @@ exit 1`; // No marker at all → not re-armed (command-grace evidence alone never // triggers this cleanup). expect(guard('', '2026-08-05T00:00:00Z')).toBe('RELEASE-ELIGIBLE'); + // No pause anchor → cannot correlate → not re-armed (fail closed). + expect(guard('2026-08-06T00:00:00Z', '')).toBe('RELEASE-ELIGIBLE'); + // A notice OLDER than the marker does not veto the cleanup. + expect( + guard( + '2026-08-06T00:00:00Z', + '2026-08-05T00:00:00Z', + '2026-08-04T00:00:00Z', + ), + ).toBe('REARMED'); + // R8-10: a cap notice NEWER than the marker means the PR re-paused + // after the re-arm — the fresh pause wins, no cleanup. + expect( + guard( + '2026-08-06T00:00:00Z', + '2026-08-05T00:00:00Z', + '2026-08-07T00:00:00Z', + ), + ).toBe('RELEASE-ELIGIBLE'); // The release gates run in blast-radius order: per-tick budget BEFORE // the PAT-backed live read, then the shared live_skip veto (fail closed // on an unreadable read; skip wins — both worded by skip_note), then the @@ -1028,7 +1135,7 @@ exit 1`; ), ).toBe('1'); expect(workflow).toMatch( - /if act "#\$\{PR\}: post auto-release summary"[\s\S]{0,1800}if act "#\$\{PR\}: auto-release takeover/, + /if act "#\$\{PR\}: post auto-release summary"[\s\S]{0,3200}if act "#\$\{PR\}: auto-release takeover/, ); // The budget is consumed BEFORE the first external write (R4-C3): a // release ATTEMPT is bounded, so a DELETE outage can't mutate many PRs @@ -1073,7 +1180,30 @@ exit 1`; // label-apply event), so a stale unlabel from an EARLIER cycle can't // heal this cycle's label, and an absent anchor skips the cleanup // (fail closed). - expect(workflow).toContain('"${CLEANUPS}" -ge "${MAX_CLEANUPS_PER_TICK}"'); + // R8-3: BOTH cleanup levers carry the budget → live_skip veto chain + // BEFORE their DELETE — a dropped veto would strip needs-human from a + // skip-frozen PR that nothing manages (the R4-3 invariant, on the + // cleanup side). R8-7: and both DELETE failure branches surface the + // failure the same way, so the dashboard distinguishes an outage from + // steady state. + expect( + workflow.match(/"\$\{CLEANUPS\}" -ge "\$\{MAX_CLEANUPS_PER_TICK\}"/g), + ).toHaveLength(2); + expect(workflow).toMatch( + /NOTE='resumed; stale-label cleanup budget reached this tick'\n\s+elif live_skip "\$\{PR\}"; then\n\s+NOTE="\$\(skip_note cleanup\)"[\s\S]{0,300}clear stale \$\{NEEDS_HUMAN_LABEL\} \(re-armed\)/, + ); + expect(workflow).toMatch( + /NOTE='cleanup budget reached this tick'\n\s+elif live_skip "\$\{PR\}"; then\n\s+NOTE="\$\(skip_note cleanup\)"[\s\S]{0,700}clear stale \$\{NEEDS_HUMAN_LABEL\} \(manual release/, + ); + expect( + workflow.match(/stale-label cleanup failed — will retry next tick/g), + ).toHaveLength(2); + // R6-16: the heal arm's R5-8 live re-apply check sits between the skip + // veto and the heal DELETE — a mid-tick re-engagement cancels the + // cleanup instead of deleting a re-managed PR's escalation label. + expect(workflow).toMatch( + /skip_note cleanup\)"[\s\S]{0,600}takeover label re-applied since the snapshot — cleanup cancelled'[\s\S]{0,300}clear stale \$\{NEEDS_HUMAN_LABEL\} \(manual release/, + ); const nhApplyJq = workflow.match( /NH_APPLY_TS="\$\(jq -r --arg nl "\$\{NEEDS_HUMAN_LABEL\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, )?.[1]; @@ -1082,6 +1212,13 @@ exit 1`; )?.[1]; expect(nhApplyJq).toBeTruthy(); expect(unlabelJq).toBeTruthy(); + // R8-19: the absent-anchor guard is pinned — without it UNLABEL_ACTOR + // would be computed against an empty anchor and admit every human + // takeover-unlabel in the window (a needs-human apply past the ~90-day + // events lookback must skip the heal, not admit everything). + expect(workflow).toMatch( + /UNLABEL_ACTOR=''\n\s+if \[\[ -n "\$\{NH_APPLY_TS\}" \]\]; then/, + ); const runUnlabel = (events) => { const ll = execFileSync( 'jq', @@ -1175,6 +1312,22 @@ exit 1`; }, ]), ).toBe(''); + // R6-27: a two-cycle history pins the LATEST anchor (`| max`): the + // human unlabel healed cycle 1 — it must NOT heal cycle 2's fresh + // label (a `min` mutant anchors on cycle 1 and admits the stale + // unlabel). + expect( + runUnlabel([ + nhLabeled('2026-08-01T00:00:00Z'), + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'wenshao' }, + created_at: '2026-08-02T00:00:00Z', + }, + nhLabeled('2026-08-03T00:00:00Z'), + ]), + ).toBe(''); // A human unlabel of an UNRELATED label must not count (pins the // `.label.name == $tl` select). expect( @@ -1242,6 +1395,18 @@ exit 1`; expect(loop3).toMatch( /\n {14}fi\n(?: {14}#[^\n]*\n)* {14}echo "🐑 #\$\{PR\} \[/, ); + // R8-15: the post-action routing — a successful release sets + // ROUTE='awaiting' so the row lands in Awaiting human, and loop 3 both + // defaults to awaiting and guards its append on ROUTE != none (a heal + // drops the row entirely). + expect(workflow).toMatch( + /ROUTE='awaiting'\n\s+NOTE="auto-released after \$\{PAUSE_D\}d paused"/, + ); + expect(workflow).toMatch( + /elif \[\[ "\$\{ROUTE\}" == 'awaiting' \]\]; then\n\s+HUMAN_ROWS=/, + ); + expect(loop3).toContain("ROUTE='awaiting'"); + expect(loop3).toContain('[[ "${ROUTE}" != \'none\' ]]'); }); it('pins the marker/headline contracts against qwen-autofix.yml — drift fails here, not in production', () => { @@ -1267,6 +1432,15 @@ exit 1`; [...p.matchAll(/contains\("()"\)/g)].map((m) => m[1]), ); expect(markerSet(rearmKey)).toEqual(markerSet(resumeTs)); + // R5-16: conflict_paused carries a THIRD copy of the resume-marker set + // — compare its resume jq against the same resume set so a marker added + // to only one side fails here too, never as a silent dispatch + // suppression. + const conflictResumeJq = workflow.match( + /resume="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/cf-ic\.json\)"/, + )?.[1]; + expect(conflictResumeJq).toBeTruthy(); + expect(markerSet(conflictResumeJq)).toEqual(markerSet(resumeTs)); // 1b. The label constants the whole integration keys on (R2-11). expect(autofix).toContain("NEEDS_HUMAN_LABEL: 'autofix/needs-human'"); expect(autofix).toContain("TAKEOVER_LABEL: 'autofix/takeover'"); @@ -1281,6 +1455,14 @@ exit 1`; expect(autofix).toContain("RETRY_COMMAND: '@qwen-code /retry'"); expect(workflow).toContain("TAKEOVER_COMMAND: '@qwen-code /takeover'"); expect(workflow).toContain("RETRY_COMMAND: '@qwen-code /retry'"); + // R8-6: the env constant is the ONLY literal command text in this file + // — the auto-release summary interpolates TAKEOVER_COMMAND like every + // ack body on the producer side, so a command-syntax rename can never + // strand a hardcoded instruction. + expect(workflow.match(/@qwen-code \/takeover/g)).toHaveLength(1); + expect(workflow).toContain( + 'comment `%s` to re-engage with a fresh round window', + ); const cmdTs = workflow.match( /cands="\$\(jq -r --arg tc "\$\{TAKEOVER_COMMAND\}" --arg rc "\$\{RETRY_COMMAND\}" '([\s\S]*?)' "\$\{ic\}"\)"/, )?.[1]; @@ -1402,7 +1584,9 @@ exit 1`; )?.[0]; expect(loop1).toBeTruthy(); expect(loop1).toContain('continue # loop 2 renders this paused PR'); - expect(loop1).toContain('pause evaluation unavailable this tick'); + expect(loop1).toContain( + 'pause evaluation unavailable this tick (truncated or unreadable paused enumeration)', + ); // R2-16: the needs-human skip filter is byte-identical to the pinned // takeover filter (an inverted population ships red, not green). The // [^']+ capture cannot span across call sites (the lazy [\s\S]*? form @@ -1415,8 +1599,8 @@ exit 1`; /jq --arg skip "\$\{SKIP_LABEL\}" \\\n\s+'([^']+)' \\\n\s+\/tmp\/takeover-raw\.json/, )?.[1]; expect(humanFilter).toBe(takeoverFilterP); - // R2-24/R2-28: the needs-human enumeration (the lever's population) - // carries the stale-first sort and the labels payload. + // R2-24/R2-28: the needs-human enumeration (the awaiting DISPLAY + // population) carries the stale-first sort and the labels payload. const humanEnum = workflow.match( /--label "\$\{NEEDS_HUMAN_LABEL\}"[\s\S]*?\/tmp\/human-raw\.json/, )?.[0]; @@ -1425,6 +1609,27 @@ exit 1`; expect(humanEnum).toContain( '--json number,author,updatedAt,mergeable,labels', ); + // R8-12: the release lever's OWN paused enumeration (R5-7) is pinned on + // the same axes as its siblings: the skip filter, the stale-first sort, + // the labels payload, the saturation warning, and loop 2's feed + gate. + const pausedEnum = workflow.match( + /--label "\$\{TAKEOVER_LABEL\}" --label "\$\{NEEDS_HUMAN_LABEL\}"[\s\S]*?\/tmp\/paused-raw\.json/, + )?.[0]; + expect(pausedEnum).toBeTruthy(); + expect(pausedEnum).toContain("--search 'sort:updated-asc'"); + expect(pausedEnum).toContain( + '--json number,author,updatedAt,mergeable,labels', + ); + expect(workflow).toContain( + 'paused-takeover pool at the 100-PR enumeration limit', + ); + const pausedFilter = workflow.match( + /jq --arg skip "\$\{SKIP_LABEL\}" \\\n\s+'([^']+)' \\\n\s+\/tmp\/paused-raw\.json/, + )?.[1]; + expect(pausedFilter).toBe(takeoverFilterP); + expect(workflow).toMatch( + /if \[\[ "\$\{PAUSED_OK\}" == "true" \]\]; then\n\s+while IFS= read -r ROW; do[\s\S]*?done < <\(jq -c '\.\[\]' \/tmp\/paused\.json\)/, + ); // R2-22: neither enumeration failure may terminate the tick — any // spelling of exit — because the dashboard write carries the liveness // watermark. @@ -1439,11 +1644,62 @@ exit 1`; // R4-28: pin the page-MERGE program too — GitHub returns comments // oldest-first, so `.[0] // []` would anchor TERM_TS to a stale page-1 // cycle while the current cycle's re-arm marker sits on page 2. + // R8-16: the events fetch exists TWICE — loop 3's heal read shares the + // shape; pin both occurrences, and pin loop 3's site INSIDE its own + // extraction so dropping --paginate only there (loop 3's population is + // the long-lived one that outgrows a page first) fails. + expect( + workflow.match( + /issues\/\$\{PR\}\/events" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ev\.json/g, + ), + ).toHaveLength(2); expect(workflow).toMatch( + /issues\/\$\{PR\}\/comments" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ic\.json/, + ); + const loop3Pag = workflow.match( + /# Loop 3: needs-human-ONLY PRs[\s\S]*?done < <\(jq -c '\.\[\]' \/tmp\/human\.json\)/, + )?.[0]; + expect(loop3Pag).toBeTruthy(); + expect(loop3Pag).toMatch( /issues\/\$\{PR\}\/events" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ev\.json/, ); + // R2-3: a re-arm landing BETWEEN the summary post and the DELETE is + // caught by a final evidence re-fetch immediately before the removal — + // the re-arm's label re-apply is a no-op while the label still rides + // the PR, so no live label read can catch it. Pin the shape and the + // veto's ordering ahead of the DELETE. expect(workflow).toMatch( - /issues\/\$\{PR\}\/comments" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ic\.json/, + /issues\/\$\{PR\}\/comments" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ic3\.json[\s\S]{0,200}issues\/\$\{PR\}\/events" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ev3\.json/, + ); + expect(workflow).toContain( + 're-arm landed during the release — takeover kept', + ); + // R6-8: the pre-write re-fetches paginate too — a page-1-only re-check + // reads the OLDEST comments and would miss a just-landed re-arm marker + // on exactly the high-comment paused population. R6-14: and the + // re-check's fail-closed arm, its veto comparison, and the cancellation + // NOTE are pinned (both re-check sites share the fail-closed wording). + expect(workflow).toMatch( + /issues\/\$\{PR\}\/comments" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ic2\.json/, + ); + expect(workflow).toMatch( + /issues\/\$\{PR\}\/events" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ev2\.json/, + ); + expect( + workflow.match( + /evidence re-check unreadable — release deferred \(fail closed\)/g, + ), + ).toHaveLength(2); + expect(workflow).toContain( + 'resume evidence appeared during evaluation — release cancelled', + ); + expect( + workflow.match( + /\[\[ -n "\$\{RESUME_NOW\}" && ! "\$\{TERM_TS\}" > "\$\{RESUME_NOW\}" \]\]/g, + ), + ).toHaveLength(2); + expect(workflow).toMatch( + /re-arm landed during the release — takeover kept'[\s\S]{0,400}auto-release takeover \(paused/, ); // R2-19: the release scope check requires BOTH labels — the --arg // bindings are the load-bearing part. R3-12: the pin runs through the @@ -1461,19 +1717,25 @@ exit 1`; expect(workflow).toMatch( /if \[\[ -n "\$\{evt\}" && ! "\$\{resume\}" > "\$\{evt\}" \]\]; then\n\s+resume="\$\{evt\}"/, ); - // R2-21: the compute_resume_ts call runs BEFORE the evaluation chain - // consumes RESUME_TS — and the R5-6 pre-write recompute exists. Both - // call sites use the global-return form (a `$(...)` subshell would drop - // PERM_READ_FAILED — the R5-4 defer would be dead code). - expect(workflow).toMatch( - /compute_resume_ts \/tmp\/tk-ic\.json \/tmp\/tk-ev\.json\n\s+RESUME_TS="\$\{RESUME_OUT\}"[\s\S]*?if \[\[ -z "\$\{TERM_TS\}" \]\]/, - ); + // R2-21/R8-5: the resume evidence is computed only at RELEASE time — + // the pre-summary recompute (R5-6) and the pre-DELETE re-check (R2-3) + // — in the global-return form (a `$(...)` subshell would drop + // PERM_READ_FAILED, making the R5-4 defer dead code). The old + // loop-entry call left a dead RESUME_TS store and spent discarded + // permission reads every tick; it must not return. expect(workflow).toContain( 'compute_resume_ts /tmp/tk-ic2.json /tmp/tk-ev2.json; RESUME_NOW="${RESUME_OUT}"', ); expect(workflow).not.toContain('$(compute_resume_ts'); + expect(workflow).not.toContain('RESUME_TS='); + // R8-1: the re-armed cleanup branch PRECEDES the empty-TERM_TS + // deferral, so a lost cap notice cannot make it unreachable. + expect(workflow).toMatch( + /STATE='managed \(re-armed\)'[\s\S]*?elif \[\[ -z "\$\{TERM_TS\}" \]\]; then/, + ); // R5-4: a failed permission read defers the release (fail closed), - // pinned at both evaluation points. + // pinned at BOTH release-time evaluation points (the pre-summary veto + // and the pre-DELETE R2-3 re-check). expect( workflow.match(/command-permission read failed — release deferred/g) ?.length, From b055a8d00d3da756458050fb447b56b38e0eca51 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Fri, 14 Aug 2026 15:46:14 +0000 Subject: [PATCH 15/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?9=20=E2=80=94=20honest=20release-failed=20ack,=20cleanup=20atte?= =?UTF-8?q?mpt=20budgets,=20dashboard=20single-owner=20routing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - /takeover stop whose label DELETE failed no longer posts a 'Takeover released' ack: a release-failed variant names the retry (R9-4), and the R7-7 echo pair gains symmetric log pins (R9-3) - stale-label cleanups count ATTEMPTS like the release budget, so a DELETE outage trips the cap instead of leaving it inert (R9-5) - dashboard renders each both-label PR exactly once: loop 1 defers by paused membership, loop 3 is the render of last resort (R9-1/R9-13) - a 404 from the collaborators-permission endpoint classifies the author read-only instead of renewably deferring the release (R9-10) - cap-branch release evidence reuses the per-iteration events fetch under a success flag (R9-18); release-clock comment corrected (R9-11) - harness gates end-anchor the --json field list (R9-14/R9-15); the escalation POST and the ack-body census gain count pins (R9-16); the date shim answers only the +%s shape it emulates (R9-9) --- .github/workflows/qwen-autofix.yml | 45 ++++++++++--- .github/workflows/qwen-fleet-shepherd.yml | 64 +++++++++++++------ scripts/tests/qwen-autofix-workflow.test.js | 50 ++++++++++++--- .../qwen-fleet-shepherd-workflow.test.js | 40 ++++++++++-- 4 files changed, 153 insertions(+), 46 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 06c3808e0f3..ee24e87cbe8 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -2060,7 +2060,14 @@ jobs: # the unlabeled-path ack when the label sender is the bot. REL_AUTHOR="$(jq -r '.author.login // ""' <<< "${PR_INFO}")" REL_HAS_SKIP="$(jq -r --arg t "${SKIP_LABEL}" '[.labels[].name] | index($t) != null' <<< "${PR_INFO}")" - if [[ "${REL_AUTHOR}" == "${AUTOFIX_BOT}" && "${REL_HAS_SKIP}" == "true" ]]; then + if [[ "${REMOVED_OK}" != "true" ]]; then + # The DELETE did not land — the label is still on and the + # loop still manages this PR. A "released" ack (and its + # marker) would record a release that never happened: no + # unlabeled event fires, nothing retries, and no human + # re-issues the command. Own the failure and name the retry. + REL_BODY="$(printf '⚠️ Takeover release did not land: removing the `%s` label failed transiently, so it is still present and the autofix loop keeps managing this PR under the round cap. Comment `%s stop` to retry the release.\n\n
\n中文说明\n\n⚠️ 释放未生效：移除 `%s` 标签时瞬时失败，标签仍在，autofix 循环仍按轮次上限继续托管此 PR。评论 `%s stop` 可重试释放。\n\n
\n\n' "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}")" + elif [[ "${REL_AUTHOR}" == "${AUTOFIX_BOT}" && "${REL_HAS_SKIP}" == "true" ]]; then REL_BODY="$(printf '👋 Takeover mode ended. This bot-authored PR also carries `%s`, which opts it out of standard bot management entirely — nothing will engage it until that label is removed.\n\n
\n中文说明\n\n👋 接管模式结束。本 bot 创建的 PR 同时带有 `%s`，已完全退出常规 bot 管理 —— 移除该标签前不会有任何介入。\n\n
\n\n' "${SKIP_LABEL}" "${SKIP_LABEL}")" elif [[ "${REL_AUTHOR}" == "${AUTOFIX_BOT}" ]]; then REL_BODY="$(printf '👋 Takeover mode ended: the raised round cap no longer applies. This is a bot-authored PR, so STANDARD bot management continues under the strict cap (apply `%s` to opt it out entirely). Re-apply `%s` (or comment `%s`) for the raised cap again.\n\n
\n中文说明\n\n👋 接管模式结束：提升的轮次上限不再适用。这是 bot 创建的 PR，常规 bot 管理仍将继续（严格上限；如需完全退出请打 `%s`）。重新打上 `%s` 标签（或评论 `%s`）可恢复提升上限。\n\n
\n\n' "${SKIP_LABEL}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}" "${SKIP_LABEL}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}")" @@ -3028,9 +3035,13 @@ jobs: add | [.[] | select((.user.login // "") == $ab) | select(.body // "" | contains("")) | .created_at] | sort | last // ""' "${WORKDIR}/ic.json")" - gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null \ - | jq -s 'add // []' > "${WORKDIR}/pr-events.json" \ - || echo '[]' > "${WORKDIR}/pr-events.json" + PR_EVENTS_OK='' + if gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null \ + | jq -s 'add // []' > "${WORKDIR}/pr-events.json"; then + PR_EVENTS_OK=true + else + echo '[]' > "${WORKDIR}/pr-events.json" + fi LAST_LABELED_TS="$(jq -rs --arg lb "${TAKEOVER_LABEL}" ' add | [.[] | select(.event == "labeled") | select((.label.name // "") == $lb) @@ -3270,9 +3281,22 @@ jobs: # re-escalate a completed release (R5-9). Fail closed: an # unreadable event history suppresses the re-label rather than # risk the ping-pong. - if ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > "${WORKDIR}/ev.json"; then + # A capped takeover candidate already paid for this paginated + # endpoint earlier in the same iteration (pr-events.json, + # gated by PR_EVENTS_OK) — reuse that fetch instead of paying + # it twice per capped takeover PR per scan. Non-takeover + # candidates keep the standalone fetch, and the fail-closed + # semantics ride the flag: the engage-side '[]' fallback must + # never read here as "no releases". + RELEASE_ACKED='' + if [[ "${HAS_TAKEOVER}" == "true" && "${PR_EVENTS_OK}" == "true" ]]; then + if ! cp "${WORKDIR}/pr-events.json" "${WORKDIR}/ev.json"; then + RELEASE_ACKED='unreadable' + fi + elif ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > "${WORKDIR}/ev.json"; then RELEASE_ACKED='unreadable' - else + fi + if [[ "${RELEASE_ACKED}" != 'unreadable' ]]; then RELEASE_ACKED="$(jq -r --arg tl "${TAKEOVER_LABEL}" --arg rt "${NOTICE_RT}" ' [ .[] | select((.event // "") == "unlabeled") | select((.label.name // "") == $tl) @@ -3327,10 +3351,11 @@ jobs: # The escalation label rides EVERY cap detection, noticed # or not: the once-per-window dedup suppresses repeat # comments, but the label is what makes a paused PR - # filterable and what the shepherd's auto-release ages - # from — and applying it unconditionally backfills the - # already-paused fleet via the scan rotation after this - # ships (idle backoff: expect hours, not the first scan). + # filterable (the shepherd's auto-release ages from the + # cap notice itself, not from the label) — and applying + # it unconditionally backfills the already-paused fleet + # via the scan rotation after this ships (idle backoff: + # expect hours, not the first scan). gh label create "${NEEDS_HUMAN_LABEL}" --repo "${REPO}" --color 'D93F0B' \ --description 'The autofix loop stopped on this PR — a human must re-arm, split, merge, or close it' \ 2> /dev/null || true diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index 8faf0fc6098..648440c357f 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -447,7 +447,12 @@ jobs: # trusted" — that fails OPEN. Flag it so the release defers. [[ "${reads}" -ge 2 ]] && { PERM_READ_FAILED=true; break; } reads=$(( reads + 1 )) - if ! cperm="$(gh api "repos/${REPO}/collaborators/${cauthor}/permission" --jq '.permission // ""' 2> /dev/null)"; then + if ! cperm="$(gh api "repos/${REPO}/collaborators/${cauthor}/permission" --jq '.permission // ""' 2> /tmp/cperm-err)"; then + # 404 is GitHub's decisive "not a collaborator" answer — + # classify read-only instead of deferring: a defer here is + # renewable by any stranger's exact command each grace + # window, and it would pose a classification as an outage. + grep -q '404' /tmp/cperm-err && continue PERM_READ_FAILED=true continue fi @@ -772,10 +777,14 @@ jobs: STATE='managed · idle' NOTE='—' + # Defer by paused-enumeration MEMBERSHIP, not by this + # snapshot's label state: a needs-human label landing between + # the two enumerations must not render the PR in both loops — + # loop 2 owns every paused member. + if [[ "${HUMAN_IDS}" == *",${PR},"* ]]; then + continue # loop 2 renders this paused PR + fi if [[ "${HAS_NH}" == "true" ]]; then - if [[ "${HUMAN_IDS}" == *",${PR},"* ]]; then - continue # loop 2 renders this paused PR - fi STATE='🛑 needs-human' NOTE='pause evaluation unavailable this tick (truncated or unreadable paused enumeration)' elif [[ "${MERGEABLE}" == "CONFLICTING" ]]; then @@ -908,12 +917,16 @@ jobs: NOTE='resumed; stale-label cleanup budget reached this tick' elif live_skip "${PR}"; then NOTE="$(skip_note cleanup)" - elif act "#${PR}: clear stale ${NEEDS_HUMAN_LABEL} (re-armed)" \ - gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')"; then - CLEANUPS=$(( CLEANUPS + 1 )) - NOTE='resumed — cleared stale escalation label' else - NOTE='stale-label cleanup failed — will retry next tick' + # Count ATTEMPTS like the release arm: a success-only + # counter never trips during a DELETE outage. + CLEANUPS=$(( CLEANUPS + 1 )) + if act "#${PR}: clear stale ${NEEDS_HUMAN_LABEL} (re-armed)" \ + gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')"; then + NOTE='resumed — cleared stale escalation label' + else + NOTE='stale-label cleanup failed — will retry next tick' + fi fi elif [[ -z "${TERM_TS}" ]]; then NOTE='pause timestamp unreadable — release deferred (fail closed)' @@ -1026,10 +1039,16 @@ jobs: [[ -z "${ROW}" ]] && continue PR="$(jq -r '.number' <<< "${ROW}")" HAS_TK="$(jq -r --arg l "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" - # Both-label PRs render in loop 2 — but only when loop 2 ran; - # on a paused-enumeration outage they must still appear here - # (the heal's live takeover check vetoes any mutation, R5-8). - [[ "${HAS_TK}" == "true" && "${PAUSED_OK}" == "true" ]] && continue + # Both-label PRs render elsewhere: loop 2 owns the paused + # members, and loop 1 owns the rest while its enumeration ran + # (its note names the pause-evaluation gap — the accurate + # render on a paused-enumeration outage). Loop 3 is the render + # of LAST RESORT — both owners blind — so a both-label PR + # never renders twice nor vanishes (the heal's live takeover + # check vetoes any mutation, R5-8). + if [[ "${HAS_TK}" == "true" ]]; then + [[ "${HUMAN_IDS}" == *",${PR},"* || "${TK_OK}" == "true" ]] && continue + fi AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" UPD_AGO='?' @@ -1077,15 +1096,18 @@ jobs: # snapshot and now — the PR is managed again, so nothing # here is stale. NOTE='takeover label re-applied since the snapshot — cleanup cancelled' - elif act "#${PR}: clear stale ${NEEDS_HUMAN_LABEL} (manual release by @${UNLABEL_ACTOR})" \ - gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')"; then - CLEANUPS=$(( CLEANUPS + 1 )) - # Both labels are gone now — the PR is fully released and - # must drop out of the dashboard entirely. - ROUTE='none' - NOTE="cleared stale escalation label (released by @${UNLABEL_ACTOR})" else - NOTE='stale-label cleanup failed — will retry next tick' + # Count ATTEMPTS, mirroring the release arm (twin above). + CLEANUPS=$(( CLEANUPS + 1 )) + if act "#${PR}: clear stale ${NEEDS_HUMAN_LABEL} (manual release by @${UNLABEL_ACTOR})" \ + gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')"; then + # Both labels are gone now — the PR is fully released and + # must drop out of the dashboard entirely. + ROUTE='none' + NOTE="cleared stale escalation label (released by @${UNLABEL_ACTOR})" + else + NOTE='stale-label cleanup failed — will retry next tick' + fi fi fi fi diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 5aef8ad3006..519ebb6befa 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -2425,11 +2425,13 @@ describe('qwen-autofix workflow', () => { // variants, mirroring the ack job — a loud add next to a mute stop // would re-create the lost-event ambiguity on the release side), and // the scan-side first-pickup engage ack (fork label events carry no - // secrets, so the scan anchors the window itself). + // secrets, so the scan anchors the window itself), and the + // command-path release-failed ack (a failed release must say so — a + // 'released' marker would record a release that never happened). const ackBodies = workflow.match( /printf '[^']*takeover-(?:ack|cap)[^']*'/g, ); - expect(ackBodies).toHaveLength(19); + expect(ackBodies).toHaveLength(20); for (const body of ackBodies) { expect(body).toContain('中文说明'); } @@ -3562,6 +3564,15 @@ describe('qwen-autofix workflow', () => { /gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{NEEDS_HUMAN_LABEL\}"/g, ) ?? []; expect(removals).toHaveLength(6); + // R9-16: the escalation POST rides the cap detection exactly ONCE — a + // duplicate outside the gate would re-add the label on every cap + // detection of a released human PR (the R3-1/R4-1 ping-pong with the + // shepherd's cleanup that this test cites). + expect( + workflow.match( + /gh api -X POST "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels" -f "labels\[\]=\$\{NEEDS_HUMAN_LABEL\}"/g, + ) ?? [], + ).toHaveLength(1); // R4-17: the scan-side first-pickup DELETE must live in the engage-ack // SUCCESS branch — moved to the ack-failure path it would clear // needs-human when no engagement happened. @@ -4006,21 +4017,31 @@ describe('qwen-autofix workflow', () => { ); expect(releaseRace.writes).toContain('takeover-ack released'); expect(releaseRace.log).not.toContain('::warning::'); + // R9-3: pin BOTH echo arms — a swap ships a lying log in either + // direction while the separate ::warning:: line keeps the old pins + // green. + expect(releaseRace.log).toContain('removed autofix/takeover'); + expect(releaseRace.log).not.toContain('removal did not land'); // R5-1: a 404 counts as landed, so the needs-human cleanup still runs… expect(releaseRace.writes).toContain('labels/autofix%2Fneeds-human'); - // Any other DELETE failure must not drop the release ack either — a - // later `/takeover stop` retries the removal — but it MUST warn: - // masked, the ack reads "released" while the loop keeps managing the - // PR. + // Any other DELETE failure keeps the label ON — the ack must say the + // release did NOT land (a 'released' marker would record a release + // that never happened: no unlabeled event fires, nothing retries, no + // human re-issues the command) and the loud warning keeps the failure + // diagnosable in the log. const releaseFailed = runToggle({ cmd: 'remove', labels: ['autofix/takeover'], deleteFails: 'HTTP 500', }); - expect(releaseFailed.writes).toContain('takeover-ack released'); + expect(releaseFailed.writes).toContain('takeover-ack release-failed'); + expect(releaseFailed.writes).toContain('release did not land'); + expect(releaseFailed.writes).not.toContain('takeover-ack released'); expect(releaseFailed.log).toContain('::warning::'); expect(releaseFailed.log).toContain('removal failed'); expect(releaseFailed.log).toContain('HTTP 500'); + expect(releaseFailed.log).toContain('removal did not land'); + expect(releaseFailed.log).not.toContain('removed autofix/takeover from'); // R5-1: …but a failed takeover release keeps needs-human — landed vs // not-landed is keyed on the DELETE's EXIT STATUS (404 counts as // landed; any other non-zero does not — R6-1's transport failures and @@ -4037,7 +4058,12 @@ describe('qwen-autofix workflow', () => { labels: ['autofix/takeover'], deleteFails: 'gh: dial tcp 140.82.121.4:443: connect: connection refused', }); - expect(releaseTransportFailed.writes).toContain('takeover-ack released'); + expect(releaseTransportFailed.writes).toContain( + 'takeover-ack release-failed', + ); + expect(releaseTransportFailed.writes).not.toContain( + 'takeover-ack released', + ); expect(releaseTransportFailed.writes).not.toContain( 'labels/autofix%2Fneeds-human', ); @@ -5501,7 +5527,10 @@ exit 1 ? // R6-22: serve the payload only for the EXACT field list // the ack job requests (--json labels,author) — a drifted // field selection gets an empty object, like production. - `if [[ "$*" == *'--json labels,author'* ]]; then printf '%s' '${JSON.stringify({ labels, author: { login: 'wenshao' } })}'; else printf '%s' '{}'; fi; exit 0` + // R9-14: anchored at the END of the token (the field list + // is the final argument at the production call site), so a + // superset drift is served the empty object too. + `if [[ "$*" == *' --json labels,author' ]]; then printf '%s' '${JSON.stringify({ labels, author: { login: 'wenshao' } })}'; else printf '%s' '{}'; fi; exit 0` : 'exit 1' }; fi`, `if [[ "$1" == 'pr' && "$2" == 'comment' ]]; then printf '%s' "$7" > ${JSON.stringify(join(dir, 'comment.md'))}; exit 0; fi`, @@ -11337,7 +11366,8 @@ exit 1 // R6-12/R2-7: the gate matches the EXACT production field list; // a drifted `--json labels` (no author) must not be served the // author metadata production would omit. - ' if [[ "$*" == *"--json labels,author"* ]]; then', + // R9-15: end-anchored like its runAck twin (R9-14). + ' if [[ "$*" == *" --json labels,author" ]]; then', ` printf '%s' '{"labels":${labels},"author":{"login":"${prAuthor}"}}'`, ' else', ` printf '%s' '{"number":7354}'`, diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index 270b62534d6..4f3288eb9f8 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -24,7 +24,10 @@ const workflow = readFileSync( // GNU `date -u -d` shim for the behavioral replays — on macOS hosts BSD date // lacks `-d`, so route it through node. Shared by every date-shim replay // (R7-4): a fix to the shim must land once, not diverge across copies. -const gnuDateShim = `date() { if [[ "$1" == '-u' && "$2" == '-d' ]]; then node -e 'console.log(Math.floor(new Date(process.argv[1]).getTime()/1000))' "$3"; else command date "$@"; fi; }`; +// Answers ONLY the +%s call shape it emulates (R9-9): any other format +// falls through to the system date, so a format-string mutation cannot +// hide behind the shim. +const gnuDateShim = `date() { if [[ "$1" == '-u' && "$2" == '-d' && "$4" == '+%s' ]]; then node -e 'console.log(Math.floor(new Date(process.argv[1]).getTime()/1000))' "$3"; else command date "$@"; fi; }`; describe('fleet shepherd workflow', () => { it('runs checkout-free — every read goes through the API', () => { @@ -764,6 +767,7 @@ exit 1`; events = [], perm = 'write', permFail = false, + permError = 'HTTP 502', now = '2026-08-06T00:30:00Z', }) => { const dir = mkdtempSync(join(tmpdir(), 'resume-')); @@ -775,7 +779,7 @@ exit 1`; [ '#!/bin/bash', 'if [[ "$1" == "api" ]]; then', - ` if [[ "${permFail}" == "true" ]]; then echo "HTTP 502" >&2; exit 1; fi`, + ` if [[ "${permFail}" == "true" ]]; then echo "${permError}" >&2; exit 1; fi`, // Per-user permission: the collaborators path ($2) names the // user; `perm` is the default, `stranger` always reads `read`. ` case "$2" in *collaborators/stranger/*) printf '%s' "read" ;; *) printf '%s' "${perm}" ;; esac`, @@ -903,6 +907,17 @@ exit 1`; }); expect(permFailed.ts).toBe(''); expect(permFailed.flag).toBe('true'); + // R9-10: a 404 is GitHub's decisive "not a collaborator" answer — the + // command is classified read-only WITHOUT PERM_READ_FAILED: failing + // closed here would let any stranger renewably defer the release with + // one exact command comment per grace window, reported as an outage. + const notCollab = computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'stranger')], + permFail: true, + permError: 'HTTP 404: Not Found', + }); + expect(notCollab.ts).toBe(''); + expect(notCollab.flag).toBe(''); // R8-4: budget exhaustion — with THREE distinct-author fresh commands // the 2-read budget runs out before the oldest is examined; that must // surface as PERM_READ_FAILED (defer), never as "no trusted command" @@ -1483,8 +1498,10 @@ exit 1`; (m) => m[1], ), ); + // release-failed is a release-side outcome ack (posted only when the + // release DELETE did NOT land), not a command-superseding refusal. const producerRefusals = [...producerAckVariants].filter( - (v) => v !== 'engaged' && v !== 'released', + (v) => v !== 'engaged' && v !== 'released' && v !== 'release-failed', ); expect(producerRefusals.sort()).toEqual([ 'base-refused', @@ -1740,9 +1757,15 @@ exit 1`; workflow.match(/command-permission read failed — release deferred/g) ?.length, ).toBe(2); - // R2-13: CLEANUPS increments only after the heal DELETE succeeded. + // R9-5: the cleanups budget counts ATTEMPTS like the release budget — + // incremented BEFORE the act() call in both arms, so a DELETE outage + // trips the cap instead of leaving it at 0 while every candidate burns + // a live read plus a failing DELETE each tick. + expect(workflow).toMatch( + /CLEANUPS=\$\(\( CLEANUPS \+ 1 \)\)\n\s+if act "#\$\{PR\}: clear stale \$\{NEEDS_HUMAN_LABEL\} \(re-armed\)/, + ); expect(workflow).toMatch( - /clear stale \$\{NEEDS_HUMAN_LABEL\}[\s\S]{0,400}labels\/\$\(jq -rn --arg l "\$\{NEEDS_HUMAN_LABEL\}" '\$l\|@uri'\)"; then\n\s+CLEANUPS=\$\(\( CLEANUPS \+ 1 \)\)/, + /CLEANUPS=\$\(\( CLEANUPS \+ 1 \)\)\n\s+if act "#\$\{PR\}: clear stale \$\{NEEDS_HUMAN_LABEL\} \(manual release/, ); // R2-23: both shepherd DELETEs are URI-encoded (the encoded path is the // addressable route). @@ -1856,6 +1879,13 @@ exit 1`; 'HUMAN_IDS=",$(jq -r \'[.[].number | tostring] | join(",")\' /tmp/paused.json),"', ); expect(workflow).toContain('if [[ "${HUMAN_IDS}" == *",${PR},"* ]]; then'); + // R9-1/R9-13: loop 3 renders a both-label PR only when BOTH other + // owners are blind — not a paused member AND the takeover enumeration + // failed — so a paused-enum outage never duplicates the row and a + // takeover-enum outage never drops it. + expect(workflow).toContain( + '[[ "${HUMAN_IDS}" == *",${PR},"* || "${TK_OK}" == "true" ]] && continue', + ); // R4-10: loop 1's row escapes STATE (the ci-red detailsUrl) as well as // NOTE — a `|` or `\c` in the URL must not break the printf '%b' table. expect(workflow).toContain('SAFE_STATE="${STATE//\\\\/\\\\\\\\}"'); From 7c76edb267448e695b50f5d6b622902db85888f9 Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot Date: Fri, 14 Aug 2026 21:22:04 +0000 Subject: [PATCH 16/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?10=20Criticals=20=E2=80=94=20exact=20HTTP=20404=20release=20cla?= =?UTF-8?q?ssification,=20isolated=20replay=20fixtures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Qwen-Coder --- .github/workflows/pr-self-report-label.yml | 6 ++-- .github/workflows/qwen-autofix.yml | 25 +++++++++-------- scripts/tests/pr-self-report-label.test.js | 16 +++++++++++ scripts/tests/qwen-autofix-workflow.test.js | 28 +++++++++++++++++++ .../qwen-fleet-shepherd-workflow.test.js | 20 +++++++++++-- 5 files changed, 80 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pr-self-report-label.yml b/.github/workflows/pr-self-report-label.yml index 2db86ce5884..b3b123bdfaf 100644 --- a/.github/workflows/pr-self-report-label.yml +++ b/.github/workflows/pr-self-report-label.yml @@ -101,11 +101,13 @@ jobs: # green — the job only re-runs on the next PR event — but must # not disappear: without the warning, the log claims "removed" # while the label stays on the PR. LBL_DEL_FAILED is keyed on the - # exit status (never the output text) and mirrors the identical + # exit status, with the exact "HTTP 404" token as the only + # text-derived exception (a bare 404 substring also rides + # transport failures' embedded URLs), and mirrors the identical # idiom in qwen-autofix.yml (a contract test pins them in sync). LBL_DEL_FAILED=false if ! REMOVE_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${REMOVE_ERR}" == *404* ]] || { LBL_DEL_FAILED=true; echo "::warning::#${PR}: ${LABEL} removal failed — ${REMOVE_ERR}"; } + [[ "${REMOVE_ERR}" == *"HTTP 404"* ]] || { LBL_DEL_FAILED=true; echo "::warning::#${PR}: ${LABEL} removal failed — ${REMOVE_ERR}"; } fi if [[ "${LBL_DEL_FAILED}" == "true" ]]; then echo "#${PR}: ${LABEL} removal did not land — will retry on the next PR event" diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index ee24e87cbe8..bda6bc14827 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -1947,7 +1947,7 @@ jobs: # Management resumed — the escalation label is stale. 404 is # the common case (the PR was never paused). if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" fi else # REST for consistency and runner-version independence: `gh pr @@ -1989,7 +1989,7 @@ jobs: # Engaged (possibly re-engaging an auto-released PR) — the # escalation label is stale. 404 is the common case. if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" fi fi else @@ -2015,13 +2015,16 @@ jobs: # NOT land and the needs-human removal below must NOT run # (R4-32) — or the PR keeps the takeover label (still capped, # nothing manages it now) while losing the only filterable - # escalation state. The flag is keyed on the EXIT STATUS, not - # the output text: a 200 success carries the remaining-labels - # body and a network error carries no "HTTP" token, so any - # text sniff fails one direction or the other (R6-1/R6-19). + # escalation state. The flag is keyed on the EXIT STATUS, with + # one text-derived exception: a failed DELETE whose error + # carries the exact "HTTP 404" token is the already-off case. + # The match must stay that precise token, not a bare "404" + # substring: transport failures embed the request URL — a PR + # number containing 404 would flip the classification — while + # no transport error carries an "HTTP" token (R6-1/R6-19). LBL_DEL_FAILED=false if ! REMOVE_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${REMOVE_ERR}" == *404* ]] || { LBL_DEL_FAILED=true; echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}"; } + [[ "${REMOVE_ERR}" == *"HTTP 404"* ]] || { LBL_DEL_FAILED=true; echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}"; } fi REMOVED_OK=true if [[ "${LBL_DEL_FAILED}" == "true" ]]; then @@ -2046,7 +2049,7 @@ jobs: elif [[ "${SKIP_STATE}" == "true" ]]; then echo "🧭 ${NEEDS_HUMAN_LABEL} removal skipped: ${SKIP_LABEL} present on #${PR}" elif ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" fi # Release ack, direct from the command — the exact mirror of # the engage side above, for the same reason: the unlabeled @@ -2150,7 +2153,7 @@ jobs: ' <<< "${RETRY_INFO}")" != "true" ]]; then echo "🧭 keeping ${NEEDS_HUMAN_LABEL} on #${PR}: nothing manages it until re-engaged (no takeover label, not bot-authored)" elif ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" fi takeover-ack: @@ -2247,7 +2250,7 @@ jobs: # under set -e must not strand the stale label. if [[ ( "${ACK}" == 'released' || "${ACK}" == 'engaged' ) && "${HAS_SKIP}" != 'true' ]]; then if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" fi fi gh pr comment "${PR}" --repo "${REPO}" --body "${BODY}" @@ -3094,7 +3097,7 @@ jobs: # Engaged — the escalation label is stale. 404 is the # common case (the PR was never paused). if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${NH_ERR}" == *404* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" fi # Atomic re-fetch: ic.json already holds a successful full # fetch from above; a truncated stream must not leave it 0 diff --git a/scripts/tests/pr-self-report-label.test.js b/scripts/tests/pr-self-report-label.test.js index 56e9485b4d8..603e594d37c 100644 --- a/scripts/tests/pr-self-report-label.test.js +++ b/scripts/tests/pr-self-report-label.test.js @@ -208,6 +208,22 @@ describe('pr-self-report-label', () => { // the label was removed (the lying-log shape this change exists to kill). expect(failed.out).toContain('removal did not land'); expect(failed.out).not.toContain('removed'); + // R10-1/R10-3: the tolerated token is exactly "HTTP 404" — a transport + // failure embedding a 404-bearing request URL must warn and report + // not-landed, never read as the already-off case (the loose *404* + // match hit the URL's PR number and claimed "removed"). + const transport = run({ + prAuthor: 'alice', + issueAuthors: 'bob', + hasLabel: true, + deleteFails: + 'gh: Delete "https://api.github.com/repos/o/r/issues/4041/labels/review%2Fself-reported": dial tcp 140.82.121.4:443: connect: connection refused', + }); + expect(transport.removed).toBe(true); + expect(transport.out).toContain('::warning::'); + expect(transport.out).toContain('removal failed'); + expect(transport.out).toContain('removal did not land'); + expect(transport.out).not.toContain('removed'); // POST carries NO tolerance: a label that fails to apply must fail the // step (the runBlock sets -e), not leave the PR unlabeled behind green. expect(() => diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 519ebb6befa..f7628a24b25 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -4068,6 +4068,24 @@ describe('qwen-autofix workflow', () => { 'labels/autofix%2Fneeds-human', ); expect(releaseTransportFailed.log).toContain('release did not land'); + // R10-1: the tolerance matches the EXACT "HTTP 404" token, never a bare + // 404 substring — a transport failure embeds the request URL, so a + // 404-bearing PR number in the path (modeled here on #4041) must + // classify as not-landed, not as the already-off case. + const releaseUrlFourOhFour = runToggle({ + cmd: 'remove', + labels: ['autofix/takeover'], + deleteFails: + 'gh: Delete "https://api.github.com/repos/QwenLM/qwen-code/issues/4041/labels/autofix%2Ftakeover": dial tcp 140.82.121.4:443: connect: connection refused', + }); + expect(releaseUrlFourOhFour.writes).toContain( + 'takeover-ack release-failed', + ); + expect(releaseUrlFourOhFour.writes).not.toContain('takeover-ack released'); + expect(releaseUrlFourOhFour.writes).not.toContain( + 'labels/autofix%2Fneeds-human', + ); + expect(releaseUrlFourOhFour.log).toContain('release did not land'); // Both ack posts keep their non-fatal fallback: under bash -e a failed // gh pr comment would otherwise abort the step RED after the label was // already toggled — a worse signal than the silence being fixed. A @@ -5679,6 +5697,16 @@ exit 1 expect(relDelete404.status).toBe(0); expect(relDelete404.stdout).not.toContain('removal failed'); expect(relDelete404.body).toContain(''); + // R10-3: …but the tolerated token is exactly "HTTP 404" — an error that + // carries 404 only as a substring must still warn (the loose *404* + // match swallowed it; transport errors embed 404-bearing request URLs). + const relDeleteSubstring = runAck({ + ack: 'released', + deleteFails: 'HTTP 502: Bad Gateway (upstream retry of request 404)', + }); + expect(relDeleteSubstring.status).toBe(0); + expect(relDeleteSubstring.stdout).toContain('removal failed'); + expect(relDeleteSubstring.body).toContain(''); // R4-16: fork-refused and skip-blocked acks — management never resumed, // zero DELETEs (with total-count assertions, not just toContain). expect(runAck({ ack: 'fork-refused' }).calls).not.toContain(nhDelete); diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index 4f3288eb9f8..06e06deee7d 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -787,11 +787,20 @@ exit 1`; ].join('\n'), ); chmodSync(join(dir, 'gh'), 0o755); + // R10-2: the verbatim function carries production's shared + // /tmp/cperm-err path — rewrite it into THIS case's temp dir so + // concurrent suite runs on one machine (shared self-hosted + // runners) cannot corrupt each other's classification fixtures + // through the one global file. + const replayFn = resumeFn + .replace(/\n {10}/g, '\n') + .replaceAll('/tmp/cperm-err', join(dir, 'cperm-err')); + expect(replayFn).not.toContain('/tmp/cperm-err'); const out = execFileSync( 'bash', [ '-c', - `${gnuDateShim}\n${resumeFn.replace(/\n {10}/g, '\n')}\ncompute_resume_ts "${dir}/ic.json" "${dir}/ev.json"\necho "TS=$RESUME_OUT"\necho "PERM_FAILED=$PERM_READ_FAILED"`, + `${gnuDateShim}\n${replayFn}\ncompute_resume_ts "${dir}/ic.json" "${dir}/ev.json"\necho "TS=$RESUME_OUT"\necho "PERM_FAILED=$PERM_READ_FAILED"`, ], { env: { @@ -1819,11 +1828,18 @@ exit 1`; ].join('\n'), ); chmodSync(join(dir, 'gh'), 0o755); + // R10-2: same isolation as the compute_resume_ts replay — the + // verbatim function writes /tmp/cf-ic.json, a path shared with + // every concurrent run on the machine. + const replayFn = conflictFn + .replace(/\n {10}/g, '\n') + .replaceAll('/tmp/cf-ic.json', join(dir, 'cf-ic.json')); + expect(replayFn).not.toContain('/tmp/cf-ic.json'); const res = spawnSync( 'bash', [ '-c', - `${conflictFn.replace(/\n {10}/g, '\n')}\nif conflict_paused 7354; then echo PAUSED; else echo ARMED; fi`, + `${replayFn}\nif conflict_paused 7354; then echo PAUSED; else echo ARMED; fi`, ], { env: { From 5f24cd5dc9bdd2be85e26bc85f0884d05708b332 Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot Date: Sat, 15 Aug 2026 02:33:15 +0000 Subject: [PATCH 17/17] =?UTF-8?q?fix(autofix):=20close=20review=20round=20?= =?UTF-8?q?11=20Criticals=20=E2=80=94=20engaged=20stale-ack=20guard,=20exa?= =?UTF-8?q?ct=20HTTP=20404=20permission=20classification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/qwen-autofix.yml | 43 ++++++++ .github/workflows/qwen-fleet-shepherd.yml | 15 ++- scripts/tests/qwen-autofix-workflow.test.js | 97 +++++++++++++++++-- .../qwen-fleet-shepherd-workflow.test.js | 12 +++ 4 files changed, 156 insertions(+), 11 deletions(-) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index bda6bc14827..eb276063e35 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -2160,6 +2160,13 @@ jobs: needs: 'route' if: |- ${{ needs.route.outputs.takeover_ack != '' }} + # A delayed or overlapping ack must not race a newer cycle's ack: + # queued (never cancelled) per-PR execution serializes the runs so + # the staleness reads below see each predecessor's writes (mirrors + # takeover-command). + concurrency: + group: 'qwen-autofix-takeover-ack-${{ needs.route.outputs.ack_pr }}' + cancel-in-progress: false runs-on: 'ubuntu-latest' timeout-minutes: 5 permissions: @@ -2225,6 +2232,42 @@ jobs: echo "⚠️ released ack skipped on #${PR}: ${TAKEOVER_LABEL} re-applied since the release (stale ack)" exit 0 fi + # R2-4 mirror for the engaged direction: a delayed engaged ack — a + # red run re-run later, or overlapping runs from quick label + # toggles — must not DELETE a fresh cycle's needs-human and must + # not post a marker that resets the round window (REARM_KEY is + # the newest engage marker). Label absent → the engagement ended + # after this event. A bot engage ack at/after the newest labeled + # event → the current cycle is already acked. Both skip without + # posting or touching labels; an unreadable history skips too + # (fail closed — the scan's NEED_ENGAGE_ACK dedup heals a + # genuinely missed ack, while nothing heals a stale marker). + if [[ "${ACK}" == 'engaged' ]]; then + if [[ "$(jq -r --arg t "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${PR_STATE_INFO}")" != "true" ]]; then + echo "⚠️ engaged ack skipped on #${PR}: ${TAKEOVER_LABEL} removed since the label event (stale ack)" + exit 0 + fi + if ! ack_ic="$(gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null)"; then + echo "::warning::engaged ack skipped on #${PR}: comment history unreadable (fail closed)" + exit 0 + fi + if ! ack_ev="$(gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null)"; then + echo "::warning::engaged ack skipped on #${PR}: event history unreadable (fail closed)" + exit 0 + fi + ack_aced_ts="$(jq -rs --arg ab "${AUTOFIX_BOT}" ' + add // [] | [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | contains("")) + | .created_at ] | max // ""' <<< "${ack_ic}")" + ack_labeled_ts="$(jq -rs --arg tl "${TAKEOVER_LABEL}" ' + add // [] | [ .[] | select((.event // "") == "labeled") + | select((.label.name // "") == $tl) + | .created_at ] | max // ""' <<< "${ack_ev}")" + if [[ -n "${ack_aced_ts}" && ! "${ack_labeled_ts}" > "${ack_aced_ts}" ]]; then + echo "⚠️ engaged ack skipped on #${PR}: a bot engage ack already landed after the newest ${TAKEOVER_LABEL} label event (stale ack)" + exit 0 + fi + fi if [[ "${ACK}" == 'base-refused' ]]; then BODY="$(printf '🚫 Takeover not engaged: the loop only manages PRs that target `main`, and this one targets `%s`. A stacked PR moves whenever its base branch does, so "new feedback since the last round" and base-conflict resolution are not well defined until the base lands. Two ways forward: retarget this PR to `main` once the base PR merges — the `%s` label is left in place and the scan lists by label, so the next scan engages it with no re-labelling — or take over the base PR instead.\n\n
\n中文说明\n\n🚫 未接管：循环只管理以 `main` 为 base 的 PR，而本 PR 的 base 是 `%s`。堆叠 PR 会随 base 分支移动，因此“自上一轮以来的新反馈”与 base 冲突处理都无法良定义。两条路：待 base 的 PR 合入后把本 PR 改为面向 `main` —— `%s` 标签予以保留，扫描按标签枚举，下一次扫描即会自动接管，无需重新打标签；或改为接管 base 那个 PR。\n\n
\n\n' "${ACK_BASE}" "${TAKEOVER_LABEL}" "${ACK_BASE}" "${TAKEOVER_LABEL}")" elif [[ "${ACK}" == 'engaged' && "${HAS_SKIP}" == "true" ]]; then diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index 648440c357f..2672b39f770 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -448,11 +448,16 @@ jobs: [[ "${reads}" -ge 2 ]] && { PERM_READ_FAILED=true; break; } reads=$(( reads + 1 )) if ! cperm="$(gh api "repos/${REPO}/collaborators/${cauthor}/permission" --jq '.permission // ""' 2> /tmp/cperm-err)"; then - # 404 is GitHub's decisive "not a collaborator" answer — - # classify read-only instead of deferring: a defer here is - # renewable by any stranger's exact command each grace - # window, and it would pose a classification as an outage. - grep -q '404' /tmp/cperm-err && continue + # An exact "HTTP 404" is GitHub's decisive "not a + # collaborator" answer — classify read-only instead of + # deferring: a defer here is renewable by any stranger's + # exact command each grace window, and it would pose a + # classification as an outage. Match the token, not the + # bare number: a transport failure embeds the request URL, + # which carries the commenter login, so a login containing + # "404" would classify an outage as read-only — the same + # exact token every label DELETE tolerates (R10-1). + grep -q 'HTTP 404' /tmp/cperm-err && continue PERM_READ_FAILED=true continue fi diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index f7628a24b25..708b84a317e 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -2469,6 +2469,12 @@ describe('qwen-autofix workflow', () => { expect(workflow).toContain( "group: 'qwen-autofix-takeover-cmd-${{ needs.route.outputs.cmd_pr }}'", ); + // R11-2: ack runs serialize per PR the same way, so a delayed engaged + // ack queues behind the newer cycle's ack and its staleness read sees + // that ack's marker. + expect(workflow).toContain( + "group: 'qwen-autofix-takeover-ack-${{ needs.route.outputs.ack_pr }}'", + ); // Fork PRs can never produce a red ack run or a stuck label: the // unlabeled branch log-and-drops forks (fork pull_request events carry // no secrets, so emitting the ack would fail the PAT identity check), @@ -2919,7 +2925,7 @@ describe('qwen-autofix workflow', () => { // forces a deliberate test update, however it is spaced or line-wrapped: // bump this count AND pipe the new site through the normalizer (bumping // the count below too) — bumping this pin alone leaves toBe(10) green. - expect(workflow.split('--paginate').length - 1).toBe(16); + expect(workflow.split('--paginate').length - 1).toBe(18); // scan ic + pr-events + ic re-fetch + scan rv/rc + prepare rv/rc/ic + // report COMMENTS_JSON fallback + the cap-branch release-evidence events // fetch (R4-1) = ten normalized fetch sites. The @@ -2930,6 +2936,10 @@ describe('qwen-autofix workflow', () => { // array and break the tail-1 consumer. The #8888 deferred-review ack // dedup is the same class: `--jq '.[].body'` feeds a grep, never a // WORKDIR file, so it bumps the total pin but not the normalizer count. + // The R11-2 engaged-stale guard's comments/events reads in + // takeover-ack are the same class too: captured into shell variables + // and consumed by slurp-style `jq -rs 'add // [] | …'` (idempotent + // over a single flat array), never a WORKDIR file. expect(workflow.split("jq -s 'add // []'").length - 1).toBe(10); // Empty-input semantics: a total gh failure feeds the fallback an EMPTY // stream, where the normalizer filter must yield '[]' and not 'null' — @@ -5524,6 +5534,9 @@ exit 1 prViewOk = true, labels = [], deleteFails = '', + comments = [], + events = [], + historyFails = false, }) => { const dir = mkdtempSync(join(tmpdir(), 'ack-')); const bin = join(dir, 'bin'); @@ -5534,6 +5547,10 @@ exit 1 '#!/usr/bin/env bash', `echo "$@" >> ${JSON.stringify(join(dir, 'calls.log'))}`, `if [[ "$1" == 'api' && "$2" == 'user' ]]; then printf 'qwen-code-dev-bot'; exit 0; fi`, + // R11-2: the engaged-stale dedup reads — served per knob + // (empty by default, so a legacy vector sees no history). + `if [[ "$1" == 'api' && "$2" == *'/comments' ]]; then ${historyFails ? 'exit 1' : `printf '%s' '${JSON.stringify(comments)}'`}; fi`, + `if [[ "$1" == 'api' && "$2" == *'/events' ]]; then ${historyFails ? 'exit 1' : `printf '%s' '${JSON.stringify(events)}'`}; fi`, // Other api calls (the needs-human DELETE) succeed silently but // are recorded above — the ack-job conditional removal is // observable. R6-33: with a failure knob — a non-404 must @@ -5612,7 +5629,12 @@ exit 1 // Unchanged for every other ack: the live read happens and still fails // CLOSED, so a transient API error cannot turn into a wrong ack. - const engaged = runAck({ ack: 'engaged' }); + // R11-2: an engaged ack posts only while the takeover label is LIVE — + // the engagement it acknowledges must still exist at ack time. + const engaged = runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/takeover' }], + }); expect(engaged.readPr).toBe(true); expect(engaged.body).toContain(''); const broken = runAck({ ack: 'engaged', prViewOk: false }); @@ -5624,9 +5646,14 @@ exit 1 const nhDelete = 'api -X DELETE repos/QwenLM/qwen-code/issues/7368/labels/autofix%2Fneeds-human'; expect( - runAck({ ack: 'engaged' }).calls.match(/api -X DELETE/gm) ?? [], + runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/takeover' }], + }).calls.match(/api -X DELETE/gm) ?? [], ).toHaveLength(1); - expect(runAck({ ack: 'engaged' }).calls).toContain(nhDelete); + expect( + runAck({ ack: 'engaged', labels: [{ name: 'autofix/takeover' }] }).calls, + ).toContain(nhDelete); expect( runAck({ ack: 'released' }).calls.match(/api -X DELETE/gm) ?? [], ).toHaveLength(1); @@ -5639,14 +5666,17 @@ exit 1 releasedOrder.calls.indexOf('pr comment'), ); expect( - runAck({ ack: 'engaged', labels: [{ name: 'autofix/skip' }] }).calls, + runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/skip' }, { name: 'autofix/takeover' }], + }).calls, ).not.toContain(nhDelete); // R6-5: assert the TOTAL DELETE count for engaged-with-skip too — // `not.toContain` alone lets a mutant add an unrelated second DELETE. expect( runAck({ ack: 'engaged', - labels: [{ name: 'autofix/skip' }], + labels: [{ name: 'autofix/skip' }, { name: 'autofix/takeover' }], }).calls.match(/api -X DELETE/gm) ?? [], ).toHaveLength(0); expect( @@ -5680,6 +5710,61 @@ exit 1 expect(staleRelease.calls).not.toContain('api -X DELETE'); expect(staleRelease.body).toBe(''); expect(staleRelease.stdout).toContain('stale ack'); + // R11-2: the R2-4 mirror for the engaged direction — a delayed engaged + // ack is stale in two shapes. Label removed since the label event: the + // engagement ended, so post nothing and touch nothing… + const staleEngagedRemoved = runAck({ ack: 'engaged' }); + expect(staleEngagedRemoved.status).toBe(0); + expect(staleEngagedRemoved.calls).not.toContain('api -X DELETE'); + expect(staleEngagedRemoved.body).toBe(''); + expect(staleEngagedRemoved.stdout).toContain('stale ack'); + // …and label LIVE but the current cycle already acked: a bot engage + // marker at/after the newest takeover labeled event means this run's + // event is superseded — it must not DELETE the fresh cycle's + // needs-human nor post a marker that would reset the round window. + const engagedMarker = (created_at) => ({ + user: { login: 'qwen-code-dev-bot' }, + created_at, + body: '🤝 … ', + }); + const takeoverLabeled = (created_at) => ({ + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at, + }); + const staleEngagedAcked = runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/takeover' }, { name: 'autofix/needs-human' }], + comments: [engagedMarker('2026-08-06T02:00:00Z')], + events: [takeoverLabeled('2026-08-06T01:00:00Z')], + }); + expect(staleEngagedAcked.status).toBe(0); + expect(staleEngagedAcked.calls).not.toContain('api -X DELETE'); + expect(staleEngagedAcked.body).toBe(''); + expect(staleEngagedAcked.stdout).toContain('stale ack'); + // Comparator: a label event NEWER than the last engage marker is a + // live unacked engagement (a re-label after release) — it posts and + // cleans the stale escalation label. + const freshEngagement = runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/takeover' }], + comments: [engagedMarker('2026-08-06T00:00:00Z')], + events: [takeoverLabeled('2026-08-06T01:00:00Z')], + }); + expect(freshEngagement.status).toBe(0); + expect(freshEngagement.body).toContain(''); + expect(freshEngagement.calls).toContain(nhDelete); + // Unreadable history fails CLOSED: a stale marker is irreparable, a + // missed ack is healed by the scan's NEED_ENGAGE_ACK dedup. + const blindEngaged = runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/takeover' }], + historyFails: true, + }); + expect(blindEngaged.status).toBe(0); + expect(blindEngaged.body).toBe(''); + expect(blindEngaged.calls).not.toContain('api -X DELETE'); + expect(blindEngaged.stdout).toContain('fail closed'); // R6-33: a non-404 needs-human DELETE failure warns and STILL posts the // ack (bash -e must not abort before the comment)… const relDeleteFailed = runAck({ diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index 06e06deee7d..a3148790708 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -927,6 +927,18 @@ exit 1`; }); expect(notCollab.ts).toBe(''); expect(notCollab.flag).toBe(''); + // R11-1: …but the match is the exact "HTTP 404" token: a transport + // failure embeds the request URL, which carries the commenter login — + // a login containing "404" must classify as an outage (defer), never + // as "not a collaborator". + const transportErr = computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'maintainer4041')], + permFail: true, + permError: + 'Get https://api.github.com/repos/QwenLM/qwen-code/collaborators/maintainer4041/permission: connection refused', + }); + expect(transportErr.ts).toBe(''); + expect(transportErr.flag).toBe('true'); // R8-4: budget exhaustion — with THREE distinct-author fresh commands // the 2-read budget runs out before the oldest is examined; that must // surface as PERM_READ_FAILED (defer), never as "no trusted command"