diff --git a/.github/workflows/pr-self-report-label.yml b/.github/workflows/pr-self-report-label.yml index cccaadf243d..b3b123bdfaf 100644 --- a/.github/workflows/pr-self-report-label.yml +++ b/.github/workflows/pr-self-report-label.yml @@ -100,11 +100,20 @@ jobs: # failure (403 rate limit, 5xx, network) also keeps the step # green — the job only re-runs on the next PR event — but must # not disappear: without the warning, the log claims "removed" - # while the label stays on the PR. + # while the label stays on the PR. LBL_DEL_FAILED is keyed on the + # exit status, with the exact "HTTP 404" token as the only + # text-derived exception (a bare 404 substring also rides + # transport failures' embedded URLs), and mirrors the identical + # idiom in qwen-autofix.yml (a contract test pins them in sync). + LBL_DEL_FAILED=false if ! REMOVE_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${REMOVE_ERR}" == *404* ]] || echo "::warning::#${PR}: ${LABEL} removal failed — ${REMOVE_ERR}" + [[ "${REMOVE_ERR}" == *"HTTP 404"* ]] || { LBL_DEL_FAILED=true; echo "::warning::#${PR}: ${LABEL} removal failed — ${REMOVE_ERR}"; } + fi + if [[ "${LBL_DEL_FAILED}" == "true" ]]; then + echo "#${PR}: ${LABEL} removal did not land — will retry on the next PR event" + else + echo "🏷️ #${PR}: removed ${LABEL} — no self-reported linked issue" fi - echo "🏷️ #${PR}: removed ${LABEL} — no self-reported linked issue" else echo "#${PR}: no change (self=${SELF}, labeled=${HAS_LABEL}, api=${API_OK})" fi diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 57018b1e17c..eb276063e35 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -225,6 +225,12 @@ env: # Comment-command sugar over TAKEOVER_LABEL ('' applies it, ' # stop' removes it). Matched EXACTLY against the trimmed comment body. TAKEOVER_COMMAND: '@qwen-code /takeover' + # Escalation label: applied when the loop STOPS on a PR (round cap, + # consecutive-failure or time-budget breaker) and a human must act — + # re-arm, split, merge, or close. It makes paused PRs filterable and lets + # the fleet shepherd age out unanswered stops. Removed wherever management + # resumes (engage/re-arm) or a human releases the PR. + NEEDS_HUMAN_LABEL: 'autofix/needs-human' # Re-arm sugar. Recovering a stranded PR previously meant DELETING the # bot's autofix-eval marker comment by hand (undiscoverable, destructive, # and it erases the audit trail). This command instead posts an @@ -1938,6 +1944,11 @@ jobs: # already addressed is never replayed. gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '🔄 Takeover re-armed: the round counter starts a fresh window (previous rounds no longer count toward the cap); management continues.\n\n
\n中文说明\n\n🔄 已重新武装：轮次计数开启新窗口（此前轮次不再计入上限），托管继续。\n\n
\n\n')" echo "🔄 re-armed ${TAKEOVER_LABEL} window on #${PR}" + # Management resumed — the escalation label is stale. 404 is + # the common case (the PR was never paused). + if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + fi else # REST for consistency and runner-version independence: `gh pr # edit`'s GraphQL lookup requests @@ -1975,6 +1986,11 @@ jobs: fi gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '🤝 Takeover engaged: the autofix loop now manages this PR — it will address new review feedback and resolve base conflicts until the label is removed or the round cap is reached.%s Remove the `%s` label (or comment `%s stop`) to release.\n\n
\n中文说明\n\n🤝 已接管：autofix 循环现在管理此 PR —— 将持续处理新的评审反馈与 base 冲突，直到移除标签或达到轮次上限。%s移除 `%s` 标签（或评论 `%s stop`）即可释放。\n\n
\n\n' "${FORK_NOTE}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}" "${FORK_NOTE_ZH}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}")" \ || echo "::warning::engage ack comment failed on #${PR}; the scan's first-pickup ack heals it" + # Engaged (possibly re-engaging an auto-released PR) — the + # escalation label is stale. 404 is the common case. + if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + fi fi else if [[ "${HAS}" != 'true' ]]; then @@ -1989,10 +2005,52 @@ jobs: # `/takeover stop` retries the removal — but must not disappear # silently either: masked, the ack reads "released" while the # loop keeps managing the PR. + # The 404-tolerance block is pinned byte-identical to the other + # workflows' label DELETE (a contract test), so REMOVED_OK — + # whether the takeover release actually LANDED — is derived + # AFTER the idiom from the captured stream: gh api prints the + # remaining-labels JSON body on success (even '[]'), while a + # failure carries "HTTP " in the error text. 404 = + # already off (landed); any other HTTP error = the release did + # NOT land and the needs-human removal below must NOT run + # (R4-32) — or the PR keeps the takeover label (still capped, + # nothing manages it now) while losing the only filterable + # escalation state. The flag is keyed on the EXIT STATUS, with + # one text-derived exception: a failed DELETE whose error + # carries the exact "HTTP 404" token is the already-off case. + # The match must stay that precise token, not a bare "404" + # substring: transport failures embed the request URL — a PR + # number containing 404 would flip the classification — while + # no transport error carries an "HTTP" token (R6-1/R6-19). + LBL_DEL_FAILED=false if ! REMOVE_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')" 2>&1)"; then - [[ "${REMOVE_ERR}" == *404* ]] || echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}" + [[ "${REMOVE_ERR}" == *"HTTP 404"* ]] || { LBL_DEL_FAILED=true; echo "::warning::#${PR}: ${TAKEOVER_LABEL} removal failed — ${REMOVE_ERR}"; } + fi + REMOVED_OK=true + if [[ "${LBL_DEL_FAILED}" == "true" ]]; then + REMOVED_OK=false + fi + if [[ "${REMOVED_OK}" == "true" ]]; then + echo "🏷️ removed ${TAKEOVER_LABEL} from #${PR}" + else + # R7-7: the success claim must not fire when the DELETE did + # not land — the motivating fix for pr-self-report-label.yml + # in this very PR is precisely this lying-log shape. + echo "⚠️ #${PR}: ${TAKEOVER_LABEL} removal did not land — the next /takeover stop retries" + fi + # Released means a human is driving — the escalation label is + # stale. Remove it only when the release landed (REMOVED_OK) + # AND the PR is not frozen by skip (a frozen PR must keep its + # only filterable escalation state — R4-3). 404 is the common + # case (never paused). + SKIP_STATE="$(jq -r --arg t "${SKIP_LABEL}" '[.labels[].name] | index($t) != null' <<< "${PR_INFO}")" + if [[ "${REMOVED_OK}" != "true" ]]; then + echo "::warning::#${PR}: release did not land — keeping ${NEEDS_HUMAN_LABEL}; the next /takeover stop retries both" + elif [[ "${SKIP_STATE}" == "true" ]]; then + echo "🧭 ${NEEDS_HUMAN_LABEL} removal skipped: ${SKIP_LABEL} present on #${PR}" + elif ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" fi - echo "🏷️ removed ${TAKEOVER_LABEL} from #${PR}" # Release ack, direct from the command — the exact mirror of # the engage side above, for the same reason: the unlabeled # round-trip is the thing we no longer trust, fork unlabeled @@ -2005,7 +2063,14 @@ jobs: # the unlabeled-path ack when the label sender is the bot. REL_AUTHOR="$(jq -r '.author.login // ""' <<< "${PR_INFO}")" REL_HAS_SKIP="$(jq -r --arg t "${SKIP_LABEL}" '[.labels[].name] | index($t) != null' <<< "${PR_INFO}")" - if [[ "${REL_AUTHOR}" == "${AUTOFIX_BOT}" && "${REL_HAS_SKIP}" == "true" ]]; then + if [[ "${REMOVED_OK}" != "true" ]]; then + # The DELETE did not land — the label is still on and the + # loop still manages this PR. A "released" ack (and its + # marker) would record a release that never happened: no + # unlabeled event fires, nothing retries, and no human + # re-issues the command. Own the failure and name the retry. + REL_BODY="$(printf '⚠️ Takeover release did not land: removing the `%s` label failed transiently, so it is still present and the autofix loop keeps managing this PR under the round cap. Comment `%s stop` to retry the release.\n\n
\n中文说明\n\n⚠️ 释放未生效：移除 `%s` 标签时瞬时失败，标签仍在，autofix 循环仍按轮次上限继续托管此 PR。评论 `%s stop` 可重试释放。\n\n
\n\n' "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}")" + elif [[ "${REL_AUTHOR}" == "${AUTOFIX_BOT}" && "${REL_HAS_SKIP}" == "true" ]]; then REL_BODY="$(printf '👋 Takeover mode ended. This bot-authored PR also carries `%s`, which opts it out of standard bot management entirely — nothing will engage it until that label is removed.\n\n
\n中文说明\n\n👋 接管模式结束。本 bot 创建的 PR 同时带有 `%s`，已完全退出常规 bot 管理 —— 移除该标签前不会有任何介入。\n\n
\n\n' "${SKIP_LABEL}" "${SKIP_LABEL}")" elif [[ "${REL_AUTHOR}" == "${AUTOFIX_BOT}" ]]; then REL_BODY="$(printf '👋 Takeover mode ended: the raised round cap no longer applies. This is a bot-authored PR, so STANDARD bot management continues under the strict cap (apply `%s` to opt it out entirely). Re-apply `%s` (or comment `%s`) for the raised cap again.\n\n
\n中文说明\n\n👋 接管模式结束：提升的轮次上限不再适用。这是 bot 创建的 PR，常规 bot 管理仍将继续（严格上限；如需完全退出请打 `%s`）。重新打上 `%s` 标签（或评论 `%s`）可恢复提升上限。\n\n
\n\n' "${SKIP_LABEL}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}" "${SKIP_LABEL}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}")" @@ -2070,11 +2135,38 @@ jobs: fi gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '\U0001F504 AutoFix re-armed. The next scan re-reads this PR'"'"'s feedback from the start and the round counter resets. Nothing was deleted — this marker supersedes the evaluation markers above it.\n\n
\n中文说明\n\n\U0001F504 已重新武装 AutoFix。下一次扫描会从头重新读取本 PR 的反馈，轮次计数也已重置。未删除任何内容 —— 本标记使其上方的评估标记失效。\n\n
\n\n')" echo "🔄 re-armed PR #${PR}" + # Management resumed — the escalation label is stale. 404 is the + # common case (the PR was never paused). Remove it only when the + # PR will actually be MANAGED after this re-arm (R5-2): the scan + # candidate population is bot-authored or takeover-labeled PRs + # only, so on an auto-released human PR (no takeover label, not + # bot-authored) /retry posts a marker nothing will act on — the + # label must stay as the only filterable escalation state. Skip + # also wins over re-arm everywhere (a frozen PR keeps its label), + # and the read FAILS CLOSED (mirrors takeover-ack's exit-1). + if ! RETRY_INFO="$(gh pr view "${PR}" --repo "${REPO}" --json labels,author 2> /dev/null)"; then + echo "::warning::#${PR}: label state unreadable — keeping ${NEEDS_HUMAN_LABEL} (fail closed)" + elif [[ "$(jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${RETRY_INFO}")" == "true" ]]; then + echo "🧭 ${NEEDS_HUMAN_LABEL} removal skipped: ${SKIP_LABEL} present on #${PR}" + elif [[ "$(jq -r --arg ab "${AUTOFIX_BOT}" --arg tk "${TAKEOVER_LABEL}" ' + ((.author.login // "") == $ab) or (([.labels[]?.name] | index($tk)) != null) + ' <<< "${RETRY_INFO}")" != "true" ]]; then + echo "🧭 keeping ${NEEDS_HUMAN_LABEL} on #${PR}: nothing manages it until re-engaged (no takeover label, not bot-authored)" + elif ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + fi takeover-ack: needs: 'route' if: |- ${{ needs.route.outputs.takeover_ack != '' }} + # A delayed or overlapping ack must not race a newer cycle's ack: + # queued (never cancelled) per-PR execution serializes the runs so + # the staleness reads below see each predecessor's writes (mirrors + # takeover-command). + concurrency: + group: 'qwen-autofix-takeover-ack-${{ needs.route.outputs.ack_pr }}' + cancel-in-progress: false runs-on: 'ubuntu-latest' timeout-minutes: 5 permissions: @@ -2130,6 +2222,52 @@ jobs: HAS_SKIP="$(jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${PR_STATE_INFO}")" PR_AUTHOR_LIVE="$(jq -r '.author.login // ""' <<< "${PR_STATE_INFO}")" fi + # R2-4: a delayed release ack can run AFTER takeover was + # re-applied and the new cycle paused again — a live takeover + # label means this ack's premise is stale. Post nothing and + # touch nothing: the new cycle's own events produce their own + # acks, while a stale run here would delete the new cycle's + # needs-human and claim a release while takeover is live. + if [[ "${ACK}" == 'released' && "$(jq -r --arg t "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${PR_STATE_INFO}")" == "true" ]]; then + echo "⚠️ released ack skipped on #${PR}: ${TAKEOVER_LABEL} re-applied since the release (stale ack)" + exit 0 + fi + # R2-4 mirror for the engaged direction: a delayed engaged ack — a + # red run re-run later, or overlapping runs from quick label + # toggles — must not DELETE a fresh cycle's needs-human and must + # not post a marker that resets the round window (REARM_KEY is + # the newest engage marker). Label absent → the engagement ended + # after this event. A bot engage ack at/after the newest labeled + # event → the current cycle is already acked. Both skip without + # posting or touching labels; an unreadable history skips too + # (fail closed — the scan's NEED_ENGAGE_ACK dedup heals a + # genuinely missed ack, while nothing heals a stale marker). + if [[ "${ACK}" == 'engaged' ]]; then + if [[ "$(jq -r --arg t "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${PR_STATE_INFO}")" != "true" ]]; then + echo "⚠️ engaged ack skipped on #${PR}: ${TAKEOVER_LABEL} removed since the label event (stale ack)" + exit 0 + fi + if ! ack_ic="$(gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null)"; then + echo "::warning::engaged ack skipped on #${PR}: comment history unreadable (fail closed)" + exit 0 + fi + if ! ack_ev="$(gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null)"; then + echo "::warning::engaged ack skipped on #${PR}: event history unreadable (fail closed)" + exit 0 + fi + ack_aced_ts="$(jq -rs --arg ab "${AUTOFIX_BOT}" ' + add // [] | [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | contains("")) + | .created_at ] | max // ""' <<< "${ack_ic}")" + ack_labeled_ts="$(jq -rs --arg tl "${TAKEOVER_LABEL}" ' + add // [] | [ .[] | select((.event // "") == "labeled") + | select((.label.name // "") == $tl) + | .created_at ] | max // ""' <<< "${ack_ev}")" + if [[ -n "${ack_aced_ts}" && ! "${ack_labeled_ts}" > "${ack_aced_ts}" ]]; then + echo "⚠️ engaged ack skipped on #${PR}: a bot engage ack already landed after the newest ${TAKEOVER_LABEL} label event (stale ack)" + exit 0 + fi + fi if [[ "${ACK}" == 'base-refused' ]]; then BODY="$(printf '🚫 Takeover not engaged: the loop only manages PRs that target `main`, and this one targets `%s`. A stacked PR moves whenever its base branch does, so "new feedback since the last round" and base-conflict resolution are not well defined until the base lands. Two ways forward: retarget this PR to `main` once the base PR merges — the `%s` label is left in place and the scan lists by label, so the next scan engages it with no re-labelling — or take over the base PR instead.\n\n
\n中文说明\n\n🚫 未接管：循环只管理以 `main` 为 base 的 PR，而本 PR 的 base 是 `%s`。堆叠 PR 会随 base 分支移动，因此“自上一轮以来的新反馈”与 base 冲突处理都无法良定义。两条路：待 base 的 PR 合入后把本 PR 改为面向 `main` —— `%s` 标签予以保留，扫描按标签枚举，下一次扫描即会自动接管，无需重新打标签；或改为接管 base 那个 PR。\n\n
\n\n' "${ACK_BASE}" "${TAKEOVER_LABEL}" "${ACK_BASE}" "${TAKEOVER_LABEL}")" elif [[ "${ACK}" == 'engaged' && "${HAS_SKIP}" == "true" ]]; then @@ -2143,6 +2281,21 @@ jobs: else BODY="$(printf '👋 Takeover released: the autofix loop will no longer engage this PR (an in-flight round, if any, completes its bounded work). Re-apply `%s` (or comment `%s`) to re-engage.\n\n
\n中文说明\n\n👋 已释放：autofix 循环不再介入此 PR（在飞的一轮如有，将完成其有界工作）。重新打上 `%s` 标签（或评论 `%s`）即可再次接管。\n\n
\n\n' "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}")" fi + # The escalation label goes stale on a real engage or any release + # (a human is driving again). NOT on base-refused (nothing changed) + # and NOT on skip-blocked (management never resumed) — and a + # RELEASE onto a skip-frozen PR also keeps the label: nothing + # manages or restores that PR, so it must stay in the needs-human + # filter (R4-3). 404 is the common case (the PR was never paused). + # Runs BEFORE the ack comment: the state change already happened + # (the human toggled the label — the comment is purely + # informational), so a transient comment failure aborting this step + # under set -e must not strand the stale label. + if [[ ( "${ACK}" == 'released' || "${ACK}" == 'engaged' ) && "${HAS_SKIP}" != 'true' ]]; then + if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + fi + fi gh pr comment "${PR}" --repo "${REPO}" --body "${BODY}" # =========================================================================== @@ -2686,10 +2839,11 @@ jobs: break fi # One PR fetch for the branch name, check rollup, creation time - # (the watermark floor below), and labels (the effective round - # cap) — avoids extra round-trips per candidate PR. + # (the watermark floor below), labels (the effective round + # cap), and author (the cap branch's bot-author exemption) — + # avoids extra round-trips per candidate PR. PR_META="$(gh pr view "${PR}" --repo "${REPO}" \ - --json headRefName,headRefOid,statusCheckRollup,createdAt,labels,isCrossRepository,headRepositoryOwner,headRepository 2> /dev/null || echo '{}')" + --json headRefName,headRefOid,statusCheckRollup,createdAt,labels,isCrossRepository,headRepositoryOwner,headRepository,author 2> /dev/null || echo '{}')" HEAD_REPO_FULL="${REPO}" if [[ "$(jq -r '.isCrossRepository // false' <<< "${PR_META}")" == "true" ]]; then HR_OWNER="$(jq -r '.headRepositoryOwner.login // ""' <<< "${PR_META}")" @@ -2927,9 +3081,13 @@ jobs: add | [.[] | select((.user.login // "") == $ab) | select(.body // "" | contains("")) | .created_at] | sort | last // ""' "${WORKDIR}/ic.json")" - gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null \ - | jq -s 'add // []' > "${WORKDIR}/pr-events.json" \ - || echo '[]' > "${WORKDIR}/pr-events.json" + PR_EVENTS_OK='' + if gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null \ + | jq -s 'add // []' > "${WORKDIR}/pr-events.json"; then + PR_EVENTS_OK=true + else + echo '[]' > "${WORKDIR}/pr-events.json" + fi LAST_LABELED_TS="$(jq -rs --arg lb "${TAKEOVER_LABEL}" ' add | [.[] | select(.event == "labeled") | select((.label.name // "") == $lb) @@ -2979,6 +3137,11 @@ jobs: fi if [[ "${SCAN_BOT_ACTOR}" == "${AUTOFIX_BOT}" ]]; then if gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '🤝 Takeover engaged: the autofix loop now manages this PR — it will address new review feedback and resolve base conflicts until the label is removed or the round cap is reached. Remove the `%s` label (or comment `%s stop`) to release.\n\n
\n中文说明\n\n🤝 已接管：autofix 循环现在管理此 PR —— 将持续处理新的评审反馈与 base 冲突，直到移除标签或达到轮次上限。移除 `%s` 标签（或评论 `%s stop`）即可释放。\n\n
\n\n' "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}" "${TAKEOVER_LABEL}" "${TAKEOVER_COMMAND}")"; then + # Engaged — the escalation label is stale. 404 is the + # common case (the PR was never paused). + if ! NH_ERR="$(gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')" 2>&1)"; then + [[ "${NH_ERR}" == *"HTTP 404"* ]] || echo "::warning::#${PR}: ${NEEDS_HUMAN_LABEL} removal failed — ${NH_ERR}" + fi # Atomic re-fetch: ic.json already holds a successful full # fetch from above; a truncated stream must not leave it 0 # bytes (jq -s fails only AFTER the redirect truncates) — @@ -3153,14 +3316,61 @@ jobs: [ .[] | select((.user.login // "") == $ab) | select((.body // "") | contains("")) | select((.created_at // "") > $rt) ] | length' "${WORKDIR}/ic.json")" + # Release evidence = a takeover unlabeled EVENT at-or-newer + # than the window key. GitHub records it whenever the label + # comes off — `/takeover stop`, the ack job, or a manual UI + # removal — unlike the release-ack COMMENT, which both release + # paths tolerate losing (R4-1): the stop branch swallows a + # failed ack post, and the ack job's set -e aborts before it. + # A re-arm advances REARM_KEY past the event, re-enabling the + # label. Same-second ties resolve toward "released" — never + # re-escalate a completed release (R5-9). Fail closed: an + # unreadable event history suppresses the re-label rather than + # risk the ping-pong. + # A capped takeover candidate already paid for this paginated + # endpoint earlier in the same iteration (pr-events.json, + # gated by PR_EVENTS_OK) — reuse that fetch instead of paying + # it twice per capped takeover PR per scan. Non-takeover + # candidates keep the standalone fetch, and the fail-closed + # semantics ride the flag: the engage-side '[]' fallback must + # never read here as "no releases". + RELEASE_ACKED='' + if [[ "${HAS_TAKEOVER}" == "true" && "${PR_EVENTS_OK}" == "true" ]]; then + if ! cp "${WORKDIR}/pr-events.json" "${WORKDIR}/ev.json"; then + RELEASE_ACKED='unreadable' + fi + elif ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > "${WORKDIR}/ev.json"; then + RELEASE_ACKED='unreadable' + fi + if [[ "${RELEASE_ACKED}" != 'unreadable' ]]; then + RELEASE_ACKED="$(jq -r --arg tl "${TAKEOVER_LABEL}" --arg rt "${NOTICE_RT}" ' + [ .[] | select((.event // "") == "unlabeled") + | select((.label.name // "") == $tl) + | select((.created_at // "") >= $rt) ] | length' "${WORKDIR}/ev.json")" + fi + # A release only suppresses re-labeling for HUMAN-authored PRs. + # A bot-authored PR released from takeover returns to STANDARD + # management (its ack says so) — it is still managed, so at the + # strict cap it deserves the same notice + escalation label as + # any managed PR (R4-5). + IS_BOT_AUTHOR="$(jq -r --arg ab "${AUTOFIX_BOT}" '((.author.login // "") == $ab)' <<< "${PR_META}")" if [[ "${DRY_RUN}" == "true" ]]; then - echo "🧪 DRY-RUN: would post cap-paused notice on #${PR}" - elif [[ "${CAP_NOTICED}" == "0" ]]; then + echo "🧪 DRY-RUN: would post cap-paused notice and apply ${NEEDS_HUMAN_LABEL} on #${PR}" + else # Consent may have moved since PR_META: skip wins everywhere, # and a takeover notice additionally requires the label to # still be present — a label removed (or skip added) moments - # ago must not receive a stale 'paused' notice. - LIVE_LABELS="$(gh pr view "${PR}" --repo "${REPO}" --json labels 2> /dev/null | jq -r '[.labels[]?.name] | join(" ")' || echo '')" + # ago must not receive a stale 'paused' notice. The read + # FAILS CLOSED (mirrors takeover-ack): an unreadable label + # state must not get a notice or the escalation label — + # collapsing the failure to '' would ignore a concurrently + # added skip for standard bot PRs. + LIVE_LABELS_JSON="$(gh pr view "${PR}" --repo "${REPO}" --json labels 2> /dev/null || echo '')" + LIVE_LABELS="$(jq -r '[.labels[]?.name] | join(" ")' <<< "${LIVE_LABELS_JSON}" 2> /dev/null || echo '')" + if [[ -z "${LIVE_LABELS_JSON}" ]]; then + echo "🧭 cap notice skipped: label state unreadable (fail closed) on #${PR}" + continue + fi if [[ " ${LIVE_LABELS} " == *" ${SKIP_LABEL} "* ]] \ || [[ "${HAS_TAKEOVER}" == "true" && " ${LIVE_LABELS} " != *" ${TAKEOVER_LABEL} "* ]]; then echo "🧭 cap notice skipped: consent changed since the snapshot (labels: ${LIVE_LABELS:-unreadable})" @@ -3175,14 +3385,38 @@ jobs: fi if [[ "${SCAN_BOT_ACTOR}" != "${AUTOFIX_BOT}" ]]; then echo "::warning::cap-paused notice skipped: PAT authenticates as '${SCAN_BOT_ACTOR}', expected ${AUTOFIX_BOT}" - else - if [[ "${HAS_TAKEOVER}" == "true" ]]; then - CAP_BODY="$(printf '⏸️ Takeover paused: this PR reached its round cap (%s/%s). Comment `%s` to re-arm a fresh window and continue management, or `%s stop` to release.\n\n
\n中文说明\n\n⏸️ 托管已暂停：本 PR 达到轮次上限（%s/%s）。评论 `%s` 可重新武装、开启新窗口继续托管；或评论 `%s stop` 释放。\n\n
\n\n' "${ROUND}" "${EFF_MAX_ROUNDS}" "${TAKEOVER_COMMAND}" "${TAKEOVER_COMMAND}" "${ROUND}" "${EFF_MAX_ROUNDS}" "${TAKEOVER_COMMAND}" "${TAKEOVER_COMMAND}")" + elif [[ "${RELEASE_ACKED}" != "0" && "${IS_BOT_AUTHOR}" != "true" ]]; then + # R6-4: an events-API outage is NOT a release — report it + # as what it is, or the log lies during an incident. + if [[ "${RELEASE_ACKED}" == "unreadable" ]]; then + echo "🧭 cap label/notice skipped: release history unreadable (fail closed) on #${PR}" else - CAP_BODY="$(printf '⏸️ AutoFix paused: this PR reached its automatic round cap (%s/%s) and the loop will not manage it further — new feedback and base conflicts stay unhandled. Comment `%s` to re-arm a fresh window under the same cap, or `%s` to take it over with the raised cap.\n\n
\n中文说明\n\n⏸️ AutoFix 已暂停：本 PR 达到自动轮次上限（%s/%s），循环不再管理——新反馈与 base 冲突将无人处理。评论 `%s` 可在同一上限下重置计数窗口，或评论 `%s` 以更高上限接管。\n\n
\n\n' "${ROUND}" "${EFF_MAX_ROUNDS}" "${RETRY_COMMAND}" "${TAKEOVER_COMMAND}" "${ROUND}" "${EFF_MAX_ROUNDS}" "${RETRY_COMMAND}" "${TAKEOVER_COMMAND}")" + echo "🧭 cap label/notice skipped: PR was released after its last re-arm (#${PR})" fi - if ! gh pr comment "${PR}" --repo "${REPO}" --body "${CAP_BODY}"; then - echo "::warning::cap-paused notice failed for #${PR}; will retry next scan" + else + # The escalation label rides EVERY cap detection, noticed + # or not: the once-per-window dedup suppresses repeat + # comments, but the label is what makes a paused PR + # filterable (the shepherd's auto-release ages from the + # cap notice itself, not from the label) — and applying + # it unconditionally backfills the already-paused fleet + # via the scan rotation after this ships (idle backoff: + # expect hours, not the first scan). + gh label create "${NEEDS_HUMAN_LABEL}" --repo "${REPO}" --color 'D93F0B' \ + --description 'The autofix loop stopped on this PR — a human must re-arm, split, merge, or close it' \ + 2> /dev/null || true + if ! gh api -X POST "repos/${REPO}/issues/${PR}/labels" -f "labels[]=${NEEDS_HUMAN_LABEL}" > /dev/null; then + echo "::warning::${NEEDS_HUMAN_LABEL} add failed for #${PR}; will retry next scan" + fi + if [[ "${CAP_NOTICED}" == "0" ]]; then + if [[ "${HAS_TAKEOVER}" == "true" ]]; then + CAP_BODY="$(printf '⏸️ Takeover paused: this PR reached its round cap (%s/%s). Comment `%s` to re-arm a fresh window and continue management, or `%s stop` to release.\n\n
\n中文说明\n\n⏸️ 托管已暂停：本 PR 达到轮次上限（%s/%s）。评论 `%s` 可重新武装、开启新窗口继续托管；或评论 `%s stop` 释放。\n\n
\n\n' "${ROUND}" "${EFF_MAX_ROUNDS}" "${TAKEOVER_COMMAND}" "${TAKEOVER_COMMAND}" "${ROUND}" "${EFF_MAX_ROUNDS}" "${TAKEOVER_COMMAND}" "${TAKEOVER_COMMAND}")" + else + CAP_BODY="$(printf '⏸️ AutoFix paused: this PR reached its automatic round cap (%s/%s) and the loop will not manage it further — new feedback and base conflicts stay unhandled. Comment `%s` to re-arm a fresh window under the same cap, or `%s` to take it over with the raised cap.\n\n
\n中文说明\n\n⏸️ AutoFix 已暂停：本 PR 达到自动轮次上限（%s/%s），循环不再管理——新反馈与 base 冲突将无人处理。评论 `%s` 可在同一上限下重置计数窗口，或评论 `%s` 以更高上限接管。\n\n
\n\n' "${ROUND}" "${EFF_MAX_ROUNDS}" "${RETRY_COMMAND}" "${TAKEOVER_COMMAND}" "${ROUND}" "${EFF_MAX_ROUNDS}" "${RETRY_COMMAND}" "${TAKEOVER_COMMAND}")" + fi + if ! gh pr comment "${PR}" --repo "${REPO}" --body "${CAP_BODY}"; then + echo "::warning::cap-paused notice failed for #${PR}; will retry next scan" + fi fi fi fi diff --git a/.github/workflows/qwen-fleet-shepherd.yml b/.github/workflows/qwen-fleet-shepherd.yml index b646f2f1569..2672b39f770 100644 --- a/.github/workflows/qwen-fleet-shepherd.yml +++ b/.github/workflows/qwen-fleet-shepherd.yml @@ -14,7 +14,22 @@ name: 'Fleet Shepherd' # recently, dispatch one (GitHub cron is unreliable) # # It also maintains a single "Fleet Shepherd Dashboard" issue (edited in -# place, never comment spam) so fleet state is observable at a glance. +# place, never comment spam) so fleet state is observable at a glance. The +# dashboard additionally lists the TAKEOVER pool (open PRs carrying +# autofix/takeover, forks included) — the autofix loop manages those, but +# without a row here a paused takeover PR was invisible until someone opened +# the PR page. +# +# One lever exists for the takeover pool: AUTO-RELEASE. A PR carrying both +# autofix/takeover and autofix/needs-human whose cap pause went unanswered +# for AUTO_RELEASE_DAYS days gets one bilingual summary comment (merge / +# close / split + re-takeover) and then its takeover label removed. The +# needs-human label stays as the filterable TODO. Idempotency: the summary +# is dedup'd by its `` marker (scoped to +# the current pause cycle), so a failed label removal retries only the +# DELETE, and the scope condition (both labels) turns false once the release +# lands. Every other takeover state — conflicts, red CI, new feedback — +# remains the autofix scan's job; the shepherd only reports it. # # NON-GOAL: rerunning flaky-failed CI. That is owned by the CI Failure Patrol # (.github/workflows/qwen-ci-flaky-rerun.yml), which has its own markers, @@ -22,15 +37,16 @@ name: 'Fleet Shepherd' # rerun owner here raced it (observed live as rerun-vs-rerun cancellation # storms), so the shepherd only REPORTS red CI on the dashboard. # -# Safety rails: bot-authored main-targeting in-repo PRs only; per-action -# markers make every write idempotent; per-tick action caps bound blast -# radius; every remote read fails CLOSED (an unreadable snapshot skips the -# levers it feeds — it never masquerades as empty state); DRY-RUN via -# dispatch input; global kill switch via the FLEET_SHEPHERD_DISABLED -# repository variable. Dispatches use the workflow's own token -# (actions: write); comments, update-branch, and the dashboard use -# CI_DEV_BOT_PAT so synced branches still trigger CI and all writes carry -# the bot identity. +# Safety rails: the conflict/sync levers touch bot-authored main-targeting +# in-repo PRs only (the takeover pool gets visibility plus the label-only +# auto-release); per-action markers make every write idempotent; per-tick +# action caps bound blast radius; every remote read fails CLOSED (an +# unreadable snapshot skips the levers it feeds — it never masquerades as +# empty state); DRY-RUN via dispatch input; global kill switch via the +# FLEET_SHEPHERD_DISABLED repository variable. Dispatches use the workflow's +# own token (actions: write); comments, label edits, update-branch, and the +# dashboard use CI_DEV_BOT_PAT so synced branches still trigger CI and all +# writes carry the bot identity. on: schedule: - cron: '*/15 * * * *' @@ -60,6 +76,30 @@ env: # Maintainer opt-out label, honored at every engagement path (mirrors # qwen-autofix.yml's SKIP_LABEL). SKIP_LABEL: 'autofix/skip' + # The takeover pool these two labels define (both mirror qwen-autofix.yml): + # TAKEOVER opts a PR into the loop; NEEDS_HUMAN marks a PR the loop STOPPED + # on (round cap or a breaker) until a human re-arms, splits, merges, or + # closes it. + TAKEOVER_LABEL: 'autofix/takeover' + NEEDS_HUMAN_LABEL: 'autofix/needs-human' + # The two re-arm commands the route listens for — mirrored from + # qwen-autofix.yml (TAKEOVER_COMMAND/RETRY_COMMAND) so the resume-evidence + # matcher can never drift from the command syntax the route actually + # accepts (R7-2). + TAKEOVER_COMMAND: '@qwen-code /takeover' + RETRY_COMMAND: '@qwen-code /retry' + # Days a takeover PR may sit paused (both labels present, no re-arm since + # the cap notice) before the shepherd releases the takeover label. Tunable + # without a deploy via the repository variable. + AUTO_RELEASE_DAYS: "${{ vars.QWEN_SHEPHERD_AUTO_RELEASE_DAYS || '3' }}" + MAX_RELEASES_PER_TICK: '3' + # How long an unacked re-arm command comment counts as resume evidence. + # An accepted command is acked within minutes (the ack carries the resume + # marker); a route-ignored command gets no reply at all, so without this + # bound it would veto the release forever. + RESUME_COMMAND_GRACE_SEC: '7200' + # Stale needs-human cleanups (manual UI releases on fork PRs) per tick. + MAX_CLEANUPS_PER_TICK: '5' jobs: shepherd: @@ -287,19 +327,23 @@ jobs: # maintainer adding autofix/skip mid-tick must still win before a # dispatch or branch sync. Fail closed: an unreadable label state # counts as skipped. + # The fetched payload is exported as LIVE_LABELS_JSON so a caller + # with follow-up label checks (the auto-release scope condition) + # rides the SAME read instead of spending a second one. # Returns 0 when the mutation must NOT proceed, with the reason in # LIVE_SKIP_REASON: 'label' (consent withdrawn) vs 'unreadable' # (fail closed on an API failure) — callers word their notes # accordingly so an outage is never reported as a maintainer # decision. live_skip() { - local pr="$1" labels + local pr="$1" LIVE_SKIP_REASON='' - if ! labels="$(gh pr view "${pr}" --repo "${REPO}" --json labels 2> /dev/null)"; then + LIVE_LABELS_JSON='' + if ! LIVE_LABELS_JSON="$(gh pr view "${pr}" --repo "${REPO}" --json labels 2> /dev/null)"; then LIVE_SKIP_REASON='unreadable' return 0 fi - if [[ "$(jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${labels}")" == "true" ]]; then + if [[ "$(jq -r --arg t "${SKIP_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${LIVE_LABELS_JSON}")" == "true" ]]; then LIVE_SKIP_REASON='label' return 0 fi @@ -313,6 +357,142 @@ jobs: fi } + # RFC3339 timestamp → whole days ago (both dashboards show ages). + days_since() { + echo "$(( (NOW_EPOCH - $(date -u -d "$1" +%s)) / 86400 ))" + } + + # Is this needs-human PR genuinely PAUSED (so the scan would refuse + # a conflict dispatch)? The label alone is only a proxy: an ARMED + # PR that kept a stale label (a resume-side removal failed) is NOT + # paused and the scan would accept the dispatch. Check marker truth + # — a re-arm/engage marker newer than the last cap notice means + # re-armed, not paused (R4-6). Fail closed toward "paused": an + # unreadable history suppresses the dispatch rather than waste it. + conflict_paused() { + local pr="$1" + if ! gh api "repos/${REPO}/issues/${pr}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/cf-ic.json; then + return 0 + fi + local term resume + term="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | contains("")) + | .created_at ] | max // ""' /tmp/cf-ic.json)" + resume="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select(((.body // "") | contains("")) + or ((.body // "") | contains(""))) + | .created_at ] | max // ""' /tmp/cf-ic.json)" + # Only marker-confirmed re-arm clears "paused" — and only against + # a REAL cap notice. The state "label present but no notice" is + # reachable (the cap path applies the label before posting the + # notice and tolerates a lost notice; a human can delete it too): + # every takeover PR carries an older engage ack, so `term` empty + # + `resume` non-empty must NOT read as "re-armed" — that fails + # OPEN onto a genuinely paused PR (R5-3). + if [[ -n "${resume}" && -n "${term}" && "${resume}" > "${term}" ]]; then + return 1 + fi + return 0 + } + + # Latest resume evidence (bot re-arm/engage markers, a trusted + # human's fresh command comment, or a fresh takeover labeled + # event) as an RFC3339 timestamp, or '' when none. A command counts + # only while FRESH (grace window), UNSUPERSEDED by a refusal ack, + # and from a write/maintain/admin author — scanned newest-first, + # deduped BY AUTHOR so a stranger posting two commands can't burn + # the 2-permission-read budget and shadow a maintainer's command + # (R5-5/R6-3). Sets PERM_READ_FAILED when a command's permission + # check could not be evaluated — the caller fails CLOSED on that + # (defer; never trust-on-error, R5-4). Results are returned via the + # RESUME_OUT global — NOT stdout — because a `$(...)` call site + # would run the function in a subshell and silently drop + # PERM_READ_FAILED (the R5-4 defer was dead code that way). + compute_resume_ts() { + local ic="$1" ev="$2" + RESUME_OUT='' + PERM_READ_FAILED='' + local resume refusal + resume="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select(((.body // "") | contains("")) + or ((.body // "") | contains(""))) + | .created_at ] | max // ""' "${ic}")" + refusal="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | test("")) + | .created_at ] | max // ""' "${ic}")" + # Fresh command candidates, newest-first, ONE row per author: + # group_by(.login) + max_by(.ts) keeps each author's newest command, + # then re-sort by ts desc restores recency order (unique_by would + # emit authors in LOGIN order, walking alphabetically and letting + # two read-only strangers shadow a maintainer's newer command, + # R6-3). + local cands cts cauthor cage cperm reads=0 + cands="$(jq -r --arg tc "${TAKEOVER_COMMAND}" --arg rc "${RETRY_COMMAND}" ' + [ .[] | select(((.body // "") | gsub("^\\s+|\\s+$"; "")) == $tc + or ((.body // "") | gsub("^\\s+|\\s+$"; "")) == $rc) + | {ts: (.created_at // ""), login: (.user.login // "")} ] + | group_by(.login) | map(max_by(.ts)) | sort_by(.ts) | reverse | .[] | "\(.ts)\t\(.login)"' "${ic}")" + while IFS=$'\t' read -r cts cauthor; do + [[ -z "${cts}" || -z "${cauthor}" ]] && continue + [[ "${cts}" > "${resume}" ]] || continue + [[ -z "${refusal}" || "${cts}" > "${refusal}" ]] || continue + cage=$(( NOW_EPOCH - $(date -u -d "${cts}" +%s) )) + [[ "${cage}" -lt "${RESUME_COMMAND_GRACE_SEC}" ]] || continue + # Budget exhausted with candidates still unexamined: the caller + # must not read this as "all candidates were read-only/none + # trusted" — that fails OPEN. Flag it so the release defers. + [[ "${reads}" -ge 2 ]] && { PERM_READ_FAILED=true; break; } + reads=$(( reads + 1 )) + if ! cperm="$(gh api "repos/${REPO}/collaborators/${cauthor}/permission" --jq '.permission // ""' 2> /tmp/cperm-err)"; then + # An exact "HTTP 404" is GitHub's decisive "not a + # collaborator" answer — classify read-only instead of + # deferring: a defer here is renewable by any stranger's + # exact command each grace window, and it would pose a + # classification as an outage. Match the token, not the + # bare number: a transport failure embeds the request URL, + # which carries the commenter login, so a login containing + # "404" would classify an outage as read-only — the same + # exact token every label DELETE tolerates (R10-1). + grep -q 'HTTP 404' /tmp/cperm-err && continue + PERM_READ_FAILED=true + continue + fi + if [[ "${cperm}" == 'write' || "${cperm}" == 'maintain' || "${cperm}" == 'admin' ]]; then + resume="${cts}" + break + fi + done <<< "${cands}" + local evt + evt="$(jq -r --arg tl "${TAKEOVER_LABEL}" ' + [ .[] | select((.event // "") == "labeled") + | select((.label.name // "") == $tl) + | .created_at ] | max // ""' "${ev}")" + # A tie (same-second) resolves toward the RESUME side: suppressing + # a release is always the safer direction (R5-9). + if [[ -n "${evt}" && ! "${resume}" > "${evt}" ]]; then + resume="${evt}" + fi + RESUME_OUT="${resume}" + } + + # CI-status classifiers shared by BOTH dashboard loops, so a + # check-naming or status-set change lands in one place and the two + # tables can never classify the same PR differently. + # WAITING and REQUESTED are also not-yet-final check states. + pending_checks() { + jq -r '[.statusCheckRollup[]? | select((.status // "") | IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))] | length' <<< "$1" + } + # Platform-blind on purpose: the dashboard is a health VIEW, and a + # Windows- or macOS-only regression is just as red as an Ubuntu one + # (reruns stay with the Patrol either way). + failed_test_url() { + jq -r '[.statusCheckRollup[]? | select(.conclusion == "FAILURE") | select(.name | startswith("Test (")) | .detailsUrl][0] // ""' <<< "$1" + } + # ---- walk the bot fleet (one list call carries all per-PR meta) -- # autofix/skip is the maintainer opt-out honored at every # engagement path — a skip-labeled PR gets no shepherd levers and @@ -321,16 +501,22 @@ jobs: if ! gh pr list --repo "${REPO}" --state open --author "${AUTOFIX_BOT}" --base main \ --limit 50 --json number,headRefName,headRefOid,mergeable,isCrossRepository,statusCheckRollup,labels \ > /tmp/fleet-raw.json 2> /dev/null; then - # A failed fleet fetch must not masquerade as an empty fleet: the - # walk is skipped AND the dashboard keeps its previous body rather - # than being overwritten with a misleading empty table. - echo "::warning::fleet enumeration failed; skipping this tick's walk and dashboard update" - exit 0 + # A failed fleet fetch must not masquerade as an empty fleet — + # but it also must not exit (B5): the takeover/needs-human + # processing below is fed by its OWN enumerations, so it keeps + # running, and the dashboard write still happens (carrying the + # liveness watermark). Degrade to a loud error row instead. + FLEET_OK=false + echo "::warning::fleet enumeration failed; the bot-fleet table shows an error row this tick" + DASH_ROWS='| — | — | ⚠️ fleet enumeration unreadable this tick | fail closed — fleet levers skipped |\n' + else + FLEET_OK=true + jq --arg skip "${SKIP_LABEL}" \ + '[.[] | select(.isCrossRepository == false) | select([.labels[]?.name] | index($skip) | not)]' \ + /tmp/fleet-raw.json > /tmp/fleet.json fi - jq --arg skip "${SKIP_LABEL}" \ - '[.[] | select(.isCrossRepository == false) | select([.labels[]?.name] | index($skip) | not)]' \ - /tmp/fleet-raw.json > /tmp/fleet.json + if [[ "${FLEET_OK}" == "true" ]]; then while IFS= read -r ROW; do [[ -z "${ROW}" ]] && continue PR="$(jq -r '.number' <<< "${ROW}")" @@ -342,16 +528,19 @@ jobs: DASH_ROWS="${DASH_ROWS}| #${PR} | ? | incomplete metadata | — |\n" continue fi - # WAITING and REQUESTED are also not-yet-final check states. - PENDING="$(jq -r '[.statusCheckRollup[]? | select((.status // "") | IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))] | length' <<< "${ROW}")" - # Platform-blind on purpose: the dashboard is a health VIEW, and - # a Windows- or macOS-only regression is just as red as an Ubuntu - # one (reruns stay with the Patrol either way). - FAILED_TEST_URL="$(jq -r '[.statusCheckRollup[]? | select(.conclusion == "FAILURE") | select(.name | startswith("Test (")) | .detailsUrl][0] // ""' <<< "${ROW}")" + PENDING="$(pending_checks "${ROW}")" + FAILED_TEST_URL="$(failed_test_url "${ROW}")" BEHIND="$(gh api "repos/${REPO}/compare/main...${HEAD}" --jq '.behind_by // 0' 2> /dev/null || echo 0)" STATUS_NOTE='idle' ACTION_NOTE='—' + # A bot PR the loop stopped on (round cap or a breaker) carries + # the escalation label — prefix its state so the pause is visible + # on the dashboard instead of only on the PR page. + NH_PREFIX='' + if [[ "$(jq -r --arg l "${NEEDS_HUMAN_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" == "true" ]]; then + NH_PREFIX='🛑 ' + fi # 1) conflict → dispatch the loop for this PR, once per head SHA. # The dedup marker is posted ONLY when the dispatch succeeded — a @@ -395,6 +584,13 @@ jobs: # Budget first: once it is exhausted no mutation is possible, # so the PAT-backed live label read would be pure waste. ACTION_NOTE="$(skip_note dispatch)" + elif [[ "$(jq -r --arg l "${NEEDS_HUMAN_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${LIVE_LABELS_JSON}")" == "true" ]] && conflict_paused "${PR}"; then + # needs-human from the LIVE payload (a label applied after + # enumeration is caught) AND marker-confirmed genuinely + # paused — an armed PR with a stale label falls through to + # the dispatch below (R4-6). The conflict stays unhandled + # until a human re-arms. + ACTION_NOTE='paused (needs-human) — conflict stays unhandled until re-arm' else if act "#${PR}: dispatch autofix for conflict resolution" \ env GITHUB_TOKEN="${ACTIONS_TOKEN}" gh workflow run qwen-autofix.yml --repo "${REPO}" -f pr_number="${PR}"; then @@ -452,18 +648,509 @@ jobs: fi echo "🐑 #${PR} [${STATUS_NOTE}] → ${ACTION_NOTE}" - DASH_ROWS="${DASH_ROWS}| #${PR} | \`${HEAD:0:9}\` | ${STATUS_NOTE} | ${ACTION_NOTE} |\n" + DASH_ROWS="${DASH_ROWS}| #${PR} | \`${HEAD:0:9}\` | ${NH_PREFIX}${STATUS_NOTE} | ${ACTION_NOTE} |\n" done < <(jq -c '.[]' /tmp/fleet.json) + fi # FLEET_OK + + # ---- takeover pool: dashboard rows + the auto-release lever ----- + # These PRs are managed by the autofix loop, not the shepherd — + # the only mutating lever here releases a takeover whose pause + # went unanswered. Enumeration failures degrade to a loud error + # row instead of exiting: the dashboard write must still happen, + # because it carries the liveness watermark the next tick's + # duplicate-dispatch guard reads. + RELEASES=0 + CLEANUPS=0 + TAKEOVER_ROWS='' + HUMAN_ROWS='' + # The variable is operator-tunable, so it is also operator- + # breakable: a non-numeric value would fail the -ge comparison + # under set -e mid-tick. Fall back to the default instead. The + # digit-only regex still admits values above Bash's signed-int + # range (e.g. 9223372036854775808 wraps negative, so EVERY pause + # age would pass the -ge check) — bound by string LENGTH before any + # arithmetic, so a huge value can never reach the comparison. + if [[ ! "${AUTO_RELEASE_DAYS}" =~ ^[0-9]+$ ]] || [[ ${#AUTO_RELEASE_DAYS} -gt 2 ]]; then + echo "::warning::AUTO_RELEASE_DAYS '${AUTO_RELEASE_DAYS}' is not numeric or is too large; using 3" + AUTO_RELEASE_DAYS=3 + fi + # Leading zeros pass the regex but bash reads 08/09 as bad octal at + # the -ge comparison — normalize to base 10 so a zero-padded + # variable cannot silently kill the lever. + AUTO_RELEASE_DAYS=$((10#${AUTO_RELEASE_DAYS})) + # sort:updated-asc keeps the stalest PRs in view when a pool + # outgrows the window; the saturation warning keeps the residual + # truncation loud. The LEVER's population deliberately comes from + # its OWN paused enumeration (always small), never from this + # display window — see the paused enumeration below. + TK_OK=true + if ! gh pr list --repo "${REPO}" --state open --label "${TAKEOVER_LABEL}" \ + --search 'sort:updated-asc' \ + --limit 100 --json number,author,updatedAt,mergeable,statusCheckRollup,labels \ + > /tmp/takeover-raw.json 2> /dev/null; then + TK_OK=false + echo "::warning::takeover enumeration failed; the takeover table shows an error row this tick" + # The error-row wording is finalized AFTER the paused + # enumeration (R4-13): the release lever is fed by THAT + # enumeration, and both reads share one PAT, so a correlated + # mid-tick outage can fail both — the row must not claim + # evaluation proceeds when the feed is unreadable too. + else + if [[ "$(jq length /tmp/takeover-raw.json)" -ge 100 ]]; then + echo "::warning::takeover pool at the 100-PR enumeration limit — recently-active PRs (paused-but-discussed ones included) may be missing from this table" + fi + jq --arg skip "${SKIP_LABEL}" \ + '[.[] | select([.labels[]?.name] | index($skip) | not)]' \ + /tmp/takeover-raw.json > /tmp/takeover.json + fi + # The needs-human enumeration drives the "awaiting human" DISPLAY + # table (loop 3). It is a long-lived population — every released PR + # keeps the label — so it can exceed its 100-window. + HM_OK=true + if ! gh pr list --repo "${REPO}" --state open --label "${NEEDS_HUMAN_LABEL}" \ + --search 'sort:updated-asc' \ + --limit 100 --json number,author,updatedAt,mergeable,labels \ + > /tmp/human-raw.json 2> /dev/null; then + HM_OK=false + echo "::warning::needs-human enumeration failed; the awaiting-human table shows an error row this tick" + HUMAN_ROWS='| — | — | — | ⚠️ enumeration unreadable this tick — previous entries may be stale |\n' + else + if [[ "$(jq length /tmp/human-raw.json)" -ge 100 ]]; then + echo "::warning::needs-human pool at the 100-PR enumeration limit — the freshest awaiting-human entries may be missing from the table" + fi + jq --arg skip "${SKIP_LABEL}" \ + '[.[] | select([.labels[]?.name] | index($skip) | not)]' \ + /tmp/human-raw.json > /tmp/human.json + fi + # The RELEASE LEVER gets its OWN enumeration of the paused + # population (both labels), sorted stale-first (R5-7): released + # awaiting PRs age back into the needs-human window, so feeding the + # lever from that display window would truncate exactly the fresh + # pauses that become release-eligible, starving the lever and + # making the zombie state permanent and self-feeding. The paused + # population is small (only currently-paused takeover PRs). + PAUSED_OK=true + if ! gh pr list --repo "${REPO}" --state open --label "${TAKEOVER_LABEL}" --label "${NEEDS_HUMAN_LABEL}" \ + --search 'sort:updated-asc' \ + --limit 100 --json number,author,updatedAt,mergeable,labels \ + > /tmp/paused-raw.json 2> /dev/null; then + PAUSED_OK=false + echo "::warning::paused-takeover enumeration failed; the release lever is skipped this tick" + else + if [[ "$(jq length /tmp/paused-raw.json)" -ge 100 ]]; then + echo "::warning::paused-takeover pool at the 100-PR enumeration limit — the freshest paused entries may miss release evaluation" + fi + jq --arg skip "${SKIP_LABEL}" \ + '[.[] | select([.labels[]?.name] | index($skip) | not)]' \ + /tmp/paused-raw.json > /tmp/paused.json + fi + + # Membership set of the paused enumeration (R4-9): loop 1 must + # only defer a paused PR to loop 2 when loop 2 will actually see + # it — a PR truncated out of the window must still render here. + HUMAN_IDS='' + if [[ "${PAUSED_OK}" == "true" ]]; then + HUMAN_IDS=",$(jq -r '[.[].number | tostring] | join(",")' /tmp/paused.json)," + fi + # Finalize the takeover-enum error row now that PAUSED_OK is known + # (the release lever is fed by the paused enumeration, not the + # needs-human display window). + if [[ "${TK_OK}" == "false" ]]; then + if [[ "${PAUSED_OK}" == "true" ]]; then + TAKEOVER_ROWS='| — | — | — | ⚠️ takeover pool unreadable this tick | paused rows below still evaluated (paused pool fed) |\n' + else + TAKEOVER_ROWS='| — | — | — | ⚠️ takeover pool unreadable this tick | paused rows NOT evaluated — paused enumeration also failed |\n' + fi + fi + + # Loop 1: managed takeover PRs that are NOT paused — cheap payload + # states only. Paused ones are rendered by loop 2, which owns the + # pause evaluation. (If the paused enumeration failed or dropped + # them, render them here so they never vanish silently.) + if [[ "${TK_OK}" == "true" ]]; then + while IFS= read -r ROW; do + [[ -z "${ROW}" ]] && continue + PR="$(jq -r '.number' <<< "${ROW}")" + AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" + UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" + MERGEABLE="$(jq -r '.mergeable // "UNKNOWN"' <<< "${ROW}")" + PENDING="$(pending_checks "${ROW}")" + FAILED_TEST_URL="$(failed_test_url "${ROW}")" + HAS_NH="$(jq -r --arg l "${NEEDS_HUMAN_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" + UPD_AGO='?' + [[ -n "${UPDATED}" ]] && UPD_AGO="$(days_since "${UPDATED}")d" + + STATE='managed · idle' + NOTE='—' + # Defer by paused-enumeration MEMBERSHIP, not by this + # snapshot's label state: a needs-human label landing between + # the two enumerations must not render the PR in both loops — + # loop 2 owns every paused member. + if [[ "${HUMAN_IDS}" == *",${PR},"* ]]; then + continue # loop 2 renders this paused PR + fi + if [[ "${HAS_NH}" == "true" ]]; then + STATE='🛑 needs-human' + NOTE='pause evaluation unavailable this tick (truncated or unreadable paused enumeration)' + elif [[ "${MERGEABLE}" == "CONFLICTING" ]]; then + STATE='conflicting' + NOTE='resolution owned by the autofix scan' + elif [[ -n "${FAILED_TEST_URL}" && "${PENDING}" == "0" ]]; then + STATE="[ci red](${FAILED_TEST_URL})" + NOTE='reruns owned by CI Failure Patrol' + elif [[ "${PENDING}" != "0" ]]; then + STATE="checks in flight (${PENDING})" + fi + + echo "🐑 #${PR} [takeover: ${STATE}] → ${NOTE}" + # STATE can carry the ci-red detailsUrl (check-run creator + # controlled) and NOTE the stop-reason headline — escape + # backslashes before printf '%b' and pipes before the table + # (R4-10). + SAFE_STATE="${STATE//\\/\\\\}" + SAFE_NOTE="${NOTE//\\/\\\\}" + TAKEOVER_ROWS="${TAKEOVER_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${SAFE_STATE//|/\\|} | ${SAFE_NOTE//|/\\|} |\n" + done < <(jq -c '.[]' /tmp/takeover.json) + fi + + # Loop 2: paused takeover PRs (both labels) — the release lever + # and the takeover-table paused rows. Fed by the dedicated paused + # enumeration (R5-7), sorted stale-first, so the lever never + # starves behind the long-lived awaiting display population. The + # paginated comment/event reads live in this loop only (paused PRs + # are few). Fail closed everywhere: an unreadable history defers + # the lever instead of acting on partial state. + if [[ "${PAUSED_OK}" == "true" ]]; then + while IFS= read -r ROW; do + [[ -z "${ROW}" ]] && continue + PR="$(jq -r '.number' <<< "${ROW}")" + AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" + UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" + MERGEABLE="$(jq -r '.mergeable // "UNKNOWN"' <<< "${ROW}")" + UPD_AGO='?' + [[ -n "${UPDATED}" ]] && UPD_AGO="$(days_since "${UPDATED}")d" + # Row routing follows POST-ACTION label state, not the + # pre-mutation snapshot: a successful release moves the PR to + # Awaiting human. + ROUTE='takeover' + + STATE='🛑 needs-human' + NOTE='—' + if ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev.json; then + NOTE='event read failed — evaluation deferred this tick (fail closed)' + elif ! gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ic.json; then + NOTE='comment read failed — release deferred this tick' + else + # The pause clock starts at the LATEST cap notice. + TERM_TS="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | contains("")) + | .created_at ] | max // ""' /tmp/tk-ic.json)" + # Resume evidence — anything at-or-newer than TERM_TS means + # a human already acted and the release must never fire; + # ties resolve toward resume (the safe direction, R5-9): + # 1. the bot's resume markers (re-arm / engage ack); + # 2. a fresh re-arm COMMAND from a write/maintain/admin + # commenter, scanned newest-first so a stranger's echo + # can't shadow a maintainer's (R4-14/R5-5), superseded + # by a refusal ack; a route-ignored command simply + # expires at the grace window; + # 3. a fresh `labeled` event — a UI re-apply is + # timestamped immediately, while its ack rides the + # queue (or, on a lost event, the scan's idle-backoff + # pickup — an hours-wide window). + # The marker variant is derived below for the cleanup + # anchor; the full evidence is evaluated only at release + # time, from a FRESH fetch (R5-6), where its permission + # read fails CLOSED: PERM_READ_FAILED defers the release + # instead of trusting on error (R5-4). Direct call, NOT + # $(...) — the function returns its result in the + # RESUME_OUT/PERM_READ_FAILED globals, and a subshell + # would drop them. + # The stop reason comes from the terminal round's headline — + # the scan-side notice always says "round cap" even when a + # breaker (consecutive failures, time budget) fired first. + # Only TERMINAL headlines match: the transient "could not + # start — a setup step failed" variant retries next scan and + # says nothing about why the loop stopped. The terminal set + # is cross-pinned against qwen-autofix.yml's HEADLINE= sites + # by the shepherd test — drift fails CI, not the dashboard. + REASON="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | test("🤖 AutoFix (stopped|could not start (evaluation|— reached the round cap))|— this was the last automatic attempt")) + | (.body | gsub("\r"; "") | split("\n")[0]) ] | last // ""' /tmp/tk-ic.json)" + [[ -z "${REASON}" ]] && REASON='round cap reached' + REASON="${REASON:0:120}" + # Re-armed detection for the stale-label cleanup is anchored + # to the CURRENT pause boundary (latest needs-human apply, + # same shape as the heal's NH_APPLY_TS) and is MARKER- + # confirmed only (bot re-arm/engage marker) — not the + # command-grace or labeled-event evidence the release veto + # carries. The cap path applies needs-human BEFORE the + # dedup'd notice and tolerates a lost notice, so keying the + # cleanup on TERM_TS (latest notice) would read a re-paused + # PR whose cycle-2 notice was lost as "re-armed" on stale + # cycle-1 evidence and DELETE its fresh label (R7-1). + # Command/label evidence still vetoes the RELEASE below via + # the pre-write RESUME_NOW recompute — it just never + # triggers this cleanup. + MARKER_RESUME="$(jq -r --arg ab "${AUTOFIX_BOT}" ' + [ .[] | select((.user.login // "") == $ab) + | select(((.body // "") | contains("")) + or ((.body // "") | contains(""))) + | .created_at ] | max // ""' /tmp/tk-ic.json)" + PAUSE_APPLY_TS="$(jq -r --arg nl "${NEEDS_HUMAN_LABEL}" ' + [ .[] | select((.event // "") == "labeled") + | select((.label.name // "") == $nl) + | .created_at ] | max // ""' /tmp/tk-ev.json)" + # The re-armed cleanup runs FIRST so it stays reachable + # when the cap notice was lost (TERM_TS empty) — exactly + # the label-without-notice state this branch exists for + # (R8-1). A cap notice NEWER than the marker means the PR + # re-paused after the re-arm: the fresh pause wins and the + # cleanup must not fire — including the case where the + # re-cap's label POST no-op'd (label already present → no + # new labeled event) and only the notice marks the new + # cycle (R8-10). + if [[ -n "${PAUSE_APPLY_TS}" && -n "${MARKER_RESUME}" && ! "${PAUSE_APPLY_TS}" > "${MARKER_RESUME}" && ! "${TERM_TS}" > "${MARKER_RESUME}" ]]; then + STATE='managed (re-armed)' + # The PR is re-armed but still carries needs-human — a + # resume-side removal failed. Clear the stale label here + # (bounded, skip-vetoed) or the PR stays pinned in the + # paused population forever. + if [[ "${CLEANUPS}" -ge "${MAX_CLEANUPS_PER_TICK}" ]]; then + NOTE='resumed; stale-label cleanup budget reached this tick' + elif live_skip "${PR}"; then + NOTE="$(skip_note cleanup)" + else + # Count ATTEMPTS like the release arm: a success-only + # counter never trips during a DELETE outage. + CLEANUPS=$(( CLEANUPS + 1 )) + if act "#${PR}: clear stale ${NEEDS_HUMAN_LABEL} (re-armed)" \ + gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')"; then + NOTE='resumed — cleared stale escalation label' + else + NOTE='stale-label cleanup failed — will retry next tick' + fi + fi + elif [[ -z "${TERM_TS}" ]]; then + NOTE='pause timestamp unreadable — release deferred (fail closed)' + else + PAUSE_D="$(days_since "${TERM_TS}")" + STATE="🛑 needs-human ${PAUSE_D}d" + NOTE="${REASON}" + if [[ "${MERGEABLE}" == "CONFLICTING" ]]; then + NOTE="paused AND conflicting — ${REASON}" + fi + if [[ "${PAUSE_D}" -ge "${AUTO_RELEASE_DAYS}" ]]; then + # Budget first (an exhausted tick stops spending API + # calls), then ONE live label read (inside live_skip) + # carries every veto: fail closed when unreadable, skip + # wins, and the scope condition (both labels) must still + # hold — a re-arm or release since the snapshot ends the + # pause. + if [[ "${RELEASES}" -ge "${MAX_RELEASES_PER_TICK}" ]]; then + NOTE='release budget reached this tick' + elif live_skip "${PR}"; then + NOTE="$(skip_note release)" + elif [[ "$(jq -r --arg a "${TAKEOVER_LABEL}" --arg b "${NEEDS_HUMAN_LABEL}" '([.labels[]?.name] | index($a) != null) and ([.labels[]?.name] | index($b) != null)' <<< "${LIVE_LABELS_JSON}")" != "true" ]]; then + NOTE='labels changed since the snapshot — deferring release' + elif ! gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ic2.json \ + || ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev2.json; then + NOTE='evidence re-check unreadable — release deferred (fail closed)' + elif compute_resume_ts /tmp/tk-ic2.json /tmp/tk-ev2.json; RESUME_NOW="${RESUME_OUT}"; [[ -n "${RESUME_NOW}" && ! "${TERM_TS}" > "${RESUME_NOW}" ]]; then + # R5-6: the resume evidence above was read at loop + # entry — a re-arm landing since must veto NOW. The + # re-fetch + recompute runs immediately before the + # first write, bounded by the per-tick budget. + NOTE='resume evidence appeared during evaluation — release cancelled' + elif [[ "${PERM_READ_FAILED}" == "true" ]]; then + NOTE='command-permission read failed — release deferred this tick (fail closed)' + else + # Consume the budget BEFORE the first external write: + # a release ATTEMPT is what is bounded. Counting only + # successful DELETEs would let a DELETE outage mutate + # many PRs in one tick while RELEASES stayed 0. + RELEASES=$(( RELEASES + 1 )) + # Summary FIRST (dedup'd by its own marker — the + # comment stream is already loaded), THEN the label + # removal: a failed summary leaves both labels in + # place so the whole release retries next tick, and a + # failed removal finds the marker and only retries the + # DELETE. Neither half can strand the other. + SUMMARY_POSTED="$(jq -r --arg ab "${AUTOFIX_BOT}" --arg term "${TERM_TS}" ' + [ .[] | select((.user.login // "") == $ab) + | select((.body // "") | contains("")) + | select((.created_at // "") > $term) + ] | length' /tmp/tk-ic.json)" + if [[ "${SUMMARY_POSTED}" == "0" ]]; then + if act "#${PR}: post auto-release summary" \ + gh pr comment "${PR}" --repo "${REPO}" --body "$(printf '🔓 Takeover auto-released: the autofix loop paused on this PR %s day(s) ago (%s) and no re-arm followed, so the `%s` label is removed to keep the managed pool honest. `%s` stays as the reminder that this PR needs a human decision: merge it, close it, or split/reduce it and comment `%s` to re-engage with a fresh round window.\n\n
\n中文说明\n\n🔓 已自动释放接管：autofix 循环在 %s 天前暂停于此 PR（%s），此后无人重新武装，现移除 `%s` 标签以保持托管池真实可用。保留 `%s` 作为待办提醒 —— 本 PR 需要人工决策：合并、关闭，或拆分/缩小后评论 `%s` 以全新轮次窗口重新接管。\n\n
\n\n' "${PAUSE_D}" "${REASON}" "${TAKEOVER_LABEL}" "${NEEDS_HUMAN_LABEL}" "${TAKEOVER_COMMAND}" "${PAUSE_D}" "${REASON}" "${TAKEOVER_LABEL}" "${NEEDS_HUMAN_LABEL}" "${TAKEOVER_COMMAND}")"; then + SUMMARY_POSTED=1 + else + NOTE='summary post failed — release deferred to next tick' + fi + fi + if [[ "${SUMMARY_POSTED}" != "0" ]]; then + # R2-3: a re-arm can land BETWEEN the summary post + # and this DELETE — its label re-apply is a no-op + # while the label still rides the PR, so no live + # label read can catch it; only fresh marker/event + # evidence can. Re-fetch and veto immediately + # before the removal (fail closed on unreadable). + if ! gh api "repos/${REPO}/issues/${PR}/comments" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ic3.json \ + || ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev3.json; then + NOTE='evidence re-check unreadable — release deferred (fail closed)' + elif compute_resume_ts /tmp/tk-ic3.json /tmp/tk-ev3.json; RESUME_NOW="${RESUME_OUT}"; [[ -n "${RESUME_NOW}" && ! "${TERM_TS}" > "${RESUME_NOW}" ]]; then + NOTE='re-arm landed during the release — takeover kept' + elif [[ "${PERM_READ_FAILED}" == "true" ]]; then + NOTE='command-permission read failed — release deferred this tick (fail closed)' + elif act "#${PR}: auto-release takeover (paused ${PAUSE_D}d)" \ + gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${TAKEOVER_LABEL}" '$l|@uri')"; then + # Released — only needs-human remains, so the PR + # belongs in Awaiting human NOW, not the takeover + # pool (route from post-action state, not the + # pre-mutation snapshot). + ROUTE='awaiting' + NOTE="auto-released after ${PAUSE_D}d paused" + else + NOTE='takeover label removal failed — will retry next tick' + fi + fi + fi + fi + fi + fi + # The row is appended OUTSIDE the evaluation arms: a fail-closed + # deferral must still render, or the PR would vanish from the + # dashboard for exactly the tick something went wrong. + echo "🐑 #${PR} [takeover: ${STATE}] → ${NOTE}" + SAFE_NOTE="${NOTE//\\/\\\\}" + if [[ "${ROUTE}" == 'takeover' ]]; then + TAKEOVER_ROWS="${TAKEOVER_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${STATE} | ${SAFE_NOTE//|/\\|} |\n" + elif [[ "${ROUTE}" == 'awaiting' ]]; then + HUMAN_ROWS="${HUMAN_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${SAFE_NOTE//|/\\|} |\n" + fi + done < <(jq -c '.[]' /tmp/paused.json) + fi + + # Loop 3: needs-human-ONLY PRs — the awaiting-human display rows + # and the manual-release heal. Needs-human is a long-lived label + # (released PRs keep it), so this is the display-only population; + # the release lever is loop 2's. Both-label PRs were already + # rendered by loop 2 and are skipped here when loop 2 ran. + if [[ "${HM_OK}" == "true" ]]; then + while IFS= read -r ROW; do + [[ -z "${ROW}" ]] && continue + PR="$(jq -r '.number' <<< "${ROW}")" + HAS_TK="$(jq -r --arg l "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($l) != null' <<< "${ROW}")" + # Both-label PRs render elsewhere: loop 2 owns the paused + # members, and loop 1 owns the rest while its enumeration ran + # (its note names the pause-evaluation gap — the accurate + # render on a paused-enumeration outage). Loop 3 is the render + # of LAST RESORT — both owners blind — so a both-label PR + # never renders twice nor vanishes (the heal's live takeover + # check vetoes any mutation, R5-8). + if [[ "${HAS_TK}" == "true" ]]; then + [[ "${HUMAN_IDS}" == *",${PR},"* || "${TK_OK}" == "true" ]] && continue + fi + AUTHOR="$(jq -r '.author.login // "?"' <<< "${ROW}")" + UPDATED="$(jq -r '.updatedAt // ""' <<< "${ROW}")" + UPD_AGO='?' + [[ -n "${UPDATED}" ]] && UPD_AGO="$(days_since "${UPDATED}")d" + ROUTE='awaiting' + STATE='🛑 needs-human' + # No takeover label: the loop stopped and management was + # released (or never took over). Keep the PR visible — and + # when a HUMAN removed the takeover label by hand (fork PRs + # get no release ack from the route, so nothing else clears + # the escalation label there), clear the stale label. The + # shepherd's own auto-release authenticates as the bot and + # is NOT a heal trigger. + # Anchor to the CURRENT pause boundary: the latest + # needs-human label-apply EVENT (the cap branch applies the + # label at pause time). A human takeover-unlabel counts only + # when NEWER than that — anything older belongs to an earlier + # cycle (R5-10) — and an absent anchor (e.g. past the ~90-day + # events lookback) means "cannot correlate" → skip the + # cleanup rather than admit everything (fail closed). + NOTE='loop stopped — needs a human decision (merge / close / split / re-engage)' + if ! gh api "repos/${REPO}/issues/${PR}/events" --paginate 2> /dev/null | jq -s 'add // []' > /tmp/tk-ev.json; then + NOTE='event read failed — heal deferred this tick (fail closed)' + else + NH_APPLY_TS="$(jq -r --arg nl "${NEEDS_HUMAN_LABEL}" ' + [ .[] | select((.event // "") == "labeled") + | select((.label.name // "") == $nl) + | .created_at ] | max // ""' /tmp/tk-ev.json)" + UNLABEL_ACTOR='' + if [[ -n "${NH_APPLY_TS}" ]]; then + UNLABEL_ACTOR="$(jq -r --arg tl "${TAKEOVER_LABEL}" --arg ab "${AUTOFIX_BOT}" --arg ll "${NH_APPLY_TS}" ' + [ .[] | select((.event // "") == "unlabeled") + | select((.label.name // "") == $tl) + | select((.actor.login // "") != $ab) + | select((.created_at // "") > $ll) + | .actor.login ] | last // ""' /tmp/tk-ev.json)" + fi + if [[ -n "${UNLABEL_ACTOR}" ]]; then + if [[ "${CLEANUPS}" -ge "${MAX_CLEANUPS_PER_TICK}" ]]; then + NOTE='cleanup budget reached this tick' + elif live_skip "${PR}"; then + NOTE="$(skip_note cleanup)" + elif [[ "$(jq -r --arg t "${TAKEOVER_LABEL}" '[.labels[]?.name] | index($t) != null' <<< "${LIVE_LABELS_JSON}")" == "true" ]]; then + # R5-8: takeover was re-applied between the tick-start + # snapshot and now — the PR is managed again, so nothing + # here is stale. + NOTE='takeover label re-applied since the snapshot — cleanup cancelled' + else + # Count ATTEMPTS, mirroring the release arm (twin above). + CLEANUPS=$(( CLEANUPS + 1 )) + if act "#${PR}: clear stale ${NEEDS_HUMAN_LABEL} (manual release by @${UNLABEL_ACTOR})" \ + gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}" '$l|@uri')"; then + # Both labels are gone now — the PR is fully released and + # must drop out of the dashboard entirely. + ROUTE='none' + NOTE="cleared stale escalation label (released by @${UNLABEL_ACTOR})" + else + NOTE='stale-label cleanup failed — will retry next tick' + fi + fi + fi + fi + echo "🐑 #${PR} [awaiting: ${STATE}] → ${NOTE}" + SAFE_NOTE="${NOTE//\\/\\\\}" + if [[ "${ROUTE}" != 'none' ]]; then + HUMAN_ROWS="${HUMAN_ROWS}| #${PR} | ${AUTHOR} | ${UPD_AGO} ago | ${SAFE_NOTE//|/\\|} |\n" + fi + done < <(jq -c '.[]' /tmp/human.json) + fi # ---- dashboard: one issue, edited in place ---------------------- { echo "Auto-maintained by the Fleet Shepherd workflow — do not edit by hand." echo - echo "Last tick: $(date -u +%Y-%m-%dT%H:%M:%SZ) · scan-signal age: ${SCAN_AGE_MIN}m · syncs: ${SYNCS} · dispatches: ${DISPATCHES}" + echo "Last tick: $(date -u +%Y-%m-%dT%H:%M:%SZ) · scan-signal age: ${SCAN_AGE_MIN}m · syncs: ${SYNCS} · dispatches: ${DISPATCHES} · releases: ${RELEASES} · cleanups: ${CLEANUPS}" + echo + echo '## Bot fleet' echo echo '| PR | Head | State | Action this tick |' echo '| --- | --- | --- | --- |' printf '%b' "${DASH_ROWS}" + echo + echo '## Takeover pool' + echo + echo 'Managed by the autofix loop; the shepherd only reports, and auto-releases a takeover whose pause went unanswered.' + echo + echo '| PR | Author | Updated | State | Note |' + echo '| --- | --- | --- | --- | --- |' + printf '%b' "${TAKEOVER_ROWS}" + echo + echo '## Awaiting human' + echo + echo 'The loop stopped on these PRs and no takeover is active; a human needs to decide (merge / close / split / re-engage).' + echo + echo '| PR | Author | Updated | Note |' + echo '| --- | --- | --- | --- |' + printf '%b' "${HUMAN_ROWS}" if [[ -n "${LIVENESS_OUT}" ]]; then echo echo "" @@ -485,4 +1172,4 @@ jobs: echo "::warning::dashboard update failed; will retry next tick" fi fi - echo "✅ tick complete (syncs=${SYNCS} dispatches=${DISPATCHES})" + echo "✅ tick complete (syncs=${SYNCS} dispatches=${DISPATCHES} releases=${RELEASES} cleanups=${CLEANUPS})" diff --git a/docs/design/takeover-fleet-visibility.md b/docs/design/takeover-fleet-visibility.md new file mode 100644 index 00000000000..593027b653d --- /dev/null +++ b/docs/design/takeover-fleet-visibility.md @@ -0,0 +1,178 @@ +# Takeover fleet visibility and cap-hit escalation + +## Problem statement + +As of 2026-08-11, 35 open PRs carry `autofix/takeover`. Two structural gaps: + +1. **The takeover pool is invisible.** The Fleet Shepherd + (`qwen-fleet-shepherd.yml`) enumerates only bot-authored PRs (3 today). + The 35 human-authored takeover PRs appear on no dashboard; their state + (working / paused / conflicting / idle-for-days) is knowable only by + opening each PR. + +2. **Cap-hit PRs die silently.** When a takeover PR reaches its round cap + (100/100), or a circuit breaker (consecutive-failure, time-budget) stops + it, the loop posts one comment and goes quiet. Five PRs have been paused + since 2026-08-06 with no re-arm: #8213, #8396, #8416, #8439, #8443. + Nothing escalates them — no label, no dashboard entry, no auto-release — + so they hold the takeover label forever ("zombie takeover"). + +## Proposed changes + +### A. `autofix/needs-human` label (qwen-autofix.yml) + +A new maintainer-facing label meaning: _the loop has stopped on this PR; a +human must act (re-arm, split, merge, or close)_. + +**Applied** in the review scan's cap-notice path (the single funnel every +terminal state passes through: round cap, consecutive-failure cap, and +time-budget cap all write a terminal `autofix-eval` marker with +`round=EFF_MAX_ROUNDS`, which the next scan sees as `ROUND >= EFF_MAX_ROUNDS` +and lands in the cap-notice branch). The label write is placed so it runs +even when the once-per-window notice comment is dedup'd — this backfills the +label onto the already-paused fleet via the regular scan rotation after +deploy (idle backoff defers PRs idle >24h to ~1 scan in 4 — expect hours, +not the first scan). + +**Removed** wherever management resumes or a human takes over: + +| Path | Site | +| ------------------------------------------------------ | -------------------- | +| `/takeover` re-arm on a managed PR | takeover-command job | +| `/takeover` fresh engage | takeover-command job | +| `/takeover stop` | takeover-command job | +| Manual label engage / release acks | takeover-ack job | +| `/retry` re-arm marker | retry-command job | +| Scan first-pickup engage ack (direct-label engagement) | review-scan job | + +Removal is best-effort with a warning on failure, mirroring the existing +`TAKEOVER_LABEL` DELETE pattern (404 tolerated). A stale `needs-human` left +behind by a failed removal is cosmetically wrong but harmless; the next +cap-stop reapplies it anyway. + +A PR closed or merged while paused keeps `needs-human` — deliberately. No +closure removal path exists (the route drops commands on non-open PRs, every +enumeration is `--state open`, and there is no `pull_request: closed` +trigger), and the residue is inert: all consumers filter on open state, so +the label only marks the resolved escalation in the closed PR's own history. +All-state label queries should pair the label with a state filter. + +Label creation follows the existing convention: `gh label create` (idempotent, +fixed color) before the first REST add, so a missing label never gets a random +color. + +### B. Shepherd covers the takeover pool (qwen-fleet-shepherd.yml) + +A second enumeration — open PRs with `autofix/takeover`, including forks — +drives a **second dashboard table** in the same edited-in-place issue: + +| PR | Author | Updated | State | Note | +| --- | ------ | ------- | ----- | ---- | + +State comes from the list payload (conflicting / ci red / checks in flight / +idle). PRs carrying `autofix/needs-human` get a `🛑 needs-human` state; for +those few PRs the shepherd additionally reads the comment stream (fail-closed) +to recover the terminal timestamp (latest `` +notice) and the stop reason (first line of the latest terminal "AutoFix +stopped" headline, else "round cap reached"). + +**NON-GOAL:** the existing levers (conflict dispatch, stale-base sync) stay +scoped to the bot fleet. Takeover-PR conflicts are already the autofix scan's +job (`HAS_CONFLICT` selects them as targets), and `update-branch` on +contributor branches is out of scope for this change. + +### C. Auto-release lever (qwen-fleet-shepherd.yml) + +When a PR carries **both** `autofix/takeover` and `autofix/needs-human` and +its terminal timestamp is older than `AUTO_RELEASE_DAYS` (default 3, tunable +via the `QWEN_SHEPHERD_AUTO_RELEASE_DAYS` repo variable): + +1. Post one bilingual summary — dedup'd by its + `` marker (scoped to the current + pause cycle): why it was released, the stop reason, and the human's + options (merge / close / split + re-takeover). +2. Remove `autofix/takeover` (the loop disengages). A failed removal finds + the marker and retries only the DELETE; a failed summary leaves both + labels in place so the whole release retries next tick. +3. Keep `autofix/needs-human`: the PR still needs a human decision, and the + label remains the filterable TODO list. It clears on re-engage/re-arm via + the paths in (A). + +Idempotency needs no marker comment: the lever's scope condition (both labels) +is false after the release, so it cannot re-fire. Per-tick cap +(`MAX_RELEASES_PER_TICK`, default 3) bounds blast radius; `live_skip` is +re-checked immediately before the mutation, mirroring every existing lever. + +## Key design decisions + +- **Label write lives in the scan, not the address leg.** Every terminal stop + converges on `round=EFF_MAX_ROUNDS` markers, which the scan's cap branch + already observes with comments loaded and PAT identity verified. One hook + point covers all stop reasons, including future ones. +- **Pause reason comes from the terminal marker headline**, because the + scan-side notice always says "round cap (N/N)" even when a breaker fired + (observed on #8443: both comments present). +- **Bootstrap without a backfill job:** the label write runs even when the + notice comment is dedup'd, so currently-paused PRs are labeled by the + regular scan rotation after deploy — note the scan's idle backoff defers + PRs idle >24h (exactly the paused population) to ~1 scan in 4, so expect + the backfill within a few hours (median ~2h, p90 ~6h), not minutes. +- **Auto-release keyed on the notice timestamp**, not label age: labels carry + no timestamps, and the notice is written by the same identity-verified path + that applies the label. Resume evidence newer than the notice vetoes the + release — the bot's re-arm/engage markers, a re-arm command comment, or a + fresh `labeled` event. Command comments count only while FRESH + (`RESUME_COMMAND_GRACE_SEC`, 2h) and UNSUPERSEDED by a refusal ack + (`fork-refused` / `base-refused` / `skip-blocked`): an accepted command is + acked within minutes; an ignored one (no route permission) simply expires; + and no permission check is mirrored into the shepherd — the route's + collaborator check is the authorization gate, and a mirrored copy would + only drift. +- **The release lever gets its own enumeration** of the paused population + (needs-human ∩ takeover, stalest-first) — not the takeover display window + and not the needs-human display window: released PRs keep `needs-human` + and age back into that display window, so feeding the lever from it would + truncate exactly the fresh pauses that become release-eligible. All three + enumerations cap at 100 with loud saturation warnings; a display + enumeration failure degrades to an error row, and a paused-enumeration + failure skips the lever for that tick — the dashboard write (which + carries the liveness watermark) always runs. +- **The summary posts before the label removal**, dedup'd by its own marker + scoped to the current pause cycle (only markers newer than the latest cap + notice count), so a failed comment leaves both labels in place and the + whole release retries next tick; a failed removal finds the marker and + retries only the DELETE; and a re-armed-and-re-capped PR still gets its + second summary. +- **Stale-label heal:** a fork PR released by hand gets no release ack (the + route suppresses fork `unlabeled` events), so nothing else clears its + `needs-human`. The shepherd watches the awaiting-human pool for a + human-actor `unlabeled` event on the takeover label that is NEWER than the + latest label-apply (a stale unlabel from an earlier takeover cycle must + never heal this cycle's label), and clears the stale label — bounded per + tick, skip-vetoed, and never triggered by the bot's own auto-release. +- **Shepherd timing:** 15-minute tick with a per-tick release cap — a backlog + of expired PRs drains over a few ticks rather than one burst. + +## Files affected + +- `.github/workflows/qwen-autofix.yml` — env, cap-notice branch, six + label-removal sites. +- `.github/workflows/qwen-fleet-shepherd.yml` — env, takeover enumeration, + dashboard takeover table plus a read-only "Awaiting human" section + (released PRs keep `needs-human` and would otherwise vanish from every + surface), auto-release lever. + +## Scope boundaries + +- No changes to round caps, breakers, or review-bot behavior. +- No shepherd levers on takeover PRs other than auto-release. +- No notification/@-mention of maintainers (comment + label + dashboard only). +- `autofix/needs-human` on plain (non-takeover) bot PRs is applied by the same + scan path and shown on the dashboard, but the auto-release lever never + touches them (they have no takeover label to release). + +## Open questions + +- Default `AUTO_RELEASE_DAYS=3` — short enough to keep the pool clean, long + enough for a maintainer to re-arm over a weekend? Adjustable without a + deploy via the repo variable. diff --git a/scripts/tests/pr-self-report-label.test.js b/scripts/tests/pr-self-report-label.test.js index 82d28c5df60..603e594d37c 100644 --- a/scripts/tests/pr-self-report-label.test.js +++ b/scripts/tests/pr-self-report-label.test.js @@ -187,6 +187,10 @@ describe('pr-self-report-label', () => { }); expect(race.removed).toBe(true); expect(race.out).not.toContain('::warning::'); + // R7-5: a 404 race means the end state holds → the success claim fires, + // the "did not land" claim must not. + expect(race.out).toContain('removed'); + expect(race.out).not.toContain('removal did not land'); // Any other DELETE failure keeps the step green (the job only re-runs // on the next PR event) but MUST surface — a silent run would claim // "removed" while the label stays on the PR. @@ -200,6 +204,26 @@ describe('pr-self-report-label', () => { expect(failed.out).toContain('::warning::'); expect(failed.out).toContain('removal failed'); expect(failed.out).toContain('HTTP 500'); + // R7-5: the LBL_DEL_FAILED branching — a failed DELETE must NOT claim + // the label was removed (the lying-log shape this change exists to kill). + expect(failed.out).toContain('removal did not land'); + expect(failed.out).not.toContain('removed'); + // R10-1/R10-3: the tolerated token is exactly "HTTP 404" — a transport + // failure embedding a 404-bearing request URL must warn and report + // not-landed, never read as the already-off case (the loose *404* + // match hit the URL's PR number and claimed "removed"). + const transport = run({ + prAuthor: 'alice', + issueAuthors: 'bob', + hasLabel: true, + deleteFails: + 'gh: Delete "https://api.github.com/repos/o/r/issues/4041/labels/review%2Fself-reported": dial tcp 140.82.121.4:443: connect: connection refused', + }); + expect(transport.removed).toBe(true); + expect(transport.out).toContain('::warning::'); + expect(transport.out).toContain('removal failed'); + expect(transport.out).toContain('removal did not land'); + expect(transport.out).not.toContain('removed'); // POST carries NO tolerance: a label that fails to apply must fail the // step (the runBlock sets -e), not leave the PR unlabeled behind green. expect(() => diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 0caa373b012..708b84a317e 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -2425,11 +2425,13 @@ describe('qwen-autofix workflow', () => { // variants, mirroring the ack job — a loud add next to a mute stop // would re-create the lost-event ambiguity on the release side), and // the scan-side first-pickup engage ack (fork label events carry no - // secrets, so the scan anchors the window itself). + // secrets, so the scan anchors the window itself), and the + // command-path release-failed ack (a failed release must say so — a + // 'released' marker would record a release that never happened). const ackBodies = workflow.match( /printf '[^']*takeover-(?:ack|cap)[^']*'/g, ); - expect(ackBodies).toHaveLength(19); + expect(ackBodies).toHaveLength(20); for (const body of ackBodies) { expect(body).toContain('中文说明'); } @@ -2467,6 +2469,12 @@ describe('qwen-autofix workflow', () => { expect(workflow).toContain( "group: 'qwen-autofix-takeover-cmd-${{ needs.route.outputs.cmd_pr }}'", ); + // R11-2: ack runs serialize per PR the same way, so a delayed engaged + // ack queues behind the newer cycle's ack and its staleness read sees + // that ack's marker. + expect(workflow).toContain( + "group: 'qwen-autofix-takeover-ack-${{ needs.route.outputs.ack_pr }}'", + ); // Fork PRs can never produce a red ack run or a stuck label: the // unlabeled branch log-and-drops forks (fork pull_request events carry // no secrets, so emitting the ack would fail the PAT identity check), @@ -2879,7 +2887,7 @@ describe('qwen-autofix workflow', () => { ).length - 1, ).toBe(2); expect(reviewScanJob).toContain( - '--json headRefName,headRefOid,statusCheckRollup,createdAt,labels', + '--json headRefName,headRefOid,statusCheckRollup,createdAt,labels,isCrossRepository,headRepositoryOwner,headRepository,author', ); // Command-style comments are instructions, not feedback — excluded at // ALL FIVE feedback sites (scan count via $cf; NEWEST, LIVE_NEW, @@ -2916,10 +2924,11 @@ describe('qwen-autofix workflow', () => { // Pin the total --paginate code-site count so ANY new paginated site // forces a deliberate test update, however it is spaced or line-wrapped: // bump this count AND pipe the new site through the normalizer (bumping - // the count below too) — bumping this pin alone leaves toBe(9) green. - expect(workflow.split('--paginate').length - 1).toBe(15); + // the count below too) — bumping this pin alone leaves toBe(10) green. + expect(workflow.split('--paginate').length - 1).toBe(18); // scan ic + pr-events + ic re-fetch + scan rv/rc + prepare rv/rc/ic + - // report COMMENTS_JSON fallback = nine normalized fetch sites. The + // report COMMENTS_JSON fallback + the cap-branch release-evidence events + // fetch (R4-1) = ten normalized fetch sites. The // blocked-takeover status lookup is deliberately NOT among them: like the // sibling STATUS_ID read, it consumes the page stream inline via // `--jq ... | .id` into `tail -1` and never lands in a WORKDIR json file, @@ -2927,7 +2936,11 @@ describe('qwen-autofix workflow', () => { // array and break the tail-1 consumer. The #8888 deferred-review ack // dedup is the same class: `--jq '.[].body'` feeds a grep, never a // WORKDIR file, so it bumps the total pin but not the normalizer count. - expect(workflow.split("jq -s 'add // []'").length - 1).toBe(9); + // The R11-2 engaged-stale guard's comments/events reads in + // takeover-ack are the same class too: captured into shell variables + // and consumed by slurp-style `jq -rs 'add // [] | …'` (idempotent + // over a single flat array), never a WORKDIR file. + expect(workflow.split("jq -s 'add // []'").length - 1).toBe(10); // Empty-input semantics: a total gh failure feeds the fallback an EMPTY // stream, where the normalizer filter must yield '[]' and not 'null' — // the PRIOR_HEADS consumer below iterates the result with .[], which @@ -3353,6 +3366,27 @@ describe('qwen-autofix workflow', () => { expect(gate('autofix/managed autofix/takeover', 'true')).toContain( 'PROCEED', ); + // B12: the consent re-read FAILS CLOSED for standard bot PRs too — an + // unreadable `gh pr view` must skip the write entirely (a collapse to + // '' would ignore a concurrently added skip). The block sits BEFORE the + // consent gate. + expect(reviewScanJob).toContain( + 'cap notice skipped: label state unreadable (fail closed)', + ); + const unreadableIdx = reviewScanJob.indexOf( + 'if [[ -z "${LIVE_LABELS_JSON}" ]]; then', + ); + expect(unreadableIdx).toBeGreaterThan(-1); + expect(unreadableIdx).toBeLessThan( + reviewScanJob.indexOf('if [[ " ${LIVE_LABELS} " == *" ${SKIP_LABEL} "*'), + ); + // R8-17: the fail-closed gate actually SKIPS the writes — the + // `continue` is load-bearing (without it an unreadable state falls + // through the consent gate, whose empty-string disjuncts are both + // false, and POSTs over a concurrently added skip). + expect(reviewScanJob).toMatch( + /if \[\[ -z "\$\{LIVE_LABELS_JSON\}" \]\]; then\n\s+echo "🧭 cap notice skipped: label state unreadable \(fail closed\) on #\$\{PR\}"\n\s+continue/, + ); // Candidates drain newest-first, and the free busy skip never consumes // inspection budget. expect(reviewScanJob).toContain('sort_by(-.number)'); @@ -3373,6 +3407,205 @@ describe('qwen-autofix workflow', () => { expect(loopHead).not.toContain('INSPECTED='); }); + it('applies the needs-human escalation label at the cap and removes it on resume/release', () => { + // The label contract: one env definition, one idempotent create with a + // fixed color (a REST add of a missing label would mint a random one). + // The create's `|| true` and the POST's `if !` guard are load-bearing + // under the runner's `bash -e`: a create that fails once the label + // exists, or a POST that aborts the scan on a transient 502, must not + // ship green (R2-25). + expect(workflow).toContain("NEEDS_HUMAN_LABEL: 'autofix/needs-human'"); + expect(reviewScanJob).toContain( + 'gh label create "${NEEDS_HUMAN_LABEL}" --repo "${REPO}" --color', + ); + expect(reviewScanJob).toMatch( + /gh label create "\$\{NEEDS_HUMAN_LABEL\}"[\s\S]{0,250}?2> \/dev\/null \|\| true/, + ); + expect(reviewScanJob).toContain( + 'if ! gh api -X POST "repos/${REPO}/issues/${PR}/labels" -f "labels[]=${NEEDS_HUMAN_LABEL}"', + ); + expect(reviewScanJob).toContain( + '${NEEDS_HUMAN_LABEL} add failed for #${PR}; will retry next scan', + ); + // The label write sits OUTSIDE the once-per-window comment dedup — + // that is the bootstrap property: already-noticed PRs (paused before + // this shipped) get labeled on the first scan after deploy. Assert on + // ordering inside the cap branch: the label POST precedes the + // CAP_NOTICED gate that guards only the comment. + const labelPost = reviewScanJob.indexOf( + '-f "labels[]=${NEEDS_HUMAN_LABEL}"', + ); + const commentGate = reviewScanJob.indexOf('"${CAP_NOTICED}" == "0"'); + expect(labelPost).toBeGreaterThan(-1); + expect(commentGate).toBeGreaterThan(-1); + expect(labelPost).toBeLessThan(commentGate); + // …but never BEFORE the live-consent recheck: a takeover label pulled + // moments ago gets neither the notice nor the escalation label. + const consentRecheck = reviewScanJob.indexOf( + 'cap notice skipped: consent changed since the snapshot', + ); + expect(consentRecheck).toBeGreaterThan(-1); + expect(consentRecheck).toBeLessThan(labelPost); + // R3-1/R4-1: a takeover `unlabeled` EVENT newer than the window + // suppresses the every-scan label POST — a released PR stays in the + // scan candidate set with ROUND >= cap, so an unconditional POST would + // re-add the escalation label the release just removed and ping-pong + // with the shepherd's cleanup. R4-5: the gate only fires for + // HUMAN-authored PRs (a bot PR released from takeover is still managed, + // so it keeps the label at the strict cap). The gate sits BEFORE the + // label POST. + expect(reviewScanJob).toContain('RELEASE_ACKED='); + expect(reviewScanJob).toContain('IS_BOT_AUTHOR='); + expect(reviewScanJob).toContain( + 'cap label/notice skipped: PR was released after its last re-arm', + ); + const releaseGate = reviewScanJob.indexOf( + 'elif [[ "${RELEASE_ACKED}" != "0" && "${IS_BOT_AUTHOR}" != "true" ]]; then', + ); + expect(releaseGate).toBeGreaterThan(-1); + expect(releaseGate).toBeLessThan(labelPost); + // R4-5 residual: IS_BOT_AUTHOR must actually resolve from PR_META — the + // fetch's field list must carry `author` and the jq must read + // .author.login. Replay both so a field-list drift fails (a dead + // IS_BOT_AUTHOR=false would suppress the label for bot PRs forever). + const botAuthorJq = reviewScanJob.match( + /IS_BOT_AUTHOR="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([^']+)' <<< "\$\{PR_META\}"\)"/, + )?.[1]; + expect(botAuthorJq).toBeTruthy(); + const runBotAuthor = (login) => + execFileSync( + 'jq', + ['-r', '--arg', 'ab', 'qwen-code-dev-bot', botAuthorJq], + { + encoding: 'utf8', + input: JSON.stringify({ author: { login } }), + }, + ).trim(); + expect(runBotAuthor('qwen-code-dev-bot')).toBe('true'); + expect(runBotAuthor('wenshao')).toBe('false'); + expect(runBotAuthor('')).toBe('false'); + // R4-15: the RELEASE_ACKED jq body (event marker + time direction) is + // replayed — a `> $rt` → `< $rt` flip must fail, not ship green. + const relAckJq = reviewScanJob.match( + /RELEASE_ACKED="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" --arg rt "\$\{NOTICE_RT\}" '([\s\S]*?)' "\$\{WORKDIR\}\/ev\.json"\)"/, + )?.[1]; + expect(relAckJq).toBeTruthy(); + const runRelAck = (events, rt) => + execFileSync( + 'jq', + ['-r', '--arg', 'tl', 'autofix/takeover', '--arg', 'rt', rt, relAckJq], + { encoding: 'utf8', input: JSON.stringify(events) }, + ).trim(); + expect( + runRelAck( + [ + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-06T00:00:00Z', + }, + ], + '2026-08-05T00:00:00Z', + ), + ).toBe('1'); + // ack older than the window → 0; no ack → 0. + expect( + runRelAck( + [ + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-04T00:00:00Z', + }, + ], + '2026-08-05T00:00:00Z', + ), + ).toBe('0'); + expect(runRelAck([{ event: 'labeled' }], '2026-08-05T00:00:00Z')).toBe('0'); + // R6-6: same-second boundary — a release at EXACTLY the window key + // counts (>= tie-toward-released: a completed release must never be + // re-escalated, R5-9). + expect( + runRelAck( + [ + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-05T00:00:00Z', + }, + ], + '2026-08-05T00:00:00Z', + ), + ).toBe('1'); + // R4-24: the gate's control-flow nesting is replayed — the label POST + // must fire ONLY in the outer else arm (RELEASE_ACKED=0 or bot author), + // never in the released arm. R7-3: elseIdx anchors on the OUTER else + // (16-space indent — the inner else at 18 spaces introduces the + // released echo, not the POST arm). R8-18: releasedIdx anchors on the + // FULL released echo — the shared prefix alone resolves to the earlier + // 'release history unreadable' echo, silently migrating the anchor. + const gateBlock = reviewScanJob.match( + /if \[\[ "\$\{SCAN_BOT_ACTOR\}" != "\$\{AUTOFIX_BOT\}" \]\]; then[\s\S]*?cap label\/notice skipped[\s\S]*?\n {16}fi/, + )?.[0]; + expect(gateBlock).toBeTruthy(); + const releasedIdx = gateBlock.indexOf( + 'cap label/notice skipped: PR was released after its last re-arm', + ); + const elseIdx = gateBlock.indexOf( + '\n' + ' '.repeat(16) + 'else', + releasedIdx, + ); + const postIdx = gateBlock.indexOf( + 'gh api -X POST "repos/${REPO}/issues/${PR}/labels"', + ); + expect(releasedIdx).toBeGreaterThan(-1); + expect(elseIdx).toBeGreaterThan(releasedIdx); + expect(postIdx).toBeGreaterThan(elseIdx); + // The dry-run line covers both writes. + expect(reviewScanJob).toContain( + 'DRY-RUN: would post cap-paused notice and apply ${NEEDS_HUMAN_LABEL}', + ); + // Removal at every resume/release point — the URI-encoded REST DELETE + // with 404 tolerance, same shape as the TAKEOVER_LABEL removal. Six + // sites: takeover-command re-arm, fresh engage, and stop; the ack + // job's engage/release; the /retry marker; the scan's first-pickup ack. + const removals = + workflow.match( + /gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{NEEDS_HUMAN_LABEL\}"/g, + ) ?? []; + expect(removals).toHaveLength(6); + // R9-16: the escalation POST rides the cap detection exactly ONCE — a + // duplicate outside the gate would re-add the label on every cap + // detection of a released human PR (the R3-1/R4-1 ping-pong with the + // shepherd's cleanup that this test cites). + expect( + workflow.match( + /gh api -X POST "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels" -f "labels\[\]=\$\{NEEDS_HUMAN_LABEL\}"/g, + ) ?? [], + ).toHaveLength(1); + // R4-17: the scan-side first-pickup DELETE must live in the engage-ack + // SUCCESS branch — moved to the ack-failure path it would clear + // needs-human when no engagement happened. + const firstPickup = reviewScanJob.match( + /if gh pr comment "\$\{PR\}" --repo "\$\{REPO\}" --body "\$\(printf '🤝 Takeover engaged[\s\S]*?; then([\s\S]*?)\n {16}else/, + )?.[1]; + expect(firstPickup).toBeTruthy(); + expect(firstPickup).toContain( + 'gh api -X DELETE "repos/${REPO}/issues/${PR}/labels/$(jq -rn --arg l "${NEEDS_HUMAN_LABEL}"', + ); + // Every removal tolerates the common 404 (never-paused PR). + expect( + workflow.match(/\$\{NEEDS_HUMAN_LABEL\} removal failed/g)?.length, + ).toBe(6); + // The ack job removes on a real engage or any release — but NOT on + // base-refused (nothing changed), NOT on skip-blocked (management never + // resumed), and NOT on a release onto a skip-frozen PR (nothing manages + // or restores it — R4-3): both arms now require HAS_SKIP != 'true'. + expect(workflow).toContain( + '( "${ACK}" == \'released\' || "${ACK}" == \'engaged\' ) && "${HAS_SKIP}" != \'true\'', + ); + }); + it('backs off idle candidates without spending inspection budget', () => { // Measured 2026-07-29: 28 open takeover PRs, 8 idle in "nothing new" // state for 10+ hours. Every idle inspection costs a unit of the @@ -3557,7 +3790,7 @@ describe('qwen-autofix workflow', () => { // the block's two failure policies instead of the stub // universally exiting 0. `elif [[ "$1" == "label" && "$2" == "create" ]]; then echo "LABEL-CREATE $*" >> '${join(dir, 'writes.log')}';`, - `elif [[ "$1" == "api" ]]; then echo "API $*" >> '${join(dir, 'writes.log')}'; if [[ "$2" == "-X" && "$3" == "POST" && -n "\${TOGGLE_POST_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_POST_FAILS}" >&2; exit 1; fi; if [[ "$2" == "-X" && "$3" == "DELETE" && -n "\${TOGGLE_DELETE_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_DELETE_FAILS}" >&2; exit 1; fi`, + `elif [[ "$1" == "api" ]]; then echo "API $*" >> '${join(dir, 'writes.log')}'; if [[ "$2" == "-X" && "$3" == "POST" && -n "\${TOGGLE_POST_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_POST_FAILS}" >&2; exit 1; fi; if [[ "$2" == "-X" && "$3" == "DELETE" && -n "\${TOGGLE_DELETE_FAILS:-}" ]]; then printf '%s' "\${TOGGLE_DELETE_FAILS}" >&2; exit 1; fi; if [[ "$2" == "-X" && "$3" == "DELETE" ]]; then printf '%s' '[{"name":"Tracks HTTP 5xx flakes"}]'; fi`, `elif [[ "$1" == "pr" && "$2" == "comment" ]]; then echo "COMMENT $*" >> '${join(dir, 'writes.log')}';`, 'fi', ].join('\n'), @@ -3585,6 +3818,7 @@ describe('qwen-autofix workflow', () => { REPO: 'QwenLM/qwen-code', TAKEOVER_LABEL: 'autofix/takeover', SKIP_LABEL: 'autofix/skip', + NEEDS_HUMAN_LABEL: 'autofix/needs-human', TAKEOVER_COMMAND: '@qwen-code /takeover', AUTOFIX_BOT: 'qwen-code-dev-bot', GITHUB_TOKEN: 'x', @@ -3627,12 +3861,31 @@ describe('qwen-autofix workflow', () => { ); expect(addAbsent.writes).toContain('labels[]=autofix/takeover'); expect(addAbsent.writes).toContain(''); + // …and the engage arm clears any stale escalation label (R2-27: the + // removal must live in THIS branch, not just anywhere in the file). + expect(addAbsent.writes).toContain('labels/autofix%2Fneeds-human'); + // R4-18: the loud takeover POST precedes the tolerant needs-human + // DELETE — a transient POST failure then aborts the step before the + // DELETE, keeping the paused PR visible (reversed order would drop the + // escalation label even though the engagement never landed). + expect(addAbsent.writes.indexOf('API api -X POST')).toBeLessThan( + addAbsent.writes.indexOf('labels/autofix%2Fneeds-human'), + ); expect(addAbsent.writes).not.toContain('next scheduled scan'); expect(addAbsent.writes).not.toContain('定时扫描'); - // add + present → re-arm ack, label untouched. + // add + present → re-arm ack, takeover label untouched — the only API + // write is the stale needs-human cleanup (404-tolerant; this PR was + // never paused). Pin the full DELETE line (method + encoded label) and + // that it is the ONLY API write — a fragment match would let a refactor + // swap the DELETE for a probe GET or add a stray label POST (R3-9). const rearm = runToggle({ cmd: 'add', labels: ['autofix/takeover'] }); expect(rearm.writes).toContain('COMMENT'); - expect(rearm.writes).not.toContain('API'); + expect(rearm.writes).not.toContain('labels[]=autofix/takeover'); + expect(rearm.writes).not.toContain('labels/autofix%2Ftakeover'); + expect(rearm.writes).toContain( + 'API api -X DELETE repos/QwenLM/qwen-code/issues/7165/labels/autofix%2Fneeds-human', + ); + expect(rearm.writes.match(/^API /gm) ?? []).toHaveLength(1); expect(rearm.log).toContain('re-armed'); // remove + present → label removed, through the URI-encoded path segment // (real jq runs in the substitution, so the %2F is the executed truth). @@ -3643,6 +3896,9 @@ describe('qwen-autofix workflow', () => { expect(removePresent.writes).toContain( 'API api -X DELETE repos/QwenLM/qwen-code/issues/7165/labels/autofix%2Ftakeover', ); + // …and the stop arm clears the escalation label in the same breath + // (R2-27): a stop on a paused PR must not leave needs-human behind. + expect(removePresent.writes).toContain('labels/autofix%2Fneeds-human'); // Release acks directly too — the exact mirror of the engage side: a // loud add next to a mute stop re-creates the lost-event ambiguity on // the release side (and fork/non-main releases have no other ack path @@ -3665,6 +3921,10 @@ describe('qwen-autofix workflow', () => { expect(botSkipRelease.writes).toContain( 'opts it out of standard bot management entirely', ); + // R7-6: the skip veto keeps needs-human on a skip-frozen release — + // /takeover stop must NOT strip the only filterable escalation state + // from a PR nothing manages. + expect(botSkipRelease.writes).not.toContain('labels/autofix%2Fneeds-human'); // remove + absent → explicit no-op, no writes at all. const removeAbsent = runToggle({ cmd: 'remove' }); expect(removeAbsent.writes.trim()).toBe(''); @@ -3680,6 +3940,7 @@ describe('qwen-autofix workflow', () => { expect(forkRefused.writes).not.toContain('API'); const forkManaged = runToggle({ cmd: 'add', fork: true }); expect(forkManaged.writes).toContain('labels[]=autofix/takeover'); + expect(forkManaged.writes).toContain('labels/autofix%2Fneeds-human'); // Fork label events carry no secrets, so no other job could ever ack a // fork engage — the command's own ack is the ONLY one, and it sets the // expectation that the first round comes from the next scheduled scan. @@ -3766,19 +4027,75 @@ describe('qwen-autofix workflow', () => { ); expect(releaseRace.writes).toContain('takeover-ack released'); expect(releaseRace.log).not.toContain('::warning::'); - // Any other DELETE failure must not drop the release ack either — a - // later `/takeover stop` retries the removal — but it MUST warn: - // masked, the ack reads "released" while the loop keeps managing the - // PR. + // R9-3: pin BOTH echo arms — a swap ships a lying log in either + // direction while the separate ::warning:: line keeps the old pins + // green. + expect(releaseRace.log).toContain('removed autofix/takeover'); + expect(releaseRace.log).not.toContain('removal did not land'); + // R5-1: a 404 counts as landed, so the needs-human cleanup still runs… + expect(releaseRace.writes).toContain('labels/autofix%2Fneeds-human'); + // Any other DELETE failure keeps the label ON — the ack must say the + // release did NOT land (a 'released' marker would record a release + // that never happened: no unlabeled event fires, nothing retries, no + // human re-issues the command) and the loud warning keeps the failure + // diagnosable in the log. const releaseFailed = runToggle({ cmd: 'remove', labels: ['autofix/takeover'], deleteFails: 'HTTP 500', }); - expect(releaseFailed.writes).toContain('takeover-ack released'); + expect(releaseFailed.writes).toContain('takeover-ack release-failed'); + expect(releaseFailed.writes).toContain('release did not land'); + expect(releaseFailed.writes).not.toContain('takeover-ack released'); expect(releaseFailed.log).toContain('::warning::'); expect(releaseFailed.log).toContain('removal failed'); expect(releaseFailed.log).toContain('HTTP 500'); + expect(releaseFailed.log).toContain('removal did not land'); + expect(releaseFailed.log).not.toContain('removed autofix/takeover from'); + // R5-1: …but a failed takeover release keeps needs-human — landed vs + // not-landed is keyed on the DELETE's EXIT STATUS (404 counts as + // landed; any other non-zero does not — R6-1's transport failures and + // R6-19's "HTTP "-bearing success bodies both mislead a text sniff), + // and the stub's success body carries a label name containing "HTTP " + // to model the real remaining-labels API contract. + expect(releaseFailed.writes).not.toContain('labels/autofix%2Fneeds-human'); + expect(releaseFailed.log).toContain('release did not land'); + // R6-9: a transport-level failure carries no "HTTP " token anywhere — + // the exit-status derivation still classifies it as not-landed (the + // class the old text sniff missed, R6-1). + const releaseTransportFailed = runToggle({ + cmd: 'remove', + labels: ['autofix/takeover'], + deleteFails: 'gh: dial tcp 140.82.121.4:443: connect: connection refused', + }); + expect(releaseTransportFailed.writes).toContain( + 'takeover-ack release-failed', + ); + expect(releaseTransportFailed.writes).not.toContain( + 'takeover-ack released', + ); + expect(releaseTransportFailed.writes).not.toContain( + 'labels/autofix%2Fneeds-human', + ); + expect(releaseTransportFailed.log).toContain('release did not land'); + // R10-1: the tolerance matches the EXACT "HTTP 404" token, never a bare + // 404 substring — a transport failure embeds the request URL, so a + // 404-bearing PR number in the path (modeled here on #4041) must + // classify as not-landed, not as the already-off case. + const releaseUrlFourOhFour = runToggle({ + cmd: 'remove', + labels: ['autofix/takeover'], + deleteFails: + 'gh: Delete "https://api.github.com/repos/QwenLM/qwen-code/issues/4041/labels/autofix%2Ftakeover": dial tcp 140.82.121.4:443: connect: connection refused', + }); + expect(releaseUrlFourOhFour.writes).toContain( + 'takeover-ack release-failed', + ); + expect(releaseUrlFourOhFour.writes).not.toContain('takeover-ack released'); + expect(releaseUrlFourOhFour.writes).not.toContain( + 'labels/autofix%2Fneeds-human', + ); + expect(releaseUrlFourOhFour.log).toContain('release did not land'); // Both ack posts keep their non-fatal fallback: under bash -e a failed // gh pr comment would otherwise abort the step RED after the label was // already toggled — a worse signal than the silence being fixed. A @@ -5211,7 +5528,16 @@ exit 1 .join('\n'); expect(ackBlock).toBeTruthy(); - const runAck = ({ ack, base = '', prViewOk = true }) => { + const runAck = ({ + ack, + base = '', + prViewOk = true, + labels = [], + deleteFails = '', + comments = [], + events = [], + historyFails = false, + }) => { const dir = mkdtempSync(join(tmpdir(), 'ack-')); const bin = join(dir, 'bin'); mkdirSync(bin); @@ -5219,10 +5545,27 @@ exit 1 join(bin, 'gh'), [ '#!/usr/bin/env bash', - `if [[ "$1" == 'api' ]]; then printf 'qwen-code-dev-bot'; exit 0; fi`, + `echo "$@" >> ${JSON.stringify(join(dir, 'calls.log'))}`, + `if [[ "$1" == 'api' && "$2" == 'user' ]]; then printf 'qwen-code-dev-bot'; exit 0; fi`, + // R11-2: the engaged-stale dedup reads — served per knob + // (empty by default, so a legacy vector sees no history). + `if [[ "$1" == 'api' && "$2" == *'/comments' ]]; then ${historyFails ? 'exit 1' : `printf '%s' '${JSON.stringify(comments)}'`}; fi`, + `if [[ "$1" == 'api' && "$2" == *'/events' ]]; then ${historyFails ? 'exit 1' : `printf '%s' '${JSON.stringify(events)}'`}; fi`, + // Other api calls (the needs-human DELETE) succeed silently but + // are recorded above — the ack-job conditional removal is + // observable. R6-33: with a failure knob — a non-404 must + // warn-and-continue onto the ack comment, and the harness's + // bash -e discriminates an aborting guard-strip mutant. + `if [[ "$1" == 'api' ]]; then if [[ -n "${deleteFails}" ]]; then echo "${deleteFails}" >&2; exit 1; fi; exit 0; fi`, `if [[ "$1" == 'pr' && "$2" == 'view' ]]; then : > ${JSON.stringify(join(dir, 'pr-view-called'))}; ${ prViewOk - ? `printf '%s' '{"labels":[],"author":{"login":"wenshao"}}'; exit 0` + ? // R6-22: serve the payload only for the EXACT field list + // the ack job requests (--json labels,author) — a drifted + // field selection gets an empty object, like production. + // R9-14: anchored at the END of the token (the field list + // is the final argument at the production call site), so a + // superset drift is served the empty object too. + `if [[ "$*" == *' --json labels,author' ]]; then printf '%s' '${JSON.stringify({ labels, author: { login: 'wenshao' } })}'; else printf '%s' '{}'; fi; exit 0` : 'exit 1' }; fi`, `if [[ "$1" == 'pr' && "$2" == 'comment' ]]; then printf '%s' "$7" > ${JSON.stringify(join(dir, 'comment.md'))}; exit 0; fi`, @@ -5240,6 +5583,7 @@ exit 1 SKIP_LABEL: 'autofix/skip', TAKEOVER_LABEL: 'autofix/takeover', TAKEOVER_COMMAND: '@qwen-code /takeover', + NEEDS_HUMAN_LABEL: 'autofix/needs-human', ACK: ack, PR: '7368', ACK_BASE: base, @@ -5247,9 +5591,12 @@ exit 1 encoding: 'utf8', }); const commentPath = join(dir, 'comment.md'); + const callsPath = join(dir, 'calls.log'); const result = { status: proc.status, + stdout: proc.stdout ?? '', body: existsSync(commentPath) ? readFileSync(commentPath, 'utf8') : '', + calls: existsSync(callsPath) ? readFileSync(callsPath, 'utf8') : '', readPr: existsSync(join(dir, 'pr-view-called')), }; rmSync(dir, { recursive: true, force: true }); @@ -5282,12 +5629,184 @@ exit 1 // Unchanged for every other ack: the live read happens and still fails // CLOSED, so a transient API error cannot turn into a wrong ack. - const engaged = runAck({ ack: 'engaged' }); + // R11-2: an engaged ack posts only while the takeover label is LIVE — + // the engagement it acknowledges must still exist at ack time. + const engaged = runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/takeover' }], + }); expect(engaged.readPr).toBe(true); expect(engaged.body).toContain(''); const broken = runAck({ ack: 'engaged', prViewOk: false }); expect(broken.status).not.toBe(0); expect(broken.body).toBe(''); + // R4-16: the conditional needs-human removal is observable per branch — + // engaged (no skip) and released perform exactly one DELETE of the + // encoded label; base-refused / engaged-with-skip perform none. + const nhDelete = + 'api -X DELETE repos/QwenLM/qwen-code/issues/7368/labels/autofix%2Fneeds-human'; + expect( + runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/takeover' }], + }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(1); + expect( + runAck({ ack: 'engaged', labels: [{ name: 'autofix/takeover' }] }).calls, + ).toContain(nhDelete); + expect( + runAck({ ack: 'released' }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(1); + expect(runAck({ ack: 'released' }).calls).toContain(nhDelete); + // R6-11: …and the cleanup runs BEFORE the unguarded final ack comment — + // a transient comment failure aborting under bash -e must not strand + // the stale label. + const releasedOrder = runAck({ ack: 'released' }); + expect(releasedOrder.calls.indexOf('api -X DELETE')).toBeLessThan( + releasedOrder.calls.indexOf('pr comment'), + ); + expect( + runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/skip' }, { name: 'autofix/takeover' }], + }).calls, + ).not.toContain(nhDelete); + // R6-5: assert the TOTAL DELETE count for engaged-with-skip too — + // `not.toContain` alone lets a mutant add an unrelated second DELETE. + expect( + runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/skip' }, { name: 'autofix/takeover' }], + }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(0); + expect( + runAck({ ack: 'base-refused', base: 'release' }).calls, + ).not.toContain(nhDelete); + expect( + runAck({ ack: 'base-refused', base: 'release' }).calls.match( + /api -X DELETE/gm, + ) ?? [], + ).toHaveLength(0); + // R7-6: the ack-job matrix's released-with-skip cell — a released PR + // frozen by skip must keep needs-human (narrowing the live-read gate to + // engaged-only would leave HAS_SKIP='' on released acks and strip it). + expect( + runAck({ ack: 'released', labels: [{ name: 'autofix/skip' }] }).calls, + ).not.toContain(nhDelete); + expect( + runAck({ + ack: 'released', + labels: [{ name: 'autofix/skip' }], + }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(0); + // R2-4: a DELAYED release ack that lands after takeover was re-applied + // is stale — it must not touch the new cycle's needs-human and posts + // nothing (the new cycle's own events produce their own acks). + const staleRelease = runAck({ + ack: 'released', + labels: [{ name: 'autofix/takeover' }, { name: 'autofix/needs-human' }], + }); + expect(staleRelease.status).toBe(0); + expect(staleRelease.calls).not.toContain('api -X DELETE'); + expect(staleRelease.body).toBe(''); + expect(staleRelease.stdout).toContain('stale ack'); + // R11-2: the R2-4 mirror for the engaged direction — a delayed engaged + // ack is stale in two shapes. Label removed since the label event: the + // engagement ended, so post nothing and touch nothing… + const staleEngagedRemoved = runAck({ ack: 'engaged' }); + expect(staleEngagedRemoved.status).toBe(0); + expect(staleEngagedRemoved.calls).not.toContain('api -X DELETE'); + expect(staleEngagedRemoved.body).toBe(''); + expect(staleEngagedRemoved.stdout).toContain('stale ack'); + // …and label LIVE but the current cycle already acked: a bot engage + // marker at/after the newest takeover labeled event means this run's + // event is superseded — it must not DELETE the fresh cycle's + // needs-human nor post a marker that would reset the round window. + const engagedMarker = (created_at) => ({ + user: { login: 'qwen-code-dev-bot' }, + created_at, + body: '🤝 … ', + }); + const takeoverLabeled = (created_at) => ({ + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at, + }); + const staleEngagedAcked = runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/takeover' }, { name: 'autofix/needs-human' }], + comments: [engagedMarker('2026-08-06T02:00:00Z')], + events: [takeoverLabeled('2026-08-06T01:00:00Z')], + }); + expect(staleEngagedAcked.status).toBe(0); + expect(staleEngagedAcked.calls).not.toContain('api -X DELETE'); + expect(staleEngagedAcked.body).toBe(''); + expect(staleEngagedAcked.stdout).toContain('stale ack'); + // Comparator: a label event NEWER than the last engage marker is a + // live unacked engagement (a re-label after release) — it posts and + // cleans the stale escalation label. + const freshEngagement = runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/takeover' }], + comments: [engagedMarker('2026-08-06T00:00:00Z')], + events: [takeoverLabeled('2026-08-06T01:00:00Z')], + }); + expect(freshEngagement.status).toBe(0); + expect(freshEngagement.body).toContain(''); + expect(freshEngagement.calls).toContain(nhDelete); + // Unreadable history fails CLOSED: a stale marker is irreparable, a + // missed ack is healed by the scan's NEED_ENGAGE_ACK dedup. + const blindEngaged = runAck({ + ack: 'engaged', + labels: [{ name: 'autofix/takeover' }], + historyFails: true, + }); + expect(blindEngaged.status).toBe(0); + expect(blindEngaged.body).toBe(''); + expect(blindEngaged.calls).not.toContain('api -X DELETE'); + expect(blindEngaged.stdout).toContain('fail closed'); + // R6-33: a non-404 needs-human DELETE failure warns and STILL posts the + // ack (bash -e must not abort before the comment)… + const relDeleteFailed = runAck({ + ack: 'released', + deleteFails: 'HTTP 502: Bad Gateway', + }); + expect(relDeleteFailed.status).toBe(0); + expect(relDeleteFailed.stdout).toContain('removal failed'); + expect(relDeleteFailed.body).toContain(''); + // …while a 404 (the never-paused common case) is silently tolerated. + const relDelete404 = runAck({ + ack: 'released', + deleteFails: 'HTTP 404: Not Found', + }); + expect(relDelete404.status).toBe(0); + expect(relDelete404.stdout).not.toContain('removal failed'); + expect(relDelete404.body).toContain(''); + // R10-3: …but the tolerated token is exactly "HTTP 404" — an error that + // carries 404 only as a substring must still warn (the loose *404* + // match swallowed it; transport errors embed 404-bearing request URLs). + const relDeleteSubstring = runAck({ + ack: 'released', + deleteFails: 'HTTP 502: Bad Gateway (upstream retry of request 404)', + }); + expect(relDeleteSubstring.status).toBe(0); + expect(relDeleteSubstring.stdout).toContain('removal failed'); + expect(relDeleteSubstring.body).toContain(''); + // R4-16: fork-refused and skip-blocked acks — management never resumed, + // zero DELETEs (with total-count assertions, not just toContain). + expect(runAck({ ack: 'fork-refused' }).calls).not.toContain(nhDelete); + expect( + runAck({ ack: 'fork-refused' }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(0); + expect( + runAck({ ack: 'skip-blocked', labels: [{ name: 'autofix/skip' }] }).calls, + ).not.toContain(nhDelete); + expect( + runAck({ + ack: 'skip-blocked', + labels: [{ name: 'autofix/skip' }], + }).calls.match(/api -X DELETE/gm) ?? [], + ).toHaveLength(0); }); it('narrows the agent prompt after a timeout since the last successful round', () => { @@ -10927,19 +11446,45 @@ exit 1 expect(rearmStep).toBeTruthy(); const block = rearmStep.replace(/\n {10}/g, '\n'); - const runRearm = ({ actor = BOT, apiFail = false } = {}) => { + const runRearm = ({ + actor = BOT, + apiFail = false, + labels = '[]', + prAuthor = BOT, + viewFail = false, + deleteFail = '', + } = {}) => { const dir = mkdtempSync(join(tmpdir(), 'autofix-rearm-')); try { writeFileSync( join(dir, 'gh'), [ '#!/bin/bash', - `echo "$1 $2" >> '${join(dir, 'calls.log')}'`, + // One line per call even when an argument (the re-arm body) + // carries newlines — the R6-34 allowlist walks call lines. + `printf '%s\\n' "\${*//$'\\n'/ }" >> '${join(dir, 'calls.log')}'`, 'if [[ "$1" == "api" && "$2" == "user" ]]; then', ` if [[ "${apiFail}" == "true" ]]; then echo "HTTP 401: Bad credentials" >&2; exit 1; fi`, ` printf '%s' "${actor}"`, + 'elif [[ "$1" == "api" && "$2" == "-X" ]]; then', + // R4-20: the DELETE can fail — non-404 surfaces the error text, + // 404 is the already-gone case. + ` if [[ -n "${deleteFail}" ]]; then echo "${deleteFail}" >&2; exit 1; fi`, 'elif [[ "$1" == "pr" && "$2" == "comment" ]]; then', ` printf '%s' "$7" > '${join(dir, 'body.txt')}'`, + 'elif [[ "$1" == "pr" && "$2" == "view" ]]; then', + ` if [[ "${viewFail}" == "true" ]]; then echo "GraphQL: Something went wrong" >&2; exit 1; fi`, + // R4-22: only serve the labels payload when the caller actually + // asked for it — a field-selection drift must fail the guard. + // R6-12/R2-7: the gate matches the EXACT production field list; + // a drifted `--json labels` (no author) must not be served the + // author metadata production would omit. + // R9-15: end-anchored like its runAck twin (R9-14). + ' if [[ "$*" == *" --json labels,author" ]]; then', + ` printf '%s' '{"labels":${labels},"author":{"login":"${prAuthor}"}}'`, + ' else', + ` printf '%s' '{"number":7354}'`, + ' fi', 'fi', ].join('\n'), ); @@ -10953,6 +11498,9 @@ exit 1 PR: '7354', REPO: 'QwenLM/qwen-code', AUTOFIX_BOT: BOT, + NEEDS_HUMAN_LABEL: 'autofix/needs-human', + SKIP_LABEL: 'autofix/skip', + TAKEOVER_LABEL: 'autofix/takeover', }, encoding: 'utf8', }); @@ -10980,10 +11528,95 @@ exit 1 expect(ok.body).toContain(''); expect(ok.body).toContain('
'); expect(ok.body).toContain('中文说明'); + // R3-11: the /retry path's needs-human DELETE must actually target the + // encoded label (broken @uri / renamed variable / swallowed failure all + // fail here), and it is the only label write the re-arm performs. + expect(ok.calls).toContain( + 'api -X DELETE repos/QwenLM/qwen-code/issues/7354/labels/autofix%2Fneeds-human', + ); + expect(ok.calls.match(/^api -X DELETE/gm) ?? []).toHaveLength(1); + // R4-23: …and it is the only label WRITE of any verb — the full api-call + // census is `api user` + the one DELETE (a stray label POST would make + // this 3). R6-34: …and the census is not verb-blind: EVERY recorded + // call must match the allowlist, so a stray `pr edit --remove-label` or + // `label create` write fails too. + expect(ok.calls.match(/^api /gm) ?? []).toHaveLength(2); + for (const call of ok.calls.trim().split('\n')) { + expect(call).toMatch(/^(api user|api -X DELETE|pr view|pr comment)/); + } + // R4-19: the marker comment posts BEFORE the cleanup DELETE — under + // `bash -eo pipefail` a comment failure then aborts before the DELETE, + // keeping the escalation label (the reverse order would drop the label + // without posting the re-arm marker). + expect(ok.calls.indexOf('pr comment')).toBeLessThan( + ok.calls.indexOf('api -X DELETE'), + ); // No line may be indented 4+ spaces, or the marker renders as a code block // and the scanners' marker match silently fails. expect(ok.body).not.toMatch(/^ {4,}/m); + // R4-20: a non-404 DELETE failure surfaces `removal failed` but does not + // fail the step (the marker was already posted); a 404 stays silent. + const delFail = runRearm({ + actor: BOT, + deleteFail: 'HTTP 502: Bad Gateway', + }); + expect(delFail.status).toBe(0); + expect(delFail.calls).toContain('api -X DELETE'); + expect(delFail.stdout).toContain('removal failed'); + const del404 = runRearm({ + actor: BOT, + deleteFail: 'HTTP 404: Not Found', + }); + expect(del404.status).toBe(0); + expect(del404.stdout).not.toContain('removal failed'); + + // R3-7: skip wins over re-arm — with autofix/skip present the stale-label + // DELETE is skipped, so a frozen PR keeps its only filterable escalation + // state (no scan manages it and the fresh window won't re-apply it). + // R4-25: the fixture uses the production multi-label shape (takeover + + // needs-human + skip) — a single-label payload would miss order- + // dependent guard mutants. + const skipped = runRearm({ + actor: BOT, + labels: + '[{"name":"autofix/takeover"},{"name":"autofix/needs-human"},{"name":"autofix/skip"}]', + }); + expect(skipped.status).toBe(0); + expect(skipped.calls).toContain('pr comment'); + expect(skipped.calls).not.toContain('api -X DELETE'); + expect(skipped.stdout).toContain('removal skipped'); + + // R4-2: the skip check FAILS CLOSED — a transient `gh pr view` failure + // keeps the label rather than dropping it (the marker still posts; only + // the cleanup is withheld). + const readFailed = runRearm({ actor: BOT, viewFail: true }); + expect(readFailed.status).toBe(0); + expect(readFailed.calls).toContain('pr comment'); + expect(readFailed.calls).not.toContain('api -X DELETE'); + expect(readFailed.stdout).toContain('label state unreadable'); + + // R5-2: an auto-released HUMAN-authored PR (no takeover label, not + // bot-authored) is in no scan candidate set — /retry must keep the + // escalation label, since nothing will re-apply it. + const orphan = runRearm({ actor: BOT, prAuthor: 'wenshao' }); + expect(orphan.status).toBe(0); + expect(orphan.calls).toContain('pr comment'); + expect(orphan.calls).not.toContain('api -X DELETE'); + expect(orphan.stdout).toContain('nothing manages it until re-engaged'); + // R6-7: a HUMAN-authored takeover PR — the managed-PR guard's takeover + // disjunct deletes the stale label (a mutant killing that disjunct + // keeps every other fixture green). + const humanTakeover = runRearm({ + actor: BOT, + prAuthor: 'wenshao', + labels: '[{"name":"autofix/takeover"},{"name":"autofix/needs-human"}]', + }); + expect(humanTakeover.status).toBe(0); + expect(humanTakeover.calls).toContain( + 'api -X DELETE repos/QwenLM/qwen-code/issues/7354/labels/autofix%2Fneeds-human', + ); + // Actor mismatch: the PAT authenticates as someone else -> the guard exits // non-zero and posts nothing (a mis-scoped PAT must not leave a stranded PR // behind a fake "re-armed" line). @@ -10991,6 +11624,9 @@ exit 1 expect(mismatch.status).toBe(1); expect(mismatch.calls).toContain('api user'); expect(mismatch.calls).not.toContain('pr comment'); + // R4-21: the identity failure must also withhold the needs-human DELETE — + // a mis-scoped PAT must not strip the escalation label either. + expect(mismatch.calls).not.toContain('api -X DELETE'); expect(mismatch.body).toBe(''); expect(mismatch.stdout).toContain(`expected ${BOT}`); @@ -10999,6 +11635,7 @@ exit 1 const failed = runRearm({ apiFail: true }); expect(failed.status).toBe(1); expect(failed.calls).not.toContain('pr comment'); + expect(failed.calls).not.toContain('api -X DELETE'); expect(failed.body).toBe(''); expect(failed.stdout).toContain('Failed to verify CI_DEV_BOT_PAT identity'); expect(failed.stdout).toContain('Bad credentials'); diff --git a/scripts/tests/qwen-fleet-shepherd-workflow.test.js b/scripts/tests/qwen-fleet-shepherd-workflow.test.js index dbaca67e030..a3148790708 100644 --- a/scripts/tests/qwen-fleet-shepherd-workflow.test.js +++ b/scripts/tests/qwen-fleet-shepherd-workflow.test.js @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { execFileSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { chmodSync, mkdtempSync, @@ -21,6 +21,14 @@ const workflow = readFileSync( 'utf8', ); +// GNU `date -u -d` shim for the behavioral replays — on macOS hosts BSD date +// lacks `-d`, so route it through node. Shared by every date-shim replay +// (R7-4): a fix to the shim must land once, not diverge across copies. +// Answers ONLY the +%s call shape it emulates (R9-9): any other format +// falls through to the system date, so a format-string mutation cannot +// hide behind the shim. +const gnuDateShim = `date() { if [[ "$1" == '-u' && "$2" == '-d' && "$4" == '+%s' ]]; then node -e 'console.log(Math.floor(new Date(process.argv[1]).getTime()/1000))' "$3"; else command date "$@"; fi; }`; + describe('fleet shepherd workflow', () => { it('runs checkout-free — every read goes through the API', () => { // The run step reads no repo files (the flake registry is gone with the @@ -63,12 +71,16 @@ describe('fleet shepherd workflow', () => { expect(workflow).toContain( '--json number,headRefName,headRefOid,mergeable,isCrossRepository,statusCheckRollup', ); - // Per-PR metadata still rides the ONE list call — the sole gh pr view - // is the labels-only live-skip recheck immediately before a mutation. + // Per-PR metadata still rides the list calls — the sole gh pr view is + // live_skip's labels-only recheck immediately before a mutation, whose + // exported payload the auto-release scope check rides (no second read). expect(workflow.split('gh pr view').length - 1).toBe(1); expect(workflow).toContain( 'gh pr view "${pr}" --repo "${REPO}" --json labels', ); + expect(workflow).toContain( + 'index($a) != null) and ([.labels[]?.name] | index($b) != null)\' <<< "${LIVE_LABELS_JSON}"', + ); // autofix/skip is the maintainer opt-out honored at every engagement // path: a skip-labeled PR gets no shepherd levers and no dashboard row. // Replay the filter VERBATIM to prove the label actually excludes. @@ -112,7 +124,7 @@ describe('fleet shepherd workflow', () => { expect(workflow).toContain('consent withdrawn, no %s'); expect(workflow).toContain('fail closed, no %s'); expect(workflow).toMatch( - /DISPATCHES\}" -ge "\$\{MAX_CONFLICT_DISPATCHES_PER_TICK\}" \]\]; then[\s\S]{0,220}elif live_skip "\$\{PR\}"; then/, + /DISPATCHES\}" -ge "\$\{MAX_CONFLICT_DISPATCHES_PER_TICK\}" \]\]; then[\s\S]{0,200}elif live_skip "\$\{PR\}"; then[\s\S]{0,800}conflict_paused "\$\{PR\}"; then/, ); expect(workflow).toMatch( /SYNCS\}" -ge "\$\{MAX_SYNCS_PER_TICK\}" \]\]; then[\s\S]{0,160}elif live_skip "\$\{PR\}"; then/, @@ -204,9 +216,36 @@ describe('fleet shepherd workflow', () => { expect(workflow).toMatch( /if \[\[ "\$\{MERGEABLE\}" == "CONFLICTING" \]\]; then[\s\S]{0,900}gh api "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/comments" --paginate/, ); - // A failed fleet fetch skips the walk AND preserves the previous - // dashboard body — never an empty-table overwrite. - expect(workflow).toContain('fleet enumeration failed; skipping this tick'); + // A failed fleet fetch must not masquerade as an empty fleet — and it + // must not exit either (B5): the takeover/needs-human processing runs + // off its OWN enumerations, so the fleet failure degrades to a loud + // error row and the tick falls through to the dashboard write (which + // carries the liveness watermark). Pin the WHOLE failure branch — an + // exit statement BEFORE the warning escapes a forward-only scan + // (R8-11). + const fleetFetch = workflow.match( + /if ! gh pr list --repo "\$\{REPO\}" --state open --author "\$\{AUTOFIX_BOT\}" --base main[\s\S]*?\n {10}else\n {12}FLEET_OK=true/, + )?.[0]; + expect(fleetFetch).toBeTruthy(); + expect(fleetFetch).not.toMatch(/\n\s*exit( 0| 1)?\s*($|;)/m); + expect(fleetFetch).toContain( + 'fleet enumeration failed; the bot-fleet table shows an error row this tick', + ); + expect(fleetFetch).toContain( + '⚠️ fleet enumeration unreadable this tick | fail closed — fleet levers skipped', + ); + expect(fleetFetch).toContain('FLEET_OK=false'); + // The fleet walk is gated on FLEET_OK so the error row is the only row + // — the success side assigns true, and the gate CLOSES before the + // takeover section, so a fleet fetch failure can never skip the + // independently-fed levers (R8-14). + expect(workflow).toContain('FLEET_OK=true'); + expect(workflow).toMatch( + /if \[\[ "\$\{FLEET_OK\}" == "true" \]\]; then\n\s+while IFS= read -r ROW; do/, + ); + expect(workflow).toMatch( + /done < <\(jq -c '\.\[\]' \/tmp\/fleet\.json\)\n {10}fi # FLEET_OK\n\n {10}# ---- takeover pool/, + ); // Per-tick blast-radius caps. expect(workflow).toContain("MAX_SYNCS_PER_TICK: '3'"); expect(workflow).toContain("MAX_CONFLICT_DISPATCHES_PER_TICK: '2'"); @@ -575,4 +614,1374 @@ exit 1`; execFileSync('bash', ['-c', dryScript], { encoding: 'utf8' }), ).toContain('DRY-OK'); }); + + it('covers the takeover pool: enumeration, dashboard rows, and a bounded auto-release', () => { + // Label contract, mirroring qwen-autofix.yml. + expect(workflow).toContain("TAKEOVER_LABEL: 'autofix/takeover'"); + expect(workflow).toContain("NEEDS_HUMAN_LABEL: 'autofix/needs-human'"); + expect(workflow).toContain( + 'AUTO_RELEASE_DAYS: "${{ vars.QWEN_SHEPHERD_AUTO_RELEASE_DAYS || \'3\' }}"', + ); + expect(workflow).toContain("MAX_RELEASES_PER_TICK: '3'"); + expect(workflow).toContain("RESUME_COMMAND_GRACE_SEC: '7200'"); + expect(workflow).toContain("MAX_CLEANUPS_PER_TICK: '5'"); + // One list call carries the takeover row payload — forks INCLUDED (no + // isCrossRepository filter: visibility covers fork takeovers, and the + // release lever needs no push access). sort:updated-asc keeps the stale + // tail in view; the saturation warnings keep truncation loud on BOTH + // enumerations. + expect(workflow).toContain('--state open --label "${TAKEOVER_LABEL}"'); + expect(workflow).toContain( + '--limit 100 --json number,author,updatedAt,mergeable,statusCheckRollup,labels', + ); + expect(workflow).toContain("--search 'sort:updated-asc'"); + expect(workflow).toContain('takeover pool at the 100-PR enumeration limit'); + expect(workflow).toContain( + 'needs-human pool at the 100-PR enumeration limit', + ); + const takeoverEnum = workflow.match( + /--label "\$\{TAKEOVER_LABEL\}"[\s\S]*?\/tmp\/takeover-raw\.json/, + )?.[0]; + expect(takeoverEnum).toBeTruthy(); + expect(takeoverEnum).not.toContain('isCrossRepository'); + // R4-30: pin the stale-first sort on THIS enumeration (not just the + // needs-human one) — over the 100 cap, unsorted order drops exactly the + // stale tail the dashboard exists to surface. + expect(takeoverEnum).toContain("--search 'sort:updated-asc'"); + // Enumeration failures degrade to a loud error row and FALL THROUGH to + // the dashboard write (which carries the liveness watermark) — they + // must not exit the tick. The release lever gets its OWN enumeration of + // the paused population (both labels), never the long-lived needs-human + // display window (R5-7). + expect(workflow).toContain( + 'takeover enumeration failed; the takeover table shows an error row this tick', + ); + expect(workflow).toContain( + 'needs-human enumeration failed; the awaiting-human table shows an error row this tick', + ); + expect(workflow).toContain( + 'paused-takeover enumeration failed; the release lever is skipped this tick', + ); + expect(workflow).toContain( + '--state open --label "${TAKEOVER_LABEL}" --label "${NEEDS_HUMAN_LABEL}"', + ); + expect(workflow).not.toMatch( + /takeover enumeration failed[\s\S]{0,200}exit 0/, + ); + expect(workflow).toContain('enumeration unreadable this tick'); + // The released-but-unresolved tail (needs-human WITHOUT takeover) gets + // its own read-only section with neutral wording — capped bot PRs land + // here too, and they never had a takeover to release. + expect(workflow).toContain('--state open --label "${NEEDS_HUMAN_LABEL}"'); + expect(workflow).toContain("'## Awaiting human'"); + expect(workflow).toContain( + 'loop stopped — needs a human decision (merge / close / split / re-engage)', + ); + // autofix/skip wins in the pool too — replay the filter VERBATIM. The + // program-prefix anchor keeps this from capturing the bot-fleet filter + // (same jq shape, different file and a leading isCrossRepository leg). + const takeoverFilter = workflow.match( + /jq --arg skip "\$\{SKIP_LABEL\}" \\\n\s+'(\[\.\[\] \| select\(\[\.labels[\s\S]*?)' \\\n\s+\/tmp\/takeover-raw\.json/, + )?.[1]; + expect(takeoverFilter).toBeTruthy(); + const kept = JSON.parse( + execFileSync('jq', ['--arg', 'skip', 'autofix/skip', takeoverFilter], { + encoding: 'utf8', + input: JSON.stringify([ + { number: 1, labels: [] }, + { number: 2, labels: [{ name: 'autofix/skip' }] }, + { number: 3, labels: [{ name: 'unrelated' }] }, + ]), + }), + ).map((r) => r.number); + expect(kept).toEqual([1, 3]); + // Dashboard: the takeover table is rendered (edited in place with the + // bot-fleet table), and the header line reports releases. + expect(workflow).toContain("'## Takeover pool'"); + expect(workflow).toContain("'| PR | Author | Updated | State | Note |'"); + expect(workflow).toContain('releases: ${RELEASES}'); + // needs-human bot PRs surface on the bot-fleet table too. + expect(workflow).toContain("NH_PREFIX='🛑 '"); + }); + + it('behaviorally proves pause detection, the re-arm guard, and the release age gate', () => { + // Extract the three pause-detection jq programs VERBATIM (drift fails + // the test) and replay them against comment-stream fixtures. + const termJq = workflow.match( + /TERM_TS="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + const resumeJq = workflow.match( + /local resume refusal\n\s+resume="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' "\$\{ic\}"\)"/, + )?.[1]; + const reasonJq = workflow.match( + /REASON="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + expect(termJq).toBeTruthy(); + expect(resumeJq).toBeTruthy(); + expect(reasonJq).toBeTruthy(); + const run = (program, comments) => + execFileSync('jq', ['-r', '--arg', 'ab', 'qwen-code-dev-bot', program], { + encoding: 'utf8', + input: JSON.stringify(comments), + }).trim(); + const bot = (created_at, body) => ({ + user: { login: 'qwen-code-dev-bot' }, + created_at, + body, + }); + const human = (created_at, body) => ({ + user: { login: 'wenshao' }, + created_at, + body, + }); + // TERM_TS: latest bot cap notice only — a human forging the marker text + // does not move the pause clock. + expect( + run(termJq, [ + bot('2026-08-01T00:00:00Z', '⏸️ … '), + bot('2026-08-05T00:00:00Z', '⏸️ … '), + human('2026-08-06T00:00:00Z', ''), + ]), + ).toBe('2026-08-05T00:00:00Z'); + expect(run(termJq, [human('2026-08-06T00:00:00Z', 'x')])).toBe(''); + // RESUME_TS: re-arm markers and engage acks both count as a resume. + expect( + run(resumeJq, [ + bot('2026-08-02T00:00:00Z', '🔄 … '), + bot('2026-08-03T00:00:00Z', '🤝 … '), + ]), + ).toBe('2026-08-03T00:00:00Z'); + expect(run(resumeJq, [bot('2026-08-02T00:00:00Z', '⏸️ unrelated')])).toBe( + '', + ); + // Resume evidence is computed by compute_resume_ts — replay the WHOLE + // function verbatim (markers + trusted commands + labeled events + + // permission gate + tie-break), with `gh` and GNU date stubbed. Each + // case pins one of R4-14 / R5-4 / R5-5 / R5-9. + const resumeFn = workflow.match( + /(compute_resume_ts\(\) \{[\s\S]*?\n {10}\})/, + )?.[1]; + expect(resumeFn).toBeTruthy(); + const computeResume = ({ + comments = [], + events = [], + perm = 'write', + permFail = false, + permError = 'HTTP 502', + now = '2026-08-06T00:30:00Z', + }) => { + const dir = mkdtempSync(join(tmpdir(), 'resume-')); + try { + writeFileSync(join(dir, 'ic.json'), JSON.stringify(comments)); + writeFileSync(join(dir, 'ev.json'), JSON.stringify(events)); + writeFileSync( + join(dir, 'gh'), + [ + '#!/bin/bash', + 'if [[ "$1" == "api" ]]; then', + ` if [[ "${permFail}" == "true" ]]; then echo "${permError}" >&2; exit 1; fi`, + // Per-user permission: the collaborators path ($2) names the + // user; `perm` is the default, `stranger` always reads `read`. + ` case "$2" in *collaborators/stranger/*) printf '%s' "read" ;; *) printf '%s' "${perm}" ;; esac`, + 'fi', + ].join('\n'), + ); + chmodSync(join(dir, 'gh'), 0o755); + // R10-2: the verbatim function carries production's shared + // /tmp/cperm-err path — rewrite it into THIS case's temp dir so + // concurrent suite runs on one machine (shared self-hosted + // runners) cannot corrupt each other's classification fixtures + // through the one global file. + const replayFn = resumeFn + .replace(/\n {10}/g, '\n') + .replaceAll('/tmp/cperm-err', join(dir, 'cperm-err')); + expect(replayFn).not.toContain('/tmp/cperm-err'); + const out = execFileSync( + 'bash', + [ + '-c', + `${gnuDateShim}\n${replayFn}\ncompute_resume_ts "${dir}/ic.json" "${dir}/ev.json"\necho "TS=$RESUME_OUT"\necho "PERM_FAILED=$PERM_READ_FAILED"`, + ], + { + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + REPO: 'QwenLM/qwen-code', + AUTOFIX_BOT: 'qwen-code-dev-bot', + TAKEOVER_LABEL: 'autofix/takeover', + TAKEOVER_COMMAND: '@qwen-code /takeover', + RETRY_COMMAND: '@qwen-code /retry', + RESUME_COMMAND_GRACE_SEC: '7200', + NOW_EPOCH: String(Date.parse(now) / 1000), + }, + encoding: 'utf8', + }, + ).trim(); + const ts = out.match(/^TS=(.*)$/m)?.[1] ?? ''; + const flag = out.match(/^PERM_FAILED=(.*)$/m)?.[1] ?? ''; + return { ts, flag }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; + const cmd = (ts, login, body = '@qwen-code /takeover') => ({ + user: { login }, + created_at: ts, + body, + }); + const evt = (ts, name = 'autofix/takeover', kind = 'labeled') => ({ + event: kind, + label: { name }, + created_at: ts, + }); + // bot markers: newest re-arm/engage wins. + expect( + computeResume({ + comments: [ + bot('2026-08-02T00:00:00Z', '🔄 … '), + bot('2026-08-03T00:00:00Z', '🤝 … '), + ], + }).ts, + ).toBe('2026-08-03T00:00:00Z'); + // a fresh trusted command counts (the R1 race fix)… + expect( + computeResume({ comments: [cmd('2026-08-06T00:00:00Z', 'wenshao')] }).ts, + ).toBe('2026-08-06T00:00:00Z'); + // …but a stranger's command does not (R4-14)… + expect( + computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'stranger')], + perm: 'read', + }).ts, + ).toBe(''); + // …and a stale command past the grace window expires (R1-1)… + expect( + computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'wenshao')], + now: '2026-08-06T03:00:00Z', + }).ts, + ).toBe(''); + // …a refusal ack NEWER than the command supersedes it (R1-2)… + expect( + computeResume({ + comments: [ + cmd('2026-08-06T00:00:00Z', 'wenshao'), + bot( + '2026-08-06T00:01:00Z', + '🚫 … ', + ), + ], + }).ts, + ).toBe(''); + // …but a refusal OLDER than the fresh command does not (R4-29)… + expect( + computeResume({ + comments: [ + bot( + '2026-08-06T00:01:00Z', + '🚫 … ', + ), + cmd('2026-08-06T01:00:00Z', 'wenshao'), + ], + now: '2026-08-06T01:30:00Z', + }).ts, + ).toBe('2026-08-06T01:00:00Z'); + // …and a stranger's echo NEWER than the maintainer's command cannot + // shadow it — every in-grace command is permission-checked (R5-5), so + // the maintainer's still promotes. + expect( + computeResume({ + comments: [ + cmd('2026-08-06T00:00:00Z', 'wenshao'), + cmd('2026-08-06T00:10:00Z', 'stranger'), + ], + }).ts, + ).toBe('2026-08-06T00:00:00Z'); + // a labeled event counts (UI re-apply)… + expect(computeResume({ events: [evt('2026-08-06T00:00:00Z')] }).ts).toBe( + '2026-08-06T00:00:00Z', + ); + // …and a same-second tie resolves toward RESUME (R5-9). + expect( + computeResume({ + comments: [bot('2026-08-06T00:00:00Z', '🔄 … ')], + events: [evt('2026-08-06T00:00:00Z')], + }).ts, + ).toBe('2026-08-06T00:00:00Z'); + // a failed permission read sets PERM_READ_FAILED and does not promote + // the command (R5-4). + const permFailed = computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'wenshao')], + permFail: true, + }); + expect(permFailed.ts).toBe(''); + expect(permFailed.flag).toBe('true'); + // R9-10: a 404 is GitHub's decisive "not a collaborator" answer — the + // command is classified read-only WITHOUT PERM_READ_FAILED: failing + // closed here would let any stranger renewably defer the release with + // one exact command comment per grace window, reported as an outage. + const notCollab = computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'stranger')], + permFail: true, + permError: 'HTTP 404: Not Found', + }); + expect(notCollab.ts).toBe(''); + expect(notCollab.flag).toBe(''); + // R11-1: …but the match is the exact "HTTP 404" token: a transport + // failure embeds the request URL, which carries the commenter login — + // a login containing "404" must classify as an outage (defer), never + // as "not a collaborator". + const transportErr = computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'maintainer4041')], + permFail: true, + permError: + 'Get https://api.github.com/repos/QwenLM/qwen-code/collaborators/maintainer4041/permission: connection refused', + }); + expect(transportErr.ts).toBe(''); + expect(transportErr.flag).toBe('true'); + // R8-4: budget exhaustion — with THREE distinct-author fresh commands + // the 2-read budget runs out before the oldest is examined; that must + // surface as PERM_READ_FAILED (defer), never as "no trusted command" + // (fail open). + const budgeted = computeResume({ + comments: [ + cmd('2026-08-06T00:30:00Z', 'reader-a'), + cmd('2026-08-06T00:20:00Z', 'reader-b'), + cmd('2026-08-06T00:10:00Z', 'reader-c'), + ], + perm: 'read', + }); + expect(budgeted.ts).toBe(''); + expect(budgeted.flag).toBe('true'); + // R8-4: the grace-window boundary — at EXACTLY RESUME_COMMAND_GRACE_SEC + // the command has expired (strict -lt; a still-unacked command that old + // is route-ignored, so the boundary resolves toward release), and one + // second inside the window still counts. + expect( + computeResume({ + comments: [cmd('2026-08-06T00:00:00Z', 'wenshao')], + now: '2026-08-06T02:00:00Z', + }).ts, + ).toBe(''); + expect( + computeResume({ + comments: [cmd('2026-08-06T00:00:01Z', 'wenshao')], + now: '2026-08-06T02:00:00Z', + }).ts, + ).toBe('2026-08-06T00:00:01Z'); + // R6-15: the resume/refusal author filters — a HUMAN-forged re-arm + // marker never counts (only the bot posts real ones)… + expect( + computeResume({ + comments: [ + { + user: { login: 'wenshao' }, + created_at: '2026-08-06T00:00:00Z', + body: 'forged ', + }, + ], + }).ts, + ).toBe(''); + // …and a forged refusal ack cannot supersede a fresh maintainer + // command. + expect( + computeResume({ + comments: [ + { + user: { login: 'wenshao' }, + created_at: '2026-08-06T00:01:00Z', + body: 'forged ', + }, + cmd('2026-08-06T01:00:00Z', 'wenshao'), + ], + now: '2026-08-06T01:30:00Z', + }).ts, + ).toBe('2026-08-06T01:00:00Z'); + // R6-28: a command OLDER than the newest resume marker never counts — + // the ordering guard keeps the marker (without it, a cap notice posted + // between the command and its processing would release a PR re-armed + // minutes earlier). + expect( + computeResume({ + comments: [ + cmd('2026-08-06T00:00:00Z', 'wenshao'), + bot('2026-08-06T01:00:00Z', '🔄 … '), + ], + }).ts, + ).toBe('2026-08-06T01:00:00Z'); + // The event read fails closed, like the comment read. + expect(workflow).toContain( + 'event read failed — evaluation deferred this tick (fail closed)', + ); + // REASON: the terminal round's own headline beats the scan-side notice, + // which always says "round cap" even when a breaker fired first. + const reason = run(reasonJq, [ + bot( + '2026-08-04T00:00:00Z', + '🤖 AutoFix stopped: this counting window now contains 3 time-budget exhaustions …\ndetails', + ), + ]); + expect(reason).toContain('time-budget exhaustions'); + expect(reason).not.toContain('details'); + // Only TERMINAL headlines count: the transient setup-failure variant + // retries on the next scan and says nothing about why the loop stopped. + expect( + run(reasonJq, [ + bot( + '2026-08-04T00:00:00Z', + '🤖 AutoFix could not start — a setup step failed …', + ), + ]), + ).toBe(''); + // R4-31: two terminal headlines (a re-armed PR stopped twice) — the + // NEWEST is reported (pins `last`; a stale round-1 reason must not win). + expect( + run(reasonJq, [ + bot( + '2026-08-01T00:00:00Z', + '🤖 AutoFix stopped after 3 consecutive rounds that failed to push anything …', + ), + bot( + '2026-08-04T00:00:00Z', + '🤖 AutoFix stopped: this counting window now contains 3 time-budget exhaustions …', + ), + ]), + ).toContain('time-budget exhaustions'); + // The re-arm guard (R7-1): the stale-label cleanup fires only when a + // MARKER-confirmed re-arm is at-or-newer than the CURRENT pause boundary + // (latest needs-human apply), never on command/label evidence, and a + // cap notice NEWER than the marker (a re-pause) vetoes it (R8-10). + // Replay the gate VERBATIM so a dropped comparison fails the test. + const rearmGuard = workflow.match( + /\n {16}(if \[\[ -n "\$\{PAUSE_APPLY_TS\}" && -n "\$\{MARKER_RESUME\}" && ! "\$\{PAUSE_APPLY_TS\}" > "\$\{MARKER_RESUME\}" && ! "\$\{TERM_TS\}" > "\$\{MARKER_RESUME\}" \]\]; then\n {18}STATE='managed \(re-armed\)')/, + )?.[1]; + expect(rearmGuard).toBeTruthy(); + const guard = (marker, apply, term = '') => + execFileSync( + 'bash', + [ + '-c', + `${rearmGuard.replace(/\n {18}/g, '\n')}\necho REARMED\nelse\necho RELEASE-ELIGIBLE\nfi`, + ], + { + env: { + ...process.env, + MARKER_RESUME: marker, + PAUSE_APPLY_TS: apply, + TERM_TS: term, + }, + encoding: 'utf8', + }, + ).trim(); + // Marker-confirmed re-arm newer than the current pause apply → re-armed + // — also with NO cap notice: a lost notice must not make the cleanup + // unreachable (R8-1). + expect(guard('2026-08-06T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( + 'REARMED', + ); + // A same-second tie resolves toward REARMED (R5-9 / R7-1). + expect(guard('2026-08-05T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( + 'REARMED', + ); + // Marker older than the pause apply (stale cycle-1 evidence) → NOT + // re-armed; the fresh pause stands. + expect(guard('2026-08-04T00:00:00Z', '2026-08-05T00:00:00Z')).toBe( + 'RELEASE-ELIGIBLE', + ); + // No marker at all → not re-armed (command-grace evidence alone never + // triggers this cleanup). + expect(guard('', '2026-08-05T00:00:00Z')).toBe('RELEASE-ELIGIBLE'); + // No pause anchor → cannot correlate → not re-armed (fail closed). + expect(guard('2026-08-06T00:00:00Z', '')).toBe('RELEASE-ELIGIBLE'); + // A notice OLDER than the marker does not veto the cleanup. + expect( + guard( + '2026-08-06T00:00:00Z', + '2026-08-05T00:00:00Z', + '2026-08-04T00:00:00Z', + ), + ).toBe('REARMED'); + // R8-10: a cap notice NEWER than the marker means the PR re-paused + // after the re-arm — the fresh pause wins, no cleanup. + expect( + guard( + '2026-08-06T00:00:00Z', + '2026-08-05T00:00:00Z', + '2026-08-07T00:00:00Z', + ), + ).toBe('RELEASE-ELIGIBLE'); + // The release gates run in blast-radius order: per-tick budget BEFORE + // the PAT-backed live read, then the shared live_skip veto (fail closed + // on an unreadable read; skip wins — both worded by skip_note), then the + // both-labels scope condition riding live_skip's exported payload — and + // the writes are act()-wrapped so dry-run makes no mutation. + expect(workflow).toMatch( + /RELEASES\}" -ge "\$\{MAX_RELEASES_PER_TICK\}" \]\]; then[\s\S]{0,200}elif live_skip "\$\{PR\}"; then\n\s+NOTE="\$\(skip_note release\)"[\s\S]{0,400}labels changed since the snapshot — deferring release/, + ); + // The summary posts FIRST (dedup'd by its own marker), then the label + // DELETE: a failed summary leaves both labels in place and the whole + // release retries next tick; a failed DELETE finds the marker and + // retries only itself. Neither half can strand the other. + expect(workflow).toMatch( + /SUMMARY_POSTED="\$\(jq[\s\S]{0,500}fleet-shepherd auto-release/, + ); + // R4-31: replay the dedup counter's aggregation — empty stream and a + // stale (pre-term) marker both yield "0", a fresh one yields "1" (pins + // `| length`; a `last` mutant would emit null and skip the summary). + const summaryJq = workflow.match( + /SUMMARY_POSTED="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" --arg term "\$\{TERM_TS\}" '([\s\S]*?)' \/tmp\/tk-ic\.json\)"/, + )?.[1]; + expect(summaryJq).toBeTruthy(); + const runSummary = (comments, term) => + execFileSync( + 'jq', + [ + '-r', + '--arg', + 'ab', + 'qwen-code-dev-bot', + '--arg', + 'term', + term, + summaryJq, + ], + { encoding: 'utf8', input: JSON.stringify(comments) }, + ).trim(); + expect(runSummary([], '2026-08-05T00:00:00Z')).toBe('0'); + expect( + runSummary( + [ + bot( + '2026-08-04T00:00:00Z', + '🔓 … ', + ), + ], + '2026-08-05T00:00:00Z', + ), + ).toBe('0'); + expect( + runSummary( + [ + bot( + '2026-08-06T00:00:00Z', + '🔓 … ', + ), + ], + '2026-08-05T00:00:00Z', + ), + ).toBe('1'); + expect(workflow).toMatch( + /if act "#\$\{PR\}: post auto-release summary"[\s\S]{0,3200}if act "#\$\{PR\}: auto-release takeover/, + ); + // The budget is consumed BEFORE the first external write (R4-C3): a + // release ATTEMPT is bounded, so a DELETE outage can't mutate many PRs + // in one tick while RELEASES stays 0. + expect(workflow).toMatch( + /RELEASES=\$\(\( RELEASES \+ 1 \)\)[\s\S]{0,1200}if act "#\$\{PR\}: post auto-release summary"/, + ); + // And the DELETE removes the TAKEOVER label — never needs-human (the + // filterable TODO must survive the release). + expect(workflow).toMatch( + /auto-release takeover \(paused \$\{PAUSE_D\}d\)" \\\n\s+gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{TAKEOVER_LABEL\}"/, + ); + // The fail-closed deferral guards are load-bearing (a missing one runs + // `date -d ""` under set -e and kills the tick) — pin their wording. + expect(workflow).toContain( + 'pause timestamp unreadable — release deferred (fail closed)', + ); + expect(workflow).toContain( + 'comment read failed — release deferred this tick', + ); + // The release is idempotent: the summary dedups on its own marker and + // the label side's scope condition (both labels) goes false once the + // takeover label lands off. + expect(workflow).toContain( + 'index($a) != null) and ([.labels[]?.name] | index($b) != null)', + ); + expect(workflow).toContain(''); + // The age gate compares whole days against the tunable. + expect(workflow).toContain('"${PAUSE_D}" -ge "${AUTO_RELEASE_DAYS}"'); + // The tunable is operator-breakable — a non-numeric repo variable falls + // back to the default instead of dying mid-tick under set -e. Pin the + // fallback ASSIGNMENT with the message (newline-anchored: a bare + // toContain('AUTO_RELEASE_DAYS=3') would substring-match '=30'). + expect(workflow).toContain('=~ ^[0-9]+$'); + expect(workflow).toMatch( + /is not numeric or is too large; using 3"\n\s+AUTO_RELEASE_DAYS=3\n/, + ); + // The stale-label heal (R1-13): a HUMAN unlabeled event on an + // awaiting-human PR clears the stale escalation label — budgeted, + // skip-vetoed, never triggered by the bot's own auto-release, and (R5-10) + // anchored to the CURRENT pause boundary (the latest needs-human + // label-apply event), so a stale unlabel from an EARLIER cycle can't + // heal this cycle's label, and an absent anchor skips the cleanup + // (fail closed). + // R8-3: BOTH cleanup levers carry the budget → live_skip veto chain + // BEFORE their DELETE — a dropped veto would strip needs-human from a + // skip-frozen PR that nothing manages (the R4-3 invariant, on the + // cleanup side). R8-7: and both DELETE failure branches surface the + // failure the same way, so the dashboard distinguishes an outage from + // steady state. + expect( + workflow.match(/"\$\{CLEANUPS\}" -ge "\$\{MAX_CLEANUPS_PER_TICK\}"/g), + ).toHaveLength(2); + expect(workflow).toMatch( + /NOTE='resumed; stale-label cleanup budget reached this tick'\n\s+elif live_skip "\$\{PR\}"; then\n\s+NOTE="\$\(skip_note cleanup\)"[\s\S]{0,300}clear stale \$\{NEEDS_HUMAN_LABEL\} \(re-armed\)/, + ); + expect(workflow).toMatch( + /NOTE='cleanup budget reached this tick'\n\s+elif live_skip "\$\{PR\}"; then\n\s+NOTE="\$\(skip_note cleanup\)"[\s\S]{0,700}clear stale \$\{NEEDS_HUMAN_LABEL\} \(manual release/, + ); + expect( + workflow.match(/stale-label cleanup failed — will retry next tick/g), + ).toHaveLength(2); + // R6-16: the heal arm's R5-8 live re-apply check sits between the skip + // veto and the heal DELETE — a mid-tick re-engagement cancels the + // cleanup instead of deleting a re-managed PR's escalation label. + expect(workflow).toMatch( + /skip_note cleanup\)"[\s\S]{0,600}takeover label re-applied since the snapshot — cleanup cancelled'[\s\S]{0,300}clear stale \$\{NEEDS_HUMAN_LABEL\} \(manual release/, + ); + const nhApplyJq = workflow.match( + /NH_APPLY_TS="\$\(jq -r --arg nl "\$\{NEEDS_HUMAN_LABEL\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, + )?.[1]; + const unlabelJq = workflow.match( + /UNLABEL_ACTOR="\$\(jq -r --arg tl "\$\{TAKEOVER_LABEL\}" --arg ab "\$\{AUTOFIX_BOT\}" --arg ll "\$\{NH_APPLY_TS\}" '([\s\S]*?)' \/tmp\/tk-ev\.json\)"/, + )?.[1]; + expect(nhApplyJq).toBeTruthy(); + expect(unlabelJq).toBeTruthy(); + // R8-19: the absent-anchor guard is pinned — without it UNLABEL_ACTOR + // would be computed against an empty anchor and admit every human + // takeover-unlabel in the window (a needs-human apply past the ~90-day + // events lookback must skip the heal, not admit everything). + expect(workflow).toMatch( + /UNLABEL_ACTOR=''\n\s+if \[\[ -n "\$\{NH_APPLY_TS\}" \]\]; then/, + ); + const runUnlabel = (events) => { + const ll = execFileSync( + 'jq', + ['-r', '--arg', 'nl', 'autofix/needs-human', nhApplyJq], + { encoding: 'utf8', input: JSON.stringify(events) }, + ).trim(); + if (ll === '') return ''; + return execFileSync( + 'jq', + [ + '-r', + '--arg', + 'tl', + 'autofix/takeover', + '--arg', + 'ab', + 'qwen-code-dev-bot', + '--arg', + 'll', + ll, + unlabelJq, + ], + { encoding: 'utf8', input: JSON.stringify(events) }, + ).trim(); + }; + const nhLabeled = (ts) => ({ + event: 'labeled', + label: { name: 'autofix/needs-human' }, + created_at: ts, + }); + // Same-cycle manual release → heal fires (human unlabel newer than the + // current pause's label-apply). + expect( + runUnlabel([ + nhLabeled('2026-08-01T12:00:00Z'), + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'wenshao' }, + created_at: '2026-08-02T00:00:00Z', + }, + ]), + ).toBe('wenshao'); + // The bot's own unlabel (the auto-release DELETE) must NOT trigger a + // cleanup — needs-human survives auto-release by design. + expect( + runUnlabel([ + nhLabeled('2026-08-01T12:00:00Z'), + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'qwen-code-dev-bot' }, + created_at: '2026-08-02T00:00:00Z', + }, + ]), + ).toBe(''); + // R5-10's core scenario: a stale human unlabel OLDER than the current + // pause's needs-human apply must not heal (cycle-1 human release, then + // /retry re-arm + re-cap re-applied needs-human). + expect( + runUnlabel([ + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'wenshao' }, + created_at: '2026-08-01T00:00:00Z', + }, + { + event: 'labeled', + label: { name: 'autofix/takeover' }, + created_at: '2026-08-02T00:00:00Z', + }, + nhLabeled('2026-08-03T00:00:00Z'), + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'qwen-code-dev-bot' }, + created_at: '2026-08-05T00:00:00Z', + }, + ]), + ).toBe(''); + // An absent anchor (needs-human apply beyond the ~90-day events + // lookback) → cannot correlate → cleanup skipped (fail closed). + expect( + runUnlabel([ + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'wenshao' }, + created_at: '2026-08-02T00:00:00Z', + }, + ]), + ).toBe(''); + // R6-27: a two-cycle history pins the LATEST anchor (`| max`): the + // human unlabel healed cycle 1 — it must NOT heal cycle 2's fresh + // label (a `min` mutant anchors on cycle 1 and admits the stale + // unlabel). + expect( + runUnlabel([ + nhLabeled('2026-08-01T00:00:00Z'), + { + event: 'unlabeled', + label: { name: 'autofix/takeover' }, + actor: { login: 'wenshao' }, + created_at: '2026-08-02T00:00:00Z', + }, + nhLabeled('2026-08-03T00:00:00Z'), + ]), + ).toBe(''); + // A human unlabel of an UNRELATED label must not count (pins the + // `.label.name == $tl` select). + expect( + runUnlabel([ + nhLabeled('2026-08-01T12:00:00Z'), + { + event: 'unlabeled', + label: { name: 'autofix/skip' }, + actor: { login: 'wenshao' }, + created_at: '2026-08-02T00:00:00Z', + }, + ]), + ).toBe(''); + // The heal removes NEEDS_HUMAN (never takeover), act-wrapped. + expect(workflow).toMatch( + /clear stale \$\{NEEDS_HUMAN_LABEL\}[\s\S]{0,300}gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{NEEDS_HUMAN_LABEL\}"/, + ); + // act() echoes its own DRY-RUN preview / failure warning — the two new + // DELETE levers must not swallow that with a redirect (R2-5). + const actDeleteSites = workflow.match( + /act "[^\n]+" \\\n\s+gh api -X DELETE[^\n]+/g, + ); + expect(actDeleteSites?.length).toBeGreaterThanOrEqual(2); + for (const site of actDeleteSites ?? []) { + expect(site).not.toContain('> /dev/null'); + } + // A zero-padded variable must not kill the lever: base-10 normalize + // AFTER the numeric guard (R2-6). Pin the ORDER, anchored on the + // AUTO_RELEASE_DAYS-specific guard line — a looser regex would match the + // unrelated COUNT guard earlier in the file, and moving the normalize + // above the guard would let `$((10#abc))` abort the tick under set -e + // (R3-10). R4-C5: the guard ALSO rejects over-long digit strings before + // any arithmetic (Bash-int overflow would wrap negative and pass -ge). + expect(workflow).toContain('is not numeric or is too large; using 3'); + expect(workflow).toMatch( + /if \[\[ ! "\$\{AUTO_RELEASE_DAYS\}" =~ \^\[0-9\]\+\$ \]\] \|\| \[\[ \$\{#AUTO_RELEASE_DAYS\} -gt 2 \]\]; then[\s\S]*?AUTO_RELEASE_DAYS=\$\(\(10#\$\{AUTO_RELEASE_DAYS\}\)\)/, + ); + // The summary dedup is scoped to THIS pause cycle — markers older than + // the latest cap notice don't suppress a second release's summary + // (R2-4). + expect(workflow).toContain('select((.created_at // "") > $term)'); + // Tick summary and dashboard header report the same counters (R1-12). + expect(workflow).toContain( + '✅ tick complete (syncs=${SYNCS} dispatches=${DISPATCHES} releases=${RELEASES} cleanups=${CLEANUPS})', + ); + expect(workflow).toContain('cleanups: ${CLEANUPS}'); + // Dynamic note text is sanitized before it reaches the printf '%b' + // table: backslashes first (else %b expands them), then pipes. + expect(workflow).toContain('SAFE_NOTE="${NOTE//\\\\/\\\\\\\\}"'); + expect(workflow).toContain('${SAFE_NOTE//|/\\\\|}'); + // Shared day-math helper (R1-10) instead of pasted epoch arithmetic. + expect(workflow).toContain('days_since() {'); + // Loop 3 renders exactly one row per needs-human-only PR — appended + // OUTSIDE the evaluation arms, so a fail-closed deferral (event read + // failure) still renders instead of silently vanishing from the + // dashboard for the tick something went wrong. + const loop3 = workflow.match( + /# Loop 3: needs-human-ONLY PRs[\s\S]*?done < <\(jq -c '\.\[\]' \/tmp\/human\.json\)/, + )?.[0]; + expect(loop3).toBeTruthy(); + expect(loop3.match(/echo "🐑 #\$\{PR\} \[/g)).toHaveLength(1); + expect(loop3.match(/HUMAN_ROWS="\$\{HUMAN_ROWS\}/g)).toHaveLength(1); + // R2-17: …and the append is POSITIONALLY outside the evaluation arms + // (a bare occurrence count can't tell arm-level from loop-body level). + expect(loop3).toMatch( + /\n {14}fi\n(?: {14}#[^\n]*\n)* {14}echo "🐑 #\$\{PR\} \[/, + ); + // R8-15: the post-action routing — a successful release sets + // ROUTE='awaiting' so the row lands in Awaiting human, and loop 3 both + // defaults to awaiting and guards its append on ROUTE != none (a heal + // drops the row entirely). + expect(workflow).toMatch( + /ROUTE='awaiting'\n\s+NOTE="auto-released after \$\{PAUSE_D\}d paused"/, + ); + expect(workflow).toMatch( + /elif \[\[ "\$\{ROUTE\}" == 'awaiting' \]\]; then\n\s+HUMAN_ROWS=/, + ); + expect(loop3).toContain("ROUTE='awaiting'"); + expect(loop3).toContain('[[ "${ROUTE}" != \'none\' ]]'); + }); + + it('pins the marker/headline contracts against qwen-autofix.yml — drift fails here, not in production', () => { + // The resume-marker set, the trusted re-arm commands, and the terminal + // headline formats are all PRODUCED by qwen-autofix.yml and CONSUMED by + // the shepherd. Each side's own suite pins its own file; only a + // cross-file assertion fails when one side moves without the other. + const autofix = readFileSync('.github/workflows/qwen-autofix.yml', 'utf8'); + // 1. Resume markers: the scan's REARM_KEY (the write side's own idea of + // "management resumed") and the shepherd's RESUME_TS must name the + // same markers — compare the FULL sets, so a resume path added to + // only one file fails here, never as a wrongful auto-release (R2-10). + const rearmKey = autofix.match( + /REARM_KEY="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)'/, + )?.[1]; + expect(rearmKey).toBeTruthy(); + const resumeTs = workflow.match( + /local resume refusal\n\s+resume="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' "\$\{ic\}"\)"/, + )?.[1]; + expect(resumeTs).toBeTruthy(); + const markerSet = (p) => + new Set( + [...p.matchAll(/contains\("()"\)/g)].map((m) => m[1]), + ); + expect(markerSet(rearmKey)).toEqual(markerSet(resumeTs)); + // R5-16: conflict_paused carries a THIRD copy of the resume-marker set + // — compare its resume jq against the same resume set so a marker added + // to only one side fails here too, never as a silent dispatch + // suppression. + const conflictResumeJq = workflow.match( + /resume="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' \/tmp\/cf-ic\.json\)"/, + )?.[1]; + expect(conflictResumeJq).toBeTruthy(); + expect(markerSet(conflictResumeJq)).toEqual(markerSet(resumeTs)); + // 1b. The label constants the whole integration keys on (R2-11). + expect(autofix).toContain("NEEDS_HUMAN_LABEL: 'autofix/needs-human'"); + expect(autofix).toContain("TAKEOVER_LABEL: 'autofix/takeover'"); + // 2. The re-arm commands the shepherd greps for are MIRRORED constants + // (R7-2): the shepherd env defines TAKEOVER_COMMAND/RETRY_COMMAND and + // passes them to the matcher via --arg, so they must equal the + // qwen-autofix.yml route constants exactly — a command-syntax edit on + // the autofix side desyncs here, not silently in production. The + // refusal-ack variants that supersede a refused command are exactly + // the ones the producer can emit. + expect(autofix).toContain("TAKEOVER_COMMAND: '@qwen-code /takeover'"); + expect(autofix).toContain("RETRY_COMMAND: '@qwen-code /retry'"); + expect(workflow).toContain("TAKEOVER_COMMAND: '@qwen-code /takeover'"); + expect(workflow).toContain("RETRY_COMMAND: '@qwen-code /retry'"); + // R8-6: the env constant is the ONLY literal command text in this file + // — the auto-release summary interpolates TAKEOVER_COMMAND like every + // ack body on the producer side, so a command-syntax rename can never + // strand a hardcoded instruction. + expect(workflow.match(/@qwen-code \/takeover/g)).toHaveLength(1); + expect(workflow).toContain( + 'comment `%s` to re-engage with a fresh round window', + ); + const cmdTs = workflow.match( + /cands="\$\(jq -r --arg tc "\$\{TAKEOVER_COMMAND\}" --arg rc "\$\{RETRY_COMMAND\}" '([\s\S]*?)' "\$\{ic\}"\)"/, + )?.[1]; + expect(cmdTs).toBeTruthy(); + // The matcher compares against the --arg'd constants, not hardcoded + // literals — so it can never drift from the env constants above. + expect(cmdTs).toContain('== $tc'); + expect(cmdTs).toContain('== $rc'); + const refusalTs = workflow.match( + /refusal="\$\(jq -r --arg ab "\$\{AUTOFIX_BOT\}" '([\s\S]*?)' "\$\{ic\}"\)"/, + )?.[1]; + expect(refusalTs).toBeTruthy(); + // Compare the refusal-variant SET against the producer's full ack + // taxonomy (minus engaged/released) — a one-sided addition on either + // side fails here (R2-10). + const producerAckVariants = new Set( + [...autofix.matchAll(//g)].map( + (m) => m[1], + ), + ); + // release-failed is a release-side outcome ack (posted only when the + // release DELETE did NOT land), not a command-superseding refusal. + const producerRefusals = [...producerAckVariants].filter( + (v) => v !== 'engaged' && v !== 'released' && v !== 'release-failed', + ); + expect(producerRefusals.sort()).toEqual([ + 'base-refused', + 'fork-refused', + 'skip-blocked', + ]); + const shepherdRefusals = ( + refusalTs.match(/takeover-ack \(([^)]+)\)/)?.[1] ?? '' + ) + .split('|') + .filter(Boolean) + .sort(); + expect(shepherdRefusals).toEqual(producerRefusals.sort()); + // 3. Every TERMINAL headline the address leg can print matches the + // shepherd's REASON regex, and the transient setup-failure headline + // does not. Templates are extracted from the producer and exercised + // against the consumer's verbatim regex — a reworded headline on + // either side fails this test. + const reasonRe = workflow.match( + /REASON="\$\(jq -r[\s\S]*?\| test\("([^"]+)"\)\)/, + )?.[1]; + expect(reasonRe).toBeTruthy(); + const headlines = [...autofix.matchAll(/HEADLINE="(🤖 [^"]+)"/g)].map( + (m) => m[1], + ); + expect(headlines.length).toBeGreaterThanOrEqual(5); + const terminal = headlines.filter( + (h) => + h.includes('stopped after') || + h.includes('stopped: this counting window') || + h.includes('could not start evaluation') || + h.includes('reached the round cap') || + h.includes('this was the last automatic attempt'), + ); + // R4-S1: BOTH sets must be explicit and exhaustive. A headline in + // neither list means the producer added wording nobody classified — and + // if that wording is terminal, the dashboard reason silently degrades. + // So unclassified must be EMPTY, not "asserted false": require every + // producer headline to be named terminal or transient here. + const transient = headlines.filter( + (h) => + h.includes('a setup step failed') || + h.includes('will retry on the next scan') || + h.includes('Could not produce a passing fix for this feedback'), + ); + const unclassified = headlines.filter( + (h) => !terminal.includes(h) && !transient.includes(h), + ); + expect(terminal.length).toBe(5); + expect(transient).toHaveLength(4); + expect(unclassified).toEqual([]); + const matches = (h) => + execFileSync('jq', ['-rn', '--arg', 'b', h, `$b | test("${reasonRe}")`], { + encoding: 'utf8', + }).trim(); + for (const h of terminal) expect(matches(h)).toBe('true'); + for (const h of transient) expect(matches(h)).toBe('false'); + }); + + it('pins the previously mutation-tested gaps from review round 2', () => { + // R2-7: days_since() is behaviorally replayed (the /86400 → /3600 + // mutation must fail) — the shared module-scope GNU date shim applies. + const daysSince = workflow.match(/days_since\(\) \{[\s\S]*?\n {10}\}/)?.[0]; + expect(daysSince).toBeTruthy(); + const runDays = (now, ts) => + execFileSync( + 'bash', + [ + '-c', + `${gnuDateShim}\n${daysSince.replace(/\n {10}/g, '\n')}\ndays_since "\${TS}"`, + ], + { + env: { ...process.env, NOW_EPOCH: String(now), TS: ts }, + encoding: 'utf8', + }, + ).trim(); + const NOW = Date.parse('2026-08-10T00:00:00Z') / 1000; + expect(runDays(NOW, '2026-08-07T01:00:00Z')).toBe('2'); + expect(runDays(NOW, '2026-08-06T23:00:00Z')).toBe('3'); + // R2-8: the 🛑 prefix must reach the bot-fleet row, not just exist. + expect(workflow).toContain('${NH_PREFIX}${STATUS_NOTE} | ${ACTION_NOTE} |'); + // R2-18: the prefix condition's truth mapping — replay verbatim. + const nhCond = workflow.match( + /if \[\[ "\$\(jq -r --arg l "\$\{NEEDS_HUMAN_LABEL\}" '([^']*)' <<< "\$\{ROW\}"\)" == "true" \]\]/, + )?.[1]; + expect(nhCond).toBeTruthy(); + const nhRun = (labels) => + execFileSync('jq', ['-r', '--arg', 'l', 'autofix/needs-human', nhCond], { + encoding: 'utf8', + input: JSON.stringify({ labels }), + }).trim(); + expect(nhRun([{ name: 'autofix/needs-human' }])).toBe('true'); + expect(nhRun([])).toBe('false'); + // R2-9: loop 1's deferral and its HM_OK=false fallback are pinned. + const loop1 = workflow.match( + /# Loop 1: managed takeover PRs[\s\S]*?done < <\(jq -c '\.\[\]' \/tmp\/takeover\.json\)/, + )?.[0]; + expect(loop1).toBeTruthy(); + expect(loop1).toContain('continue # loop 2 renders this paused PR'); + expect(loop1).toContain( + 'pause evaluation unavailable this tick (truncated or unreadable paused enumeration)', + ); + // R2-16: the needs-human skip filter is byte-identical to the pinned + // takeover filter (an inverted population ships red, not green). The + // [^']+ capture cannot span across call sites (the lazy [\s\S]*? form + // silently would). + const humanFilter = workflow.match( + /jq --arg skip "\$\{SKIP_LABEL\}" \\\n\s+'([^']+)' \\\n\s+\/tmp\/human-raw\.json/, + )?.[1]; + expect(humanFilter).toBeTruthy(); + const takeoverFilterP = workflow.match( + /jq --arg skip "\$\{SKIP_LABEL\}" \\\n\s+'([^']+)' \\\n\s+\/tmp\/takeover-raw\.json/, + )?.[1]; + expect(humanFilter).toBe(takeoverFilterP); + // R2-24/R2-28: the needs-human enumeration (the awaiting DISPLAY + // population) carries the stale-first sort and the labels payload. + const humanEnum = workflow.match( + /--label "\$\{NEEDS_HUMAN_LABEL\}"[\s\S]*?\/tmp\/human-raw\.json/, + )?.[0]; + expect(humanEnum).toBeTruthy(); + expect(humanEnum).toContain("--search 'sort:updated-asc'"); + expect(humanEnum).toContain( + '--json number,author,updatedAt,mergeable,labels', + ); + // R8-12: the release lever's OWN paused enumeration (R5-7) is pinned on + // the same axes as its siblings: the skip filter, the stale-first sort, + // the labels payload, the saturation warning, and loop 2's feed + gate. + const pausedEnum = workflow.match( + /--label "\$\{TAKEOVER_LABEL\}" --label "\$\{NEEDS_HUMAN_LABEL\}"[\s\S]*?\/tmp\/paused-raw\.json/, + )?.[0]; + expect(pausedEnum).toBeTruthy(); + expect(pausedEnum).toContain("--search 'sort:updated-asc'"); + expect(pausedEnum).toContain( + '--json number,author,updatedAt,mergeable,labels', + ); + expect(workflow).toContain( + 'paused-takeover pool at the 100-PR enumeration limit', + ); + const pausedFilter = workflow.match( + /jq --arg skip "\$\{SKIP_LABEL\}" \\\n\s+'([^']+)' \\\n\s+\/tmp\/paused-raw\.json/, + )?.[1]; + expect(pausedFilter).toBe(takeoverFilterP); + expect(workflow).toMatch( + /if \[\[ "\$\{PAUSED_OK\}" == "true" \]\]; then\n\s+while IFS= read -r ROW; do[\s\S]*?done < <\(jq -c '\.\[\]' \/tmp\/paused\.json\)/, + ); + // R2-22: neither enumeration failure may terminate the tick — any + // spelling of exit — because the dashboard write carries the liveness + // watermark. + expect(workflow).not.toMatch( + /takeover enumeration failed[\s\S]{0,400}exit( 0| 1)?\b/, + ); + expect(workflow).not.toMatch( + /needs-human enumeration failed[\s\S]{0,400}exit( 0| 1)?\b/, + ); + // R2-29: both loop-2 evidence fetches paginate (paused PRs are the + // high-comment population — page 1 alone misses the newest notice). + // R4-28: pin the page-MERGE program too — GitHub returns comments + // oldest-first, so `.[0] // []` would anchor TERM_TS to a stale page-1 + // cycle while the current cycle's re-arm marker sits on page 2. + // R8-16: the events fetch exists TWICE — loop 3's heal read shares the + // shape; pin both occurrences, and pin loop 3's site INSIDE its own + // extraction so dropping --paginate only there (loop 3's population is + // the long-lived one that outgrows a page first) fails. + expect( + workflow.match( + /issues\/\$\{PR\}\/events" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ev\.json/g, + ), + ).toHaveLength(2); + expect(workflow).toMatch( + /issues\/\$\{PR\}\/comments" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ic\.json/, + ); + const loop3Pag = workflow.match( + /# Loop 3: needs-human-ONLY PRs[\s\S]*?done < <\(jq -c '\.\[\]' \/tmp\/human\.json\)/, + )?.[0]; + expect(loop3Pag).toBeTruthy(); + expect(loop3Pag).toMatch( + /issues\/\$\{PR\}\/events" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ev\.json/, + ); + // R2-3: a re-arm landing BETWEEN the summary post and the DELETE is + // caught by a final evidence re-fetch immediately before the removal — + // the re-arm's label re-apply is a no-op while the label still rides + // the PR, so no live label read can catch it. Pin the shape and the + // veto's ordering ahead of the DELETE. + expect(workflow).toMatch( + /issues\/\$\{PR\}\/comments" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ic3\.json[\s\S]{0,200}issues\/\$\{PR\}\/events" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ev3\.json/, + ); + expect(workflow).toContain( + 're-arm landed during the release — takeover kept', + ); + // R6-8: the pre-write re-fetches paginate too — a page-1-only re-check + // reads the OLDEST comments and would miss a just-landed re-arm marker + // on exactly the high-comment paused population. R6-14: and the + // re-check's fail-closed arm, its veto comparison, and the cancellation + // NOTE are pinned (both re-check sites share the fail-closed wording). + expect(workflow).toMatch( + /issues\/\$\{PR\}\/comments" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ic2\.json/, + ); + expect(workflow).toMatch( + /issues\/\$\{PR\}\/events" --paginate 2> \/dev\/null \| jq -s 'add \/\/ \[\]' > \/tmp\/tk-ev2\.json/, + ); + expect( + workflow.match( + /evidence re-check unreadable — release deferred \(fail closed\)/g, + ), + ).toHaveLength(2); + expect(workflow).toContain( + 'resume evidence appeared during evaluation — release cancelled', + ); + expect( + workflow.match( + /\[\[ -n "\$\{RESUME_NOW\}" && ! "\$\{TERM_TS\}" > "\$\{RESUME_NOW\}" \]\]/g, + ), + ).toHaveLength(2); + expect(workflow).toMatch( + /re-arm landed during the release — takeover kept'[\s\S]{0,400}auto-release takeover \(paused/, + ); + // R2-19: the release scope check requires BOTH labels — the --arg + // bindings are the load-bearing part. R3-12: the pin runs through the + // comparison operator, so a polarity flip (`!=` → `==`) of the scope + // guard fails instead of silently releasing label-changed PRs. + expect(workflow).toContain( + 'jq -r --arg a "${TAKEOVER_LABEL}" --arg b "${NEEDS_HUMAN_LABEL}" \'([.labels[]?.name] | index($a) != null) and ([.labels[]?.name] | index($b) != null)\' <<< "${LIVE_LABELS_JSON}")" != "true" ]]; then', + ); + // R2-20: live_skip exports the payload the scope check rides. + expect(workflow).toContain( + 'LIVE_LABELS_JSON="$(gh pr view "${pr}" --repo "${REPO}" --json labels', + ); + // R2-12/R5-9: the labeled-event merge lives in compute_resume_ts with a + // tie-safe comparison — an event at-or-newer than other evidence wins. + expect(workflow).toMatch( + /if \[\[ -n "\$\{evt\}" && ! "\$\{resume\}" > "\$\{evt\}" \]\]; then\n\s+resume="\$\{evt\}"/, + ); + // R2-21/R8-5: the resume evidence is computed only at RELEASE time — + // the pre-summary recompute (R5-6) and the pre-DELETE re-check (R2-3) + // — in the global-return form (a `$(...)` subshell would drop + // PERM_READ_FAILED, making the R5-4 defer dead code). The old + // loop-entry call left a dead RESUME_TS store and spent discarded + // permission reads every tick; it must not return. + expect(workflow).toContain( + 'compute_resume_ts /tmp/tk-ic2.json /tmp/tk-ev2.json; RESUME_NOW="${RESUME_OUT}"', + ); + expect(workflow).not.toContain('$(compute_resume_ts'); + expect(workflow).not.toContain('RESUME_TS='); + // R8-1: the re-armed cleanup branch PRECEDES the empty-TERM_TS + // deferral, so a lost cap notice cannot make it unreachable. + expect(workflow).toMatch( + /STATE='managed \(re-armed\)'[\s\S]*?elif \[\[ -z "\$\{TERM_TS\}" \]\]; then/, + ); + // R5-4: a failed permission read defers the release (fail closed), + // pinned at BOTH release-time evaluation points (the pre-summary veto + // and the pre-DELETE R2-3 re-check). + expect( + workflow.match(/command-permission read failed — release deferred/g) + ?.length, + ).toBe(2); + // R9-5: the cleanups budget counts ATTEMPTS like the release budget — + // incremented BEFORE the act() call in both arms, so a DELETE outage + // trips the cap instead of leaving it at 0 while every candidate burns + // a live read plus a failing DELETE each tick. + expect(workflow).toMatch( + /CLEANUPS=\$\(\( CLEANUPS \+ 1 \)\)\n\s+if act "#\$\{PR\}: clear stale \$\{NEEDS_HUMAN_LABEL\} \(re-armed\)/, + ); + expect(workflow).toMatch( + /CLEANUPS=\$\(\( CLEANUPS \+ 1 \)\)\n\s+if act "#\$\{PR\}: clear stale \$\{NEEDS_HUMAN_LABEL\} \(manual release/, + ); + // R2-23: both shepherd DELETEs are URI-encoded (the encoded path is the + // addressable route). + expect(workflow).toMatch( + /auto-release takeover \(paused \$\{PAUSE_D\}d\)" \\\n\s+gh api -X DELETE "repos\/\$\{REPO\}\/issues\/\$\{PR\}\/labels\/\$\(jq -rn --arg l "\$\{TAKEOVER_LABEL\}" '\$l\|@uri'\)"/, + ); + // R3-2: the takeover-enum error row must NOT claim "no release + // evaluation ran" — the lever is fed by the paused enumeration. + expect(workflow).toContain( + 'paused rows below still evaluated (paused pool fed)', + ); + expect(workflow).not.toContain('fail closed — no release evaluation ran'); + // R3-8: the conflict-dispatch lever refuses a paused (needs-human) PR + // instead of spending a dispatch slot the scan would refuse. R4-C1: the + // check reads the LIVE label payload (after live_skip), not the + // tick-start snapshot — a needs-human applied after enumeration is + // still caught. + expect(workflow).toContain( + 'paused (needs-human) — conflict stays unhandled until re-arm', + ); + expect(workflow).toMatch( + /elif \[\[ "\$\(jq -r --arg l "\$\{NEEDS_HUMAN_LABEL\}" '\[\.labels\[\]\?\.name\] \| index\(\$l\) != null' <<< "\$\{LIVE_LABELS_JSON\}"\)" == "true" \]\] && conflict_paused "\$\{PR\}"; then\n\s+#[\s\S]*?conflict stays unhandled until re-arm/, + ); + // R4-6: the marker-truth helper exists and fails closed (an unreadable + // comment history suppresses the dispatch, not wastes it). + expect(workflow).toContain('conflict_paused() {'); + expect(workflow).toMatch( + /conflict_paused\(\) \{[\s\S]*?return 0[\s\S]*?return 1[\s\S]*?return 0/, + ); + // R5-14: replay conflict_paused's decision condition (a structural pin + // alone lets `-n term &&` drop and every label-without-notice PR read as + // "armed"). The helper reads /tmp/cf-ic.json via gh, so stub gh to write + // it and run the extracted function verbatim. + const conflictFn = workflow.match( + /(conflict_paused\(\) \{[\s\S]*?\n {10}\})/, + )?.[1]; + expect(conflictFn).toBeTruthy(); + const conflictPaused = ({ comments, failRead = false }) => { + const dir = mkdtempSync(join(tmpdir(), 'conflict-')); + try { + // gh api --paginate emits the paginated array to stdout; the helper + // itself pipes it through `jq -s 'add // []'`. + writeFileSync( + join(dir, 'gh'), + [ + '#!/usr/bin/env bash', + 'if [[ "$1" == "api" ]]; then', + ` if [[ "${failRead}" == "true" ]]; then echo "HTTP 502" >&2; exit 1; fi`, + ` printf '%s' '${JSON.stringify(comments).replace(/'/g, `'\\''`)}'`, + 'fi', + ].join('\n'), + ); + chmodSync(join(dir, 'gh'), 0o755); + // R10-2: same isolation as the compute_resume_ts replay — the + // verbatim function writes /tmp/cf-ic.json, a path shared with + // every concurrent run on the machine. + const replayFn = conflictFn + .replace(/\n {10}/g, '\n') + .replaceAll('/tmp/cf-ic.json', join(dir, 'cf-ic.json')); + expect(replayFn).not.toContain('/tmp/cf-ic.json'); + const res = spawnSync( + 'bash', + [ + '-c', + `${replayFn}\nif conflict_paused 7354; then echo PAUSED; else echo ARMED; fi`, + ], + { + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + REPO: 'QwenLM/qwen-code', + AUTOFIX_BOT: 'qwen-code-dev-bot', + }, + encoding: 'utf8', + }, + ); + return res.stdout.trim(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; + const bot = (ts, body) => ({ + user: { login: 'qwen-code-dev-bot' }, + created_at: ts, + body, + }); + // No cap notice + old engage ack → paused (fail closed, R5-3). + expect( + conflictPaused({ + comments: [ + bot('2026-08-01T00:00:00Z', '🤝 … '), + ], + }), + ).toBe('PAUSED'); + // Re-arm newer than the cap notice → armed. + expect( + conflictPaused({ + comments: [ + bot('2026-08-01T00:00:00Z', '⏸️ … '), + bot('2026-08-02T00:00:00Z', '🔄 … '), + ], + }), + ).toBe('ARMED'); + // Cap notice newer than the re-arm → paused. + expect( + conflictPaused({ + comments: [ + bot('2026-08-02T00:00:00Z', '🔄 … '), + bot('2026-08-03T00:00:00Z', '⏸️ … '), + ], + }), + ).toBe('PAUSED'); + // Unreadable comments → paused (fail closed). + expect(conflictPaused({ comments: [], failRead: true })).toBe('PAUSED'); + // R4-9: loop 1 defers a paused PR to loop 2 ONLY when it is actually in + // the paused enumeration (membership), so a truncated PR still renders. + expect(workflow).toContain( + 'HUMAN_IDS=",$(jq -r \'[.[].number | tostring] | join(",")\' /tmp/paused.json),"', + ); + expect(workflow).toContain('if [[ "${HUMAN_IDS}" == *",${PR},"* ]]; then'); + // R9-1/R9-13: loop 3 renders a both-label PR only when BOTH other + // owners are blind — not a paused member AND the takeover enumeration + // failed — so a paused-enum outage never duplicates the row and a + // takeover-enum outage never drops it. + expect(workflow).toContain( + '[[ "${HUMAN_IDS}" == *",${PR},"* || "${TK_OK}" == "true" ]] && continue', + ); + // R4-10: loop 1's row escapes STATE (the ci-red detailsUrl) as well as + // NOTE — a `|` or `\c` in the URL must not break the printf '%b' table. + expect(workflow).toContain('SAFE_STATE="${STATE//\\\\/\\\\\\\\}"'); + expect(workflow).toContain('${SAFE_STATE//|/\\\\|}'); + // R4-13: the takeover-enum error row is finalized AFTER the paused + // result and branches on PAUSED_OK — it must not claim evaluation + // proceeds when the paused feed also failed. + expect(workflow).toContain( + 'paused rows NOT evaluated — paused enumeration also failed', + ); + // R4-14: a command comment counts only after a write/maintain/admin + // permission read on its author. + expect(workflow).toContain( + 'gh api "repos/${REPO}/collaborators/${cauthor}/permission"', + ); + expect(workflow).toMatch( + /cperm}" == 'write' \|\| "\$\{cperm\}" == 'maintain' \|\| "\$\{cperm\}" == 'admin'/, + ); + // R1-10: the CI-status classifiers are shared helpers, and NEITHER loop + // inlines its own copy of the jq (a check-naming change must land once). + expect(workflow).toContain('pending_checks() {'); + expect(workflow).toContain('failed_test_url() {'); + expect( + workflow.match(/PENDING="\$\(pending_checks "\$\{ROW\}"\)/g), + ).toHaveLength(2); + expect( + workflow.match(/FAILED_TEST_URL="\$\(failed_test_url "\$\{ROW\}"\)/g), + ).toHaveLength(2); + expect(workflow).not.toContain( + 'IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED"))] | length\' <<< "${ROW}"', + ); + // R4-11: the two classifiers are replayed behaviorally (a polarity or + // arithmetic mutation of the jq body must fail, not ship green). + const pendingJq = workflow.match( + /pending_checks\(\) \{\n\s+jq -r '([^']+)'/, + )?.[1]; + const failedJq = workflow.match( + /failed_test_url\(\) \{\n\s+jq -r '([^']+)'/, + )?.[1]; + expect(pendingJq).toBeTruthy(); + expect(failedJq).toBeTruthy(); + const runClassifier = (prog, rollup) => + execFileSync('jq', ['-r', prog], { + encoding: 'utf8', + input: JSON.stringify({ statusCheckRollup: rollup }), + }).trim(); + expect(runClassifier(pendingJq, [])).toBe('0'); + expect( + runClassifier(pendingJq, [ + { status: 'QUEUED' }, + { status: 'IN_PROGRESS' }, + { status: 'COMPLETED' }, + ]), + ).toBe('2'); + expect( + runClassifier(failedJq, [ + { conclusion: 'FAILURE', name: 'Test (x)', detailsUrl: 'u1' }, + ]), + ).toBe('u1'); + expect( + runClassifier(failedJq, [ + { conclusion: 'SUCCESS', name: 'Test (x)', detailsUrl: 'u1' }, + ]), + ).toBe(''); + expect( + runClassifier(failedJq, [ + { conclusion: 'FAILURE', name: 'Lint', detailsUrl: 'u1' }, + ]), + ).toBe(''); + }); });