From b3aa87230fac79d01113ff763d4e9792efe15ec9 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sun, 12 Jul 2026 15:23:09 +0000 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20implement=20issue=20#677=20?= =?UTF-8?q?=E2=80=94=20canary=20#668=20increment=203:=20opt-in=20human=20c?= =?UTF-8?q?onfirmation=20at=20ring1=E2=86=92stable=20(require=5Fconfirmati?= =?UTF-8?q?on,=20dev-lead)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/canary-rollout.yml | 9 ++ scripts/canary-rollout.sh | 123 ++++++++++++++++++- standards/canary-rings.json | 4 +- tests/canary_rollout.bats | 170 +++++++++++++++++++++++++++ 4 files changed, 299 insertions(+), 7 deletions(-) diff --git a/.github/workflows/canary-rollout.yml b/.github/workflows/canary-rollout.yml index 4decef39..3c84bc0f 100644 --- a/.github/workflows/canary-rollout.yml +++ b/.github/workflows/canary-rollout.yml @@ -59,6 +59,10 @@ on: description: "promote: advance even if the gate is not PROMOTE (records a warning)" type: boolean default: false + confirm: + description: "promote: clear an AWAITING_CONFIRMATION hold at a require_confirmation ring boundary (#668 Layer 3). This is the human go/no-go — NOT --override: it advances only a reliability-clean state and cannot bypass a failing gate." + type: boolean + default: false dry_run: description: "promote/rollback: show the tag move but do not push" type: boolean @@ -152,6 +156,10 @@ jobs: CMD: ${{ github.event_name == 'schedule' && (vars.CANARY_AUTO_PROMOTE == 'true' && 'promote-all' || 'evaluate-all') || github.event.inputs.command }} AGENT: ${{ github.event.inputs.agent || 'dev-lead' }} OVERRIDE: ${{ github.event.inputs.override }} + # Human go/no-go at a require_confirmation ring boundary (#668 Layer 3). Only meaningful + # on a `promote` dispatch — the scheduled sweep never sets it, so AWAITING_CONFIRMATION + # holds until a human deliberately confirms. + CONFIRM: ${{ github.event.inputs.confirm }} # Scheduled promote-all is a REAL move (DRY_RUN=false); dispatch honours the input # (which defaults to true — dispatch promote/promote-all are dry unless set false). DRY_RUN: ${{ github.event_name == 'schedule' && 'false' || github.event.inputs.dry_run }} @@ -176,6 +184,7 @@ jobs: promote) args=(promote "$AGENT") [ "$OVERRIDE" = "true" ] && args+=(--override) + [ "$CONFIRM" = "true" ] && args+=(--confirm) # human go/no-go for AWAITING_CONFIRMATION (#668 Layer 3) [ "$DRY_RUN" != "false" ] && args+=(--dry-run) # default-safe: dry unless explicitly false ;; promote-all) # gated fleet auto-promote (the SCHEDULED arm, #1045b) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 1dc3a8b3..23b1d380 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -21,7 +21,7 @@ set -euo pipefail # canary-rollout.sh autocut [--dry-run] # cut+seed new candidates for changed reusables (gated by CANARY_AUTO_CUT) # canary-rollout.sh evaluate # read-only gate + health report (also the #502 report) # canary-rollout.sh evaluate-all # read-only evaluate for EVERY registry agent (fleet-wide; the 4h timer) -# canary-rollout.sh promote [--override] [--allow-pre-existing] [--dry-run] +# canary-rollout.sh promote [--override] [--confirm] [--allow-pre-existing] [--dry-run] # --confirm clears an AWAITING_CONFIRMATION hold (#668 Layer 3); NOT --override # canary-rollout.sh promote-all [--override] [--allow-pre-existing] [--dry-run] # gated fleet auto-promote (the SCHEDULED arm, #1045b) # canary-rollout.sh rollback --to [--dry-run] # canary-rollout.sh resolve # debug: print resolved member repos @@ -518,6 +518,16 @@ _frontier_state() { local state; state="$(decide_graduated "$dwell_h" "$dwell_floor" "$sample" "$target" "$waived" "$cum_fail" "$cum_startup")" + # Layer 3 (#668 increment 3): opt-in human confirmation at a ring boundary. When the + # reliability verdict is PROMOTE but the frontier transition sets require_confirmation, hold + # in a distinct AWAITING_CONFIRMATION state — SOAKING-like: the scheduled promote-all never + # auto-advances it. It is cleared only by a deliberate `promote --confirm` (the + # dispatch IS the confirmation; no state store). The overlay fires ONLY from an otherwise- + # PROMOTE (reliability-clean) verdict, so `--confirm` can never bypass a BLOCKED gate. + if [ "$state" = "PROMOTE" ] && [ "$(_gate_knob "$agent" "$transition" require_confirmation)" = "true" ]; then + state="AWAITING_CONFIRMATION" + fi + local triage="-" if [ "$state" = "BLOCKED" ]; then triage="$(classify_failure "$differs" "${CANARY_FAILURE_CATEGORY:-unknown}" "$cum_suspect")" @@ -554,6 +564,8 @@ cmd_evaluate() { else echo "::warning::triage=PRE_EXISTING — failure is pre-existing/environmental. Report only; do NOT rollback, do NOT advance." fi + elif [ "$state" = "AWAITING_CONFIRMATION" ]; then + echo "::notice::state=AWAITING_CONFIRMATION — reliability PASSED; holding for an opt-in human go/no-go at $transition (#668 Layer 3). Review the canary-confirm issue, then dispatch: promote $agent --confirm (not --override)." fi fi } @@ -579,10 +591,11 @@ cmd_evaluate_all() { cmd_promote() { local agent="$1"; shift - local override=false dry=false allow_pre_flag=false + local override=false dry=false allow_pre_flag=false confirm=false while [ $# -gt 0 ]; do case "$1" in --override) override=true ;; + --confirm) confirm=true ;; --dry-run) dry=true ;; --allow-pre-existing) allow_pre_flag=true ;; *) echo "::error::unknown promote flag: $1" >&2; return 2 ;; @@ -610,11 +623,24 @@ cmd_promote() { [ "$state" = "PROMOTE" ] && advance=true [ "$override" = true ] && advance=true [ "$state" = "BLOCKED" ] && [ "$triage" = "PRE_EXISTING" ] && [ "$allow_pre" = true ] && advance=true + # Layer 3 (#668 increment 3): a human --confirm advances an AWAITING_CONFIRMATION frontier + # (reliability is already PROMOTE — the state is only ever set from an otherwise-PROMOTE + # verdict). --confirm is NOT --override: it clears ONLY this state and can never advance a + # BLOCKED gate, so it cannot bypass reliability. + [ "$state" = "AWAITING_CONFIRMATION" ] && [ "$confirm" = true ] && advance=true if [ "$advance" != true ]; then - echo "gate=$state for ring '$frontier' [$transition] (cum_fail=$cum_fail, triage=$triage) — not promoting. (use --override, or --allow-pre-existing for a PRE_EXISTING triage, after investigating)" + if [ "$state" = "AWAITING_CONFIRMATION" ]; then + echo "gate=AWAITING_CONFIRMATION for ring '$frontier' [$transition] — reliability PASSED; holding for an opt-in human go/no-go. Review the canary-confirm issue, then dispatch: promote $agent --confirm (--confirm advances ONLY this reliability-clean state; it is NOT --override)." + else + echo "gate=$state for ring '$frontier' [$transition] (cum_fail=$cum_fail, triage=$triage) — not promoting. (use --override, or --allow-pre-existing for a PRE_EXISTING triage, after investigating)" + fi return 0 fi - [ "$state" != "PROMOTE" ] && echo "::warning::advancing $agent/$frontier despite gate state '$state' (triage=$triage)" + if [ "$state" = "AWAITING_CONFIRMATION" ] && [ "$confirm" = true ]; then + echo "::notice::human confirmation received (--confirm) — advancing $agent/$frontier [$transition] past the ring1->stable go/no-go (reliability was already PROMOTE)." + elif [ "$state" != "PROMOTE" ]; then + echo "::warning::advancing $agent/$frontier despite gate state '$state' (triage=$triage)" + fi # Consistent move (#1076): EVERY agent moves its channel tag via `gh api` on its HOST # repo — never a local `git push`. A local force-push is NOT granted the release-manager # App's ruleset bypass for a tag UPDATE, so it 013s on a protected channel tag such as @@ -807,6 +833,46 @@ _Whole-fleet status is in the Canary Rollout workflow run's job summary (Actions EOF } +# _confirm_body — the body of the +# evidence-carrying human-confirmation issue for an AWAITING_CONFIRMATION frontier (#668 Layer 3). +# Reliability has PASSED; a human confirms the candidate is behaving CORRECTLY (not merely exiting +# green) before it reaches the stable tier (all consumers). Marker-keyed (canary-confirm:) +# so sync-issues upserts it idempotently and auto-closes it on promote/candidate change. +_confirm_body() { + local agent="$1" transition="$2" cand="$3" prior="$4" host="$5" sample="$6" target="$7" + local suspect; suspect="$(_suspect_guidance "$agent")" + local watch="" + [ -n "$suspect" ] && watch=" + +### Watch for (suspect classes registered for this agent) +$suspect" + cat < +**Canary rollout — human confirmation requested (\`$transition\`).** + +The release gate holds \`$agent\` in **AWAITING_CONFIRMATION**: reliability has PASSED (dwell + sample + cumulative health all clean), but this transition is flagged \`require_confirmation\` (#668 Layer 3), so a human confirms the candidate is behaving *correctly* — not merely exiting green — before it reaches the stable tier (all consumers). Filed + maintained by the Canary Rollout workflow; **regenerated each run and auto-closes** when the promotion is confirmed or the candidate changes — do not edit by hand. + +| field | value | +|---|---| +| agent | \`$agent\` | +| transition | \`$transition\` | +| candidate | \`${cand:0:12}\` | +| current stable | \`${prior:0:12}\` | +| host repo | \`$host\` | +| reliability | ✅ PROMOTE — source-tier sample **$sample** (target ≥ $target), cumulative health clean | + +### Review the candidate +- **Diff (current stable → candidate):** https://github.com/$host/compare/${prior}...${cand} +- Skim the changed reusable logic and the recent green runs on the ring1 tier before confirming.$watch + +### Confirm or hold +- **Go:** dispatch the **Canary Rollout** workflow → command \`promote\`, agent \`$agent\`, **confirm = true** (or run \`promote $agent --confirm\` locally). \`--confirm\` advances ONLY this reliability-clean state; it is **not** \`--override\` and cannot bypass a failing gate. +- **No-go:** if the candidate looks wrong, roll back instead of confirming (\`rollback $agent --to \`). +--- +_Whole-fleet status is in the Canary Rollout workflow run's job summary (Actions → Canary Rollout → latest run → Summary)._ +EOF +} + # _dashboard_md — the fleet-status table. This is a read-only snapshot # (not a work item), so it is rendered into the GitHub Actions run's job summary — NOT filed # as an issue: it needs no Issues:write, spawns no synthetic pinned issue, and shows on the @@ -822,7 +888,7 @@ Last updated: \`$2\` · auto-promote armed: \`${CANARY_AUTO_PROMOTE:-unset}\` · |---|---|---|---|---|---| $1 -\`PROMOTE\`/\`COMPLETE\`/\`SOAKING\` need no action. \`BLOCKED\` opens a per-agent issue (label \`canary-blocker\`) with the failing-run evidence; it auto-closes when the gate clears. +\`PROMOTE\`/\`COMPLETE\`/\`SOAKING\` need no action. \`BLOCKED\` opens a per-agent issue (label \`canary-blocker\`) with the failing-run evidence; it auto-closes when the gate clears. \`AWAITING_CONFIRMATION\` (reliability passed, held for a human go/no-go at a \`require_confirmation\` boundary, #668 Layer 3) opens a \`canary-confirm\` issue with the diff link; confirm via \`promote --confirm\`. EOF } @@ -845,6 +911,9 @@ cmd_sync_issues() { # question then --override, rather than a blind --override). gh label create dev-lead --repo "$ISSUE_REPO" --color 5319e7 --description "Route to the dev-lead agent for action" >/dev/null 2>&1 || true gh label create needs-human --repo "$ISSUE_REPO" --color d93f0b --description "Requires human judgement (canary regression)" >/dev/null 2>&1 || true + # canary-confirm (#668 Layer 3): the human go/no-go issue for an AWAITING_CONFIRMATION + # frontier — a SEPARATE label/issue from canary-blocker so the two concerns never collide. + gh label create canary-confirm --repo "$ISSUE_REPO" --color fbca04 --description "canary-rollout: human go/no-go at a require_confirmation ring boundary" >/dev/null 2>&1 || true fi local rows="" agent while IFS= read -r agent; do @@ -894,6 +963,48 @@ cmd_sync_issues() { blk="#$num (closed)" fi fi + + # Layer 3 (#668 increment 3): the human go/no-go issue for an AWAITING_CONFIRMATION frontier. + # A SEPARATE marker/label from the blocker issue (the two concerns are independent), upserted + # idempotently and auto-closed once the agent is no longer awaiting confirmation (promoted, + # rolled back, or a new candidate cut). Labelled needs-human — a human confirms, dev-lead does + # not action it. Best-effort under set -e, like the blocker path (`|| true`). + local cnum_state cnum cistate + cnum_state="$(_issue_find canary-confirm "" || true)" + cnum="${cnum_state%%$'\t'*}"; cistate="${cnum_state##*$'\t'}" + if [ "$state" = "AWAITING_CONFIRMATION" ]; then + local prior cbody ctitle + prior="$(channel_commit "$agent" "$frontier" || true)" + cbody="$(_confirm_body "$agent" "$transition" "$cand" "$prior" "$host" "$_s" "$_t" || true)" + ctitle="Canary confirm: $agent $transition — human go/no-go before stable" + if [ -z "$cnum" ]; then + if [ "$dry" = true ]; then echo " [DRY] would OPEN confirm issue for $agent"; blk="(new confirm)"; else + cnum="$(_gh_issue_create "$ctitle" "$cbody" "canary-confirm" || true)" + if [ -n "$cnum" ]; then + gh issue edit "$cnum" --repo "$ISSUE_REPO" --add-label needs-human >/dev/null 2>&1 || true + echo " opened confirm issue #$cnum for $agent"; blk="#$cnum (confirm)" + else echo "::warning::could not open confirm issue for $agent (Issues:write on the App?)"; fi + fi + else + if [ "$dry" = true ]; then echo " [DRY] would UPDATE confirm issue #$cnum for $agent"; blk="#$cnum (confirm)"; else + [ "$cistate" = "OPEN" ] || gh issue reopen "$cnum" --repo "$ISSUE_REPO" >/dev/null 2>&1 || true + gh issue edit "$cnum" --repo "$ISSUE_REPO" --title "$ctitle" --body "$cbody" >/dev/null 2>&1 \ + || echo "::warning::could not update confirm issue #$cnum for $agent" + gh issue edit "$cnum" --repo "$ISSUE_REPO" --add-label needs-human >/dev/null 2>&1 || true + echo " updated confirm issue #$cnum for $agent"; blk="#$cnum (confirm)" + fi + fi + else + # No longer awaiting — close a stale open confirm issue (confirmed, rolled back, or recut). + if [ -n "$cnum" ] && [ "$cistate" = "OPEN" ]; then + if [ "$dry" = true ]; then echo " [DRY] would CLOSE cleared confirm issue #$cnum for $agent ($state)"; else + gh issue close "$cnum" --repo "$ISSUE_REPO" \ + --comment "✅ No longer awaiting confirmation — \`$agent\` is now \`$state\`. Closed automatically by canary-rollout." >/dev/null 2>&1 || true + echo " closed cleared confirm issue #$cnum for $agent" + fi + fi + fi + rows+="| \`$agent\` | $state | \`$transition\` | $cum_fail | $triage | $blk |"$'\n' done <<< "$agents" @@ -1236,7 +1347,7 @@ main() { case "$sub" in evaluate) [ $# -ge 1 ] || { echo "usage: evaluate " >&2; return 2; }; cmd_evaluate "$@" ;; evaluate-all) cmd_evaluate_all ;; - promote) [ $# -ge 1 ] || { echo "usage: promote [--override] [--allow-pre-existing] [--dry-run]" >&2; return 2; }; cmd_promote "$@" ;; + promote) [ $# -ge 1 ] || { echo "usage: promote [--override] [--confirm] [--allow-pre-existing] [--dry-run]" >&2; return 2; }; cmd_promote "$@" ;; promote-all) cmd_promote_all "$@" ;; rollback) [ $# -ge 2 ] || { echo "usage: rollback --to " >&2; return 2; }; cmd_rollback "$@" ;; resolve) [ $# -ge 2 ] || { echo "usage: resolve " >&2; return 2; }; resolve_members "$@" ;; diff --git a/standards/canary-rings.json b/standards/canary-rings.json index 05cd2261..5d6638ad 100644 --- a/standards/canary-rings.json +++ b/standards/canary-rings.json @@ -80,6 +80,7 @@ } ], "_suspect_note": "suspect_failure_classes (#668 increment 2): same shape as benign classes (`id`, `workflow` ERE, `step` ERE) plus a `guidance` discriminating question. A counted (non-benign) failure that matches a suspect class at differs=1 is triaged SUSPECT instead of REGRESSION \u2014 it still BLOCKS and still needs a human (labelled `needs-human`, NOT auto-cleared like `version_independent`), but the blocker issue renders the `guidance` so the human confirm is a 30-second check rather than a from-scratch investigation. Default-absent (this key omitted) = today's behavior, byte-identical for agents that do not opt in.", + "_confirm_note": "require_confirmation (#668 increment 3, Layer 3): opt-in human go/no-go at ONE ring boundary for a correctness-sensitive agent. When a transition sets `require_confirmation: true` and the reliability verdict is otherwise PROMOTE, the frontier holds in state AWAITING_CONFIRMATION \u2014 the scheduled promote-all never auto-advances it; a deliberate `promote --confirm` dispatch (Actions-audit-logged) does. The confirmation IS the dispatch (no state store). `--confirm` is NOT `--override`: it advances ONLY a reliability-clean AWAITING_CONFIRMATION state and can never bypass a BLOCKED gate. sync-issues files an evidence-carrying `canary-confirm:` issue (compare diff link + run stats; labelled `needs-human`) that auto-closes on promote or candidate change. dev-lead only initially; reliability still clears next/ring0/ring1 unattended. Default-absent (this key omitted) = fully autonomous, byte-identical for agents that do not opt in.", "transitions": { "next->ring0": { "dwell_hours": 4, @@ -94,7 +95,8 @@ }, "ring1->stable": { "dwell_hours": 12, - "sample_min": 1 + "sample_min": 1, + "require_confirmation": true } } } diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index 43a374b4..a76c16fc 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -1572,3 +1572,173 @@ GHEOF [[ "$output" == *"updated blocker issue #501 for dev-lead"* ]] grep -q -- "--add-label needs-human" "$ISSUE_LOG" } + +# ── AWAITING_CONFIRMATION: opt-in human go/no-go at ring1->stable (#668 increment 3, #677) ─ +# Layer 3 of the #668 design. A correctness-sensitive agent (dev-lead) flagged +# require_confirmation on its ring1->stable transition holds in AWAITING_CONFIRMATION once +# reliability PASSES: the scheduled promote-all never auto-advances it; a deliberate +# `promote --confirm` dispatch (the confirmation IS the dispatch — no state store) +# clears it. `--confirm` is NOT `--override`: it advances ONLY a reliability-clean +# AWAITING_CONFIRMATION state and can never bypass a BLOCKED gate. sync-issues files an +# evidence-carrying `canary-confirm` issue (compare diff link; needs-human). Default-absent +# (transition without the key) = fully autonomous, byte-identical for opted-out agents. + +@test "canary-rings.json: dev-lead ring1->stable opts into require_confirmation; opted-out agents default-off" { + run jq -e '.agents["dev-lead"].gate.transitions["ring1->stable"].require_confirmation == true' "$RINGS" + [ "$status" -eq 0 ] + # Every OTHER agent's ring1->stable must NOT carry the key (absent → today's autonomous behaviour). + run jq -e '[.agents | to_entries[] | select(.key != "dev-lead") + | .value.gate.transitions["ring1->stable"].require_confirmation] | all(. == null)' "$RINGS" + [ "$status" -eq 0 ] +} + +# _confirm_stub [failed_step] — dev-lead (cross-repo) laid out with +# next/ring0/ring1 = candidate (cccc) and stable = prior (bbbb): frontier = stable, transition +# ring1->stable (which opts into require_confirmation). cut 2 days ago (dwell >> 12h), runs 1 day +# ago. conclusion=success + reusable_diff=0 → reliability PROMOTE → overlaid to AWAITING_CONFIRMATION; +# conclusion=failure + reusable_diff=1 + a non-suspect step → BLOCKED + REGRESSION. +_confirm_stub() { + local conclusion="${1:-success}" reusable_diff="${2:-0}" failed_step="${3:-Compile TypeScript}" + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + local cut_iso run_iso cand_blob="reuseAAAA" prior_blob="reuseAAAA" + [ "$reusable_diff" = "1" ] && prior_blob="reuseBBBB" + cut_iso="$(date -u -d '-2 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-2d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + run_iso="$(date -u -d '-1 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-1d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + cat > "$STUB_BIN/gh" < "$STUB_BIN/git" <<'GITEOF' +#!/usr/bin/env bash +: # dev-lead is cross-repo; all tag/blob resolution goes via gh api above +GITEOF + chmod +x "$STUB_BIN/git" +} + +@test "orchestrator: reliability PROMOTE at ring1->stable holds as AWAITING_CONFIRMATION (not PROMOTE)" { + # Clean window, dwell >> 12h, sample >= 1 → reliability PROMOTE — but require_confirmation + # overlays it to AWAITING_CONFIRMATION so the scheduled sweep will not auto-advance. + _confirm_stub success 0 + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"ring1->stable"* ]] + [[ "$output" == *"AWAITING_CONFIRMATION"* ]] + [[ "$output" == *"--confirm"* ]] # the report names the exact clearing action +} + +@test "orchestrator: AWAITING_CONFIRMATION does NOT advance without --confirm (scheduled sweep holds)" { + # promote with no flags (the promote-all sweep forwards none) must NOT move the tag. + _confirm_stub success 0 + run env CANARY_RINGS="$RINGS" bash "$ORCH" promote dev-lead --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"AWAITING_CONFIRMATION"* ]] + [[ "$output" != *"DRY-RUN"* ]] # no move planned — held for confirmation + [[ "$output" == *"--confirm"* ]] # tells the human how to confirm +} + +@test "orchestrator: promote --confirm advances a reliability-clean AWAITING_CONFIRMATION frontier" { + _confirm_stub success 0 + run env CANARY_RINGS="$RINGS" bash "$ORCH" promote dev-lead --confirm --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"DRY-RUN"* ]] # the move IS planned once confirmed + [[ "$output" == *"stable"* ]] + [[ "$output" == *"gh api PATCH"* ]] +} + +@test "orchestrator: --confirm is NOT --override — it cannot advance a BLOCKED gate" { + # A real regression (failure + differs=1, non-suspect step) is BLOCKED+REGRESSION. --confirm + # must refuse it: confirmation only clears a reliability-clean AWAITING_CONFIRMATION state. + _confirm_stub failure 1 "Compile TypeScript" + run env CANARY_RINGS="$RINGS" bash "$ORCH" promote dev-lead --confirm --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"REGRESSION"* ]] + [[ "$output" != *"DRY-RUN"* ]] # no move — --confirm did not bypass reliability +} + +@test "_confirm_body: renders the compare diff link + the promote --confirm instruction" { + run env CANARY_RINGS="$RINGS" bash -c \ + "source '$ORCH' && _confirm_body dev-lead 'ring1->stable' cccccccccccc bbbbbbbbbbbb petry-projects/.github-private 5 1" + [ "$status" -eq 0 ] + [[ "$output" == *"canary-confirm:dev-lead"* ]] # idempotency marker + [[ "$output" == *"compare/bbbbbbbbbbbb...cccccccccccc"* ]] # stable -> candidate diff + [[ "$output" == *"--confirm"* ]] # the go action + [[ "$output" == *"AWAITING_CONFIRMATION"* ]] +} + +# _confirm_sync_stub — dev-lead-only sync fixture at the ring1->stable frontier +# (next/ring0/ring1 = cand, stable = prior), clean success runs → AWAITING_CONFIRMATION; logs +# every gh issue op to ISSUE_LOG so a test can assert the confirmation-issue upsert. +_confirm_sync_stub() { + local issue_list="${1:-[]}" + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + export ISSUE_LOG="$STUB_BIN/issue.log"; : > "$ISSUE_LOG" + local cut_iso run_iso + cut_iso="$(date -u -d '-2 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-2d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + run_iso="$(date -u -d '-1 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-1d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + cat > "$STUB_BIN/git" <<'GITEOF' +#!/usr/bin/env bash +: # dev-lead is cross-repo; all tag/blob resolution goes via gh api +GITEOF + chmod +x "$STUB_BIN/git" + cat > "$STUB_BIN/gh" <> "$ISSUE_LOG"; echo "https://github.com/petry-projects/.github-private/issues/777" ;; + "issue edit"*) echo "EDIT|\$*" >> "$ISSUE_LOG" ;; + "issue close"*) echo "CLOSE|\$*" >> "$ISSUE_LOG" ;; + "issue reopen"*) echo "REOPEN|\$*" >> "$ISSUE_LOG" ;; + "label create"*) : ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + SYNC_RINGS="$BATS_TEST_TMPDIR/sync-rings-confirm.json" + jq '{org_infra_repos, agents: {"dev-lead": .agents["dev-lead"]}}' "$RINGS" > "$SYNC_RINGS" +} + +@test "orchestrator: sync-issues opens a canary-confirm issue for an AWAITING_CONFIRMATION agent" { + _confirm_sync_stub '[]' + local summ="$BATS_TEST_TMPDIR/summary-confirm.md"; : > "$summ" + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" GITHUB_STEP_SUMMARY="$summ" bash "$ORCH" sync-issues + [ "$status" -eq 0 ] + [[ "$output" == *"opened confirm issue #777 for dev-lead"* ]] + # A SEPARATE issue keyed by the canary-confirm label + marker (not the blocker issue). + grep -q -- "--label canary-confirm" "$ISSUE_LOG" + grep -q -- "--add-label needs-human" "$ISSUE_LOG" + grep -q "compare/" "$ISSUE_LOG" # evidence: the stable -> candidate diff link + grep -q "AWAITING_CONFIRMATION" "$summ" # the fleet dashboard surfaces the new state +} + +@test "orchestrator: sync-issues auto-closes a stale canary-confirm issue once the agent is no longer awaiting" { + # dev-lead is at next->ring0 (PROMOTE, NOT require_confirmation) but an OPEN confirm issue + # #502 lingers → it must be closed (the go/no-go no longer applies). + _sync_stub success '[{"number":502,"state":"OPEN","body":""}]' + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" bash "$ORCH" sync-issues + [ "$status" -eq 0 ] + [[ "$output" == *"closed cleared confirm issue #502 for dev-lead"* ]] + grep -q "CLOSE|.*502" "$ISSUE_LOG" +} From a3e18afc78210a59451181f23a149fbfcbb2915b Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sun, 12 Jul 2026 15:28:28 +0000 Subject: [PATCH 2/4] chore: dev-lead update (review-changes) [skip ci-relay] --- .github/workflows/canary-rollout.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/canary-rollout.yml b/.github/workflows/canary-rollout.yml index 3c84bc0f..8be095b0 100644 --- a/.github/workflows/canary-rollout.yml +++ b/.github/workflows/canary-rollout.yml @@ -60,7 +60,10 @@ on: type: boolean default: false confirm: - description: "promote: clear an AWAITING_CONFIRMATION hold at a require_confirmation ring boundary (#668 Layer 3). This is the human go/no-go — NOT --override: it advances only a reliability-clean state and cannot bypass a failing gate." + description: >- + promote: clear an AWAITING_CONFIRMATION hold at a require_confirmation ring boundary + (#668 Layer 3). This is the human go/no-go — NOT --override: it advances only a + reliability-clean state and cannot bypass a failing gate. type: boolean default: false dry_run: From a30bf012c623d2586167eb73cd6b7c843d841c2e Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sun, 12 Jul 2026 15:33:59 +0000 Subject: [PATCH 3/4] chore: dev-lead update (review-changes) [skip ci-relay] --- scripts/canary-rollout.sh | 12 ++++++++++-- tests/canary_rollout.bats | 10 ++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 23b1d380..4022cb40 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -846,6 +846,14 @@ _confirm_body() { ### Watch for (suspect classes registered for this agent) $suspect" + local diff_link + if [ -n "$prior" ]; then + diff_link="https://github.com/$host/compare/${prior}...${cand}" + else + diff_link="(no prior stable release)" + fi + local display_prior="${prior:0:12}" + display_prior="${display_prior:-none}" cat < **Canary rollout — human confirmation requested (\`$transition\`).** @@ -857,12 +865,12 @@ The release gate holds \`$agent\` in **AWAITING_CONFIRMATION**: reliability has | agent | \`$agent\` | | transition | \`$transition\` | | candidate | \`${cand:0:12}\` | -| current stable | \`${prior:0:12}\` | +| current stable | \`$display_prior\` | | host repo | \`$host\` | | reliability | ✅ PROMOTE — source-tier sample **$sample** (target ≥ $target), cumulative health clean | ### Review the candidate -- **Diff (current stable → candidate):** https://github.com/$host/compare/${prior}...${cand} +- **Diff (current stable → candidate):** $diff_link - Skim the changed reusable logic and the recent green runs on the ring1 tier before confirming.$watch ### Confirm or hold diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index a76c16fc..e7487f00 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -1678,6 +1678,16 @@ GITEOF [[ "$output" == *"AWAITING_CONFIRMATION"* ]] } +@test "_confirm_body: gracefully handles empty prior (no prior stable release)" { + run env CANARY_RINGS="$RINGS" bash -c \ + "source '$ORCH' && _confirm_body dev-lead 'ring1->stable' cccccccccccc '' petry-projects/.github-private 5 1" + [ "$status" -eq 0 ] + [[ "$output" == *"canary-confirm:dev-lead"* ]] + [[ "$output" == *"(no prior stable release)"* ]] # fallback diff link + [[ "$output" != *"compare/..."* ]] # no broken URL + [[ "$output" == *"none"* ]] # fallback display_prior +} + # _confirm_sync_stub — dev-lead-only sync fixture at the ring1->stable frontier # (next/ring0/ring1 = cand, stable = prior), clean success runs → AWAITING_CONFIRMATION; logs # every gh issue op to ISSUE_LOG so a test can assert the confirmation-issue upsert. From 983a19c1ae264866730bbd22ea95372341b47e89 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sun, 12 Jul 2026 15:47:00 +0000 Subject: [PATCH 4/4] fix(reviews): address review comments [skip ci-relay] --- scripts/canary-rollout.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 4022cb40..1e3bf293 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -637,7 +637,7 @@ cmd_promote() { return 0 fi if [ "$state" = "AWAITING_CONFIRMATION" ] && [ "$confirm" = true ]; then - echo "::notice::human confirmation received (--confirm) — advancing $agent/$frontier [$transition] past the ring1->stable go/no-go (reliability was already PROMOTE)." + echo "::notice::human confirmation received (--confirm) — advancing $agent/$frontier [$transition] past the confirmation go/no-go (reliability was already PROMOTE)." elif [ "$state" != "PROMOTE" ]; then echo "::warning::advancing $agent/$frontier despite gate state '$state' (triage=$triage)" fi @@ -1010,6 +1010,7 @@ cmd_sync_issues() { --comment "✅ No longer awaiting confirmation — \`$agent\` is now \`$state\`. Closed automatically by canary-rollout." >/dev/null 2>&1 || true echo " closed cleared confirm issue #$cnum for $agent" fi + blk="#$cnum (confirm closed)" fi fi