From 1d3987d009ca177fd781db3a64bed7909f30e28c Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 13 Sep 2026 03:47:40 +0000 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20implement=20issue=20#1103=20?= =?UTF-8?q?=E2=80=94=20cmd=5Frollback=20moves=20the=20bare=20/=20tag,=20not=20/v-=20=E2=80=94=20the=20emerge?= =?UTF-8?q?ncy=20lever=20is=20a=20silent=20no-op=20for=20every=20v-scoped?= =?UTF-8?q?=20agent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/canary-rollout.sh | 26 +++++++++--- tests/canary_rollout.bats | 83 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 1a212c20c..a493439af 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -1360,14 +1360,30 @@ cmd_rollback() { local target target="$(_gh_tag_commit "$host" "$agent/$to")" [ -z "$target" ] && { echo "::error::release tag $agent/$to not found on $host" >&2; return 1; } - echo "rolling back $agent/$ring -> $to (${target:0:12}) on $host" + # Resolve the RING tag the SAME way cmd_promote/_resolved_channel do (major-scoped-channels + # epic #657 F4): PREFER the v-scoped `/v-` within the agent's current major + # line when it exists, else the legacy bare `/`. Moving the bare tag on a + # v-scoped fleet relocated a tag no caller pins and still exited 0 — a silent no-op on the + # emergency lever (#1103). _resolved_channel returns the resolved tag AND its current commit. + # (The `--to` release tag above is always the bare immutable `/vX.Y.Z`, so it is + # correctly resolved without a major.) + local ring_tag ring_commit + IFS=$'\t' read -r ring_tag ring_commit < <(_resolved_channel "$agent" "$ring") + # Fail loudly rather than no-op (#1103): a resolved ring tag with no commit does not exist on + # the host. Creating-and-moving it would relocate a channel tag nobody pins while reporting + # success — the worst failure mode for a rollback. Refuse and say so. + if [ -z "$ring_commit" ]; then + echo "::error::rollback target ring tag $ring_tag does not exist on $host — refusing to create-and-move a channel tag nobody pins (that would be a silent no-op). Check the ring name and the agent's major line." >&2 + return 1 + fi + echo "rolling back $ring_tag -> $to (${target:0:12}) on $host" if [ "$dry" = true ]; then - echo "[DRY-RUN] would: gh api PATCH repos/$host/git/refs/tags/$agent/$ring sha=$target (force)" + echo "[DRY-RUN] would: gh api PATCH repos/$host/git/refs/tags/$ring_tag sha=$target (force)" return 0 fi - _gh_move_tag "$host" "$agent/$ring" "$target" \ - || { echo "::error::failed to move $agent/$ring -> $to on $host" >&2; return 1; } - echo "rolled back $agent/$ring -> $to" + _gh_move_tag "$host" "$ring_tag" "$target" \ + || { echo "::error::failed to move $ring_tag -> $to on $host" >&2; return 1; } + echo "rolled back $ring_tag -> $to" } # ── blocker-issue + fleet-status automation (auto-triage of held promotions) ──── diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index 45d8085a8..e51087513 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -1012,6 +1012,89 @@ GITEOF ! grep -Ev '^[[:space:]]*#' "$ORCH" | grep -Eq '\bgit[[:space:]]+(tag|push)\b' } +# ── orchestrator: rollback resolves the ring tag v-scope-aware, like promote (#1103) ── +# cmd_rollback used to move the BARE / tag regardless of the agent's major +# line. On a v-scoped fleet (all stubs pin /v-) that moved a tag nobody +# reads and still exited 0 — a silent no-op on the emergency lever. It must resolve the +# ring tag the SAME way cmd_promote/_resolved_channel do: prefer /v-. + +@test "orchestrator: rollback targets the v-scoped ring tag on a v-scoped agent, not the bare tier (#1103)" { + # The real shape from the issue: dev-lead on major 139, the v139-* family present. + # `rollback dev-lead stable --to v139.8.0` MUST target dev-lead/v139-stable — the tag the + # fleet stubs actually pin — never the bare dev-lead/stable that nobody reads. + _make_stub_bin + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"matching-refs/tags/dev-lead/v"*) echo "refs/tags/dev-lead/v139.8.0" ;; # → current major = 139 + *"git/ref/tags/dev-lead/v139-stable"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # the pinned channel tag + *"git/ref/tags/dev-lead/v139.8.0"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # the release --to target + *"git/ref/tags/dev-lead/stable"*) echo "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef commit" ;; # bare tier — must NOT be targeted + *) echo "" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + + # GITHUB_REPOSITORY=.github forces THIS_REPO=.github → dev-lead (host=.github-private) is cross-repo. + run env GITHUB_REPOSITORY="petry-projects/.github" CANARY_RINGS="$RINGS" \ + bash "$ORCH" rollback dev-lead stable --to v139.8.0 --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"rolling back dev-lead/v139-stable"* ]] + [[ "$output" == *"refs/tags/dev-lead/v139-stable"* ]] + # The exact defect: the bare ring tag must never be the PATCH target. + [[ "$output" != *"refs/tags/dev-lead/stable"* ]] +} + +@test "orchestrator: rollback fails loudly when the resolved ring tag does not exist (no silent no-op) (#1103)" { + # dev-lead on major 139, release v139.8.0 present, but NEITHER the v-scoped channel tag + # v139-stable NOR the bare stable exists. Rollback must NOT create-and-move a tag nobody + # pins and exit 0 — it must fail with a clear error. + _make_stub_bin + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"matching-refs/tags/dev-lead/v"*) echo "refs/tags/dev-lead/v139.8.0" ;; # → current major = 139 + *"git/ref/tags/dev-lead/v139.8.0"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # release --to resolves + *) echo "" ;; # every channel tag is absent +esac +GHEOF + chmod +x "$STUB_BIN/gh" + + run env GITHUB_REPOSITORY="petry-projects/.github" CANARY_RINGS="$RINGS" \ + bash "$ORCH" rollback dev-lead stable --to v139.8.0 --dry-run + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] + # The resolution falls back to the bare tier (v-scoped absent), so the loud error names the + # resolved-but-absent tag and refuses — never a "[DRY-RUN] would … (force)" success line. + [[ "$output" == *"dev-lead/stable does not exist"* ]] + [[ "$output" != *"DRY-RUN"* ]] +} + +@test "orchestrator: rollback falls back to the bare ring tag when no v-scoped channel tag exists (#1103)" { + # Pre-F4 shape: releases exist (so a current major is derived) but the CHANNEL tags are + # still bare — v139-stable is absent, dev-lead/stable is what the fleet pins. Rollback must + # then move the bare tag, byte-identical to _resolved_channel's fall-back (matches promote). + _make_stub_bin + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"matching-refs/tags/dev-lead/v"*) echo "refs/tags/dev-lead/v139.8.0" ;; # → current major = 139 + *"git/ref/tags/dev-lead/v139.8.0"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # release --to target + *"git/ref/tags/dev-lead/v139-stable"*) echo "" ;; # v-scoped channel tag ABSENT + *"git/ref/tags/dev-lead/stable"*) echo "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef commit" ;; # bare channel tag present + *) echo "" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + + run env GITHUB_REPOSITORY="petry-projects/.github" CANARY_RINGS="$RINGS" \ + bash "$ORCH" rollback dev-lead stable --to v139.8.0 --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"rolling back dev-lead/stable"* ]] + [[ "$output" == *"refs/tags/dev-lead/stable"* ]] + [[ "$output" != *"v139-stable"* ]] +} + # ── orchestrator: full graduated verdicts (cut date + gh run data → gate state) ─ # Lay out next = candidate (cccc); ring0/ring1/stable = prior (bbbb): frontier = ring0, # transition next->ring0, source = next. The release tag cccc is dated `cut_days` ago, From a547522d4ad3435a3ae1819fcc42d1d43ef4f168 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 13 Sep 2026 03:55:24 +0000 Subject: [PATCH 2/4] fix(reviews): address review comments [skip ci-relay] --- tests/canary_rollout.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index e51087513..40e137e33 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -1062,7 +1062,7 @@ GHEOF run env GITHUB_REPOSITORY="petry-projects/.github" CANARY_RINGS="$RINGS" \ bash "$ORCH" rollback dev-lead stable --to v139.8.0 --dry-run - [ "$status" -ne 0 ] + [ "$status" -eq 1 ] [[ "$output" == *"::error::"* ]] # The resolution falls back to the bare tier (v-scoped absent), so the loud error names the # resolved-but-absent tag and refuses — never a "[DRY-RUN] would … (force)" success line. From dbac61d1457b79f973f68d95d513975fe0ae1607 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 13 Sep 2026 04:10:29 +0000 Subject: [PATCH 3/4] fix(tests): add v-scoped channel anchors to rollback test fixtures The three new rollback tests were missing the v-next channel anchor tags in their gh stubs. These anchors are required for _agent_current_channel_major to detect the release major and enable v-scoped channel tag resolution. Without them, _resolved_channel would fall back to bare tags, causing test 112 to fail. Added v139-next stubs and handlers to all three tests so they properly exercise v-scoped channel resolution as intended. Co-Authored-By: Claude Haiku 4.5 --- tests/canary_rollout.bats | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index d5cae9c25..36f1d9c61 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -1026,7 +1026,8 @@ GITEOF cat > "$STUB_BIN/gh" <<'GHEOF' #!/usr/bin/env bash case "$*" in - *"matching-refs/tags/dev-lead/v"*) echo "refs/tags/dev-lead/v139.8.0" ;; # → current major = 139 + *"matching-refs/tags/dev-lead/v"*) printf 'refs/tags/dev-lead/v139-next\nrefs/tags/dev-lead/v139.8.0\n' ;; # channel anchor + latest release + *"git/ref/tags/dev-lead/v139-next"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # channel anchor exists → major=139 *"git/ref/tags/dev-lead/v139-stable"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # the pinned channel tag *"git/ref/tags/dev-lead/v139.8.0"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # the release --to target *"git/ref/tags/dev-lead/stable"*) echo "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef commit" ;; # bare tier — must NOT be targeted @@ -1053,7 +1054,8 @@ GHEOF cat > "$STUB_BIN/gh" <<'GHEOF' #!/usr/bin/env bash case "$*" in - *"matching-refs/tags/dev-lead/v"*) echo "refs/tags/dev-lead/v139.8.0" ;; # → current major = 139 + *"matching-refs/tags/dev-lead/v"*) printf 'refs/tags/dev-lead/v139-next\nrefs/tags/dev-lead/v139.8.0\n' ;; # channel anchor + release + *"git/ref/tags/dev-lead/v139-next"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # channel anchor exists → major=139 *"git/ref/tags/dev-lead/v139.8.0"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # release --to resolves *) echo "" ;; # every channel tag is absent esac @@ -1078,7 +1080,8 @@ GHEOF cat > "$STUB_BIN/gh" <<'GHEOF' #!/usr/bin/env bash case "$*" in - *"matching-refs/tags/dev-lead/v"*) echo "refs/tags/dev-lead/v139.8.0" ;; # → current major = 139 + *"matching-refs/tags/dev-lead/v"*) printf 'refs/tags/dev-lead/v139-next\nrefs/tags/dev-lead/v139.8.0\n' ;; # channel anchor + release + *"git/ref/tags/dev-lead/v139-next"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # channel anchor exists → major=139 *"git/ref/tags/dev-lead/v139.8.0"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # release --to target *"git/ref/tags/dev-lead/v139-stable"*) echo "" ;; # v-scoped channel tag ABSENT *"git/ref/tags/dev-lead/stable"*) echo "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef commit" ;; # bare channel tag present From e2850dd3e5dff6ffdf1901d7b0726c0cf98073cd Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 13 Sep 2026 08:30:09 +0000 Subject: [PATCH 4/4] fix: cmd_rollback falls back to bare ring tag when v-scoped tag doesn't exist When an agent has an established channel major but the v-scoped ring tag doesn't exist, cmd_rollback should fall back to the bare tag as an intermediate step (the agent may have released vM.*.* but not yet seeded all channel tier tags). This prevents silent no-ops on the emergency lever and ensures we target the tag that's actually pinned. Implements fallback logic in cmd_rollback that: 1. Prefers v-scoped tag if it exists 2. Falls back to bare tag if v-scoped doesn't exist 3. Fails loudly with clear error if neither exists (preventing silent no-op) Fixes tests 113 and 114 in the rollback test suite (#1103). Co-Authored-By: Claude Haiku 4.5 --- scripts/canary-rollout.sh | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index a410a456e..8df0fccfb 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -1403,18 +1403,34 @@ cmd_rollback() { local target target="$(_gh_tag_commit "$host" "$agent/$to")" [ -z "$target" ] && { echo "::error::release tag $agent/$to not found on $host" >&2; return 1; } - # Resolve the RING tag the SAME way cmd_promote/_resolved_channel do (major-scoped-channels - # epic #657 F4): PREFER the v-scoped `/v-` within the agent's current major - # line when it exists, else the legacy bare `/`. Moving the bare tag on a - # v-scoped fleet relocated a tag no caller pins and still exited 0 — a silent no-op on the - # emergency lever (#1103). _resolved_channel returns the resolved tag AND its current commit. + # Resolve the RING tag with fallback (major-scoped-channels epic #657 F4): PREFER the + # v-scoped `/v-` within the agent's current major line when it exists, + # else the legacy bare `/`. Moving the bare tag on a v-scoped fleet relocated + # a tag no caller pins and still exited 0 — a silent no-op on the emergency lever (#1103). + # First check if the v-scoped tier exists; if not, fall back to bare. # (The `--to` release tag above is always the bare immutable `/vX.Y.Z`, so it is # correctly resolved without a major.) local ring_tag ring_commit IFS=$'\t' read -r ring_tag ring_commit < <(_resolved_channel "$agent" "$ring") - # Fail loudly rather than no-op (#1103): a resolved ring tag with no commit does not exist on - # the host. Creating-and-moving it would relocate a channel tag nobody pins while reporting - # success — the worst failure mode for a rollback. Refuse and say so. + # If the resolved tag (v-scoped or bare from _resolved_channel's logic) doesn't exist, + # and we got v-scoped, try the bare tag as a fallback. The agent may have released v.*.* + # but not yet seeded channel tier tags, so bare still matters as an intermediate step. + if [ -z "$ring_commit" ] && [[ "$ring_tag" == *"/v"[0-9]* ]]; then + # Constructed ring_tag was v-scoped and doesn't exist; try bare. + local bare_commit bare_tag + bare_tag="$agent/$ring" + bare_commit="$(_channel_tag_commit "$agent" "$ring")" + if [ -n "$bare_commit" ]; then + ring_tag="$bare_tag" + ring_commit="$bare_commit" + else + # Both v-scoped and bare are absent; report the bare tag in the error message. + ring_tag="$bare_tag" + fi + fi + # Fail loudly rather than no-op (#1103): if BOTH v-scoped and bare (if tried) don't exist, + # refuse to create-and-move a channel tag nobody pins. The worst failure mode for rollback + # is a silent no-op that exits 0, so make it loud. if [ -z "$ring_commit" ]; then echo "::error::rollback target ring tag $ring_tag does not exist on $host — refusing to create-and-move a channel tag nobody pins (that would be a silent no-op). Check the ring name and the agent's major line." >&2 return 1