fix(canary-rollout): autocut cuts inline via gh-api — no sibling cut-release.sh (#633)

.github/workflows/canary-rollout.yml

@@ -157,8 +157,8 @@ jobs:
           DRY_RUN: ${{ github.event_name == 'schedule' && 'false' || github.event.inputs.dry_run }}
           RING: ${{ github.event.inputs.ring }}
           TO: ${{ github.event.inputs.to }}
-          # cut-release.sh (cross-repo tag moves on petry-projects/.github) and the
-          # gate's cross-repo run-history reads both authenticate with the App token.
+          # The inline autocut cut (create+move tags on each agent's host) and the gate's
+          # cross-repo run-history reads both authenticate with the App token.
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
           # One TSV line per real promotion (agent, ring, sha, owning-repo) so the deployment
           # step records a deployment for EVERY move in a promote-all run, on the right host.

scripts/canary-rollout.sh

@@ -48,9 +48,12 @@ source "${_HERE}/lib/canary-rollout.sh"
 DEFAULT_RINGS="$(cd "${_HERE}/.." && pwd)/standards/canary-rings.json"
 CANARY_RINGS="${CANARY_RINGS:-$DEFAULT_RINGS}"
 
-# CUT_RELEASE — the cut-release.sh invoked by `autocut` to cut a new candidate. A
-# sibling of this script by default; overridable so tests can stub the cut (#1069).
-CUT_RELEASE="${CUT_RELEASE:-${_HERE}/cut-release.sh}"
+# The `autocut` front end (#1069) cuts new candidates INLINE via the App-token gh-api path
+# (_gh_create_annotated_tag + _gh_move_tag against the agent's registry host) — it no longer
+# shells out to a sibling cut-release.sh. That coupling broke when the engine relocated to
+# .github (#613): cut-release.sh stayed in .github-private as the human/runbook CLI and does
+# not travel with this checkout. Cutting inline keeps the engine self-contained and writes to
+# the correct host for every agent (incl. dev-lead, hosted in .github-private).
 
 # THIS_REPO — the repo this checkout belongs to. Agents hosted HERE (dev-lead, pr-review)
 # keep their channel/release tags in this checkout and resolve them via local git. A
@@ -120,6 +123,32 @@ _gh_move_tag() {
       -f ref="refs/tags/$tag" -f sha="$sha" >/dev/null 2>&1
 }
 
+# _gh_create_annotated_tag <repo> <tag> <sha> <message> — create the immutable annotated
+# tag object <tag> pointing at commit <sha> on <repo> and publish its ref, via the GitHub
+# API with the release-manager App token (mirrors cut-release.sh's gh_create_annotated_tag).
+# This is the CUT primitive for `autocut` (#1069): once the engine relocated to .github the
+# sibling cut-release.sh no longer travels with it, so the cut runs inline through the same
+# App-token path used for every other tag write — no dependency on a script left behind in
+# another repo, and it writes to the agent's registry HOST (correct for the this-repo
+# dev-lead agent whose reusable lives in .github-private, #613 relocation). Returns non-zero
+# on API failure so the caller can degrade best-effort.
+_gh_create_annotated_tag() {
+  [ $# -lt 4 ] && return 1
+  local repo="$1" tag="$2" sha="$3" message="$4" obj
+  obj="$(gh api -X POST "repos/$repo/git/tags" \
+      -f tag="$tag" -f message="$message" -f object="$sha" -f type=commit \
+      --jq '.sha // empty')"
+  if [ $? -ne 0 ] || [ -z "$obj" ]; then
+      echo "::error::_gh_create_annotated_tag: could not create the annotated release tag on $repo or read back its object SHA" >&2
+      return 1
+  fi
+  gh api -X POST "repos/$repo/git/refs" \
+      -f ref="refs/tags/$tag" -f sha="$obj" >/dev/null 2>&1 || {
+      echo "::error::_gh_create_annotated_tag: created tag object $obj on $repo but could not publish ref refs/tags/$tag" >&2
+      return 1
+  }
+}
+
 # channel_commit <agent> <channel> — commit the channel tag <agent>/<channel> resolves to
 # (empty if the tag does not exist). Agents hosted in THIS repo resolve against the local
 # checkout; a cross-repo agent's channel tags live on ITS host, so they are resolved there
@@ -810,9 +839,11 @@ cmd_sync_issues() {
 # tick — gated by CANARY_AUTO_CUT — for each registered agent it compares the reusable
 # blob at the host's default-branch HEAD against the blob at the current `next` candidate;
 # if they differ it cuts a new immutable <agent>/vX.Y.Z (patch bump by default) and moves
-# `next` onto it via cut-release.sh, seeding the candidate into the existing soak/promote
-# pipeline. Detection is done here in .github-private via the App token (which reads every
-# host), so no cross-repo Actions plumbing is needed. Best-effort: never fails the run.
+# `next` onto it INLINE via the App-token gh-api path (_gh_create_annotated_tag + _gh_move_tag),
+# seeding the candidate into the existing soak/promote pipeline. Detection AND the cut both run
+# against the agent's registry host via the App token (which reads/writes every host), so no
+# cross-repo Actions plumbing — and no sibling cut-release.sh — is needed. Best-effort: never
+# fails the run.
 #
 # Scope/limitation (v1): detection is on the reusable FILE blob only (the registry
 # `reusable` path). A change to a shared library the reusable sources — without the
@@ -891,13 +922,31 @@ _autocut_agent() {
   bump="$(_autocut_bump "$agent")"
   newver="$(_next_release_version "$agent" "$bump")"
   echo "autocut $agent: reusable changed on $host ($defbranch ${mainsha:0:12}) vs next ${next_commit:0:12} — cutting v$newver (bump=$bump), moving next."
+  local relver="$agent/v$newver"
   if [ "$dry" = true ]; then
-    echo "[DRY-RUN] would: cut-release.sh $agent $newver --ref $mainsha --channel next --push"
+    echo "[DRY-RUN] would: cut $relver at ${mainsha:0:12} on $host + move $agent/next (gh-api, App token)"
     return 0
   fi
-  bash "$CUT_RELEASE" "$agent" "$newver" --ref "$mainsha" --channel next --push \
-    || { echo "::warning::autocut $agent: cut-release failed for v$newver (best-effort, continuing)"; return 0; }
-  echo "autocut $agent: cut v$newver from ${mainsha:0:12} and moved next."
+  # Cut inline (no sibling cut-release.sh, #613): create the immutable annotated release tag,
+  # then advance `next` onto the same commit — both on the agent's HOST via the App token. If
+  # the release tag already exists (e.g. a retry after a partial move), skip the create and just
+  # re-point next so the operation stays idempotent. Guard the invariant: if the existing tag
+  # resolves to a different commit (manual retag, concurrent run, prior bad state), skip the
+  # next move entirely rather than advancing next to an untagged commit.
+  local existing_sha
+  existing_sha="$(_gh_tag_commit "$host" "$relver")"
+  if [ -n "$existing_sha" ]; then
+    if [ "$existing_sha" != "$mainsha" ]; then
+      echo "::warning::autocut $agent: release $relver on $host points to ${existing_sha:0:12}, not ${mainsha:0:12} — skipping next move to preserve invariant."
+      return 0
+    fi
+    echo "autocut $agent: release $relver already exists on $host — re-pointing next only."
+  elif ! _gh_create_annotated_tag "$host" "$relver" "$mainsha" "$agent release v$newver"; then
+    echo "::warning::autocut $agent: could not create $relver on $host (best-effort, continuing)"; return 0
+  fi
+  _gh_move_tag "$host" "$agent/next" "$mainsha" \
+    || { echo "::warning::autocut $agent: could not move $agent/next on $host (best-effort, continuing)"; return 0; }
+  echo "autocut $agent: cut v$newver from ${mainsha:0:12} and moved next (on $host)."
 }
 
 # cmd_autocut [--dry-run] — the scheduled front end. Gated by CANARY_AUTO_CUT (the single

tests/canary_rollout.bats

@@ -828,31 +828,37 @@ GHEOF
 # The front end of the canary pipeline: at each scheduled tick (gated by CANARY_AUTO_CUT), for
 # each registered agent compare the reusable blob at the host's main HEAD against the blob at the
 # current `next` candidate; if they differ, cut a new immutable vX.Y.Z (patch bump default) and
-# move `next` onto it via cut-release.sh. The stub feeds: default_branch, main HEAD, the two blob
-# SHAs, the `next` commit (git for a this-repo agent, gh api for a cross-repo one), the existing
-# release-tag versions (matching-refs), and a cut-release.sh stand-in (CUT_RELEASE) that logs args.
+# move `next` onto it INLINE via the App-token gh-api path (create annotated tag + move ref) —
+# no sibling cut-release.sh (#613). The stub feeds: default_branch, main HEAD, the two blob SHAs,
+# the `next` commit (git for a this-repo agent, gh api for a cross-repo one), the existing
+# release-tag versions (matching-refs), and LOGS every mutating gh-api call (tag/ref writes) to
+# GH_LOG so a test can assert the cut hit the right host without a cut-release.sh stand-in.
 _autocut_stub() {
-  # args: agent host reusable main_blob next_blob mainsha nextsha versions_ws [bump]
-  local agent="$1" host="$2" reusable="$3" MAIN_BLOB="$4" NEXT_BLOB="$5" MAINSHA="$6" NEXTSHA="$7" versions="$8" bump="${9:-}"
+  # args: agent host reusable main_blob next_blob mainsha nextsha versions_ws [bump [existing_tag_sha]]
+  # existing_tag_sha: if set, the release-tag existence probe returns this sha (simulates a prior
+  # partial cut); if empty (default), the probe returns nothing (fresh cut path).
+  local agent="$1" host="$2" reusable="$3" MAIN_BLOB="$4" NEXT_BLOB="$5" MAINSHA="$6" NEXTSHA="$7" versions="$8" bump="${9:-}" existing_tag_sha="${10:-}"
   STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH"
-  export CUT_LOG="$STUB_BIN/cut.log"; : > "$CUT_LOG"
-  export CUT_RELEASE="$STUB_BIN/cut-release"
-  cat > "$CUT_RELEASE" <<CUTEOF
-#!/usr/bin/env bash
-echo "\$*" >> "$CUT_LOG"
-CUTEOF
-  chmod +x "$CUT_RELEASE"
+  export GH_LOG="$STUB_BIN/gh-writes.log"; : > "$GH_LOG"
   local refs="" v
   for v in $versions; do refs+="refs/tags/$agent/v$v"$'\n'; done
+  # Build the probe response: empty → tag not yet cut; "<sha>\tcommit" → existing tag at that sha.
+  local tag_probe_resp=""
+  [ -n "$existing_tag_sha" ] && tag_probe_resp="${existing_tag_sha}"$'\t'"commit"
   cat > "$STUB_BIN/gh" <<GHEOF
 #!/usr/bin/env bash
 case "\$*" in
   *".default_branch"*) echo "main" ;;
   *"contents/"*"ref=$MAINSHA"*) echo "$MAIN_BLOB" ;;
   *"contents/"*"ref=$NEXTSHA"*) echo "$NEXT_BLOB" ;;
+  # inline cut (mutating writes) — log to GH_LOG and simulate success:
+  *"-X POST"*"git/tags"*) echo "\$*" >> "$GH_LOG"; echo "7a90000000000000000000000000000000000000" ;;  # create annotated tag object
+  *"-X PATCH"*"git/refs/tags/$agent/next"*) echo "\$*" >> "$GH_LOG"; exit 0 ;;                          # move next (force PATCH)
+  *"-X POST"*"git/refs"*) echo "\$*" >> "$GH_LOG"; echo "{}" ;;                                          # publish a ref
   *"/commits/"*) echo "$MAINSHA" ;;
   *"matching-refs/tags/$agent/v"*) printf '%s' "$refs" ;;
   *"git/ref/tags/$agent/next"*) printf '%s\tcommit\n' "$NEXTSHA" ;;
+  *"git/ref/tags/$agent/v"*) printf '%s\n' "$tag_probe_resp" ;;
   *"run list"*) echo "[]" ;;
   *) echo "{}" ;;
 esac
@@ -881,58 +887,94 @@ GITEOF
 @test "orchestrator: autocut is a no-op when CANARY_AUTO_CUT is not 'true' (kill-switch off)" {
   _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \
     blobMAIN blobNEXT aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa cccccccccccccccccccccccccccccccccccccccc "2.1.0"
-  run env CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut
+  run env CANARY_RINGS="$AUTOCUT_RINGS" bash "$ORCH" autocut
   [ "$status" -eq 0 ]
   [[ "$output" == *"DISABLED"* ]]
-  [ ! -s "$CUT_LOG" ]   # nothing cut when the kill-switch is off
+  [ ! -s "$GH_LOG" ]   # nothing cut when the kill-switch is off
 }
 
 @test "orchestrator: autocut cuts a patch-bumped version + moves next when the reusable blob differs on main" {
   _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \
     blobMAIN blobNEXT aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa cccccccccccccccccccccccccccccccccccccccc "2.1.0"
-  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut
+  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" bash "$ORCH" autocut
   [ "$status" -eq 0 ]
-  # Highest existing tag is v2.1.0 → patch bump → v2.1.1, cut from main HEAD, channel next, pushed.
-  grep -q "dev-lead 2.1.1 --ref aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa --channel next --push" "$CUT_LOG"
+  # Highest existing tag is v2.1.0 → patch bump → v2.1.1, annotated tag cut from main HEAD on the
+  # HOST repo (.github-private for dev-lead), then `next` force-moved onto the same commit — all gh-api.
+  grep -q "repos/petry-projects/.github-private/git/tags .*tag=dev-lead/v2.1.1 .*object=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "$GH_LOG"
+  grep -q "PATCH repos/petry-projects/.github-private/git/refs/tags/dev-lead/next .*sha=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "$GH_LOG"
 }
 
 @test "orchestrator: autocut is idempotent — identical blob on main and next is a clean no-op" {
   _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \
     sameBLOB sameBLOB aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa cccccccccccccccccccccccccccccccccccccccc "2.1.0"
-  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut
+  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" bash "$ORCH" autocut
   [ "$status" -eq 0 ]
   [[ "$output" == *"no cut"* ]]
-  [ ! -s "$CUT_LOG" ]   # nothing cut when the blob is unchanged
+  [ ! -s "$GH_LOG" ]   # nothing cut when the blob is unchanged
 }
 
-@test "orchestrator: autocut --dry-run prints the intended cut without invoking cut-release --push" {
+@test "orchestrator: autocut --dry-run prints the intended cut without writing any tag" {
   _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \
     blobMAIN blobNEXT aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa cccccccccccccccccccccccccccccccccccccccc "2.1.0"
-  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut --dry-run
+  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" bash "$ORCH" autocut --dry-run
   [ "$status" -eq 0 ]
   [[ "$output" == *"DRY-RUN"* ]]
   [[ "$output" == *"2.1.1"* ]]
-  [ ! -s "$CUT_LOG" ]   # dry-run never pushes a real cut
+  [ ! -s "$GH_LOG" ]   # dry-run never writes a real cut
 }
 
 @test "orchestrator: autocut honors the registry autocut.bump override (minor)" {
   _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \
     blobMAIN blobNEXT aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa cccccccccccccccccccccccccccccccccccccccc "2.1.0" minor
-  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut
+  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" bash "$ORCH" autocut
   [ "$status" -eq 0 ]
   # minor bump of v2.1.0 → v2.2.0
-  grep -q "dev-lead 2.2.0 --ref aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa --channel next --push" "$CUT_LOG"
+  grep -q "repos/petry-projects/.github-private/git/tags .*tag=dev-lead/v2.2.0 .*object=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "$GH_LOG"
 }
 
-@test "orchestrator: autocut is cross-repo aware — cuts v2.1.1 for auto-rebase from the host main HEAD (#1069)" {
+@test "orchestrator: autocut is cross-repo aware — cuts v2.1.1 for auto-rebase on the host repo (#1069)" {
   # auto-rebase is hosted in petry-projects/.github; its next candidate + release tags live there,
-  # so the next commit is resolved via gh api (not local git) and the cut is cross-repo.
+  # so the next commit is resolved via gh api (not local git) and BOTH the tag create and the
+  # next move are written to that host — not GITHUB_REPOSITORY.
   _autocut_stub auto-rebase petry-projects/.github .github/workflows/auto-rebase-reusable.yml \
     ece45480ece45480ece45480ece45480ece45480 2763750027637500276375002763750027637500 \
     ece45480ece45480ece45480ece45480ece45480 2763750027637500276375002763750027637500 "2.1.0"
-  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut
+  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" bash "$ORCH" autocut
+  [ "$status" -eq 0 ]
+  grep -q "repos/petry-projects/.github/git/tags .*tag=auto-rebase/v2.1.1 .*object=ece45480ece45480ece45480ece45480ece45480" "$GH_LOG"
+  grep -q "PATCH repos/petry-projects/.github/git/refs/tags/auto-rebase/next .*sha=ece45480ece45480ece45480ece45480ece45480" "$GH_LOG"
+}
+
+@test "orchestrator: autocut — existing release tag matching mainsha skips create and still moves next (idempotent retry)" {
+  # Simulate a partial retry: the release tag was already created on a prior run (pointing to
+  # mainsha), but `next` was not yet moved. The idempotency branch must skip POST git/tags
+  # and proceed straight to the PATCH for next without calling _gh_create_annotated_tag.
+  local mainsha="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+  _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \
+    blobMAIN blobNEXT "$mainsha" cccccccccccccccccccccccccccccccccccccccc "2.1.0" "" "$mainsha"
+  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" bash "$ORCH" autocut
+  [ "$status" -eq 0 ]
+  [[ "$output" == *"already exists"* ]]
+  [[ "$output" == *"re-pointing next only"* ]]
+  # The annotated tag create (POST git/tags) must NOT be called — tag already exists.
+  ! grep -q "POST.*git/tags" "$GH_LOG"
+  # The next move (PATCH) must still happen to complete the idempotent operation.
+  grep -q "PATCH.*git/refs/tags/dev-lead/next" "$GH_LOG"
+}
+
+@test "orchestrator: autocut — existing release tag pointing to a different commit emits warning and skips next move" {
+  # If vX.Y.Z already exists but points to a different commit (manual retag, concurrent run,
+  # prior bad state), moving next to mainsha would violate the "release tag + next → same commit"
+  # invariant. The engine must warn and skip rather than advance next to an untagged commit.
+  local mainsha="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+  local stale_sha="dddddddddddddddddddddddddddddddddddddddd"
+  _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \
+    blobMAIN blobNEXT "$mainsha" cccccccccccccccccccccccccccccccccccccccc "2.1.0" "" "$stale_sha"
+  run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" bash "$ORCH" autocut
   [ "$status" -eq 0 ]
-  grep -q "auto-rebase 2.1.1 --ref ece45480ece45480ece45480ece45480ece45480 --channel next --push" "$CUT_LOG"
+  [[ "$output" == *"::warning::"* ]]
+  # Neither tag create nor next move may be written when the invariant check fails.
+  [ ! -s "$GH_LOG" ]
 }
 
 # ── set_difference (pure set-diff core for drift detection, #1082) ─────────────