diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index efb71a4f3..b597c24eb 100644 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -187,16 +187,33 @@ _reusable_differs() { # unless state is BLOCKED (then REGRESSION | PRE_EXISTING). _frontier_state() { local agent="$1" - local cand chans frontier="" + local cand chans frontier="" prev_on=() + local soak_start use_soak_as_candidate=false next_health_signal + soak_start="$(_agent_field "$agent" soak_start_ring)" + [ "$soak_start" = "null" ] && soak_start="" + next_health_signal="$(_agent_field "$agent" next_tier_health_signal)" + [ "$next_health_signal" = "null" ] && next_health_signal="" + cand="$(channel_commit "$agent" next)" + # soak_start_ring agents have no @next channel caller — the innermost soaked ring is + # the candidate source. Fall back only when next genuinely has no tag. + if [ -z "$cand" ] && [ -n "$soak_start" ]; then + cand="$(channel_commit "$agent" "$soak_start")" + use_soak_as_candidate=true + fi chans="$(ordered_channels "$agent")" + # Collect the rings already on the candidate (starting at next / soak_start) and find the frontier. local chan_array=() IFS=, read -r -a chan_array <<< "$chans" local ch for ch in "${chan_array[@]}"; do local c; c="$(channel_commit "$agent" "$ch")" - if [ "$ch" = "next" ] || [ "$c" = "$cand" ]; then :; else frontier="$ch"; break; fi + if [ "$ch" = "next" ] || { [ "$use_soak_as_candidate" = "true" ] && [ "$ch" = "$soak_start" ]; } || [ "$c" = "$cand" ]; then + prev_on+=("$ch") + else + frontier="$ch"; break + fi done if [ -z "$frontier" ]; then echo "$cand - - COMPLETE 0 0 0 0 0 0 -"; return 0 @@ -207,18 +224,36 @@ _frontier_state() { source="${transition%%->*}" cut_z="$(candidate_cut_date "$agent" "$cand")" if [ -z "$cut_z" ]; then - # Cannot determine the per-candidate window start — fail closed to prevent unbounded history queries. - echo "$cand $frontier $transition BLOCKED 0 0 0 0 0 0 -"; return 0 + if [ -z "$soak_start" ]; then + # Cannot determine the per-candidate window start — fail closed to prevent unbounded history queries. + echo "$cand $frontier $transition BLOCKED 0 0 0 0 0 0 -"; return 0 + fi + # soak_start_ring agents may lack a vX.Y.Z release tag; proceed with an unbounded window. fi now_epoch="$(date -u +%s)" # Source-tier repos (the tier currently running the candidate). - local src_repos=() r - while IFS= read -r r; do [ -n "$r" ] && src_repos+=("$r"); done < <(resolve_members "$agent" "$source") + # When the frontier IS the soak_start_ring (transition is empty — no source ring below it), + # use the host repo with next_tier_health_signal as the pre-soak health source. + local src_repos=() r use_health_signal=false + if [ -z "$transition" ] && [ -n "$next_health_signal" ]; then + use_health_signal=true + src_repos=("$(_agent_field "$agent" host)") + else + while IFS= read -r r; do [ -n "$r" ] && src_repos+=("$r"); done < <(resolve_members "$agent" "$source") + fi # Sample on the source tier over the per-candidate window. local sample earliest - read -r sample earliest < <(_tier_sample "$agent" "$cut_z" "${src_repos[@]}") + if [ "$use_health_signal" = "true" ]; then + local hs_json + hs_json="$(_run_json "${src_repos[0]}" "$next_health_signal" "$cut_z")" + sample=$(jq '[.[]?|select(.conclusion=="success" or .conclusion=="failure")]|length' 2>/dev/null <<< "$hs_json" || echo 0) + earliest="$(jq -r '[.[]?|select(.conclusion=="success" or .conclusion=="failure")|.createdAt?]|min // empty' 2>/dev/null <<< "$hs_json")" + [ -z "$earliest" ] && earliest="-" + else + read -r sample earliest < <(_tier_sample "$agent" "$cut_z" "${src_repos[@]}") + fi # Dwell is always measured from the candidate's own cut (tagger date), per #548 spec. local dwell_h=0 diff --git a/standards/canary-rings.json b/standards/canary-rings.json index a23f868f2..d497d0810 100644 --- a/standards/canary-rings.json +++ b/standards/canary-rings.json @@ -1,12 +1,18 @@ { "version": 1, - "description": "Ring membership for agent canary rollout (initiative #495, issues #500/#501). The promotion automation (scripts/canary-rollout.sh) reads this as the single source of truth. `next` is host-relative: it resolves to the repo that HOSTS the reusable; `ring0` covers the other org-infra repo (the host already sits in `next`).", + "description": "Ring membership for agent canary rollout (initiative #495, issues #500/#501; pipeline onboarding #1008). The promotion automation (scripts/canary-rollout.sh) reads this as the single source of truth. `next` is host-relative: it resolves to the repo that HOSTS the reusable; `ring0` covers the other org-infra repo (the host already sits in `next`). An agent may set `soak_start_ring` to name the innermost ring that actually has a channel caller — used when the host dogfoods the reusable via a local `./` ref (so there is no `@next`/`ring0` channel caller) and no org-infra repo consumes it as a channel member; soak then begins at that ring. `unmanaged` records reusable-less / non-channel-promotable workflows (e.g. direct `gh workflow run` dispatch) so the automation does not try to gate them.", "org_infra_repos": ["petry-projects/.github", "petry-projects/.github-private"], "member_tokens": { "$host": "the agent's host repo (the repo that owns the reusable)", "$org_infra": "org_infra_repos minus the host (the host is already in `next`)", "*": "every other consumer not named in an earlier ring" }, + "unmanaged": { + "initiative-driver": { + "reason": "Direct `gh workflow run` dispatch to petry-projects/.github-private; no reusable workflow, so it is not channel-promotable. Recorded here (out of model, #1008) so the canary automation does not attempt to gate it.", + "health_source": ".github/workflows/initiative-driver-canary.yml" + } + }, "agents": { "dev-lead": { "host": "petry-projects/.github-private", @@ -40,6 +46,28 @@ } } } + }, + "initiative-planner": { + "host": "petry-projects/.github", + "reusable": ".github/workflows/initiative-planner-reusable.yml", + "run_workflow": "Initiative Planner — Approval Trigger", + "soak_start_ring": "ring1", + "next_tier_health_signal": ".github/workflows/initiative-planner-canary.yml", + "rings": [ + { "channel": "ring1", "order": 2, "members": ["petry-projects/TalkTerm", "petry-projects/bmad-bgreat-suite"] }, + { "channel": "stable", "order": 3, "members": ["*"] } + ] + }, + "idea-triage": { + "host": "petry-projects/.github", + "reusable": ".github/workflows/idea-triage-reusable.yml", + "run_workflow": "Idea Triage — Weekly Shortlist", + "soak_start_ring": "ring1", + "next_tier_health_signal": ".github/workflows/idea-triage-canary.yml", + "rings": [ + { "channel": "ring1", "order": 2, "members": ["petry-projects/TalkTerm", "petry-projects/bmad-bgreat-suite"] }, + { "channel": "stable", "order": 3, "members": ["*"] } + ] } } } diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index 48affe726..2c4d37aca 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -168,6 +168,69 @@ setup() { [ "$status" -eq 0 ] } +# ── canary-rings.json: idea→initiative pipeline onboarding (#1008) ───────────── +# initiative-planner + idea-triage are onboarded as managed agents whose reusable +# lives cross-repo in petry-projects/.github. Their host dogfoods via a local `./` +# ref (no @next caller) and .github-private runs a full central copy (not a channel +# consumer), so there is no next/ring0 channel caller — soak begins at ring1. +_pipeline_agent_shape() { + # $1 = agent key. Asserts the shared onboarding shape for planner/triage. + local agent="$1" + # hosted in the public repo (reusable lives there, not in .github-private) + run jq -e --arg a "$agent" '.agents[$a].host == "petry-projects/.github"' "$RINGS" + [ "$status" -eq 0 ] + # soak starts at ring1 (owner-confirmed: no channel caller below ring1) + run jq -e --arg a "$agent" '.agents[$a].soak_start_ring == "ring1"' "$RINGS" + [ "$status" -eq 0 ] + # only ring1 + stable channels are modelled, ordered ring1 -> stable + run jq -r --arg a "$agent" '.agents[$a].rings | sort_by(.order) | map(.channel) | join(",")' "$RINGS" + [ "$output" = "ring1,stable" ] + # ring1 = the two low-traffic consumers + run jq -e --arg a "$agent" '.agents[$a].rings[] | select(.channel=="ring1") | (.members | index("petry-projects/TalkTerm")) and (.members | index("petry-projects/bmad-bgreat-suite"))' "$RINGS" + [ "$status" -eq 0 ] + # stable = ["*"] (un-enrolled today; populated by the promotion automation) + run jq -e --arg a "$agent" '.agents[$a].rings[] | select(.channel=="stable") | .members == ["*"]' "$RINGS" + [ "$status" -eq 0 ] + # .github-private runs a full central copy — never a channel member of these agents + run jq -e --arg a "$agent" '[.agents[$a].rings[].members[]] | index("petry-projects/.github-private") | not' "$RINGS" + [ "$status" -eq 0 ] +} + +@test "canary-rings.json: initiative-planner onboarded (soak_start_ring=ring1, cross-repo host)" { + _pipeline_agent_shape "initiative-planner" + # its pre-ring1 health signal is the in-repo canary workflow + run jq -e '.agents["initiative-planner"].next_tier_health_signal == ".github/workflows/initiative-planner-canary.yml"' "$RINGS" + [ "$status" -eq 0 ] +} + +@test "canary-rings.json: idea-triage onboarded (soak_start_ring=ring1, cross-repo host)" { + _pipeline_agent_shape "idea-triage" + # same pattern as initiative-planner: needs next_tier_health_signal for pre-ring1 gate + run jq -e '.agents["idea-triage"].next_tier_health_signal == ".github/workflows/idea-triage-canary.yml"' "$RINGS" + [ "$status" -eq 0 ] +} + +@test "canary-rings.json: initiative-driver is recorded as unmanaged, not a channel agent" { + # no reusable -> not channel-promotable -> excluded from .agents so the + # automation never tries to gate it. + run jq -e '.agents["initiative-driver"] == null' "$RINGS" + [ "$status" -eq 0 ] + # but explicitly recorded under .unmanaged with a rationale + health source. + run jq -e '.unmanaged["initiative-driver"].reason | type == "string" and (. | length > 0)' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.unmanaged["initiative-driver"].health_source == ".github/workflows/initiative-driver-canary.yml"' "$RINGS" + [ "$status" -eq 0 ] +} + +@test "canary-rings.json: blocked/hold reusables are NOT onboarded" { + # idea-enhancer (zero tags/callers) and feature-ideation (open bug #571 + missing + # channel tags) must stay out of the SoT entirely — neither agent nor unmanaged. + run jq -e '.agents["idea-enhancer"] == null and .agents["feature-ideation"] == null' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.unmanaged["idea-enhancer"] == null and .unmanaged["feature-ideation"] == null' "$RINGS" + [ "$status" -eq 0 ] +} + # ── orchestrator: resolve_members (host-relative tokens) ────────────────────── @test "orchestrator: resolve_members expands \$host / \$org_infra / * " { run bash -c "source '$ORCH' && CANARY_RINGS='$RINGS' resolve_members dev-lead next" @@ -220,6 +283,74 @@ GHEOF [[ "$output" == *"stable"* ]] } +@test "orchestrator: evaluate uses next_tier_health_signal when prev_on is empty (soak_start_ring agent)" { + _make_stub_bin + # initiative-planner/next is the candidate; ring1 is not yet on it → prev_on=[] + cat > "$STUB_BIN/git" <<'GITEOF' +#!/usr/bin/env bash +case "$*" in + *"rev-parse"*"initiative-planner/next"*) echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ;; + *"rev-parse"*"initiative-planner/ring1"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;; + *"rev-parse"*"initiative-planner/stable"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;; + *) : ;; +esac +GITEOF + chmod +x "$STUB_BIN/git" + # Healthy runs from the host canary; ring member queries return empty (low traffic) + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"run list"*"initiative-planner-canary"*) + printf '['; for i in $(seq 1 15); do [ "$i" -gt 1 ] && printf ','; printf '{"conclusion":"success"}'; done; printf ']' + ;; + *"run list"*) echo "[]" ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate initiative-planner + [ "$status" -eq 0 ] + [[ "$output" == *"initiative-planner"* ]] + [[ "$output" == *"ring1"* ]] + # 15 healthy runs from canary, baseline=0 → ceil(0/7)=0 min needed → PROMOTE + [[ "$output" == *"PROMOTE"* ]] +} + +@test "orchestrator: evaluate uses ring1 health when soak_start_ring is the candidate source (no next tag)" { + _make_stub_bin + # initiative-planner has soak_start_ring=ring1 and no @next channel caller. + # ring1 tag exists (aaaa = candidate); stable tag is on a different commit (bbbb). + # _frontier_state must treat ring1 as the candidate source and identify stable as + # the frontier (not get stuck or falsely report COMPLETE). + cat > "$STUB_BIN/git" <<'GITEOF' +#!/usr/bin/env bash +case "$*" in + *"rev-parse"*"initiative-planner/ring1"*) echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ;; + *"rev-parse"*"initiative-planner/stable"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;; + *) : ;; +esac +GITEOF + chmod +x "$STUB_BIN/git" + # ring1 members return healthy runs; stable frontier members are wildcards (skipped) + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"run list"*) echo '[{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"},{"conclusion":"success"}]' ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate initiative-planner + [ "$status" -eq 0 ] + [[ "$output" == *"initiative-planner"* ]] + # stable is the frontier (ring1 = soak_start_ring = already on candidate) + [[ "$output" == *"stable"* ]] + # 10+ healthy ring1 runs, baseline (stable/*) = 0 → min_healthy=0, healthy>0 → PROMOTE + [[ "$output" == *"PROMOTE"* ]] +} + @test "orchestrator: promote --override --dry-run shows the move but never pushes" { _make_stub_bin cat > "$STUB_BIN/gh" <<'GHEOF'