nightly-next: new builds go to the Dev ring first and move to Nightly on a canary pass

.github/workflows/ci-guards.yml

@@ -1353,6 +1353,10 @@ jobs:
         if: ${{ !cancelled() && matrix.group == 'release-notary' }}
         run: python3 tests/test_nightly_next_feed_rollback.py
 
+      - name: Validate the nightly-next release-ring tags
+        if: ${{ !cancelled() && matrix.group == 'release-notary' }}
+        run: python3 tests/test_nightly_next_ring.py
+
       - name: Validate the nightly-next canary mode and its one-week flip
         if: ${{ !cancelled() && matrix.group == 'release-notary' }}
         run: python3 tests/test_nightly_next_canary_mode.py

.github/workflows/nightly-next-verdict.yml

@@ -13,6 +13,13 @@ name: nightly-next verdict
 # of this repository whose `canary <build>` job succeeded. `fail` and
 # `skipped` never make a build a base, so they are written as given.
 #
+# Release rings (scripts/ci/nightly_next_ring.py): with repository variable
+# NIGHTLY_NEXT_DEV_RING=on, nightly.yml publishes each new build as
+# cmux-next-dev (Dev only). After writing a verdict, this run sets every feed
+# item's channel from the verdicts again, so a build whose canary passed moves
+# to cmux-next (Nightly, and every build from before rings). With the variable
+# off, every item goes back to cmux-next.
+#
 # Run it on branch nightly-next; release-next, which holds the feed's R2
 # token, deploys only to that branch. GitHub registers a dispatch workflow
 # only from the default branch, so this file is on main too; a run there
@@ -40,8 +47,11 @@ concurrency:
   cancel-in-progress: false
 
 env:
+  # The same values as nightly.yml; nightly-next-guard.py checks every write against them.
+  NIGHTLY_NEXT_FEED_BASE: https://files-next.cmux.com/nightly-next/next
   NIGHTLY_NEXT_R2_BUCKET: cmux-nightly-next
   NIGHTLY_NEXT_R2_PREFIX: nightly-next/next/
+  NIGHTLY_APPCAST: appcast-arm64.xml
 
 jobs:
   write:
@@ -61,6 +71,7 @@ jobs:
             scripts/ci/nightly_next_train.py
             scripts/ci/nightly_next_fix_forward.py
             scripts/ci/nightly-next-guard.py
+            scripts/ci/nightly_next_ring.py
             scripts/ci/upload-r2-object.py
           sparse-checkout-cone-mode: false
 
@@ -102,3 +113,45 @@ jobs:
             cat verdict.json
             echo '```'
           } >> "$GITHUB_STEP_SUMMARY"
+
+      # Promotion: the feed's channels follow the verdicts. The write happens
+      # only while the feed still holds the bytes read (If-Match on its ETag);
+      # a publication that lands meanwhile wins, and it derives the channels
+      # from the same verdicts (republish-nightly-deltas).
+      - name: Move builds whose canary passed to Nightly
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.NIGHTLY_NEXT_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.NIGHTLY_NEXT_R2_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          R2_ENDPOINT: "https://${{ secrets.NIGHTLY_NEXT_R2_ACCOUNT_ID }}.r2.cloudflarestorage.com"
+          DEV_RING: ${{ vars.NIGHTLY_NEXT_DEV_RING == 'on' && 'on' || 'off' }}
+        run: |
+          set -euo pipefail
+          mkdir -p ring
+          curl -fsSL --retry 3 -A "cmux-nightly-next-ring/1" -H "Cache-Control: no-cache" \
+            -o ring/live.xml "${NIGHTLY_NEXT_FEED_BASE}/${NIGHTLY_APPCAST}?ring=${GITHUB_RUN_ID}"
+          summary="$(python3 scripts/ci/nightly_next_ring.py retag --feed ring/live.xml \
+            --out "ring/$NIGHTLY_APPCAST" --mode "$DEV_RING" --verdicts-base "$NIGHTLY_NEXT_FEED_BASE")"
+          echo "$summary"
+          if [ "$(python3 -I -c 'import json, sys; print(json.loads(sys.argv[1])["changed"])' "$summary")" != True ]; then
+            echo "Release rings ($DEV_RING): the feed's channels already match the verdicts." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+          key="${NIGHTLY_NEXT_R2_PREFIX}${NIGHTLY_APPCAST}"
+          python3 scripts/ci/nightly-next-guard.py appcast "ring/$NIGHTLY_APPCAST"
+          python3 scripts/ci/nightly-next-guard.py r2-key "$NIGHTLY_NEXT_R2_BUCKET" "$key"
+          status=0
+          python3 scripts/ci/upload-r2-object.py \
+            --file "ring/$NIGHTLY_APPCAST" \
+            --endpoint-url "$R2_ENDPOINT" \
+            --bucket "$NIGHTLY_NEXT_R2_BUCKET" \
+            --key "$key" \
+            --content-type "application/xml" \
+            --cache-control "no-cache, no-store, must-revalidate" \
+            --replace-sha256 "$(shasum -a 256 ring/live.xml | awk '{print $1}')" --conflict-exit-code 3 || status=$?
+          if [ "$status" = 3 ]; then
+            echo "::notice title=nightly-next rings::the feed changed after this run read it; the newer write sets the channels from the same verdicts"
+            exit 0
+          fi
+          [ "$status" = 0 ] || exit "$status"
+          echo "Release rings ($DEV_RING): wrote the feed's channels $summary" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/nightly.yml

@@ -3825,9 +3825,12 @@ jobs:
           SHARED_SPARKLE_PRIVATE_KEY: ${{ needs.decide.outputs.track != 'nightly-next' && secrets.SPARKLE_PRIVATE_KEY || '' }}
           NIGHTLY_SPARKLE_PRIVATE_KEY: ${{ needs.decide.outputs.track != 'nightly-next' && secrets.NIGHTLY_SPARKLE_PRIVATE_KEY || '' }}
           NEXT_SPARKLE_PRIVATE_KEY: ${{ needs.decide.outputs.track == 'nightly-next' && secrets.SPARKLE_NEXT_PRIVATE_KEY || '' }}
-          # cmux-next items carry sparkle:channel cmux-next, which only the
-          # cmux-next build allows; every other build ignores them.
-          SPARKLE_CHANNEL: ${{ needs.decide.outputs.track == 'nightly-next' && 'cmux-next' || '' }}
+          # cmux-next items carry a release-ring sparkle:channel, which only
+          # cmux-next builds allow; every other build ignores them. With
+          # NIGHTLY_NEXT_DEV_RING=on a new build starts on Dev (cmux-next-dev)
+          # and moves to Nightly (cmux-next) on a canary pass
+          # (scripts/ci/nightly_next_ring.py, nightly-next-verdict.yml).
+          SPARKLE_CHANNEL: ${{ needs.decide.outputs.track == 'nightly-next' && (vars.NIGHTLY_NEXT_DEV_RING == 'on' && 'cmux-next-dev' || 'cmux-next') || '' }}
         run: |
           set -euo pipefail
           SPARKLE_PRIVATE_KEY="$(./scripts/ci/nightly-sparkle-key.sh)"
@@ -4630,7 +4633,8 @@ jobs:
         if: steps.continue.outputs.accepted == 'true'
         env:
           NEXT_SPARKLE_PRIVATE_KEY: ${{ secrets.SPARKLE_NEXT_PRIVATE_KEY }}
-          SPARKLE_CHANNEL: cmux-next
+          # A recovered build is new to the feed: Dev first, as in build-sign-notarize-nightly.
+          SPARKLE_CHANNEL: ${{ vars.NIGHTLY_NEXT_DEV_RING == 'on' && 'cmux-next-dev' || 'cmux-next' }}
         run: |
           set -euo pipefail
           SPARKLE_PRIVATE_KEY="$(./scripts/ci/nightly-sparkle-key.sh)"
@@ -5508,6 +5512,17 @@ jobs:
           set -euo pipefail
           curl -fsSL --retry 3 -A "cmux-nightly-legacy/1" -o next.xml "${NIGHTLY_NEXT_FEED_BASE}/${LEGACY_APPCAST}"
           curl -fsSL --retry 3 -A "cmux-nightly-legacy/1" -o old.xml "${NIGHTLY_NEXT_LEGACY_FEED_BASE}/${LEGACY_APPCAST}"
+          # The old feed's installs predate release rings and follow Nightly:
+          # only builds already tagged cmux-next are candidates, never one
+          # still on Dev (scripts/ci/nightly_next_ring.py; a no-op while
+          # NIGHTLY_NEXT_DEV_RING is off).
+          selected="$(python3 scripts/ci/nightly_next_ring.py select --feed next.xml --out next.xml --channel cmux-next)"
+          echo "cmux NEXT feed by ring: $selected"
+          if [ "$(python3 -I -c 'import json, sys; print(len(json.loads(sys.argv[1])["kept"]))' "$selected")" = 0 ]; then
+            echo "### cmux NIGHTLY-identity: no cmux NEXT build is on Nightly yet; the old feed waits (cx-x7hwx)" >> "$GITHUB_STEP_SUMMARY"
+            { echo "action=none"; echo "pending_run="; echo "bundle_run="; echo "build="; } >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
           # A cmux NIGHTLY-identity submission still in Apple's queue is kept by
           # publish-nightly-identity-legacy as the artifacts
           # cmux-nightly-identity-pending-meta/-bundle of its run: the newest
@@ -6201,6 +6216,8 @@ jobs:
         env:
           TRACK: nightly-next
           NEXT_SPARKLE_PRIVATE_KEY: ${{ secrets.SPARKLE_NEXT_PRIVATE_KEY }}
+          # The old feed's installs predate rings and allow only cmux-next;
+          # legacy-identity-target picks only builds already on Nightly.
           SPARKLE_CHANNEL: cmux-next
           SPARKLE_MINIMUM_SYSTEM_VERSION: ${{ needs.sign-nightly-identity-legacy.outputs.minimum_system_version }}
         run: |
@@ -6450,7 +6467,9 @@ jobs:
       SHARED_SPARKLE_PRIVATE_KEY: ${{ needs.decide.outputs.track != 'nightly-next' && secrets.SPARKLE_PRIVATE_KEY || '' }}
       NIGHTLY_SPARKLE_PRIVATE_KEY: ${{ needs.decide.outputs.track != 'nightly-next' && secrets.NIGHTLY_SPARKLE_PRIVATE_KEY || '' }}
       NEXT_SPARKLE_PRIVATE_KEY: ${{ needs.decide.outputs.track == 'nightly-next' && secrets.SPARKLE_NEXT_PRIVATE_KEY || '' }}
-      SPARKLE_CHANNEL: ${{ needs.decide.outputs.track == 'nightly-next' && 'cmux-next' || '' }}
+      # Every item starts on the new build's ring; republish-nightly-deltas
+      # moves the ones whose canary passed back to Nightly.
+      SPARKLE_CHANNEL: ${{ needs.decide.outputs.track == 'nightly-next' && (vars.NIGHTLY_NEXT_DEV_RING == 'on' && 'cmux-next-dev' || 'cmux-next') || '' }}
     steps:
       - name: Checkout build ref
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -6514,6 +6533,20 @@ jobs:
           persist-credentials: false
           ref: ${{ needs.resolve-nightly-cmux-tui-client.outputs.build_sha }}
 
+      # The ring script and the guard come from the workflow commit: the build
+      # ref can predate them (publish-nightly does the same).
+      - name: Checkout publication tooling at the workflow commit
+        if: needs.decide.outputs.track == 'nightly-next'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+          path: .publish-tooling
+          sparse-checkout: |
+            scripts/ci/nightly_next_ring.py
+            scripts/ci/nightly-next-guard.py
+          sparse-checkout-cone-mode: false
+
       - name: Download delta results
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
@@ -6525,6 +6558,8 @@ jobs:
         id: republish
         env:
           GH_TOKEN: ${{ github.token }}
+          TRACK: ${{ needs.decide.outputs.track }}
+          DEV_RING: ${{ vars.NIGHTLY_NEXT_DEV_RING == 'on' && 'on' || 'off' }}
         run: |
           set -euo pipefail
           # The tag itself: `commits/<name>` resolves the nightly-next branch,
@@ -6563,6 +6598,15 @@ jobs:
             echo "publish=false" >> "$GITHUB_OUTPUT"
             exit 0
           fi
+          # The revised feed lists earlier builds again, all on the new build's
+          # ring: each goes back to Nightly when its canary verdict is a pass.
+          if [ "$TRACK" = nightly-next ]; then
+            for feed in nightly-deltas/appcast*.xml; do
+              [ -f "$feed" ] || continue
+              python3 .publish-tooling/scripts/ci/nightly_next_ring.py retag --feed "$feed" --out "$feed" \
+                --mode "$DEV_RING" --verdicts-base "$NIGHTLY_NEXT_FEED_BASE"
+            done
+          fi
           feeds=()
           for feed in nightly-deltas/appcast*.xml; do
             [ -f "$feed" ] || continue
@@ -6610,8 +6654,8 @@ jobs:
             [ -f "$feed" ] || continue
             name="$(basename "$feed")"
             key="${NIGHTLY_NEXT_R2_PREFIX}${name}"
-            python3 scripts/ci/nightly-next-guard.py appcast "$feed"
-            python3 scripts/ci/nightly-next-guard.py r2-key "$NIGHTLY_NEXT_R2_BUCKET" "$key"
+            python3 .publish-tooling/scripts/ci/nightly-next-guard.py appcast "$feed"
+            python3 .publish-tooling/scripts/ci/nightly-next-guard.py r2-key "$NIGHTLY_NEXT_R2_BUCKET" "$key"
             python3 scripts/ci/upload-r2-object.py \
               --file "$feed" \
               --endpoint-url "$R2_ENDPOINT" \

docs/ci/nightly-next-canary.md

@@ -135,6 +135,29 @@ gh variable set NIGHTLY_NEXT_CANARY_ENFORCE_AFTER -R manaflow-ai/cmux --body 202
 gh variable delete NIGHTLY_NEXT_CANARY -R manaflow-ai/cmux               # back to auto
 ```
 
+### Release rings: Dev before Nightly
+
+The app's Update Channel setting picks a ring, and each ring accepts its own
+`sparkle:channel` tag and every slower ring's (`UpdateRing.swift`): Dev
+accepts `cmux-next-dev` and `cmux-next`, Nightly and every build from before
+rings accept only `cmux-next`. Sparkle never offers an item whose tag the
+updater does not allow.
+
+Repository variable `NIGHTLY_NEXT_DEV_RING` (`scripts/ci/nightly_next_ring.py`):
+
+| `NIGHTLY_NEXT_DEV_RING` | Effect |
+| --- | --- |
+| `on` | A new build is published as `cmux-next-dev`, so only Dev gets it. It moves to `cmux-next` once its durable verdict (`verdicts/<build>.json`, the one the train uses) is `pass`: `nightly-next-verdict.yml` retags the live feed after each verdict, and `republish-nightly-deltas` tags the earlier builds it lists from the same verdicts. A build the canary failed, skipped or never checked stays on Dev. The old cmux NIGHTLY-identity feed copies only builds already on `cmux-next`. |
+| unset, `off` or anything else | Every item is `cmux-next`, as before rings. The next verdict or publication puts any Dev item back on Nightly. |
+
+With `on`, Nightly gets nothing new while the canary passes no build, so turn
+it on only while the canary passes.
+
+```bash
+gh variable set NIGHTLY_NEXT_DEV_RING -R manaflow-ai/cmux --body on
+gh variable delete NIGHTLY_NEXT_DEV_RING -R manaflow-ai/cmux   # back to one ring
+```
+
 ### Fix-forward
 
 A pull stops a bad build spreading but can't move a Mac that already has it.

scripts/ci/nightly-next-guard.py

@@ -7,7 +7,7 @@
 bakes a feed URL into the app and before every object it uploads:
 
   feed-url URL           the SUFeedURL baked into the app
-  appcast FILE           every download URL and item channel in an appcast
+  appcast FILE           every download URL and item channel (one ring tag) in an appcast
   r2-key BUCKET KEY      every object it writes
 
 The allowed values come from the workflow environment
@@ -33,10 +33,13 @@
 MAIN_OBJECT_PREFIXES = ("nightly/", "rc/", "stable/")
 NEXT_DOWNLOAD_PREFIX = "https://github.com/manaflow-ai/cmux/releases/download/nightly-next/"
 SPARKLE_NS = "http://www.andymatuschak.org/xml-namespaces/sparkle"
-# Every cmux-next item carries this sparkle:channel. Sparkle offers a
-# channel-tagged item only to an updater that allows the channel, and only the
-# cmux-next build does (UpdateFeedResolver.Channel.allowedSparkleChannels).
-NEXT_SPARKLE_CHANNEL = "cmux-next"
+# Every cmux-next item carries exactly one of these sparkle:channel tags, one
+# per release ring (UpdateRing.sparkleChannel; scripts/ci/nightly_next_ring.py
+# says which). Sparkle offers a channel-tagged item only to an updater that
+# allows the channel; only a cmux-next build allows any of them
+# (UpdateFeedResolver.Channel.allowedSparkleChannels), so main's builds ignore
+# every item, and an untagged item is what this guard must never let through.
+NEXT_SPARKLE_CHANNELS = ("cmux-next-stable", "cmux-next", "cmux-next-canary", "cmux-next-dev")
 
 
 class Refused(Exception):
@@ -75,8 +78,8 @@ def check_appcast(path):
     items = list(root.iter("item"))
     for item in items:
         channels = [c.text for c in item.findall(f"{{{SPARKLE_NS}}}channel")]
-        if channels != [NEXT_SPARKLE_CHANNEL]:
-            raise Refused(f"{path} item channel {channels} is not [{NEXT_SPARKLE_CHANNEL!r}]")
+        if len(channels) != 1 or channels[0] not in NEXT_SPARKLE_CHANNELS:
+            raise Refused(f"{path} item channel {channels} is not one of {list(NEXT_SPARKLE_CHANNELS)}")
     for url in urls:
         if not url.startswith(NEXT_DOWNLOAD_PREFIX):
             raise Refused(f"{path} enclosure {url} is not a nightly-next download")