ci(nightly-next): register the post-publish launch smoke on main (not a publish gate)

.github/workflows/nightly-next-launch-smoke.yml

@@ -0,0 +1,343 @@
+name: nightly-next launch smoke
+
+# The cmux NEXT post-publish launch smoke. It is not a publish gate: nightly.yml
+# publishes without waiting on it, and nothing reads its result before a
+# publish. After a build is on the feed, it launches that build's app the way a
+# first install does (an empty home, no defaults; scripts/ci/fresh_home_launch_smoke.py)
+# and needs it to stay up 30 s with Home rendered.
+#
+#   pass        nothing happens.
+#   cannot-run  nothing happens (no Mac, no GUI session, a download, harness or
+#               infrastructure error before the app launched); a notice says so.
+#   crash       the app exited, wrote a crash report, or never rendered Home, on
+#               both attempts. The feed rolls back through nightly-next-rollback.yml
+#               (bad_build = this build), which serves the earlier Nightly build the
+#               feed keeps (publish-nightly-next-r2.py keeps two), and one issue
+#               per build gets the evidence. A rollback is asked for only while the
+#               build is still the feed's newest (scripts/ci/nightly_next_launch_smoke.py
+#               decide), and the rollback itself re-checks that on its own read
+#               (if_newest), so a second report of the same build, or a newer build
+#               published meanwhile, is never pulled.
+#
+# Every 10 minutes it checks the feed's newest build: a pass is cached per build,
+# so a tick with no new build stops in the Linux pick job. A cannot-run is cached
+# for the rest of its UTC hour only, so a build the infrastructure could not
+# smoke is tried again at most an hour later. A crash is not cached, so it is
+# retried until its rollback lands. A schedule and a dispatch register only
+# from the default branch, so this file is on main too; every run takes its
+# tooling from feat-cmux-next (a pull request into feat-cmux-next, from its own
+# head). A pull request's run smokes the newest feed build and never rolls back
+# or files.
+on:
+  schedule:
+    - cron: "1,11,21,31,41,51 * * * *"
+  workflow_dispatch:
+    inputs:
+      build:
+        description: Smoke this build from the feed (digits). Empty takes the newest.
+        required: false
+        default: ''
+        type: string
+  pull_request:
+    paths:
+      - .github/workflows/nightly-next-launch-smoke.yml
+      - scripts/ci/fresh_home_launch_smoke.py
+      - scripts/ci/nightly_next_launch_smoke.py
+
+permissions: {}
+
+# A new push to a pull request replaces its smoke (it never rolls back or files);
+# a scheduled or dispatched smoke has a group of its own run, so it is never
+# cancelled or replaced while it waits.
+concurrency:
+  group: nightly-next-launch-smoke-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+env:
+  NIGHTLY_NEXT_FEED: https://files-next.cmux.com/nightly-next/next/appcast-arm64.xml
+  # Bumping it smokes every feed build again.
+  SMOKE_CACHE_PREFIX: nightly-next-launch-smoke-v1-
+  # A pull request into feat-cmux-next tests its own head; every other run (and a
+  # pull request into main, whose tree has none of this tooling) takes feat-cmux-next's.
+  TOOLING_REF: ${{ github.event_name == 'pull_request' && github.base_ref == 'feat-cmux-next' && github.sha || 'feat-cmux-next' }}
+
+jobs:
+  pick:
+    if: github.repository == 'manaflow-ai/cmux' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-4vcpu-ubuntu-2404' }}
+    timeout-minutes: 5
+    permissions: {}
+    outputs:
+      build: ${{ steps.feed.outputs.build }}
+      url: ${{ steps.feed.outputs.url }}
+      length: ${{ steps.feed.outputs.length }}
+      checked: ${{ steps.seen.outputs.cache-hit == 'true' || steps.seen-hour.outputs.cache-hit == 'true' }}
+      hour: ${{ steps.hour.outputs.hour }}
+    steps:
+      - name: Read the feed
+        id: feed
+        env:
+          BUILD: ${{ inputs.build }}
+        run: |
+          set -euo pipefail
+          curl -fsSL --retry 3 -A "cmux-nightly-next-launch-smoke/1" -H "Cache-Control: no-cache" -o feed.xml "$NIGHTLY_NEXT_FEED?smoke=$GITHUB_RUN_ID"
+          python3 - feed.xml "$BUILD" >> "$GITHUB_OUTPUT" <<'PY'
+          import re, sys
+          text, wanted = open(sys.argv[1], encoding="utf-8").read(), sys.argv[2]
+          items = []
+          for item in re.findall(r"<item>.*?</item>", text, re.S):
+              version = re.search(r"<sparkle:version>\s*(\d+)\s*</sparkle:version>", item)
+              enclosure = re.search(r'<enclosure\s[^>]*?url="([^"]+\.dmg)"[^>]*?length="(\d+)"', item)
+              if version and enclosure:
+                  items.append((int(version.group(1)), enclosure.group(1), enclosure.group(2)))
+          if wanted:
+              items = [i for i in items if str(i[0]) == wanted]
+          if not items:
+              sys.exit(f"no feed item {'for build ' + wanted if wanted else 'with a DMG'}")
+          build, url, length = max(items)
+          if not url.startswith(("https://github.com/manaflow-ai/cmux/releases/download/nightly-next/",
+                                 "https://files-next.cmux.com/nightly-next/next/")):
+              sys.exit(f"{url} is not a nightly-next download")
+          print(f"build={build}\nurl={url}\nlength={length}")
+          PY
+          cat "$GITHUB_OUTPUT"
+
+      # A scheduled tick skips a build an earlier tick passed, and a build an earlier
+      # tick could not smoke in this same UTC hour (a bounded retry: one transient
+      # infrastructure failure never stops the build from being smoked). A crash is
+      # never cached. A dispatch or a pull request always runs.
+      - name: Skip a build already passed
+        id: seen
+        if: github.event_name == 'schedule'
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: smoke-verdict
+          key: ${{ env.SMOKE_CACHE_PREFIX }}${{ steps.feed.outputs.build }}
+          lookup-only: true
+
+      - name: Read the hour
+        id: hour
+        run: echo "hour=$(date -u +%Y%m%d%H)" >> "$GITHUB_OUTPUT"
+
+      - name: Skip a build that could not be smoked this hour
+        id: seen-hour
+        if: github.event_name == 'schedule' && steps.seen.outputs.cache-hit != 'true'
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: smoke-verdict
+          key: ${{ env.SMOKE_CACHE_PREFIX }}${{ steps.feed.outputs.build }}-cannot-run-${{ steps.hour.outputs.hour }}
+          lookup-only: true
+
+  smoke:
+    name: smoke ${{ needs.pick.outputs.build }}
+    needs: pick
+    if: needs.pick.result == 'success' && needs.pick.outputs.checked != 'true'
+    # No secrets and no environment: it runs the published app. An owned Mac's
+    # side label on attempt 1 (owned_pool_rescue.SIDE_WORKFLOW_PATHS re-runs a
+    # refused job), Blacksmith macOS 26 after. Never a GitHub-hosted Mac.
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || vars.CI_PR_POOL_OWNED == '1' && github.run_attempt == 1 && vars.CI_SIDE_LANE_RUNNER || 'blacksmith-6vcpu-macos-26' }}
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    outputs:
+      verdict: ${{ steps.verdict.outputs.verdict }}
+    env:
+      BUILD: ${{ needs.pick.outputs.build }}
+      DMG_URL: ${{ needs.pick.outputs.url }}
+      DMG_LENGTH: ${{ needs.pick.outputs.length }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ env.TOOLING_REF }}
+          persist-credentials: false
+          sparse-checkout: |
+            scripts/ci/fresh_home_launch_smoke.py
+            scripts/ci/nightly_next_launch_smoke.py
+            scripts/ci/frame-pacing/console-session.sh
+            scripts/ci/lib/signing-runtime-guard.sh
+          sparse-checkout-cone-mode: false
+
+      - name: Download the feed's DMG
+        id: dmg
+        run: |
+          set -euo pipefail
+          curl -fsSL --retry 3 -o "$RUNNER_TEMP/smoke.dmg" "$DMG_URL"
+          actual="$(stat -f %z "$RUNNER_TEMP/smoke.dmg")"
+          [ "$actual" = "$DMG_LENGTH" ] || { echo "downloaded $actual bytes, the feed says $DMG_LENGTH" >&2; exit 1; }
+
+      # Two attempts: one flaky launch never rolls a build back. Exit 0 passes,
+      # 1 is a crash, anything else could not run (and ends the check).
+      - name: Launch the app from a fresh home
+        id: launch
+        timeout-minutes: 8
+        run: |
+          set -uo pipefail
+          source scripts/ci/lib/signing-runtime-guard.sh
+          cmux_signing_runtime_guard
+          mkdir -p "$RUNNER_TEMP/launch-smoke"
+          for attempt in 1 2; do
+            out="$RUNNER_TEMP/launch-smoke/attempt-$attempt"
+            mkdir -p "$out" && chmod 0777 "$out"
+            status=0
+            scripts/ci/frame-pacing/console-session.sh python3 -I "$GITHUB_WORKSPACE/scripts/ci/fresh_home_launch_smoke.py" \
+              --dmg "$RUNNER_TEMP/smoke.dmg" --out "$out" || status=$?
+            echo "$status" > "$RUNNER_TEMP/launch-smoke/status-$attempt"
+            echo "attempt $attempt: exit $status"
+            [ "$status" = 1 ] || break
+          done
+          exit 0
+
+      - name: Record the verdict
+        id: verdict
+        if: always()
+        run: |
+          set -euo pipefail
+          dir="$RUNNER_TEMP/launch-smoke"
+          mkdir -p "$dir"
+          # Until the tooling has landed on TOOLING_REF (a pull request into main runs
+          # before it does), there is nothing to judge with: no verdict, so nothing
+          # rolls back and nothing is cached, and the build is smoked once it lands.
+          if [ ! -f scripts/ci/nightly_next_launch_smoke.py ]; then
+            echo "::notice title=nightly-next launch smoke::$TOOLING_REF has no scripts/ci/nightly_next_launch_smoke.py yet; build $BUILD is not judged"
+            exit 0
+          fi
+          verdict="$(python3 scripts/ci/nightly_next_launch_smoke.py verdict --dir "$dir")"
+          echo "verdict=$verdict" >> "$GITHUB_OUTPUT"
+          last=""
+          for d in "$dir"/attempt-1 "$dir"/attempt-2; do [ -d "$d" ] && last="$d"; done
+          if [ -n "$last" ]; then
+            crashes=()
+            for f in "$last"/*.ips "$last"/*.crash; do [ -f "$f" ] && crashes+=(--crash "$f"); done
+            python3 scripts/ci/nightly_next_launch_smoke.py evidence --report "$last/report.json" \
+              --app-log "$last/app.log" ${crashes[@]+"${crashes[@]}"} --out "$dir/evidence.md"
+          else
+            printf 'The smoke did not reach the app (download or setup failed).\n' > "$dir/evidence.md"
+          fi
+          {
+            echo "### cmux NEXT build $BUILD: launch smoke $verdict"
+            echo ""
+            cat "$dir/evidence.md"
+          } >> "$GITHUB_STEP_SUMMARY"
+          case "$verdict" in
+            crash) echo "::error title=nightly-next launch smoke::build $BUILD crashed at launch on every attempt" ;;
+            cannot-run) echo "::notice title=nightly-next launch smoke::build $BUILD could not be smoked here; nothing is rolled back" ;;
+          esac
+
+      - name: Upload the evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: launch-smoke-${{ env.BUILD }}
+          path: ${{ runner.temp }}/launch-smoke
+          if-no-files-found: ignore
+          retention-days: 3
+
+  # A crash: roll the feed back (only while the build is still its newest) and
+  # file the evidence. A pull request's run does neither.
+  crashed:
+    needs: [pick, smoke]
+    if: always() && github.event_name != 'pull_request' && needs.smoke.outputs.verdict == 'crash'
+    # One at a time per build, so two reports of one crash never open two issues.
+    concurrency:
+      group: nightly-next-launch-crash-${{ needs.pick.outputs.build }}
+      cancel-in-progress: false
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-4vcpu-ubuntu-2404' }}
+    timeout-minutes: 5
+    permissions:
+      actions: write # dispatch nightly-next-rollback.yml
+      contents: read
+      issues: write # the launch crash issue
+    env:
+      BUILD: ${{ needs.pick.outputs.build }}
+      GH_TOKEN: ${{ github.token }}
+      RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ env.TOOLING_REF }}
+          persist-credentials: false
+          sparse-checkout: scripts/ci/nightly_next_launch_smoke.py
+          sparse-checkout-cone-mode: false
+
+      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          name: launch-smoke-${{ env.BUILD }}
+          path: evidence
+
+      - name: Roll the feed back
+        id: rollback
+        run: |
+          set -euo pipefail
+          curl -fsSL --retry 3 -A "cmux-nightly-next-launch-smoke/1" -H "Cache-Control: no-cache" -o live.xml "$NIGHTLY_NEXT_FEED?crashed=$GITHUB_RUN_ID"
+          decision="$(python3 scripts/ci/nightly_next_launch_smoke.py decide --feed live.xml --build "$BUILD" --verdict crash)"
+          echo "$decision"
+          action="$(python3 -c 'import json, sys; print(json.loads(sys.argv[1])["action"])' "$decision")"
+          why="$(python3 -c 'import json, sys; print(json.loads(sys.argv[1])["why"])' "$decision")"
+          echo "why=$why" >> "$GITHUB_OUTPUT"
+          if [ "$action" = rollback ]; then
+            reason="build $BUILD crashed at launch in the fresh-home smoke: $RUN_URL"
+            # if_newest makes the rollback re-check, on its own read, that BUILD is still the
+            # newest, so a build published while the rollback waited in a queue is never pulled.
+            # There is no dispatch without it: a nightly-next branch that predates the input
+            # rejects the dispatch, this step fails, and the issue says the rollback was not made.
+            gh workflow run nightly-next-rollback.yml -R "$GITHUB_REPOSITORY" --ref nightly-next \
+              -f bad_build="$BUILD" -f if_newest="$BUILD" -f reason="$reason"
+            echo "action=requested" >> "$GITHUB_OUTPUT"
+          else
+            echo "action=none" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Also when the dispatch failed: the issue then says the build may still be up.
+      - name: File the crash
+        if: always()
+        env:
+          ACTION: ${{ steps.rollback.outputs.action }}
+          WHY: ${{ steps.rollback.outputs.why }}
+        run: |
+          set -euo pipefail
+          title="nightly-next launch crash: build $BUILD"
+          case "${ACTION:-failed}" in
+            requested) what="The feed rollback was requested (nightly-next-rollback.yml, bad_build $BUILD): $WHY." ;;
+            none) what="No rollback: $WHY." ;;
+            *) what="Requesting the rollback FAILED, so the build may still be on the feed. Roll it back: \`gh workflow run nightly-next-rollback.yml -R manaflow-ai/cmux --ref nightly-next -f bad_build=$BUILD -f if_newest=$BUILD -f reason=\"launch crash\"\`" ;;
+          esac
+          {
+            echo "cmux NEXT build $BUILD crashed at launch from a fresh home on every attempt: $RUN_URL"
+            echo ""
+            echo "$what"
+            echo ""
+            echo "The crash report and the full app output are in the run's \`launch-smoke-$BUILD\` artifact."
+            echo ""
+            cat evidence/evidence.md 2>/dev/null || echo "(no evidence was uploaded)"
+          } > body.md
+          number="$(gh issue list -R "$GITHUB_REPOSITORY" --state all --search "\"$title\" in:title" --json number,title \
+            --jq "map(select(.title == \"$title\"))[0].number // empty")"
+          if [ -n "$number" ]; then
+            gh issue comment "$number" -R "$GITHUB_REPOSITORY" --body-file body.md
+          else
+            gh issue create -R "$GITHUB_REPOSITORY" --title "$title" --body-file body.md
+          fi
+
+  # Mark a passed build smoked, and a cannot-run build for the rest of the pick's
+  # UTC hour only. A crash is never marked: while the build is still the feed's
+  # newest, the next tick smokes it again and asks for the rollback again, so a
+  # rollback that failed is retried until the build is off.
+  record:
+    needs: [pick, smoke]
+    if: always() && github.event_name == 'schedule' && (needs.smoke.outputs.verdict == 'pass' || needs.smoke.outputs.verdict == 'cannot-run')
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-4vcpu-ubuntu-2404' }}
+    timeout-minutes: 5
+    permissions: {}
+    steps:
+      - name: Record the verdict
+        env:
+          VERDICT: ${{ needs.smoke.outputs.verdict }}
+        run: |
+          mkdir -p smoke-verdict
+          printf '%s\n' "$VERDICT" > smoke-verdict/verdict
+
+      - name: Save the smoked marker
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: smoke-verdict
+          key: ${{ env.SMOKE_CACHE_PREFIX }}${{ needs.pick.outputs.build }}${{ needs.smoke.outputs.verdict == 'cannot-run' && format('-cannot-run-{0}', needs.pick.outputs.hour) || '' }}

scripts/ci/owned_pool_rescue.py

@@ -443,6 +443,8 @@
     ".github/workflows/iroh-v2.yml",
     # The post-publication canary's Mac job takes the side label on attempt 1.
     ".github/workflows/nightly-next-canary.yml",
+    # The post-publish launch smoke's Mac job takes the side label on attempt 1.
+    ".github/workflows/nightly-next-launch-smoke.yml",
     ".github/workflows/relay-tls.yml",
     ".github/workflows/reload-build.yml",
     ".github/workflows/remote-daemon.yml",

tests/test_ci_macos_xcode_selection.py

@@ -52,6 +52,8 @@
         "runs the relay TLS verifier under Xcode 16.2 / Swift 6, below the "
         "floor on purpose",
     ("remote-daemon.yml", "remote-daemon-macos-tests"): "Go only; no Xcode",
+    ("nightly-next-launch-smoke.yml", "smoke"):
+        "launches the published nightly-next DMG; builds nothing, no Xcode",
     ("ci.yml", "claude-wrapper"): "shell wrapper tests only; no Xcode",
     ("cmux-tui.yml", "lint"): "Rust only; no Xcode",
     ("cmux-tui.yml", "test"): "Rust only; no Xcode",

tests/test_nightly_delivery_concurrency.py

@@ -32,6 +32,9 @@
         "the scheduled 'feed' tick canaries the feed's newest build when it starts; a dispatch for one "
         "build and a pull request run have their own groups"),
     ("nightly-next-feed-watch.yml", ""): "a schedule-only watch that reads the feed when it starts",
+    ("nightly-next-launch-smoke.yml", ""): (
+        "a schedule or dispatch run has a group of its own run id; only a pull request's runs share one, "
+        "per pull request, and such a run delivers nothing (it never rolls back or files)"),
     ("nightly-next-tick.yml", "request"): (
         "the tick dispatches no build itself (test_delivery_coalescers_have_fixed_backstops...); a sha "
         "request has its own group"),