ci(nightly): headroom under the size budget (14 more run blocks, step notes into scripts, expressions through env)

scripts/ci/nightly-steps/app-assemble-computer-use-helper.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# nightly.yml build-nightly-app: "Assemble the Computer Use helper". Moved out of the workflow to keep nightly.yml under
+# its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The step's notes, moved from nightly.yml:
+# The Computer Use helper that onboarding grants (cx-urd.6, D19). Only a
+# com.cmuxterm.cua app at Contents/Library with the team's Developer ID
+# signature is used, so it is assembled here for Codesign apps to sign and
+# notarize-nightly-dmg.sh to notarize. It builds with the reviewed rustc
+# that cmux-tui pins: the notices check (rust-std) needs a reviewed rustc.
+set -euo pipefail
+app="build-universal/Build/Products/Release/cmux.app"
+toolchain="$(sed -n 's/^channel = "\(.*\)"$/\1/p' cmux-tui/rust-toolchain.toml)"
+rustup toolchain install "$toolchain" --profile minimal
+# Only the slices a variant ships: nightly-next and fast builds are arm64.
+archs="${HELPER_ARCHS}"
+RUSTUP_TOOLCHAIN="$toolchain" ./scripts/build-cmux-cua.sh \
+  --output "$app/Contents/Resources/bin/cmux-cua" --helper-app --archs "$archs"
+# The helper app is what ships; build-cmux-cua.sh's bin copy is scratch.
+rm -f "$app/Contents/Resources/bin/cmux-cua" "$app/Contents/Resources/bin/cmux-cua-LICENSE.md"
+./scripts/ci/verify-computer-use-helper.sh layout "$app"

scripts/ci/nightly-steps/app-bound-rust-targets.sh

@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 # nightly.yml build-nightly-app: "Bound the kept Rust target directories (warm nightly-next builder)". Moved out of the workflow to keep nightly.yml under
 # its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The step's notes, moved from nightly.yml:
+# The warm builder keeps the Rust target directories (above). Cargo
+# never deletes old artifacts, so start them over above 30 GiB.
 set -euo pipefail
 dirs=(cmux-tui/target Native/OptChat/optchat-chief/target cmux-next-ios/rust/target)
 kib=0

scripts/ci/nightly-steps/app-build-mobile-host.sh

@@ -1,6 +1,15 @@
 #!/usr/bin/env bash
 # nightly.yml build-nightly-app: "Build the cmux-next iPhone host (cmux-mobile-host)". Moved out of the workflow to keep nightly.yml under
 # its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The step's notes, moved from nightly.yml:
+# The cmux mobile iPhone host (cmux-next-ios/rust/crates/cmux-mobile-host), which
+# Settings > Mobile runs: the "Bundle cmux-mobile-host" phase script
+# copies it to Resources/bin, and sign-cmux-bundle-helpers.sh signs it
+# with the app's Developer ID, hardened runtime and the helper
+# entitlements before notarization, like every Mach-O there.
+# It builds in the background too, and is bundled with the Chief after
+# xcodebuild. An unsigned resume restores the app and builds neither.
 set -euo pipefail
 hostd_bin="$RUNNER_TEMP/cmux-next-mobile-host/cmux-mobile-host"
 python3 scripts/ci/background_build.py start --state "$RUNNER_TEMP/nightly-helper-builds" mobile-host -- \

scripts/ci/nightly-steps/app-build-release.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+# nightly.yml build-nightly-app: "Build nightly app (Release)". Moved out of the workflow to keep nightly.yml under
+# its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+set -euo pipefail
+# cx-f58x: compile out browser-import code as `decide` chose (the
+# notary tests, and every nightly-next build while
+# NIGHTLY_NEXT_BROWSER_DATA_IMPORT=compiled-out).
+echo "browser import variant: $IMPORT_VARIANT (flags: ${SWIFT_IMPORT_FLAGS:-none}, empty browser-sources.json: $EMPTY_BROWSER_SOURCES)"
+notary_test_flags=(build)
+if [[ "$EMPTY_BROWSER_SOURCES" == true ]]; then
+  printf '%s\n' '{"schemaVersion": 1, "engines": {}, "browsers": []}' \
+    > Packages/macOS/CmuxNext/Sources/CmuxNextBrowserImport/Resources/browser-sources.json
+fi
+swift_defines=""
+if [[ -n "$SWIFT_IMPORT_FLAGS" ]]; then
+  [[ "$SWIFT_IMPORT_FLAGS" =~ ^(-DCMUX_NO_[A-Z_]+ ?)+$ ]] || { echo "unexpected import flags: $SWIFT_IMPORT_FLAGS" >&2; exit 1; }
+  swift_defines="$SWIFT_IMPORT_FLAGS"
+fi
+if [[ "${BUILD_TRACK:-}" == nightly-next ]]; then
+  # cmux NEXT Nightly carries the main-thread blocking detector, record only
+  # (scripts/cmux-next/lib/nightly-swift-defines.sh, plans/cmux-next/threading-model.md).
+  # shellcheck source=scripts/cmux-next/lib/nightly-swift-defines.sh
+  source scripts/cmux-next/lib/nightly-swift-defines.sh
+  swift_defines="${swift_defines:+$swift_defines }$CMUX_NIGHTLY_NEXT_SWIFT_DEFINES"
+fi
+if [[ -n "$swift_defines" ]]; then
+  echo "OTHER_SWIFT_FLAGS defines: $swift_defines"
+  # shellcheck disable=SC2016 # $(inherited) is an Xcode build setting, not a shell expansion
+  notary_test_flags=("OTHER_SWIFT_FLAGS=\$(inherited) $swift_defines" build)
+fi
+if [[ "$NIGHTLY_COMPILE_ARCHS" == "arm64" ]]; then
+  archs="arm64"
+  only_active="YES"
+else
+  archs="arm64 x86_64"
+  only_active="NO"
+fi
+# No -jobs 1: the two whole-module Swift compiles ran back to back (#12704).
+# The Rust helper phases are prebuilt beside the compile at low priority;
+# a failed prebuild leaves the work to the phases. Cargo, rustup and CEF
+# roots come from the job's HOME so the bytes match a Blacksmith build
+# (build 74986f8b recorded /Users/cmux paths).
+if [ "${BUILD_TRACK:-}" = nightly-next ]; then
+  export CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
+  export RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}"
+  CMUX_CEF_CACHE_DIR="${CMUX_CEF_CACHE_DIR:-$(scripts/cmux-next/cef-cache-root.sh)}"
+  export CMUX_CEF_CACHE_DIR
+fi
+# Bundle cmux-tui bundles none (deferred): "Bundle the fast cmux-tui client"
+# or build-sign-notarize-nightly's "Bundle the cmux-tui client" installs
+# this tree's client (install-cmux-tui-client.sh).
+# Reuse the managed fleet CAS; unavailable nodes keep the local cache.
+fleet_cache_settings=()
+local_cache_settings=()
+while IFS= read -r setting; do
+  fleet_cache_settings+=("$setting")
+  case "$setting" in
+    COMPILATION_CACHE_CAS_PATH=*|COMPILATION_CACHE_ENABLE_PLUGIN=*|COMPILATION_CACHE_REMOTE_SERVICE_PATH=*) ;;
+    *) local_cache_settings+=("$setting") ;;
+  esac
+done < <(CMUX_CI_DISABLE_COMPILATION_CACHE="${DECIDE_COLD_CACHE_DISABLE}" bash scripts/ci/fleet-compilation-cache.sh)
+# Keep retry classification identical to the other Release compile
+# paths: only compiler diagnostics that identify the CAS/cache as
+# failing may fall back to a local cache.
+# shellcheck source=scripts/cmux-next/lib/compilation-cache.sh
+source scripts/cmux-next/lib/compilation-cache.sh
+run_release_build() {
+  local disable_fleet_cache="${1:-0}"
+  local settings=()
+  if [[ "$disable_fleet_cache" != 1 ]]; then
+    settings=("${fleet_cache_settings[@]}")
+  else
+    settings=("${local_cache_settings[@]}")
+  fi
+  python3 scripts/ci/run-with-helper-prebuild.py \
+    --derived-data build-universal --configuration Release --archs "$archs" \
+    --log "$RUNNER_TEMP/helper-prebuild.log" -- \
+  ./scripts/ci/run-xcodebuild-with-diagnostics.sh -- \
+    env CMUX_NEXT_TUI_MODE=deferred CMUX_SKIP_ZIG_BUILD=1 xcodebuild -scheme cmux -configuration Release -derivedDataPath build-universal \
+    -destination 'generic/platform=macOS' \
+    -clonedSourcePackagesDirPath .spm-cache \
+    -packageCachePath "$PWD/.spm-cache/.package-cache" \
+    -showBuildTimingSummary \
+    ARCHS="$archs" \
+    ONLY_ACTIVE_ARCH="$only_active" \
+    COMPILATION_CACHE_ENABLE_CACHING=YES \
+    COMPILATION_CACHE_LIMIT_SIZE=3221225472 \
+    ${settings[@]+"${settings[@]}"} \
+    COMPILER_INDEX_STORE_ENABLE=NO \
+    CODE_SIGNING_ALLOWED=NO ASSETCATALOG_COMPILER_APPICON_NAME="$CHANNEL_ICON_NAME" \
+    "${notary_test_flags[@]}"
+}
+# A live CAS failure is recoverable: preserve the build's incremental inputs,
+# then retry once with Xcode's local cache. Other compile failures remain red.
+set +e
+run_release_build 0 2>&1 | tee "$RUNNER_TEMP/nightly-release-build.log"
+release_status=${PIPESTATUS[0]}
+set -e
+if [[ "$release_status" -ne 0 ]] && cmux_next_runtime_cas_failed "$RUNNER_TEMP/nightly-release-build.log"; then
+  echo "::warning::managed fleet compilation cache failed; retrying the Release build with the local cache"
+  run_release_build 1
+else
+  exit "$release_status"
+fi

scripts/ci/nightly-steps/app-bundle-chief-and-mobile-host.sh

@@ -1,6 +1,13 @@
 #!/usr/bin/env bash
 # nightly.yml build-nightly-app: "Bundle the OptChat Chief and the iPhone host". Moved out of the workflow to keep nightly.yml under
 # its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The step's notes, moved from nightly.yml:
+# The Chief and the iPhone host built beside the Release build. Their Xcode
+# phases found no CMUX_NEXT_*_BIN and bundled nothing; this joins each
+# background build (a failed one fails here) and runs the same phase
+# script on the built app, with the variables Xcode gave it, so the copy,
+# its checks and Resources/bin are what the phase made before.
 set -euo pipefail
 state="$RUNNER_TEMP/nightly-helper-builds"
 if [[ "$NIGHTLY_COMPILE_ARCHS" == "arm64" ]]; then archs="arm64"; else archs="arm64 x86_64"; fi

scripts/ci/nightly-steps/app-bundle-cmux-tui-client.sh

@@ -1,6 +1,12 @@
 #!/usr/bin/env bash
 # nightly.yml build-nightly-app: "Bundle the cmux-tui client". Moved out of the workflow to keep nightly.yml under
 # its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The step's notes, moved from nightly.yml:
+# One attested commit manifest for every variant, also on a reused app: the
+# resolved commit, or while the tree publishes the one await-tree-client.sh finds.
+# install-cmux-tui-client.sh verifies the manifest's build-provenance
+# attestation with gh before trusting it.
 set -euo pipefail
 app="build-universal/Build/Products/Release/cmux.app"
 cmux_tui_commit="$CMUX_TUI_CLIENT_COMMIT"

scripts/ci/nightly-steps/app-bundle-fast-cmux-tui-client.sh

@@ -1,6 +1,12 @@
 #!/usr/bin/env bash
 # nightly.yml build-nightly-app: "Bundle the fast cmux-tui client". Moved out of the workflow to keep nightly.yml under
 # its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The step's notes, moved from nightly.yml:
+# The fast arm64 build bundles the client it just compiled. Full builds
+# take the published client in build-sign-notarize-nightly instead: the
+# commit is resolved by resolve-nightly-cmux-tui-client while this job
+# compiles, so the history deepen (1.5 to 11 minutes) is off this path.
 set -euo pipefail
 app="build-universal/Build/Products/Release/cmux.app"
 local_client="$GITHUB_WORKSPACE/cmux-tui/target/aarch64-apple-darwin/release/cmux-tui"

scripts/ci/nightly-steps/app-compilation-cache-key.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# nightly.yml build-nightly-app: "Compute Xcode compilation cache key". Moved out of the workflow to keep nightly.yml under
+# its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+set -euo pipefail
+# The key is OS, arch and toolchain. An owned mini (runner
+# <member>-glaeda[-K]) runs the same Xcode build as an ephemeral
+# runner but compiles in another workspace path, so its cache
+# entries would mostly miss there and push the ephemeral entry out
+# of the prefix restore. Fold its workspace path into the toolchain
+# hash: each lineage restores only its own, and the ephemeral key is
+# unchanged.
+identity="$(xcodebuild -version)"
+case "${RUNNER_NAME:-}" in
+  *-glaeda | *-glaeda-[0-9]*)
+    identity="$identity"$'\n'"owned-workspace:$GITHUB_WORKSPACE"
+    if [ "${CACHE_KEY_TRACK}" = nightly-next ]; then identity="$identity"$'\n'"track:nightly-next"; fi
+    ;;
+esac
+echo "toolchain=$(printf '%s\n' "$identity" | shasum -a 256 | awk '{print $1}')" >> "$GITHUB_OUTPUT"

scripts/ci/nightly-steps/app-drop-iroh-stub.sh

@@ -1,6 +1,16 @@
 #!/usr/bin/env bash
 # nightly.yml build-nightly-app: "Drop the empty Iroh.framework stub (nightly-next)". Moved out of the workflow to keep nightly.yml under
 # its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The step's notes, moved from nightly.yml:
+# In Release Xcode merges iroh-ffi's static code into Contents/MacOS/cmux
+# and still embeds Iroh.framework: a 35 KB stub (only __TEXT and
+# __LINKEDIT, no exported symbol) that no binary of the app loads (otool
+# -L of build 3939848657701), and whose LC_ID_DYLIB is a random
+# swbuild.tmp folder, so its bytes differ in every build. The app
+# launched without it (nx-remote job 1020-050356-5d14cf). Drop it; the
+# hosted sign job signs the app afterwards. The checks refuse a
+# framework that exports code or that something loads.
 set -euo pipefail
 app="build-universal/Build/Products/Release/cmux.app"
 fw="$app/Contents/Frameworks/Iroh.framework"

scripts/ci/nightly-steps/app-fetch-chromium-engine.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# nightly.yml build-nightly-app: "Fetch the Chromium engine". Moved out of the workflow to keep nightly.yml under
+# its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The step's notes, moved from nightly.yml:
+# Fills the ensure-cef cache with the pinned Chromium engine (cef-manifest.json) from private R2 with the
+# read-only key (CMUX_CEF_READ_TOKEN's release as fallback); only this background fetch gets the key. No
+# source: the app builds without the engine ("Report the embedded Chromium engine" flags it). "Finish
+# fetching the Chromium engine" joins it before the Release build and fails where it failed.
+# x86_64: a universal build embeds both engine slices (none: arm64 only). The remote browser host
+# (Contents/Helpers, phone tabs) comes from R2 too; missing, the phase warns; a digest mismatch fails.
+python3 scripts/ci/background_build.py start --state "$RUNNER_TEMP/nightly-helper-builds" chromium-engine -- \
+  bash -c 'set -e
+    scripts/cmux-next/ensure-cef.sh --optional
+    scripts/cmux-next/ensure-cef.sh --optional --arch x86_64
+    if [[ -x scripts/cmux-next/ensure-remote-browser-host.sh ]]; then
+      CMUX_RB_HOST_NO_STORE=1 scripts/cmux-next/ensure-remote-browser-host.sh --optional >/dev/null
+    fi'

scripts/ci/nightly-steps/app-fetch-tree-companions.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 # nightly.yml build-nightly-app: "Fetch the resolved cmux-tui tree companions". Moved out of the workflow to keep nightly.yml under
 # its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The step's notes, moved from nightly.yml:
+# The resolved tree's attested acpmux companion: an input of Bundle acpmux.
 set -euo pipefail
 scripts/cmux-next/pin-cmux-tui.sh fetch-acpmux
 acpmux_bin="$(scripts/cmux-next/pin-cmux-tui.sh acpmux-path)"

scripts/ci/nightly-steps/app-finish-dsym-upload.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# nightly.yml build-nightly-app: "Finish the dSYM upload to Sentry". Moved out of the workflow to keep nightly.yml under
+# its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+set -uo pipefail
+state="$RUNNER_TEMP/sentry-dsyms"
+[ -p "$state/status.fifo" ] || { echo "the dSYM upload to Sentry was never started" >&2; exit 1; }
+status=""
+read -r status <"$state/status.fifo" || true
+cat "$state/upload.log"
+rm -rf "$state"
+[ "$status" = 0 ] || { echo "the dSYM upload to Sentry failed (status ${status:-none})" >&2; exit 1; }

scripts/ci/nightly-steps/app-resolve-daemon-metadata.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# nightly.yml build-nightly-app: "Resolve immutable SSH daemon metadata". Moved out of the workflow to keep nightly.yml under
+# its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+set -euo pipefail
+app="build-universal/Build/Products/Release/cmux.app"
+base_version="$(/usr/libexec/PlistBuddy -c 'Print :CFBundleShortVersionString' "$app/Contents/Info.plist")"
+build="${RESOLVED_DAEMON_BUILD}"
+[[ "$build" =~ ^[0-9]+$ ]]
+track="${DAEMON_DECIDE_TRACK}"
+version="$(python3 scripts/ci/nightly_version.py marketing --track "$track" \
+  --project-version "$base_version" --build "$build" --next-base "$(python3 scripts/ci/nightly_version.py next-base)")"
+if [[ "$track" == "nightly-next" && "${DECIDE_SHOULD_PUBLISH}" == "true" ]]; then
+  # cmux-next and main's NIGHTLY install as the same app, so a published
+  # cmux-next build must be above every build on both feeds: no feed
+  # ever offers a downgrade after a switch between the two.
+  feeds=()
+  for appcast in appcast.xml appcast-arm64.xml appcast-x86_64.xml; do
+    feeds+=(--feed-if-present "${NIGHTLY_NEXT_FEED_BASE}/${appcast}" --feed "${NIGHTLY_NEXT_LEGACY_FEED_BASE}/${appcast}" --feed "https://files.cmux.com/nightly/${appcast}")
+  done
+  python3 scripts/ci/nightly_version.py check-build --build "$build" "${feeds[@]}"
+fi
+echo "build=$build" >> "$GITHUB_OUTPUT"
+echo "version=$version" >> "$GITHUB_OUTPUT"

scripts/ci/nightly-steps/app-start-release-derived-data-empty.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# nightly.yml build-nightly-app: "Start Release DerivedData empty". Moved out of the workflow to keep nightly.yml under
+# its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The step's notes, moved from nightly.yml:
+# The build's settings key the kept DerivedData (nightly_build_state.py BUILD_SETTINGS).
+set -euo pipefail
+# Every runner but one starts from an empty tree. An incremental
+# build can keep stale autolink and force-load data in objects it
+# did not recompile after the module graph changed, and those objects
+# then link into the app. Warm state that stays: the compilation cache restored into it
+# below (content-keyed, with explicit module dependencies in each
+# key) and the Swift package checkouts in .spm-cache.
+# The exception is the dedicated nightly-next builder, which runs only
+# this build: nightly_build_state.py keeps its DerivedData and builds
+# each tip on it, starting clean when the last build did not finish,
+# the toolchain, a build setting, a lockfile or a submodule changed, a
+# module- or link-shaping file (Package.swift, *.pbxproj, ...), an import
+# or an #if changed, or every 10th build. Output `mode`: clean or incremental.
+scripts/ci/clear-dirs.sh build-universal
+mkdir -p build-universal
+if [ "${DECIDE_TRACK}" = nightly-next ]; then
+  # A trusted worker keeps only its package checkouts and its
+  # content-keyed compilation cache on disk, and deletes the build
+  # state that earlier revisions kept. Elsewhere NEXT restores no
+  # compilation cache, so its build is fully cold.
+  python3 scripts/ci/nightly_build_state.py prepare nightly-next
+  exit 0
+fi
+case "${RUNNER_NAME:-}" in
+  *-glaeda|*-glaeda-[0-9]*)
+    # Earlier jobs linked build-universal to a tree kept per channel
+    # and toolchain. Delete this channel's trees, so no later change
+    # can link one back and they stop using disk. Another channel's
+    # trees (for example cmux-nightly-rc-*) are not touched.
+    # The name ends in the 64-digit toolchain hash (plus the suffix
+    # clear-dirs.sh adds when a delete did not finish), so the match
+    # never reaches another name with the same prefix. The
+    # nightly-next track keeps its state under
+    # ~/Library/Caches/cmux-nightly-next, which this never touches.
+    shopt -s nullglob
+    stale=()
+    for tree in "$HOME"/Library/Developer/Xcode/DerivedData/cmux-nightly-${DECIDE_CHANNEL}-*; do
+      if [[ "${tree##*/}" =~ ^cmux-nightly-${DECIDE_CHANNEL}-[0-9a-f]{64}(\.stale\..+)?$ ]]; then
+        stale+=("$tree")
+      fi
+    done
+    if [ "${#stale[@]}" -gt 0 ]; then
+      scripts/ci/clear-dirs.sh "${stale[@]}"
+    fi
+    ;;
+esac