ci(nightly): take the web bundles and the pending-tree cmux-tui download off build-nightly-app's critical path

.github/nightly-step-coverage.json

@@ -70,9 +70,12 @@
         "Restore Xcode compilation cache": {"plumbing": true, "note": "local action: restores or saves a cache with actions/cache, WarpBuilds/cache or R2 and reports a receipt; no product check. Re-review this entry if the action gains a check"},
         "Install compilation dependencies": {"plumbing": true},
         "Check node is on PATH (cmux-next web bundles)": {"pr": "ci-guards.yml:workflow-guard-tests:Validate universal nightly workflow", "note": "tests/test_ensure_setup_node.py runs scripts/ci/ensure-setup-node.sh against a fake tool cache and dist"},
+        "Source key of the web bundles (cmux-next)": {"plumbing": true},
+        "Restore the published web bundles (cmux-next)": {"plumbing": true, "note": "Restores the same keyed bundle as cmux-next.yml; build-web-bundles.sh --from checks it before use"},
         "Build the web bundles (cmux-next)": {"pr": "cmux-next.yml:web-bundles:Build the web bundles for this source key"},
         "Stand in for the acpmux companion while the tree publishes": {"plumbing": true},
         "Fetch the resolved cmux-tui tree companions": {"pr": "cmux-next.yml:checks:Script tests (pin-cmux-tui, embed-cef, ensure-cef store)"},
+        "Prefetch the cmux-tui client while the tree publishes": {"plumbing": true, "note": "fills the client cache beside the compile; Bundle the cmux-tui client re-verifies every slice against the attested manifest"},
         "Build in-tree acpmux for the compilation cache": {"pr": "cmux-next.yml:cmux-scheme-compile:Build in-tree acpmux for the app bundle"},
         "Prepare the OptChat Chief cache key": {"plumbing": true, "note": "combines the portable source key with the Mac SDK, compiler and environment identity"},
         "Restore the OptChat Chief cache": {"plumbing": true, "note": "restores an exact source and environment keyed Chief artifact; no product check"},

.github/workflows/nightly.yml

@@ -2136,6 +2136,16 @@ jobs:
           f="$RUNNER_TEMP/acpmux-input/acpmux"; mkdir -p "${f%/*}"; rm -f "$f.ref"; echo alias > "$f"
           echo "CMUX_NEXT_ACPMUX_BIN=$f" >> "$GITHUB_ENV"
 
+      - name: Prefetch the cmux-tui client while the tree publishes
+        if: (inputs.nightly_next_unsigned_run == '' && needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.build_only != 'true') && steps.nightly-product.outputs.hit != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.tree_pending == 'true'
+        continue-on-error: true
+        env:
+          PREFETCH_GH_TOKEN: ${{ github.token }}
+          CMUX_TUI_CLIENT_KEY: ${{ needs.resolve-nightly-cmux-tui-client.outputs.key }}
+          TREE_CANDIDATES: ${{ needs.resolve-nightly-cmux-tui-client.outputs.tree_candidates }}
+          CMUX_TUI_CLIENT_CACHE: ${{ runner.temp }}/cmux-tui-client
+        run: bash scripts/ci/nightly-steps/app-prefetch-pending-cmux-tui.sh start
+
       - name: Build in-tree acpmux for the compilation cache
         if: (inputs.nightly_next_unsigned_run == '') && steps.nightly-product.outputs.hit != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.tree_pending != 'true'
         env:
@@ -2237,6 +2247,26 @@ jobs:
         timeout-minutes: 12
         run: scripts/ci/ensure-setup-node.sh 24.22.1
 
+      # build-nightly-inputs can start after this step (run 41570134320 queued it 7.6 min, so the
+      # bundles built here for 106 s); feat-cmux-next's pushes publish them to R2 by source key.
+      - name: Source key of the web bundles (cmux-next)
+        id: app-web-key
+        if: (inputs.nightly_next_unsigned_run == '' && (needs.decide.outputs.track == 'nightly-next')) && steps.nightly-product.outputs.hit != 'true'
+        run: scripts/ci/nightly-steps/inputs-web-bundles.sh key
+
+      - name: Restore the published web bundles (cmux-next)
+        id: app-web-r2
+        if: (inputs.nightly_next_unsigned_run == '' && (needs.decide.outputs.track == 'nightly-next')) && steps.nightly-product.outputs.hit != 'true'
+        continue-on-error: true
+        uses: ./.github/actions/cache-restore
+        env:
+          CI_CACHE_R2_PUBLIC_URL: https://ci-cache-next.cmux.com
+          CI_CACHE_R2_NAMESPACE: any
+        with:
+          backend: r2
+          path: web-bundles-out
+          key: cmux-next-web-bundles-v1-${{ steps.app-web-key.outputs.key }}
+
       # A pattern, not a name: an artifact that is not there yet downloads
       # nothing instead of failing.
       - name: Download the web bundles built beside the app (cmux-next)
@@ -2250,6 +2280,8 @@ jobs:
 
       - name: Build the web bundles (cmux-next)
         if: (inputs.nightly_next_unsigned_run == '' && (needs.decide.outputs.track == 'nightly-next')) && steps.nightly-product.outputs.hit != 'true'
+        env:
+          RESTORED: ${{ steps.app-web-r2.outputs.cache-hit }}
         run: bash scripts/ci/nightly-steps/app-build-web-bundles.sh
 
       - name: Finish fetching the Chromium engine
@@ -4404,9 +4436,7 @@ jobs:
           retention-days: 3
 
 
-      # cx-f58x notary tests leave text receipts only: the repository is
-      # public, so no app, DMG or recovery artifact of a diagnostic build or
-      # of the production-identity probe is ever uploaded.
+      # cx-f58x: text receipts only (notary-test-receipt.sh).
       - name: Write notary test receipt
         id: notary-receipt
         if: always() && needs.decide.outputs.notary_test == 'true' && steps.notarize-nightly.outcome != 'skipped'
@@ -4416,40 +4446,9 @@ jobs:
           EMPTY_BROWSER_SOURCES: ${{ needs.decide.outputs.empty_browser_sources }}
           NOTARY_PROBE: ${{ needs.decide.outputs.notary_probe }}
           NOTARIZE_OUTCOME: ${{ steps.notarize-nightly.outcome }}
-        run: |
-          set -uo pipefail
-          out="notary-receipt"
-          mkdir -p "$out"
-          {
-            echo "run: $GITHUB_RUN_ID attempt ${GITHUB_RUN_ATTEMPT:-1}"
-            echo "commit: ${{ needs.resolve-nightly-cmux-tui-client.outputs.build_sha }}"
-            echo "bundle id: $CHANNEL_BUNDLE_ID"
-            echo "production probe: $NOTARY_PROBE"
-            echo "import variant: $IMPORT_VARIANT"
-            echo "swift flags: ${SWIFT_IMPORT_FLAGS:-none}"
-            echo "empty browser-sources.json: $EMPTY_BROWSER_SOURCES"
-            echo "notarize step: $NOTARIZE_OUTCOME"
-            if [ -f "$NIGHTLY_DMG_RELEASE" ]; then echo "dmg sha256: $(shasum -a 256 "$NIGHTLY_DMG_RELEASE" | cut -d' ' -f1)"; fi
-            if [ -f "$NIGHTLY_DMG_RELEASE.notarization.state" ]; then grep -E '^(submission_id|status)=' "$NIGHTLY_DMG_RELEASE.notarization.state" || true; fi
-          } > "$out/receipt.txt"
-          [ -f "$NIGHTLY_DMG_RELEASE.notarization.log" ] && cp "$NIGHTLY_DMG_RELEASE.notarization.log" "$out/notarization.log"
-          /usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$CHANNEL_APP_PATH/Contents/Info.plist" > "$out/bundle-id.txt" 2>&1
-          if /usr/libexec/PlistBuddy -c "Print :SUPublicEDKey" "$CHANNEL_APP_PATH/Contents/Info.plist" >/dev/null 2>&1; then
-            echo "SUPublicEDKey present" > "$out/sparkle.txt"
-          else
-            echo "SUPublicEDKey absent (Sparkle disabled)" > "$out/sparkle.txt"
-          fi
-          codesign -dvv "$CHANNEL_APP_PATH" > "$out/codesign.txt" 2>&1
-          xcrun stapler validate "$CHANNEL_APP_PATH" > "$out/stapler-app.txt" 2>&1
-          [ -f "$NIGHTLY_DMG_RELEASE" ] && xcrun stapler validate "$NIGHTLY_DMG_RELEASE" > "$out/stapler-dmg.txt" 2>&1
-          spctl -a -vv -t exec "$CHANNEL_APP_PATH" > "$out/spctl-app.txt" 2>&1
-          [ -f "$NIGHTLY_DMG_RELEASE" ] && spctl -a -vv -t open --context context:primary-signature "$NIGHTLY_DMG_RELEASE" > "$out/spctl-dmg.txt" 2>&1
-          syspolicy_check distribution "$CHANNEL_APP_PATH" > "$out/syspolicy-check.txt" 2>&1
-          cat "$out/receipt.txt" "$out/sparkle.txt" "$out/stapler-app.txt"
-          { echo "### cx-f58x notary test receipt"; echo '```'; cat "$out/receipt.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
-          # Receipts are text only: refuse to upload anything else.
-          if find "$out" -type f ! -name '*.txt' ! -name '*.log' | grep -q .; then echo "non-text receipt file" >&2; exit 1; fi
-          exit 0
+          BUILD_SHA: ${{ needs.resolve-nightly-cmux-tui-client.outputs.build_sha }}
+        # An unsigned resume of an older tree may predate the script: the workflow commit's copy.
+        run: s=scripts/ci/nightly-steps/notary-test-receipt.sh; [ -f "$s" ] || s=".nightly-signing-control/$s"; bash "$s"
 
 
       - name: Upload notary test receipt

scripts/ci/nightly-steps/app-build-web-bundles.sh

@@ -1,11 +1,17 @@
 #!/usr/bin/env bash
 # nightly.yml build-nightly-app: "Build the web bundles (cmux-next)". Moved out of the workflow to keep nightly.yml under
 # its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+# The hand-off from build-nightly-inputs first, then the bundles R2 holds for this source key
+# (RESTORED, "Restore the published web bundles (cmux-next)"), then a build here.
 set -euo pipefail
 if python3 scripts/ci/nightly_inputs.py install-web "$RUNNER_TEMP/nightly-inputs/web"; then
   exit 0
 else
   status=$?
 fi
 [ "$status" = 3 ] || echo "::warning title=Web bundles hand-off::installing the handed-off web bundles failed (status $status); building them here"
+if [ "${RESTORED:-}" = true ] && scripts/cmux-next/build-web-bundles.sh --from web-bundles-out; then
+  echo "installed the web bundles R2 holds for this source key"
+  exit 0
+fi
 scripts/cmux-next/build-web-bundles.sh --force

scripts/ci/nightly-steps/app-bundle-cmux-tui-client.sh

@@ -15,13 +15,17 @@ if [[ "$TREE_PENDING" == true ]]; then
   cmux_tui_commit="$(scripts/cmux-next/await-tree-client.sh "$CMUX_TUI_CLIENT_KEY" "${c[@]}")"
 fi
 [[ "$cmux_tui_commit" =~ ^[0-9a-f]{40}$ ]] || { echo "resolve-nightly-cmux-tui-client produced no commit" >&2; exit 1; }
-# The prefetch "Fetch the resolved cmux-tui tree companions" started beside the compile
-# (app-fetch-tree-companions.sh). A failed or unfinished one is ended and its cache
-# dropped, so the install below downloads everything itself, as it did without it.
+# The prefetch "Fetch the resolved cmux-tui tree companions" (app-fetch-tree-companions.sh), or while the
+# tree publishes "Prefetch the cmux-tui client while the tree publishes" (app-prefetch-pending-cmux-tui.sh),
+# started beside the compile. A failed or unfinished one is ended and its cache dropped, so the install
+# below downloads everything itself, as it did without it. A pending tree's prefetch waits for the tree as
+# await-tree-client.sh above did, so by now it is at most one 20 s poll behind: 60 s more for it.
 state="$RUNNER_TEMP/nightly-helper-builds"
+join_timeout=120
+[[ "$TREE_PENDING" == true ]] && join_timeout=180
 if [[ -f "$state/cmux-tui-client.pid" ]]; then
   set +e
-  python3 scripts/ci/background_build.py join --state "$state" cmux-tui-client --timeout 120 2>&1 \
+  python3 scripts/ci/background_build.py join --state "$state" cmux-tui-client --timeout "$join_timeout" 2>&1 \
     | sed 's/^::error::/::warning::/'
   joined=${PIPESTATUS[0]}
   set -e

scripts/ci/nightly-steps/app-prefetch-pending-cmux-tui.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# nightly.yml build-nightly-app: "Prefetch the cmux-tui client while the tree publishes". Moved out of the workflow
+# to keep nightly.yml under its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The tree was pending in 10 of 15 nightly-next runs (2026-11-03), so "Fetch the resolved cmux-tui tree companions"
+# skipped its prefetch and "Bundle the cmux-tui client" waited for the tree and downloaded the client after the
+# compile: 45-139 s there against 15-33 s with the prefetch. This starts the same wait (await-tree-client.sh) and
+# the same install into CMUX_TUI_CLIENT_COMMIT's cache in the background, beside the compile.
+# app-bundle-cmux-tui-client.sh joins it and still resolves the commit and installs itself; the install reuses a
+# cached slice only when its sha256 matches the attested manifest, so a prefetch of another commit only downloads.
+#   start   start the background wait and install
+#   run     (the background command) wait for the tree's client, then install it on a scratch bundle
+set -euo pipefail
+scratch="$RUNNER_TEMP/cmux-tui-prefetch/cmux.app"
+case "${1:-}" in
+  start)
+    rm -rf "${scratch%/*}"
+    mkdir -p "$scratch/Contents"
+    GH_TOKEN="${PREFETCH_GH_TOKEN:-}" \
+      python3 scripts/ci/background_build.py start --state "$RUNNER_TEMP/nightly-helper-builds" cmux-tui-client -- \
+      bash "$0" run \
+      || echo "::warning title=cmux-tui prefetch::could not start; Bundle the cmux-tui client downloads it"
+    ;;
+  run)
+    read -r -a candidates <<<"$TREE_CANDIDATES"
+    commit="$(scripts/cmux-next/await-tree-client.sh "$CMUX_TUI_CLIENT_KEY" "${candidates[@]}")"
+    exec ./scripts/install-cmux-tui-client.sh "$scratch" \
+      --manifest-url "https://files.cmux.com/cmux-tui/${commit}/manifest.json" \
+      --expected-commit "$commit" \
+      --require-capability wireguard-hub \
+      --require-acpmux
+    ;;
+  *) echo "usage: $0 start | run" >&2; exit 2 ;;
+esac

scripts/ci/nightly-steps/notary-test-receipt.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# nightly.yml build-sign-notarize-nightly: "Write notary test receipt". Moved out of the workflow to keep nightly.yml under
+# its size budget (tests/test_workflow_file_size.py); the step's env: still sets every input.
+#
+# The step's notes, moved from nightly.yml:
+# cx-f58x notary tests leave text receipts only: the repository is
+# public, so no app, DMG or recovery artifact of a diagnostic build or
+# of the production-identity probe is ever uploaded.
+set -uo pipefail
+out="notary-receipt"
+mkdir -p "$out"
+{
+  echo "run: $GITHUB_RUN_ID attempt ${GITHUB_RUN_ATTEMPT:-1}"
+  echo "commit: $BUILD_SHA"
+  echo "bundle id: $CHANNEL_BUNDLE_ID"
+  echo "production probe: $NOTARY_PROBE"
+  echo "import variant: $IMPORT_VARIANT"
+  echo "swift flags: ${SWIFT_IMPORT_FLAGS:-none}"
+  echo "empty browser-sources.json: $EMPTY_BROWSER_SOURCES"
+  echo "notarize step: $NOTARIZE_OUTCOME"
+  if [ -f "$NIGHTLY_DMG_RELEASE" ]; then echo "dmg sha256: $(shasum -a 256 "$NIGHTLY_DMG_RELEASE" | cut -d' ' -f1)"; fi
+  if [ -f "$NIGHTLY_DMG_RELEASE.notarization.state" ]; then grep -E '^(submission_id|status)=' "$NIGHTLY_DMG_RELEASE.notarization.state" || true; fi
+} > "$out/receipt.txt"
+[ -f "$NIGHTLY_DMG_RELEASE.notarization.log" ] && cp "$NIGHTLY_DMG_RELEASE.notarization.log" "$out/notarization.log"
+/usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$CHANNEL_APP_PATH/Contents/Info.plist" > "$out/bundle-id.txt" 2>&1
+if /usr/libexec/PlistBuddy -c "Print :SUPublicEDKey" "$CHANNEL_APP_PATH/Contents/Info.plist" >/dev/null 2>&1; then
+  echo "SUPublicEDKey present" > "$out/sparkle.txt"
+else
+  echo "SUPublicEDKey absent (Sparkle disabled)" > "$out/sparkle.txt"
+fi
+codesign -dvv "$CHANNEL_APP_PATH" > "$out/codesign.txt" 2>&1
+xcrun stapler validate "$CHANNEL_APP_PATH" > "$out/stapler-app.txt" 2>&1
+[ -f "$NIGHTLY_DMG_RELEASE" ] && xcrun stapler validate "$NIGHTLY_DMG_RELEASE" > "$out/stapler-dmg.txt" 2>&1
+spctl -a -vv -t exec "$CHANNEL_APP_PATH" > "$out/spctl-app.txt" 2>&1
+[ -f "$NIGHTLY_DMG_RELEASE" ] && spctl -a -vv -t open --context context:primary-signature "$NIGHTLY_DMG_RELEASE" > "$out/spctl-dmg.txt" 2>&1
+syspolicy_check distribution "$CHANNEL_APP_PATH" > "$out/syspolicy-check.txt" 2>&1
+cat "$out/receipt.txt" "$out/sparkle.txt" "$out/stapler-app.txt"
+{ echo "### cx-f58x notary test receipt"; echo '```'; cat "$out/receipt.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
+# Receipts are text only: refuse to upload anything else.
+if find "$out" -type f ! -name '*.txt' ! -name '*.log' | grep -q .; then echo "non-text receipt file" >&2; exit 1; fi
+exit 0

scripts/ci/nightly_build_state.py

@@ -101,6 +101,17 @@
 # Top-level trees the Release app build never reads (no project or package in it references them), so
 # a change there leaves the kept DerivedData valid.
 OUTSIDE_APP_BUILD = ('Prototypes/',)
+# A Swift package's Tests/ tree holds its test targets and the support targets only they link
+# (tests/test_nightly_build_state.py checks every Package.swift for that). The Release build of the
+# app scheme compiles none of them, so an import or a structure line there leaves the kept
+# DerivedData valid: run 41570134320 compiled clean for 562 s on an import change in
+# Packages/Shared/CmuxMessagesLab/Tests/MessagesLabHomeTests/HomeAgentMessageAvatarTests.swift.
+PACKAGE_TESTS = re.compile(r'^Packages/[^/]+/[^/]+/Tests/')
+PACKAGE_TESTS_GLOB = ':(exclude,glob)Packages/*/*/Tests/**'
+
+
+def outside_app_build(name: str) -> bool:
+    return name.startswith(OUTSIDE_APP_BUILD) or PACKAGE_TESTS.match(name) is not None
 NOTICES_CACHE = 'notices-crates'
 NOTICES_CACHE_MAX_BYTES = 8 << 30
 
@@ -174,12 +185,12 @@ def structure_changed(workspace: Path, since: str) -> str | None:
     except subprocess.CalledProcessError:
         return None
     changed = next((name for name in names
-                    if STRUCTURE_PATHS.search(name) and not name.startswith(OUTSIDE_APP_BUILD)), '')
+                    if STRUCTURE_PATHS.search(name) and not outside_app_build(name)), '')
     if changed:
         return changed
     try:
         diff = git(workspace, 'diff', '-U0', '--no-renames', since, 'HEAD', '--', *IMPORT_SOURCES,
-                   *(f':(exclude){tree}' for tree in OUTSIDE_APP_BUILD))
+                   *(f':(exclude){tree}' for tree in OUTSIDE_APP_BUILD), PACKAGE_TESTS_GLOB)
     except subprocess.CalledProcessError:
         return None
     current = ''

tests/test_nightly_build_state.py

@@ -195,6 +195,38 @@ def test_a_prototype_package_or_import_change_stays_incremental(self):
         self.commit({'Prototypes/Lab/Package.swift': '// removed\n', 'Sources/a.swift': 'import Foundation\n'})
         self.assertEqual(self.build()[0], 'mode=clean')
 
+    def test_a_package_tests_import_or_structure_change_stays_incremental(self):
+        self.build()
+        self.commit({'Packages/Shared/Lab/Tests/LabTests/a.swift': 'import Testing\n#if DEBUG\n#endif\n',
+                     'Packages/macOS/App/Tests/Support/module.modulemap': 'module X {}\n'})
+        self.assertEqual(self.build()[0], 'mode=incremental')
+        self.commit({'Packages/Shared/Lab/Sources/Lab/a.swift': 'import Foundation\n'})
+        self.assertEqual(self.build()[0], 'mode=clean')
+
+    def test_package_tests_trees_feed_only_test_targets(self):
+        """PACKAGE_TESTS skips Packages/*/*/Tests/: no product and no non-test target outside Tests/ may
+        depend on a target whose sources live there, or a change there would reach the app."""
+        import re
+        for package in sorted(ROOT.glob('Packages/*/*/Package.swift')):
+            text = package.read_text()
+            pieces = re.split(r'\.(target|testTarget|executableTarget|binaryTarget|plugin|library|executable)\(', text)
+            in_tests, others = set(), []
+            for kind, body in zip(pieces[1::2], pieces[2::2]):
+                name = re.search(r'name:\s*"([^"]+)"', body)
+                path = re.search(r'path:\s*"([^"]+)"', body)
+                if name is None:
+                    continue
+                if kind in ('library', 'executable'):
+                    others.append((kind, name.group(1), body.split(')', 1)[0]))
+                elif path and re.match(r'(\./)?Tests(/|$)', path.group(1)):
+                    in_tests.add(name.group(1))
+                elif kind != 'testTarget':
+                    others.append((kind, name.group(1), body.split('swiftSettings')[0]))
+            for kind, name, body in others:
+                for target in in_tests:
+                    with self.subTest(package=str(package.relative_to(ROOT)), target=name, depends_on=target):
+                        self.assertNotRegex(body, rf'"{re.escape(target)}"')
+
     def test_an_import_change_makes_it_clean(self):
         for name, text in (('Sources/a.swift', 'import Foundation\nlet a = 1\n'),
                            ('Sources/b.swift', '@_exported import Bar\n'),