nightly: Sparkle delta updates per architecture track (77 MB → ~11 MB daily)

.github/workflows/ci.yml

@@ -296,6 +296,12 @@ jobs:
       - name: Validate nightly notarization behavior
         run: ./tests/test_notarize_nightly_dmg.sh
 
+      - name: Validate Sparkle delta finalization
+        run: ./tests/test_finalize_sparkle_deltas.sh
+
+      - name: Validate previous nightly build fetch
+        run: ./tests/test_fetch_previous_nightly_dmgs.sh
+
       - name: Validate Computer Use helper notarization behavior
         run: ./tests/test_notarize_computer_use_helper.sh
 

.github/workflows/nightly.yml

@@ -410,7 +410,8 @@ jobs:
     runs-on: ${{ vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26' }}
     # Start the standalone helper submission early so
     # Apple's first wait overlaps useful work; finish it before the final DMG.
-    timeout-minutes: 60
+    # Delta generation against two previous builds adds about 15 minutes.
+    timeout-minutes: 80
     # One universal Xcode build feeds every variant: the thin variants are cut
     # from it with lipo, so each user downloads only their architecture.
     strategy:
@@ -757,23 +758,43 @@ jobs:
             "$NIGHTLY_DMG_RELEASE" \
             "$NIGHTLY_DMG_IMMUTABLE"
 
+      - name: Fetch previous nightly builds for delta updates
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          python3 scripts/ci/fetch-previous-nightly-dmgs.py \
+            --repo manaflow-ai/cmux \
+            --release-tag nightly \
+            --variant "$NIGHTLY_VARIANT" \
+            --exclude-build "$NIGHTLY_BUILD" \
+            --count 2 \
+            --out previous-nightlies
+
       - name: Generate Sparkle appcasts (nightly)
         env:
           SPARKLE_PRIVATE_KEY: ${{ secrets.SPARKLE_PRIVATE_KEY }}
+          # Deltas from the two previous builds of this track, named
+          # cmux-nightly-macos-<variant>-<new>-<old>.delta so tracks never collide.
+          SPARKLE_PREVIOUS_ARCHIVES_DIR: previous-nightlies
+          SPARKLE_MAXIMUM_DELTAS: "2"
         run: |
           if [ -z "$SPARKLE_PRIVATE_KEY" ]; then
             echo "Missing SPARKLE_PRIVATE_KEY secret" >&2
             exit 1
           fi
-          ./scripts/sparkle_generate_appcast.sh "$NIGHTLY_DMG_IMMUTABLE" nightly "$NIGHTLY_APPCAST"
-          ls -l "$NIGHTLY_DMG_IMMUTABLE" "$NIGHTLY_DMG_RELEASE" "$NIGHTLY_APPCAST"
+          SPARKLE_DELTA_NAME_PREFIX="cmux-nightly-macos-${NIGHTLY_VARIANT}-" \
+            ./scripts/sparkle_generate_appcast.sh "$NIGHTLY_DMG_IMMUTABLE" nightly "$NIGHTLY_APPCAST"
+          ls -l "$NIGHTLY_DMG_IMMUTABLE" "$NIGHTLY_DMG_RELEASE" "$NIGHTLY_APPCAST" ./*.delta 2>/dev/null || true
+          grep -c 'sparkle:deltas' "$NIGHTLY_APPCAST" || echo "no deltas in $NIGHTLY_APPCAST (no previous build of this track)"
 
       - name: Upload nightly variant artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: cmux-nightly-variant-${{ matrix.variant }}
           path: |
             cmux-nightly-macos-*.dmg
+            cmux-nightly-macos-*.delta
             appcast*.xml
           if-no-files-found: error
           retention-days: 3
@@ -845,6 +866,7 @@ jobs:
           name: cmux-nightly-${{ needs.decide.outputs.short_sha }}
           path: |
             nightly-out/cmux-nightly-macos*.dmg
+            nightly-out/cmux-nightly-macos*.delta
             nightly-out/appcast*.xml
           if-no-files-found: error
 
@@ -876,6 +898,7 @@ jobs:
             - legacy `cmux-nightly-macos.dmg` and feed `appcast.xml` currently serve the `${{ needs.decide.outputs.legacy_variant }}` build; installs on the legacy feed move to their architecture's feed on their next update check
           files: |
             nightly-out/cmux-nightly-macos-*-${{ github.run_id }}*.dmg
+            nightly-out/cmux-nightly-macos-*-${{ github.run_id }}*.delta
             nightly-out/cmux-nightly-macos-arm64.dmg
             nightly-out/cmux-nightly-macos-x86_64.dmg
             nightly-out/cmux-nightly-macos.dmg

scripts/ci/fetch-previous-nightly-dmgs.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Download the newest immutable nightly DMGs of one track for Sparkle delta generation.
+
+    fetch-previous-nightly-dmgs.py --repo manaflow-ai/cmux --release-tag nightly \
+        --variant arm64 --exclude-build 3371353821401 --count 2 --out previous-nightlies
+
+Assets are matched by name (cmux-nightly-macos-<variant>-<build>.dmg), ordered by
+build number, and the newest <count> below --exclude-build are downloaded with
+`gh release download`. No matching asset is not an error: the first per-track
+publish simply ships without deltas.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    parser.add_argument("--repo", required=True)
+    parser.add_argument("--release-tag", required=True)
+    parser.add_argument("--variant", required=True)
+    parser.add_argument("--exclude-build", type=int, default=0)
+    parser.add_argument("--count", type=int, default=2)
+    parser.add_argument("--out", required=True)
+    args = parser.parse_args()
+
+    pattern = re.compile(rf"^cmux-nightly-macos-{re.escape(args.variant)}-(?P<build>\d+)\.dmg$")
+    proc = subprocess.run(
+        ["gh", "release", "view", args.release_tag, "--repo", args.repo, "--json", "assets"],
+        capture_output=True,
+        text=True,
+    )
+    if proc.returncode != 0:
+        print(f"warning: could not list release assets: {proc.stderr.strip()}", file=sys.stderr)
+        return 0
+    candidates: list[tuple[int, str]] = []
+    for asset in json.loads(proc.stdout or "{}").get("assets", []):
+        match = pattern.match(asset["name"])
+        if not match:
+            continue
+        build = int(match.group("build"))
+        if args.exclude_build and build >= args.exclude_build:
+            continue
+        candidates.append((build, asset["name"]))
+    candidates.sort(reverse=True)
+    chosen = candidates[: args.count]
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    for build, name in chosen:
+        print(f"downloading previous {args.variant} build {build}: {name}")
+        subprocess.run(
+            ["gh", "release", "download", args.release_tag, "--repo", args.repo, "--pattern", name, "--dir", str(out), "--clobber"],
+            check=True,
+        )
+    print(f"fetched {len(chosen)} previous {args.variant} build(s) into {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/finalize-sparkle-deltas.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Rename Sparkle delta files to release-asset-safe, track-specific names and
+# rewrite the matching enclosure URLs in the appcast.
+#
+#   scripts/ci/finalize-sparkle-deltas.sh <appcast.xml> <archives-dir> <out-dir> <name-prefix>
+#
+# generate_appcast writes "<app name><new>-<old>.delta" (e.g. "cmux NIGHTLY123-100.delta")
+# into <archives-dir>. The name carries a space, which GitHub rewrites on
+# upload, and it is identical for every per-architecture track. Each delta is
+# moved to <out-dir>/<name-prefix><new>-<old>.delta and the appcast's URL for
+# it (percent-encoded by generate_appcast) is rewritten to the new basename.
+set -euo pipefail
+
+if [ "$#" -ne 4 ]; then
+  sed -n '2,11p' "$0" >&2
+  exit 2
+fi
+APPCAST="$1"
+ARCHIVES_DIR="$2"
+OUT_DIR="$3"
+PREFIX="$4"
+[ -f "$APPCAST" ] || { echo "error: appcast not found: $APPCAST" >&2; exit 1; }
+[ -d "$ARCHIVES_DIR" ] || { echo "error: archives dir not found: $ARCHIVES_DIR" >&2; exit 1; }
+mkdir -p "$OUT_DIR"
+
+python3 - "$APPCAST" "$ARCHIVES_DIR" "$OUT_DIR" "$PREFIX" <<'EOF'
+import os, re, sys, urllib.parse
+appcast, archives, out_dir, prefix = sys.argv[1:5]
+xml = open(appcast, encoding="utf-8").read()
+renamed = 0
+for name in sorted(os.listdir(archives)):
+    if not name.endswith(".delta"):
+        continue
+    match = re.search(r"(\d+-\d+)\.delta$", name)
+    if not match:
+        print(f"error: unexpected delta name {name!r}", file=sys.stderr)
+        sys.exit(1)
+    new_name = f"{prefix}{match.group(1)}.delta"
+    encoded = urllib.parse.quote(name)
+    if encoded not in xml and name not in xml:
+        print(f"error: appcast does not reference delta {name!r}", file=sys.stderr)
+        sys.exit(1)
+    xml = xml.replace(encoded, new_name).replace(name, new_name)
+    os.replace(os.path.join(archives, name), os.path.join(out_dir, new_name))
+    print(f"delta {name} -> {new_name}")
+    renamed += 1
+open(appcast, "w", encoding="utf-8").write(xml)
+print(f"finalized {renamed} delta(s)")
+EOF

scripts/prune_nightly_release_assets.py

@@ -18,6 +18,8 @@
 IMMUTABLE_ASSET_PATTERNS = [
     re.compile(r"^cmux-nightly-macos-(?P<build>\d+)\.dmg$"),
     re.compile(r"^cmux-nightly-macos-(?:arm64|x86_64|universal)-(?P<build>\d+)\.dmg$"),
+    # Sparkle delta from an older build to <build>; pruned together with <build>.
+    re.compile(r"^cmux-nightly-macos-(?:arm64|x86_64|universal)-(?P<build>\d+)-\d+\.delta$"),
     re.compile(r"^cmux-nightly-universal-macos-(?P<build>\d+)\.dmg$"),
 ]
 

scripts/sign-cmux-bundle.sh

@@ -79,6 +79,13 @@ if [[ "$SIGN_MODE" == "all" || "$SIGN_MODE" == "all-except-computer-use" ]]; the
   for helper_dir in bin libexec; do
     for helper in "$APP_PATH/Contents/Resources/$helper_dir"/*; do
       [[ -f "$helper" && -x "$helper" ]] || continue
+      # Scripts are sealed by the bundle signature. Code-signing them directly
+      # stores the signature in an extended attribute, which Sparkle's
+      # BinaryDelta refuses to diff, so it would block delta updates.
+      if ! /usr/bin/file -b "$helper" | grep -q 'Mach-O'; then
+        echo "==> leaving non-Mach-O helper $(basename "$helper") to the bundle seal"
+        continue
+      fi
       echo "==> signing helper $(basename "$helper")"
       /usr/bin/codesign "${COMMON[@]}" --entitlements "$HELPER_ENTITLEMENTS" "$helper"
     done
@@ -171,6 +178,7 @@ done
 for helper_dir in bin libexec; do
   for helper in "$APP_PATH/Contents/Resources/$helper_dir"/*; do
     [[ -f "$helper" && -x "$helper" ]] || continue
+    /usr/bin/file -b "$helper" | grep -q 'Mach-O' || continue
     if /usr/bin/codesign -d --entitlements :- "$helper" 2>&1 \
          | grep -q "application-identifier"; then
       echo "error: helper $(basename "$helper") unexpectedly carries application-identifier" >&2

scripts/sparkle_generate_appcast.sh

@@ -62,6 +62,23 @@ archives_dir="$work_dir/archives"
 mkdir -p "$archives_dir"
 cp "$DMG_PATH" "$archives_dir/$(basename "$DMG_PATH")"
 
+# Delta updates: older archives of the same track placed next to the new one
+# make generate_appcast emit <sparkle:deltas> items plus .delta files, so a
+# machine on a recent build downloads only what changed.
+delta_args=()
+if [[ -n "${SPARKLE_PREVIOUS_ARCHIVES_DIR:-}" ]]; then
+  previous_count=0
+  for previous in "$SPARKLE_PREVIOUS_ARCHIVES_DIR"/*.dmg; do
+    [[ -f "$previous" ]] || continue
+    cp "$previous" "$archives_dir/$(basename "$previous")"
+    previous_count=$((previous_count + 1))
+  done
+  echo "Previous archives available for deltas: $previous_count"
+  if [[ "$previous_count" -gt 0 ]]; then
+    delta_args=(--maximum-deltas "${SPARKLE_MAXIMUM_DELTAS:-2}")
+  fi
+fi
+
 key_file="$work_dir/sparkle_ed_key"
 # Ensure base64 padding (keys may be stored without trailing '=')
 padded_key="$SPARKLE_PRIVATE_KEY"
@@ -76,6 +93,7 @@ generated_appcast_path="$archives_dir/$(basename "$OUT_PATH")"
   --ed-key-file "$key_file" \
   --download-url-prefix "$DOWNLOAD_URL_PREFIX" \
   --full-release-notes-url "$RELEASE_NOTES_URL" \
+  "${delta_args[@]}" \
   "$archives_dir"
 
 if [[ ! -f "$generated_appcast_path" ]]; then
@@ -101,22 +119,39 @@ if ! grep -q 'sparkle:edSignature' "$generated_appcast_path"; then
   echo "  EdDSA signature: ${SIGNATURE:0:20}..."
   echo "  DMG length: $DMG_LENGTH"
 
-  # Inject sparkle:edSignature and correct length into the enclosure element
-  python3 -c "
-import sys
-xml = open('$generated_appcast_path').read()
-sig = '$SIGNATURE'
-length = '$DMG_LENGTH'
-# Add edSignature to enclosure
-xml = xml.replace(
-    'type=\"application/octet-stream\"',
-    'sparkle:edSignature=\"' + sig + '\" length=\"' + length + '\" type=\"application/octet-stream\"'
-)
-open('$generated_appcast_path', 'w').write(xml)
-print('  Injected edSignature into appcast.xml')
-"
+  # Inject sparkle:edSignature and correct length into the full-archive
+  # enclosure only. Deltas cannot be signed here, and unsigned deltas would be
+  # rejected by Sparkle at install time, so drop them from the feed.
+  python3 - "$generated_appcast_path" "$SIGNATURE" "$DMG_LENGTH" "$(basename "$DMG_PATH")" <<'EOF'
+import re, sys, urllib.parse
+path, sig, length, dmg_name = sys.argv[1:5]
+xml = open(path, encoding="utf-8").read()
+deltas = re.compile(r"\s*<sparkle:deltas>.*?</sparkle:deltas>", re.S)
+if deltas.search(xml):
+    print("  Dropping unsigned delta entries from the appcast")
+    xml = deltas.sub("", xml)
+needle = re.compile(r'<enclosure(?P<attrs>[^>]*url="[^"]*' + re.escape(urllib.parse.quote(dmg_name)) + r'"[^>]*)/>')
+match = needle.search(xml) or re.search(r'<enclosure(?P<attrs>[^>]*url="[^"]*' + re.escape(dmg_name) + r'"[^>]*)/>', xml)
+if not match:
+    print("  error: full-archive enclosure not found in appcast", file=sys.stderr)
+    sys.exit(1)
+attrs = match.group("attrs")
+if "sparkle:edSignature" not in attrs:
+    attrs = attrs.replace('type="application/octet-stream"', 'sparkle:edSignature="' + sig + '" length="' + length + '" type="application/octet-stream"')
+    xml = xml[:match.start()] + "<enclosure" + attrs + "/>" + xml[match.end():]
+open(path, "w", encoding="utf-8").write(xml)
+print("  Injected edSignature into the full-archive enclosure")
+EOF
+  rm -f "$archives_dir"/*.delta
 fi
 
+# generate_appcast names deltas after the app ("cmux NIGHTLY<new>-<old>.delta"),
+# which collides across per-architecture tracks and gets mangled by GitHub
+# release assets. Rename them after the archive and rewrite the appcast URLs.
+delta_prefix="${SPARKLE_DELTA_NAME_PREFIX:-$(basename "$DMG_PATH" .dmg | sed -E 's/-[0-9]+$//')-}"
+"$(dirname "$0")/ci/finalize-sparkle-deltas.sh" \
+  "$generated_appcast_path" "$archives_dir" "$(cd "$(dirname "$OUT_PATH")" && pwd)" "$delta_prefix"
+
 cp "$generated_appcast_path" "$OUT_PATH"
 echo "Generated appcast at $OUT_PATH"
 

tests/test_fetch_previous_nightly_dmgs.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Behavioral check for scripts/ci/fetch-previous-nightly-dmgs.py through a fake gh.
+set -euo pipefail
+ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+TOOL="$ROOT_DIR/scripts/ci/fetch-previous-nightly-dmgs.py"
+TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/cmux-fetch-previous.XXXXXX")"
+trap 'rm -rf "$TMP_DIR"' EXIT
+mkdir -p "$TMP_DIR/bin"
+cat > "$TMP_DIR/bin/gh" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+printf 'gh %s\n' "$*" >> "$CMUX_TEST_CALL_LOG"
+case "$1 $2" in
+  "release view")
+    cat <<'JSON'
+{"assets":[
+ {"name":"cmux-nightly-macos-arm64-300.dmg"},
+ {"name":"cmux-nightly-macos-arm64-100.dmg"},
+ {"name":"cmux-nightly-macos-x86_64-200.dmg"},
+ {"name":"cmux-nightly-macos-arm64-200.dmg"},
+ {"name":"cmux-nightly-macos-arm64-300-200.delta"},
+ {"name":"cmux-nightly-macos-arm64.dmg"},
+ {"name":"cmux-nightly-macos-arm64-50.dmg"}
+]}
+JSON
+    ;;
+  "release download")
+    while [ $# -gt 0 ]; do case "$1" in --pattern) name="$2"; shift;; --dir) dir="$2"; shift;; esac; shift; done
+    printf 'fixture' > "$dir/$name"
+    ;;
+esac
+EOF
+chmod +x "$TMP_DIR/bin/gh"
+export CMUX_TEST_CALL_LOG="$TMP_DIR/calls.log"
+fail() { echo "FAIL: $*" >&2; exit 1; }
+PATH="$TMP_DIR/bin:$PATH" python3 "$TOOL" --repo o/r --release-tag nightly --variant arm64 --exclude-build 300 --count 2 --out "$TMP_DIR/prev" >/dev/null
+[ -f "$TMP_DIR/prev/cmux-nightly-macos-arm64-200.dmg" ] || fail "newest previous arm64 build was not downloaded"
+[ -f "$TMP_DIR/prev/cmux-nightly-macos-arm64-100.dmg" ] || fail "second previous arm64 build was not downloaded"
+[ ! -f "$TMP_DIR/prev/cmux-nightly-macos-arm64-300.dmg" ] || fail "the current build was downloaded as a previous build"
+[ ! -f "$TMP_DIR/prev/cmux-nightly-macos-arm64-50.dmg" ] || fail "more than --count builds were downloaded"
+[ ! -f "$TMP_DIR/prev/cmux-nightly-macos-x86_64-200.dmg" ] || fail "another track's build was downloaded"
+[ "$(grep -c '^gh release download' "$CMUX_TEST_CALL_LOG")" -eq 2 ] || fail "expected exactly two downloads"
+# First publish of a track: nothing to fetch, still exit 0 with an empty dir.
+: > "$CMUX_TEST_CALL_LOG"
+PATH="$TMP_DIR/bin:$PATH" python3 "$TOOL" --repo o/r --release-tag nightly --variant universal --exclude-build 300 --count 2 --out "$TMP_DIR/none" >/dev/null || fail "no previous build must not fail the job"
+[ -z "$(ls -A "$TMP_DIR/none")" ] || fail "unexpected download for a track with no history"
+echo "PASS: previous nightly builds are fetched per track, newest first, excluding the current build"