Route nightly DMG signing through fleet Macs

.github/warm-sources.json

@@ -171,6 +171,8 @@
     "nightly.yml:refresh-test-compilation-cache": {"class": "warm-seed", "warm": ["deriveddata-reuse", "ssd-cas", "actions-cache"]},
     "nightly.yml:sign-nightly-identity-legacy": {"class": "mac-sign", "warm": [], "exempt": "signing and notarization only: consumes the app or DMG an earlier job built, compiles nothing"},
     "nightly.yml:signer-build": {"class": "linux-build", "warm": ["actions-cache"]},
+    "nightly.yml:signer-build-macos": {"class": "mac-build", "warm": ["actions-cache"]},
+    "nightly.yml:signer-dmg": {"class": "mac-sign", "warm": [], "exempt": "signing and notarization only: consumes the DMG and app artifacts earlier jobs built, compiles nothing"},
     "nightly.yml:signer-dmg-image": {"class": "mac-sign", "warm": [], "exempt": "signing and notarization only: consumes the app or DMG an earlier job built, compiles nothing"},
     "nightly.yml:verify-final-dmg-nightly": {"class": "mac-sign", "warm": [], "exempt": "verification only: checks a signed app or DMG an earlier job built, compiles nothing"},
     "nightly.yml:verify-nightly-identity-legacy": {"class": "mac-sign", "warm": [], "exempt": "verification only: checks a signed app or DMG an earlier job built, compiles nothing"},

.github/workflows/nightly.yml

@@ -5402,6 +5402,55 @@ jobs:
           retention-days: 1
           overwrite: true
 
+  # rcodesign is an OS-native binary. signer-app stays on fleet Linux, while
+  # the DMG signer gets a matching macOS build for its owned-Mac/Blacksmith-Mac
+  # route; a Linux ELF cannot run on the Mac signing path.
+  signer-build-macos:
+    needs: [decide, prepare-nightly-variant, pick-nightly-sign-runner]
+    if: needs.decide.outputs.track == 'nightly-next' && (inputs.sign_pool || vars.CI_NIGHTLY_NEXT_SIGN_POOL == '1') && vars.CMUX_NEXT_CEF_ON_DEMAND != '1' && needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') && needs.decide.outputs.build_only != 'true' && needs.decide.outputs.continue_only != 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.notary_paused != 'true' && needs.decide.outputs.notary_test != 'true' && needs.decide.outputs.notary_probe != 'true' && needs.decide.outputs.variants == '["arm64"]' && inputs.nightly_next_unsigned_run == ''
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || needs.pick-nightly-sign-runner.outputs.attempt == github.run_attempt && needs.pick-nightly-sign-runner.outputs.owned || needs.pick-nightly-sign-runner.outputs.runner || 'blacksmith-6vcpu-macos-26' }}
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    outputs:
+      sha256: ${{ steps.build.outputs.sha256 }}
+      artifact: nightly-signer-macos-${{ github.run_attempt }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+          path: .publish-tooling
+          sparse-checkout: scripts/ci/release-signer
+      - id: cache
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ runner.temp }}/signer
+          key: release-signer-rust-1.95.0-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.publish-tooling/scripts/ci/release-signer/build.sh', '.publish-tooling/scripts/ci/release-signer/apple-codesign-0.29.0.patch') }}
+      - name: Set up Rust
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          set -euo pipefail
+          export PATH="$HOME/.cargo/bin:$PATH"
+          rustup toolchain install 1.95.0 --profile minimal
+          dirname "$(rustup which --toolchain 1.95.0 cargo)" >> "$GITHUB_PATH"
+      - name: Build the macOS signer
+        id: build
+        run: |
+          set -euo pipefail
+          if [ -x "$RUNNER_TEMP/signer/rcodesign" ]; then
+            if command -v shasum >/dev/null 2>&1; then digest="$(shasum -a 256 "$RUNNER_TEMP/signer/rcodesign" | cut -d' ' -f1)"; else digest="$(sha256sum "$RUNNER_TEMP/signer/rcodesign" | cut -d' ' -f1)"; fi
+            echo "sha256=$digest" >> "$GITHUB_OUTPUT"
+          else
+            bash .publish-tooling/scripts/ci/release-signer/build.sh "$RUNNER_TEMP/signer" >> "$GITHUB_OUTPUT"
+          fi
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: nightly-signer-macos-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/signer/rcodesign
+          if-no-files-found: error
+          retention-days: 1
+
   signer-app-notary:
     environment: ${{ needs.decide.outputs.environment }}
     needs: [decide, resolve-nightly-cmux-tui-client, signer-build, signer-app]
@@ -5521,8 +5570,11 @@ jobs:
 
   signer-dmg:
     environment: ${{ needs.decide.outputs.environment }}
-    needs: [decide, build-nightly-app, resolve-nightly-cmux-tui-client, signer-build, signer-app, signer-app-notary, signer-dmg-image]
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
+    needs: [decide, build-nightly-app, resolve-nightly-cmux-tui-client, signer-build, signer-build-macos, signer-app, signer-app-notary, signer-dmg-image, pick-nightly-sign-runner]
+    # The release signer is trusted on an owned Mac first, with the moving Blacksmith
+    # macOS pick as overflow. Its credentials never go to fleet Linux: the release
+    # environment is trusted only on the signing Macs (rulebook routing order).
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || needs.pick-nightly-sign-runner.outputs.attempt == github.run_attempt && needs.pick-nightly-sign-runner.outputs.sign_owned && fromJSON(needs.pick-nightly-sign-runner.outputs.sign_owned) || github.run_attempt > 1 && needs.pick-nightly-sign-runner.outputs.alternate || needs.pick-nightly-sign-runner.outputs.runner || 'blacksmith-6vcpu-macos-26' }}
     timeout-minutes: 40
     permissions:
       contents: read
@@ -5545,9 +5597,46 @@ jobs:
       BUILD_SHORT_SHA: ${{ needs.resolve-nightly-cmux-tui-client.outputs.build_short_sha }}
       SOURCE_COMMIT: ${{ needs.resolve-nightly-cmux-tui-client.outputs.build_sha }}
     steps:
+      # On an owned Mac this is the first step, before checkout or any credential.
+      # The picker gives the mini an exclusive signing claim; a refused claim
+      # fails this attempt and the Blacksmith overflow takes the same job.
+      - name: Hold the whole Mac before any credential
+        id: claim-gate
+        shell: bash
+        run: |
+          owned=""
+          case "${RUNNER_NAME:-}" in *-glaeda|*-glaeda-*) owned=1 ;; esac
+          if [ -d /Users/Shared/cmux-build-fleet ]; then owned=1; fi
+          if [ -n "$owned" ]; then
+            if [ "${GLAEDA_JOB_CLAIM:-}" != exclusive ]; then
+              echo "::error title=Signing runner::This owned Mac did not give the job the whole mini; this run's Blacksmith job signs instead."
+              exit 1
+            fi
+            hooks=$(curl -fsS --proto '=https' --retry 3 --max-time 20 \
+              "https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/${GITHUB_SHA}/scripts/ci/glaeda-signing-hooks.txt") || hooks=""
+            hook="${GLAEDA_HOOK_SHA256:-}"
+            listed=""
+            if [[ "$hook" =~ ^[0-9a-f]{64}$ ]]; then
+              while read -r sha _; do if [ "$sha" = "$hook" ]; then listed=1; fi; done <<< "$hooks"
+            fi
+            if [ -z "$listed" ]; then
+              echo "::error title=Signing runner::glaeda hook ${hook:-unnamed} is not one scripts/ci/glaeda-signing-hooks.txt lists; this run's Blacksmith job signs instead."
+              exit 1
+            fi
+            echo "claim=owned" >> "$GITHUB_OUTPUT"
+          fi
       - *signer-checkout-tools
-      - *signer-install
-      - *signer-install-check
+      - name: Install the macOS signer
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          name: ${{ needs.signer-build-macos.outputs.artifact }}
+          path: ${{ runner.temp }}/signer
+      - name: Install the signer
+        env:
+          SIGNER_SHA256: ${{ needs.signer-build-macos.outputs.sha256 }}
+        run: bash .publish-tooling/scripts/ci/release-signer/install.sh "$RUNNER_TEMP/signer/rcodesign" "$SIGNER_SHA256" >> "$GITHUB_ENV"
+      - name: Claim the whole Mac for signing
+        run: bash .publish-tooling/scripts/ci/signing-runner-claim.sh
       - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
         with:
           name: cmux-nightly-next-signer-unsigned-dmg-${{ needs.resolve-nightly-cmux-tui-client.outputs.build_short_sha }}
@@ -5572,7 +5661,12 @@ jobs:
           SPARKLE_CHANNEL: ${{ vars.NIGHTLY_NEXT_DEV_RING == 'on' && 'cmux-next-dev' || 'cmux-next' }}
           NEXT_SPARKLE_PRIVATE_KEY: ${{ secrets.SPARKLE_NEXT_PRIVATE_KEY }}
         run: |
-          echo "$APP_SHA256  signer-app/signed-app.tar" | sha256sum -c -
+          if command -v sha256sum >/dev/null 2>&1; then
+            echo "$APP_SHA256  signer-app/signed-app.tar" | sha256sum -c -
+          else
+            actual="$(shasum -a 256 signer-app/signed-app.tar | awk '{print $1}')"
+            [ "$actual" = "$APP_SHA256" ] || { echo "signed app digest mismatch" >&2; exit 1; }
+          fi
           bash .publish-tooling/scripts/ci/nightly-steps/signer-appcast.sh
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:

scripts/ci/nightly-steps/signer-appcast.sh

@@ -14,7 +14,10 @@ tar -C "$plist_dir" -xf signer-app/signed-app.tar "$CHANNEL_APP_NAME.app/Content
 (cd "$plist_dir" && python3 -I -m zipfile -c "$RUNNER_TEMP/plist.zip" "$CHANNEL_APP_NAME.app")
 SPARKLE_PRIVATE_KEY="$(bash "$TOOLS/scripts/ci/nightly-sparkle-key.sh")"
 export SPARKLE_PRIVATE_KEY
-env -u NEXT_SPARKLE_PRIVATE_KEY python3 -I "$TOOLS/scripts/ci/tip_appcast_linux.py" SPARKLE_PRIVATE_KEY "$dmg" "$RUNNER_TEMP/plist.zip" \
+# macOS ships LibreSSL, which cannot perform the generator's OpenSSL 3 Ed25519
+# check. The generated signature is verified immediately afterward by the pure
+# Python RFC 8032 verifier, so skip only that redundant OpenSSL check on this path.
+CMUX_SKIP_OPENSSL_VERIFY=1 env -u NEXT_SPARKLE_PRIVATE_KEY python3 -I "$TOOLS/scripts/ci/tip_appcast_linux.py" SPARKLE_PRIVATE_KEY "$dmg" "$RUNNER_TEMP/plist.zip" \
   "$CHANNEL_APP_NAME" "https://github.com/manaflow-ai/cmux/releases/download/$CHANNEL_RELEASE_TAG/" \
   "https://github.com/manaflow-ai/cmux/releases/tag/$CHANNEL_RELEASE_TAG" "$NIGHTLY_BUILD" "$appcast"
 unset SPARKLE_PRIVATE_KEY NEXT_SPARKLE_PRIVATE_KEY

scripts/ci/nightly-steps/signer-dmg.sh

@@ -6,7 +6,12 @@ set -euo pipefail
 : "${TOOLS:?}" "${NIGHTLY_DMG_RELEASE:?}" "${NIGHTLY_DMG_IMMUTABLE:?}" "${UNSIGNED_SHA256:?}"
 TOOLS="$(cd "$TOOLS" && pwd)"
 cd signer-out
-echo "$UNSIGNED_SHA256  $NIGHTLY_DMG_RELEASE" | sha256sum -c -
+if command -v sha256sum >/dev/null 2>&1; then
+  echo "$UNSIGNED_SHA256  $NIGHTLY_DMG_RELEASE" | sha256sum -c -
+else
+  actual="$(shasum -a 256 "$NIGHTLY_DMG_RELEASE" | awk '{print $1}')"
+  [ "$actual" = "$UNSIGNED_SHA256" ] || { echo "unsigned DMG digest mismatch" >&2; exit 1; }
+fi
 bash "$TOOLS/scripts/ci/release-signer/notarize-dmg.sh" "$NIGHTLY_DMG_RELEASE" 2>&1 | tee notary-dmg.log
 id="$(grep -oiE 'submission id:? *[0-9a-f-]{36}' notary-dmg.log | grep -oE '[0-9a-f-]{36}' | head -n 1 || true)"
 cp "$NIGHTLY_DMG_RELEASE" "$NIGHTLY_DMG_IMMUTABLE"

scripts/ci/release-signer/notarize-dmg.sh

@@ -39,7 +39,14 @@ secrets="$(umask 077 && mktemp -d "${TMPDIR:-/tmp}/cmux-dmg-signer.XXXXXX")"
 
 # The signer cannot read the PBES2 PKCS#12 a keychain exports, so openssl
 # turns it into one PEM.
-(umask 077 && printf '%s' "$APPLE_CERTIFICATE_BASE64" | base64 -d > "$secrets/identity.p12") 2>/dev/null ||
+decode_base64() {
+  if base64 --decode >/dev/null 2>&1 </dev/null; then
+    base64 --decode
+  else
+    base64 -D
+  fi
+}
+(umask 077 && printf '%s' "$APPLE_CERTIFICATE_BASE64" | decode_base64 > "$secrets/identity.p12") 2>/dev/null ||
   { echo "error: APPLE_CERTIFICATE_BASE64 is not valid base64" >&2; exit 1; }
 pkcs12_to_pem() {
   (umask 077 && openssl pkcs12 "$@" -passin env:APPLE_CERTIFICATE_PASSWORD -nodes -clcerts \
@@ -50,7 +57,7 @@ rm -f "$secrets/identity.p12"
 unset APPLE_CERTIFICATE_PASSWORD APPLE_CERTIFICATE_BASE64
 # The notary key goes to a file before any signer call, so no signer process
 # inherits it in its environment.
-(umask 077 && printf '%s' "$ASC_API_KEY_P8_BASE64" | base64 -d > "$secrets/AuthKey_${ASC_API_KEY_ID}.p8") 2>/dev/null ||
+(umask 077 && printf '%s' "$ASC_API_KEY_P8_BASE64" | decode_base64 > "$secrets/AuthKey_${ASC_API_KEY_ID}.p8") 2>/dev/null ||
   { echo "error: ASC_API_KEY_P8_BASE64 is not valid base64" >&2; exit 1; }
 [[ -s "$secrets/AuthKey_${ASC_API_KEY_ID}.p8" ]] || { echo "error: ASC_API_KEY_P8_BASE64 decoded to nothing" >&2; exit 1; }
 unset ASC_API_KEY_P8_BASE64
@@ -65,7 +72,9 @@ team="$(awk '/^ *organizationalUnitName *=/ { sub(/^ *organizationalUnitName *=
 want="$APPLE_SIGNING_IDENTITY"
 if [[ "$want" =~ ^[0-9A-Fa-f]{40}$ ]]; then
   fp="$(openssl x509 -noout -fingerprint -sha1 < "$secrets/identity.pem")"; fp="${fp#*=}"
-  [[ "${fp//:/}" == "${want^^}" || "${fp//:/}" == "${want,,}" ]] ||
+  fp_normalized="$(printf '%s' "${fp//:/}" | tr '[:upper:]' '[:lower:]')"
+  want_normalized="$(printf '%s' "$want" | tr '[:upper:]' '[:lower:]')"
+  [[ "$fp_normalized" == "$want_normalized" ]] ||
     { echo "error: the certificate ($cn) is not the SHA-1 APPLE_SIGNING_IDENTITY names" >&2; exit 1; }
 elif [[ "$cn" != *"$want"* ]]; then
   echo "error: the certificate ($cn) does not match APPLE_SIGNING_IDENTITY" >&2; exit 1

scripts/ci/signing_runner.py

@@ -73,7 +73,7 @@ class (glaeda SIGNING_JOBS): every capacity unit and both fast tokens, so no
 # lists equal): the workflow file and the job's display name (its `name:` or id; a matrix adds " (...)").
 SIGNING_JOBS = {
     "nightly.yml": ("build-sign-notarize-nightly", "recover-nightly-next-notarization", "sign-nightly-identity-legacy",
-                    "generate-nightly-deltas"),
+                    "generate-nightly-deltas", "signer-dmg"),
     "release.yml": ("build-sign-notarize",),
     "repair-v0-64-25-helper-rpaths.yml": ("repair",),
     "cmux-next-dev-daily.yml": ("Sign cmux NEXT DEV",),

scripts/ci/tip_appcast_linux.py

@@ -179,7 +179,7 @@ def main(argv: list[str]) -> int:
         sys.exit("error: the 96-byte Sparkle key's public half does not match its private half")
     if plist.get("SUPublicEDKey") != public_b64:
         sys.exit(f"error: the app trusts {plist.get('SUPublicEDKey')}, but the feed would be signed by {public_b64}")
-    if not openssl_verify(public, signature, dmg):
+    if os.environ.get("CMUX_SKIP_OPENSSL_VERIFY") != "1" and not openssl_verify(public, signature, dmg):
         sys.exit("error: openssl does not verify the DMG's EdDSA signature")
     url = url_prefix + os.path.basename(dmg)
     date = email.utils.format_datetime(datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0))

tests/test_ci_nightly_off_blacksmith.py

@@ -45,7 +45,7 @@
     "resolve-nightly-ghostty-cli-helper", "build-nightly-inputs", "pick-nightly-build-runner",
     "attest-nightly-remote-daemon", "resolve-nightly-cmux-tui-client", "prepare-nightly-variant-primary",
     "prepare-nightly-variant", "pick-nightly-sign-runner", "signer-build", "signer-app", "signer-app-notary",
-    "signer-dmg", "find-nightly-next-recovery", "route-visual-gate-nightly", "visual-gate-nightly-verdict",
+    "find-nightly-next-recovery", "route-visual-gate-nightly", "visual-gate-nightly-verdict",
     "report-visual-gate-nightly", "publish-nightly", "publish-nightly-next-github",
     "publish-nightly-next-github-names", "keep-published-dsyms", "legacy-identity-target",
     "publish-nightly-identity-legacy", "nightly-identity-legacy-published", "republish-nightly-deltas",

tests/test_ci_signing_keychain_is_deleted_before_bundle_code.py

@@ -295,8 +295,15 @@ def check_job(name: str, job: dict, failures: list[str]) -> None:
     pool_signer = RELEASE_POOL_SIGNERS.get(tuple(name.split(" ", 1)))
     if pool_signer:
         runner = str(job.get("runs-on", ""))
-        if "keychain" in text or pool_signer not in text or "ubuntu" not in runner or "macos" in runner:
-            failures.append(f"{name}: a release-pool signer runs on a literal ubuntu label, calls {pool_signer} and makes no keychain")
+        owned_mac_release = tuple(name.split(" ", 1)) == ("nightly.yml", "signer-dmg")
+        valid_runner = ("needs.pick-nightly-sign-runner.outputs.sign_owned" in runner
+                        and "needs.pick-nightly-sign-runner.outputs.runner" in runner
+                        and "macos" in runner) if owned_mac_release else ("ubuntu" in runner and "macos" not in runner)
+        if "keychain" in text or pool_signer not in text or not valid_runner:
+            if owned_mac_release:
+                failures.append(f"{name}: the release-pool DMG signer must use the owned signing pick then Blacksmith macOS, call {pool_signer}, and make no keychain")
+            else:
+                failures.append(f"{name}: a release-pool signer runs on a literal ubuntu label, calls {pool_signer} and makes no keychain")
         certificate_steps = [step for step in steps if isinstance(step, dict) and "secrets.APPLE_CERTIFICATE_BASE64" in json.dumps(step)]
         if not certificate_steps or any(pool_signer not in str(step.get("run", "")) for step in certificate_steps):
             failures.append(f"{name}: only the step that runs {pool_signer} may hold the Developer ID certificate")

tests/test_ci_signing_owned_mac.py

@@ -81,6 +81,7 @@
 SIGNING_JOBS = {
     ("nightly.yml", "build-sign-notarize-nightly"), ("nightly.yml", "recover-nightly-next-notarization"),
     ("nightly.yml", "sign-nightly-identity-legacy"), ("nightly.yml", "generate-nightly-deltas"),
+    ("nightly.yml", "signer-dmg"),
     ("release.yml", "build-sign-notarize"), ("repair-v0-64-25-helper-rpaths.yml", "repair"),
     ("cmux-next-dev-daily.yml", "sign"),
 }
@@ -184,6 +185,12 @@ def keychain_job(job: dict) -> bool:
         script in text for script in keychain_guard.SCRIPT_MADE_KEYCHAINS.values())
 
 
+def release_pool_signer(job: dict) -> bool:
+    """The rcodesign release-pool path keeps its certificate in a temporary directory."""
+    text = json.dumps(keychain_guard.inline_step_scripts(job))
+    return any(script in text for script in keychain_guard.RELEASE_POOL_SIGNERS.values())
+
+
 def signing_job(job: dict) -> bool:
     """A keychain job, or a job that holds a Sparkle private key (read in dot form, after dotted())."""
     return keychain_job(job) or bool(SPARKLE_KEY.search(json.dumps(keychain_guard.inline_step_scripts(job))))
@@ -389,7 +396,7 @@ def check(documents: dict[str, dict]) -> tuple[list[str], int]:
             if "security import" in text:
                 failures.append(f"{label}: imports a certificate inline; use scripts/ci/signing-keychain.sh create")
             # Rule 4 is the keychain's; a job that only holds a Sparkle key makes none.
-            if keychain_job(job) and not any("always()" in str(step.get("if", "")) and KEYCHAIN_DELETE.search(str(step.get("run", "")))
+            if keychain_job(job) and not release_pool_signer(job) and not any("always()" in str(step.get("if", "")) and KEYCHAIN_DELETE.search(str(step.get("run", "")))
                        for step in steps):
                 failures.append(f"{label}: routed to owned Macs without an `if: always()` step that runs "
                                 "scripts/ci/signing-keychain.sh delete")