From b61156f5adcf397c5b5f956757f08751d9b56024 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Thu, 29 Oct 2026 02:22:05 -0700 Subject: [PATCH 1/4] ci: owned-Mac signing accepts only listed glaeda hooks; a refused owned attempt signs on Blacksmith in the same run The first step of every owned-routed signing job now also requires GLAEDA_HOOK_SHA256 to be a hook scripts/ci/glaeda-signing-hooks.txt lists (each one waits out an earlier job's leftover processes) and sets the job's claim output when it passes on an owned Mac. Each such job gets a -blacksmith twin with the same steps, env, outputs and permissions that signs in the same run when the owned attempt never passed its gate, and every reader of the signing job reads the twin first. The guard tests check the twin's shape, the gate (refuses fb0442d3, passes 2695209c), and that every listed hook is one glaeda-disk pins. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- .github/workflows/cmux-next-dev-daily.yml | 54 +++- .github/workflows/nightly.yml | 85 +++--- .github/workflows/release.yml | 35 ++- .../repair-v0-64-25-helper-rpaths.yml | 38 ++- scripts/ci/glaeda-signing-hooks.txt | 7 + scripts/ci/signing-runner-claim.sh | 20 +- tests/test_ci_macos_xcode_selection.py | 10 +- ..._keychain_is_deleted_before_bundle_code.py | 3 + tests/test_ci_signing_owned_mac.py | 245 +++++++++++++++--- tests/test_cmux_next_dev_daily_dispatch.py | 2 +- .../upload_artifact_retention_allowlist.json | 5 + 11 files changed, 407 insertions(+), 97 deletions(-) create mode 100644 scripts/ci/glaeda-signing-hooks.txt diff --git a/.github/workflows/cmux-next-dev-daily.yml b/.github/workflows/cmux-next-dev-daily.yml index a1f97c9afcf2..0898db3b3914 100644 --- a/.github/workflows/cmux-next-dev-daily.yml +++ b/.github/workflows/cmux-next-dev-daily.yml @@ -100,13 +100,15 @@ jobs: environment: release runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.run_attempt == 1 && needs.pick-sign-runner.outputs.owned && fromJSON(needs.pick-sign-runner.outputs.owned) || github.run_attempt > 1 && needs.pick-sign-runner.outputs.alternate || needs.pick-sign-runner.outputs.runner || 'blacksmith-6vcpu-macos-26' }} timeout-minutes: 30 - permissions: + permissions: &sign-permissions actions: read contents: read - outputs: + outputs: &sign-outputs + # owned when the claim gate passed on an owned Mac; the Blacksmith twin signs when it did not. + claim: ${{ steps.claim-gate.outputs.claim }} signed: ${{ steps.package.outputs.signed }} source_sha: ${{ steps.package.outputs.source_sha }} - env: + env: &sign-env SOURCE_RUN_ID: ${{ inputs.source_run_id }} RELEASE_TAG: cmux-next-dev APP_NAME: cmux NEXT DEV @@ -118,18 +120,27 @@ jobs: cmux NEXT DEV Release compile cmux app scheme compile (Debug) cmux-next Release compile (Xcode 26) - steps: + steps: &sign-steps # First, before the checkout's token or any secret: on an owned Mac this job holds the whole # mini (glaeda's exclusive class) or fails here (tests/test_ci_signing_owned_mac.py CLAIM_GATE). - name: Hold the whole Mac before any credential + id: claim-gate shell: bash run: | owned="" case "${RUNNER_NAME:-}" in *-glaeda|*-glaeda-*) owned=1 ;; esac if [ -d /Users/Shared/cmux-build-fleet ]; then owned=1; fi - if [ -n "$owned" ] && [ "${GLAEDA_JOB_CLAIM:-}" != exclusive ]; then - echo "::error title=Signing runner::This owned Mac did not give the job the whole mini; its re-run takes Blacksmith." - exit 1 + if [ -n "$owned" ]; then + if [ "${GLAEDA_JOB_CLAIM:-}" != exclusive ]; then + echo "::error title=Signing runner::This owned Mac did not give the job the whole mini; this run's Blacksmith job signs instead." + exit 1 + fi + case "${GLAEDA_HOOK_SHA256:-}" in + 2695209c13db5fe7514daf76d3e8e547b9569ad9a545225e52f7e07dd83d69e4|41e29cd9dca8cdee88ce069b61956674bb79c2c3da90af5520e3e397bfbedeef) ;; + *) echo "::error title=Signing runner::glaeda hook ${GLAEDA_HOOK_SHA256:-unnamed} is not one scripts/ci/glaeda-signing-hooks.txt lists; this run's Blacksmith job signs instead." + exit 1 ;; + esac + echo "claim=owned" >> "$GITHUB_OUTPUT" fi # SOURCE_SHA is the commit the source run built: a dev daily run of @@ -299,6 +310,21 @@ jobs: || security delete-keychain "$CMUX_NEXT_DEV_DAILY_KEYCHAIN" >/dev/null 2>&1 || true fi + sign-blacksmith: + needs: [pick-sign-runner, sign] + # The same run signs on Blacksmith when the owned Mac took sign but its claim gate never passed + # (glaeda refused the job, or a hook scripts/ci/glaeda-signing-hooks.txt does not list admitted it). + if: "!cancelled() && github.run_attempt == 1 && needs.pick-sign-runner.outputs.owned != '' && needs.sign.result != 'skipped' && needs.sign.result != 'cancelled' && needs.sign.outputs.claim != 'owned'" + name: Sign cmux NEXT DEV on Blacksmith + environment: release + # Blacksmith only: this run's sign when the owned Mac refused the job (its claim gate). + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.run_attempt > 1 && needs.pick-sign-runner.outputs.alternate || needs.pick-sign-runner.outputs.runner || 'blacksmith-6vcpu-macos-26' }} + timeout-minutes: 30 + permissions: *sign-permissions + outputs: *sign-outputs + env: *sign-env + steps: *sign-steps + # Offers verify to the owned Macs (Leo, 2026-10-24: owned Macs before # Blacksmith). It runs e2e_runner_pool.py, test-e2e.yml's picker, as # iroh-release-gate.yml does: an owned pool on the lane's Xcode pin while @@ -307,8 +333,8 @@ jobs: # skipped pick or a failure of this job leaves verify on Blacksmith. The # routing App key stays in this Linux job; verify holds no secret. runner: - needs: sign - if: needs.sign.outputs.signed == 'true' + needs: [sign, sign-blacksmith] + if: "!cancelled() && (needs.sign-blacksmith.result == 'skipped' && needs.sign.result || needs.sign-blacksmith.result) == 'success' && ((needs.sign-blacksmith.outputs.signed || needs.sign.outputs.signed) == 'true')" environment: release runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 5 @@ -423,8 +449,8 @@ jobs: verify: name: Verify signed cmux NEXT DEV # No environment and no secrets: this job runs the app's diff sidecar. - needs: [sign, runner] - if: ${{ !cancelled() && needs.sign.result == 'success' && needs.sign.outputs.signed == 'true' }} + needs: [sign, runner, sign-blacksmith] + if: ${{ !cancelled() && (needs.sign-blacksmith.result == 'skipped' && needs.sign.result || needs.sign-blacksmith.result) == 'success' && (needs.sign-blacksmith.outputs.signed || needs.sign.outputs.signed) == 'true' }} runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.run_attempt == 1 && needs.runner.outputs.label || 'blacksmith-6vcpu-macos-26') }} timeout-minutes: 15 permissions: @@ -479,8 +505,8 @@ jobs: # nightly)." Signing and codesign verification remain the admission gate; # the sidecar verification job runs independently and never blocks the # cutting-edge DEV artifact. - needs: [sign] - if: needs.sign.outputs.signed == 'true' + needs: [sign, sign-blacksmith] + if: "!cancelled() && (needs.sign-blacksmith.result == 'skipped' && needs.sign.result || needs.sign-blacksmith.result) == 'success' && ((needs.sign-blacksmith.outputs.signed || needs.sign.outputs.signed) == 'true')" # This copy's publish updates the rolling release without a lease, so its # group still serializes it. main's copy, the one that runs, replaced the # group with an R2 lease and a newest-wins check. @@ -494,7 +520,7 @@ jobs: env: RELEASE_TAG: cmux-next-dev APP_ARCHIVE_NAME: cmux-NEXT-DEV.zip - SOURCE_SHA: ${{ needs.sign.outputs.source_sha }} + SOURCE_SHA: ${{ (needs.sign-blacksmith.outputs.source_sha || needs.sign.outputs.source_sha) }} steps: - name: Download the signed app uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0dae9b03b567..c97b4b88327d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6884,7 +6884,7 @@ jobs: # same build to Apple; the "Skip a build the old feed already has" step # here and the publish job's order check keep the old feed right, and the # extra submission only costs Apple time (cx-vws1x). - concurrency: + concurrency: &sign-nightly-identity-legacy-concurrency group: nightly-next-legacy-identity-${{ github.run_id }} cancel-in-progress: false # Signs and notarizes only. It holds signing and notarization secrets, so @@ -6892,12 +6892,14 @@ jobs: # the stapled DMG on a fresh secret-free runner, and # publish-nightly-identity-legacy publishes only the DMG whose sha256 that # job verified (cx-vws1x). - permissions: + permissions: &sign-nightly-identity-legacy-permissions contents: read # Read the pending-submission artifacts of earlier runs, and retire # metadata whose bundle is gone (nightly-next-legacy-orphan-meta.sh). actions: write - outputs: + outputs: &sign-nightly-identity-legacy-outputs + # owned when the claim gate passed on an owned Mac; the Blacksmith twin signs when it did not. + claim: ${{ steps.claim-gate.outputs.claim }} # The old feed already has this build (or a newer one): nothing ran. skip: ${{ steps.order.outputs.skip }} pending: ${{ steps.notarize.outputs.submission_pending }} @@ -6921,7 +6923,7 @@ jobs: # never counted as a failure. The DMG around the stapled app adds one more # submission with the same 20 min wait (CMUX_STAPLE_APP_IN_DMG). timeout-minutes: 160 - env: + env: &sign-nightly-identity-legacy-env CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_26 }} CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" NIGHTLY_BUILD: ${{ needs.legacy-identity-target.outputs.build }} @@ -6946,18 +6948,27 @@ jobs: # prunes them itself. LEGACY_DMG_PREFIX: cmux-nightly-next-macos-nightlyid-arm64- LEGACY_APPCAST: appcast-arm64.xml - steps: + steps: &sign-nightly-identity-legacy-steps # First, before the checkout's token or any secret: on an owned Mac this job holds the whole # mini (glaeda's exclusive class) or fails here (tests/test_ci_signing_owned_mac.py CLAIM_GATE). - name: Hold the whole Mac before any credential + id: claim-gate shell: bash run: | owned="" case "${RUNNER_NAME:-}" in *-glaeda|*-glaeda-*) owned=1 ;; esac if [ -d /Users/Shared/cmux-build-fleet ]; then owned=1; fi - if [ -n "$owned" ] && [ "${GLAEDA_JOB_CLAIM:-}" != exclusive ]; then - echo "::error title=Signing runner::This owned Mac did not give the job the whole mini; its re-run takes Blacksmith." - exit 1 + if [ -n "$owned" ]; then + if [ "${GLAEDA_JOB_CLAIM:-}" != exclusive ]; then + echo "::error title=Signing runner::This owned Mac did not give the job the whole mini; this run's Blacksmith job signs instead." + exit 1 + fi + case "${GLAEDA_HOOK_SHA256:-}" in + 2695209c13db5fe7514daf76d3e8e547b9569ad9a545225e52f7e07dd83d69e4|41e29cd9dca8cdee88ce069b61956674bb79c2c3da90af5520e3e397bfbedeef) ;; + *) echo "::error title=Signing runner::glaeda hook ${GLAEDA_HOOK_SHA256:-unnamed} is not one scripts/ci/glaeda-signing-hooks.txt lists; this run's Blacksmith job signs instead." + exit 1 ;; + esac + echo "claim=owned" >> "$GITHUB_OUTPUT" fi - name: Checkout @@ -7174,13 +7185,29 @@ jobs: ./scripts/ci/signing-keychain.sh delete --keychain build.keychain \ || security delete-keychain build.keychain >/dev/null 2>&1 || true + sign-nightly-identity-legacy-blacksmith: + needs: [decide, build-nightly-app, legacy-identity-target, pick-nightly-sign-runner, sign-nightly-identity-legacy] + # The same run signs on Blacksmith when the owned Mac took sign-nightly-identity-legacy but its claim gate never passed + # (glaeda refused the job, or a hook scripts/ci/glaeda-signing-hooks.txt does not list admitted it). + if: "!cancelled() && github.run_attempt == 1 && needs.pick-nightly-sign-runner.outputs.sign_owned != '' && needs.sign-nightly-identity-legacy.result != 'skipped' && needs.sign-nightly-identity-legacy.result != 'cancelled' && needs.sign-nightly-identity-legacy.outputs.claim != 'owned'" + continue-on-error: true + environment: release-next + concurrency: *sign-nightly-identity-legacy-concurrency + permissions: *sign-nightly-identity-legacy-permissions + outputs: *sign-nightly-identity-legacy-outputs + # Blacksmith only: this run's sign when the owned Mac refused the job (its claim gate). + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.run_attempt > 1 && needs.legacy-identity-target.outputs.sign_alternate || needs.legacy-identity-target.outputs.sign_runner || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26' }} + timeout-minutes: 160 + env: *sign-nightly-identity-legacy-env + steps: *sign-nightly-identity-legacy-steps + # cx-vws1x: static verification of the exact stapled cmux NIGHTLY DMG, on a # fresh secret-free pooled runner (no environment or secret). Its verified sha256 is the only DMG publish-nightly-identity-legacy # publishes. verify-nightly-identity-legacy: - needs: [legacy-identity-target, sign-nightly-identity-legacy, pick-nightly-sign-runner] - if: "!cancelled() && needs.sign-nightly-identity-legacy.outputs.signed_ok == 'true' && needs.sign-nightly-identity-legacy.outputs.dmg_sha256 != ''" + needs: [legacy-identity-target, sign-nightly-identity-legacy, pick-nightly-sign-runner, sign-nightly-identity-legacy-blacksmith] + if: "!cancelled() && (needs.sign-nightly-identity-legacy-blacksmith.outputs.signed_ok || needs.sign-nightly-identity-legacy.outputs.signed_ok) == 'true' && (needs.sign-nightly-identity-legacy-blacksmith.outputs.dmg_sha256 || needs.sign-nightly-identity-legacy.outputs.dmg_sha256) != ''" continue-on-error: true permissions: contents: read @@ -7198,10 +7225,10 @@ jobs: CMUX_CI_XCODE_APP: ${{ needs.pick-nightly-sign-runner.outputs.attempt == github.run_attempt && needs.pick-nightly-sign-runner.outputs.owned != '' && vars.CMUX_CI_XCODE_APP_PR || vars.CMUX_CI_XCODE_APP_MACOS_26 }} CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" NIGHTLY_BUILD: ${{ needs.legacy-identity-target.outputs.build }} - SIGNED_SHA256: ${{ needs.sign-nightly-identity-legacy.outputs.dmg_sha256 }} - BUILD_SHA: ${{ needs.sign-nightly-identity-legacy.outputs.build_sha }} + SIGNED_SHA256: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.dmg_sha256 || needs.sign-nightly-identity-legacy.outputs.dmg_sha256) }} + BUILD_SHA: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.build_sha || needs.sign-nightly-identity-legacy.outputs.build_sha) }} LEGACY_DMG_PREFIX: cmux-nightly-next-macos-nightlyid-arm64- - STAPLED_APP_FALLBACK_ID: ${{ needs.sign-nightly-identity-legacy.outputs.stapled_app_fallback_id }} + STAPLED_APP_FALLBACK_ID: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.stapled_app_fallback_id || needs.sign-nightly-identity-legacy.outputs.stapled_app_fallback_id) }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -7250,7 +7277,7 @@ jobs: retention-days: 3 publish-nightly-identity-legacy: - needs: [decide, publish-nightly, legacy-identity-target, sign-nightly-identity-legacy, verify-nightly-identity-legacy] + needs: [decide, publish-nightly, legacy-identity-target, sign-nightly-identity-legacy, verify-nightly-identity-legacy, sign-nightly-identity-legacy-blacksmith] # A signing job that a newer run's concurrency slot cancelled tried # nothing: no resolve, no failure count. # The target job may select this run's prepared candidate before the main @@ -7260,7 +7287,7 @@ jobs: # nightly_next_continue_only run intentionally skips publish-nightly; its # target came from the live NEXT feed and may continue the accepted Apple # submission without another NEXT write. - if: "!cancelled() && github.repository == 'manaflow-ai/cmux' && github.ref == 'refs/heads/nightly-next' && needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.environment == 'release-next' && (needs.decide.outputs.notary_paused != 'true' || needs.legacy-identity-target.outputs.action == 'continue') && ((needs.publish-nightly.result == 'success' && needs.publish-nightly.outputs.refused != 'true' && needs.publish-nightly.outputs.build == needs.legacy-identity-target.outputs.build) || (needs.publish-nightly.result == 'skipped' && needs.legacy-identity-target.outputs.action == 'continue')) && needs.legacy-identity-target.result == 'success' && needs.legacy-identity-target.outputs.build != '' && vars.NIGHTLY_NEXT_LEGACY_IDENTITY != 'off' && needs.sign-nightly-identity-legacy.result != 'cancelled'" + if: "!cancelled() && github.repository == 'manaflow-ai/cmux' && github.ref == 'refs/heads/nightly-next' && needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.environment == 'release-next' && (needs.decide.outputs.notary_paused != 'true' || needs.legacy-identity-target.outputs.action == 'continue') && ((needs.publish-nightly.result == 'success' && needs.publish-nightly.outputs.refused != 'true' && needs.publish-nightly.outputs.build == needs.legacy-identity-target.outputs.build) || (needs.publish-nightly.result == 'skipped' && needs.legacy-identity-target.outputs.action == 'continue')) && needs.legacy-identity-target.result == 'success' && needs.legacy-identity-target.outputs.build != '' && vars.NIGHTLY_NEXT_LEGACY_IDENTITY != 'off' && (needs.sign-nightly-identity-legacy-blacksmith.result == 'skipped' && needs.sign-nightly-identity-legacy.result || needs.sign-nightly-identity-legacy-blacksmith.result) != 'cancelled'" continue-on-error: true environment: release-next # One old-feed writer at a time across runs. A feed writer, not a build: @@ -7309,11 +7336,11 @@ jobs: # prunes them itself. LEGACY_DMG_PREFIX: cmux-nightly-next-macos-nightlyid-arm64- LEGACY_APPCAST: appcast-arm64.xml - SIGN_RESULT: ${{ needs.sign-nightly-identity-legacy.result }} - SIGN_SKIP: ${{ needs.sign-nightly-identity-legacy.outputs.skip }} - SIGN_OK: ${{ needs.sign-nightly-identity-legacy.outputs.signed_ok }} - SIGN_PENDING: ${{ needs.sign-nightly-identity-legacy.outputs.pending }} - SIGNED_SHA256: ${{ needs.sign-nightly-identity-legacy.outputs.dmg_sha256 }} + SIGN_RESULT: ${{ (needs.sign-nightly-identity-legacy-blacksmith.result == 'skipped' && needs.sign-nightly-identity-legacy.result || needs.sign-nightly-identity-legacy-blacksmith.result) }} + SIGN_SKIP: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.skip || needs.sign-nightly-identity-legacy.outputs.skip) }} + SIGN_OK: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.signed_ok || needs.sign-nightly-identity-legacy.outputs.signed_ok) }} + SIGN_PENDING: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.pending || needs.sign-nightly-identity-legacy.outputs.pending) }} + SIGNED_SHA256: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.dmg_sha256 || needs.sign-nightly-identity-legacy.outputs.dmg_sha256) }} TESTED_SHA256: ${{ needs.verify-nightly-identity-legacy.outputs.tested_sha256 }} steps: - name: Checkout @@ -7387,7 +7414,7 @@ jobs: # Rewritten from the verdicts below (nightly_next_ring.py legacy). SPARKLE_CHANNEL: cmux-next DEV_RING: ${{ steps.ring.outputs.mode || 'off' }} - SPARKLE_MINIMUM_SYSTEM_VERSION: ${{ needs.sign-nightly-identity-legacy.outputs.minimum_system_version }} + SPARKLE_MINIMUM_SYSTEM_VERSION: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.minimum_system_version || needs.sign-nightly-identity-legacy.outputs.minimum_system_version) }} run: bash scripts/ci/nightly-steps/legacy-identity-appcast.sh - name: Publish the cmux NIGHTLY DMG, release notes and the old feed @@ -7423,12 +7450,12 @@ jobs: if: always() && env.SIGN_SKIP == 'false' && steps.order.outputs.skip != 'true' env: GH_TOKEN: ${{ github.token }} - PENDING: ${{ needs.sign-nightly-identity-legacy.outputs.pending == 'true' }} - NOTARIZE: ${{ needs.sign-nightly-identity-legacy.outputs.notarize || 'failure' }} + PENDING: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.pending || needs.sign-nightly-identity-legacy.outputs.pending) == 'true' }} + NOTARIZE: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.notarize || needs.sign-nightly-identity-legacy.outputs.notarize) || 'failure' }} PUBLISHED: ${{ steps.publish.outcome == 'success' }} # Apple's answer as the signing job saw it (a notary log or a stapled ticket). - APPLE: ${{ needs.sign-nightly-identity-legacy.outputs.apple || 'unknown' }} - REQUIREMENT_REFUSED: ${{ needs.sign-nightly-identity-legacy.outputs.requirement_refused || 'false' }} + APPLE: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.apple || needs.sign-nightly-identity-legacy.outputs.apple) || 'unknown' }} + REQUIREMENT_REFUSED: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.requirement_refused || needs.sign-nightly-identity-legacy.outputs.requirement_refused) || 'false' }} run: bash scripts/ci/nightly-steps/legacy-identity-resolve.sh - name: Store the accepted submission's metadata @@ -7518,7 +7545,7 @@ jobs: # was nothing to publish (current, refused, or still in Apple's queue), or # this job turns the run red. No job needs it. nightly-identity-legacy-published: - needs: [decide, publish-nightly, legacy-identity-target, sign-nightly-identity-legacy, verify-nightly-identity-legacy, publish-nightly-identity-legacy] + needs: [decide, publish-nightly, legacy-identity-target, sign-nightly-identity-legacy, verify-nightly-identity-legacy, publish-nightly-identity-legacy, sign-nightly-identity-legacy-blacksmith] if: "always() && !cancelled() && github.repository == 'manaflow-ai/cmux' && github.ref == 'refs/heads/nightly-next' && needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.environment == 'release-next' && needs.decide.outputs.notary_test != 'true' && needs.decide.outputs.notary_probe != 'true' && needs.decide.outputs.build_only != 'true' && needs.decide.outputs.fast_build != 'true' && needs.publish-nightly.result == 'success' && needs.publish-nightly.outputs.refused != 'true' && vars.NIGHTLY_NEXT_LEGACY_IDENTITY != 'off'" runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 5 @@ -7529,9 +7556,9 @@ jobs: ACTION: ${{ needs.legacy-identity-target.outputs.action }} BUILD: ${{ needs.legacy-identity-target.outputs.build }} NEXT_BUILD: ${{ needs.publish-nightly.outputs.build }} - SIGN_SKIP: ${{ needs.sign-nightly-identity-legacy.outputs.skip }} - SIGN_PENDING: ${{ needs.sign-nightly-identity-legacy.outputs.pending }} - SIGN_OK: ${{ needs.sign-nightly-identity-legacy.outputs.signed_ok }} + SIGN_SKIP: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.skip || needs.sign-nightly-identity-legacy.outputs.skip) }} + SIGN_PENDING: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.pending || needs.sign-nightly-identity-legacy.outputs.pending) }} + SIGN_OK: ${{ (needs.sign-nightly-identity-legacy-blacksmith.outputs.signed_ok || needs.sign-nightly-identity-legacy.outputs.signed_ok) }} TESTED: ${{ needs.verify-nightly-identity-legacy.outputs.tested_sha256 }} ALREADY: ${{ needs.publish-nightly-identity-legacy.outputs.already }} PUBLISHED: ${{ needs.publish-nightly-identity-legacy.outputs.published }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f992c61b185..794a402eea0c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -196,7 +196,7 @@ jobs: run: python3 scripts/ci/signing_runner.py build-sign-notarize: - permissions: + permissions: &build-sign-notarize-permissions contents: write attestations: write id-token: write @@ -216,18 +216,30 @@ jobs: # duration. Ten more minutes are the release publisher's rate-limit budget # (publish-release-assets.py, RATE_LIMIT_BUDGET_SECONDS), which waits at the end. timeout-minutes: 100 - steps: + outputs: &build-sign-notarize-outputs + # owned when the claim gate passed on an owned Mac; the Blacksmith twin signs when it did not. + claim: ${{ steps.claim-gate.outputs.claim }} + steps: &build-sign-notarize-steps # First, before the checkout's token or any secret: on an owned Mac this job holds the whole # mini (glaeda's exclusive class) or fails here (tests/test_ci_signing_owned_mac.py CLAIM_GATE). - name: Hold the whole Mac before any credential + id: claim-gate shell: bash run: | owned="" case "${RUNNER_NAME:-}" in *-glaeda|*-glaeda-*) owned=1 ;; esac if [ -d /Users/Shared/cmux-build-fleet ]; then owned=1; fi - if [ -n "$owned" ] && [ "${GLAEDA_JOB_CLAIM:-}" != exclusive ]; then - echo "::error title=Signing runner::This owned Mac did not give the job the whole mini; its re-run takes Blacksmith." - exit 1 + if [ -n "$owned" ]; then + if [ "${GLAEDA_JOB_CLAIM:-}" != exclusive ]; then + echo "::error title=Signing runner::This owned Mac did not give the job the whole mini; this run's Blacksmith job signs instead." + exit 1 + fi + case "${GLAEDA_HOOK_SHA256:-}" in + 2695209c13db5fe7514daf76d3e8e547b9569ad9a545225e52f7e07dd83d69e4|41e29cd9dca8cdee88ce069b61956674bb79c2c3da90af5520e3e397bfbedeef) ;; + *) echo "::error title=Signing runner::glaeda hook ${GLAEDA_HOOK_SHA256:-unnamed} is not one scripts/ci/glaeda-signing-hooks.txt lists; this run's Blacksmith job signs instead." + exit 1 ;; + esac + echo "claim=owned" >> "$GITHUB_OUTPUT" fi - name: Clear stale git locks (self-hosted reused workspace) @@ -1010,3 +1022,16 @@ jobs: ./scripts/ci/signing-keychain.sh delete --keychain build.keychain \ || security delete-keychain build.keychain >/dev/null 2>&1 || true rm -f /tmp/cert.p12 + + build-sign-notarize-blacksmith: + permissions: *build-sign-notarize-permissions + environment: release + needs: [build-ghostty-cli-helper, resolve-release-cmux-tui-client, pick-sign-runner, build-sign-notarize] + # The same run signs on Blacksmith when the owned Mac took build-sign-notarize but its claim gate never passed + # (glaeda refused the job, or a hook scripts/ci/glaeda-signing-hooks.txt does not list admitted it). + if: "!cancelled() && github.run_attempt == 1 && needs.pick-sign-runner.outputs.owned != '' && needs.build-sign-notarize.result != 'skipped' && needs.build-sign-notarize.result != 'cancelled' && needs.build-sign-notarize.outputs.claim != 'owned'" + # Blacksmith only: this run's sign when the owned Mac refused the job (its claim gate). + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.run_attempt > 1 && needs.pick-sign-runner.outputs.alternate || needs.pick-sign-runner.outputs.runner || 'blacksmith-6vcpu-macos-26' }} + timeout-minutes: 100 + outputs: *build-sign-notarize-outputs + steps: *build-sign-notarize-steps diff --git a/.github/workflows/repair-v0-64-25-helper-rpaths.yml b/.github/workflows/repair-v0-64-25-helper-rpaths.yml index e9aa4a3aa67e..a6430f06546a 100644 --- a/.github/workflows/repair-v0-64-25-helper-rpaths.yml +++ b/.github/workflows/repair-v0-64-25-helper-rpaths.yml @@ -125,31 +125,43 @@ jobs: repair: needs: [verify-shipped, pick-sign-runner] - permissions: + permissions: &repair-permissions contents: write # Production secrets: GitHub releases them only to protected refs (the # environment's deployment branch policy: main, tags v*). environment: release runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.run_attempt == 1 && needs.pick-sign-runner.outputs.owned && fromJSON(needs.pick-sign-runner.outputs.owned) || github.run_attempt > 1 && needs.pick-sign-runner.outputs.alternate || needs.pick-sign-runner.outputs.runner || 'blacksmith-6vcpu-macos-26' }} timeout-minutes: 90 - env: + env: &repair-env TARGET_TAG: v0.64.25 REPAIR_ROOT: ${{ github.workspace }}/.cmux-v0.64.25-repair APP_PATH: ${{ github.workspace }}/.cmux-v0.64.25-repair/cmux.app DMG_PATH: ${{ github.workspace }}/cmux-macos.dmg APPCAST_PATH: ${{ github.workspace }}/appcast.xml - steps: + outputs: &repair-outputs + # owned when the claim gate passed on an owned Mac; the Blacksmith twin signs when it did not. + claim: ${{ steps.claim-gate.outputs.claim }} + steps: &repair-steps # First, before the checkout's token or any secret: on an owned Mac this job holds the whole # mini (glaeda's exclusive class) or fails here (tests/test_ci_signing_owned_mac.py CLAIM_GATE). - name: Hold the whole Mac before any credential + id: claim-gate shell: bash run: | owned="" case "${RUNNER_NAME:-}" in *-glaeda|*-glaeda-*) owned=1 ;; esac if [ -d /Users/Shared/cmux-build-fleet ]; then owned=1; fi - if [ -n "$owned" ] && [ "${GLAEDA_JOB_CLAIM:-}" != exclusive ]; then - echo "::error title=Signing runner::This owned Mac did not give the job the whole mini; its re-run takes Blacksmith." - exit 1 + if [ -n "$owned" ]; then + if [ "${GLAEDA_JOB_CLAIM:-}" != exclusive ]; then + echo "::error title=Signing runner::This owned Mac did not give the job the whole mini; this run's Blacksmith job signs instead." + exit 1 + fi + case "${GLAEDA_HOOK_SHA256:-}" in + 2695209c13db5fe7514daf76d3e8e547b9569ad9a545225e52f7e07dd83d69e4|41e29cd9dca8cdee88ce069b61956674bb79c2c3da90af5520e3e397bfbedeef) ;; + *) echo "::error title=Signing runner::glaeda hook ${GLAEDA_HOOK_SHA256:-unnamed} is not one scripts/ci/glaeda-signing-hooks.txt lists; this run's Blacksmith job signs instead." + exit 1 ;; + esac + echo "claim=owned" >> "$GITHUB_OUTPUT" fi - name: Checkout repair tooling @@ -413,3 +425,17 @@ jobs: ./scripts/ci/signing-keychain.sh delete --keychain build.keychain \ || security delete-keychain build.keychain >/dev/null 2>&1 || true rm -f /tmp/cert.p12 + + repair-blacksmith: + needs: [verify-shipped, pick-sign-runner, repair] + # The same run signs on Blacksmith when the owned Mac took repair but its claim gate never passed + # (glaeda refused the job, or a hook scripts/ci/glaeda-signing-hooks.txt does not list admitted it). + if: "!cancelled() && github.run_attempt == 1 && needs.pick-sign-runner.outputs.owned != '' && needs.repair.result != 'skipped' && needs.repair.result != 'cancelled' && needs.repair.outputs.claim != 'owned'" + permissions: *repair-permissions + environment: release + # Blacksmith only: this run's sign when the owned Mac refused the job (its claim gate). + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.run_attempt > 1 && needs.pick-sign-runner.outputs.alternate || needs.pick-sign-runner.outputs.runner || 'blacksmith-6vcpu-macos-26' }} + timeout-minutes: 90 + env: *repair-env + outputs: *repair-outputs + steps: *repair-steps diff --git a/scripts/ci/glaeda-signing-hooks.txt b/scripts/ci/glaeda-signing-hooks.txt new file mode 100644 index 000000000000..a8d9faf187ae --- /dev/null +++ b/scripts/ci/glaeda-signing-hooks.txt @@ -0,0 +1,7 @@ +# glaeda runner hooks (sha256 of scripts/glaeda-cmux-runner-hook in teamleaderleo/glaeda) whose whole-mini claim +# an owned-Mac signing job accepts: each one waits out an earlier job's leftover processes. A job admitted by any +# other hook fails its first step (CLAIM_GATE) and the run's Blacksmith twin signs. Every sha is one glaeda-disk +# pins in CMUX_NEXT_DRAIN_HOOK_SHA256 (tests/test_ci_signing_owned_mac.py). Add a new hook's sha when glaeda +# releases it; signing stays on Blacksmith for minis on it until then. +2695209c13db5fe7514daf76d3e8e547b9569ad9a545225e52f7e07dd83d69e4 glaeda cb2186b0 (#1738), the leftover-process check +41e29cd9dca8cdee88ce069b61956674bb79c2c3da90af5520e3e397bfbedeef glaeda #1739, the claim names its hook (GLAEDA_HOOK_SHA256) diff --git a/scripts/ci/signing-runner-claim.sh b/scripts/ci/signing-runner-claim.sh index d1f988f33b44..9d530af4c75c 100755 --- a/scripts/ci/signing-runner-claim.sh +++ b/scripts/ci/signing-runner-claim.sh @@ -7,17 +7,22 @@ # (a glaeda runner on a fleet mini) the job must hold the whole mini: # glaeda's job-started hook gives the signing jobs the exclusive class (every # capacity unit and both fast tokens) and writes GLAEDA_JOB_CLAIM=exclusive to -# $GITHUB_ENV. This step fails the job, before any secret is read, on an owned -# Mac without that claim (a hook from before the exclusive class), and removes -# a signing keychain a killed job on that Mac left behind. +# $GITHUB_ENV, with GLAEDA_HOOK_SHA256 (the hook's own sha256). This step fails +# the job, before any secret is read, on an owned Mac without that claim or +# whose hook glaeda-signing-hooks.txt does not list (a hook from before the +# leftover-process check), and removes a signing keychain a killed job on that +# Mac left behind. # # Usage: signing-runner-claim.sh (no arguments; reads RUNNER_NAME, -# GLAEDA_JOB_CLAIM and CMUX_FLEET_DIR, default /Users/Shared/cmux-build-fleet) +# GLAEDA_JOB_CLAIM, GLAEDA_HOOK_SHA256 and CMUX_FLEET_DIR, default +# /Users/Shared/cmux-build-fleet) set -euo pipefail FLEET_DIR="${CMUX_FLEET_DIR:-/Users/Shared/cmux-build-fleet}" SECURITY_TOOL="${CMUX_SECURITY_TOOL:-security}" KEYCHAIN_DIR="${CMUX_KEYCHAIN_DIR:-$HOME/Library/Keychains}" +# The glaeda hooks whose claim a signing job accepts (one sha256 per line, then a note). +HOOKS_FILE="${CMUX_SIGNING_HOOKS_FILE:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/glaeda-signing-hooks.txt}" # The keychain names the workflows give signing-keychain.sh create. STALE_KEYCHAINS=(build.keychain cmux-next-dev-daily.keychain) @@ -35,7 +40,12 @@ if [ -z "$owned" ]; then fi if [ "${GLAEDA_JOB_CLAIM:-}" != "exclusive" ]; then - echo "::error title=Signing runner::This owned Mac ($owned) did not give the job the whole mini (GLAEDA_JOB_CLAIM='${GLAEDA_JOB_CLAIM:-}'). A signing job never runs beside another job; its re-run takes Blacksmith." + echo "::error title=Signing runner::This owned Mac ($owned) did not give the job the whole mini (GLAEDA_JOB_CLAIM='${GLAEDA_JOB_CLAIM:-}'). A signing job never runs beside another job; this run's Blacksmith job signs instead." + exit 1 +fi +hook_sha="${GLAEDA_HOOK_SHA256:-}" +if [ -z "$hook_sha" ] || ! grep -Eq "^${hook_sha}( |\$)" "$HOOKS_FILE" 2>/dev/null; then + echo "::error title=Signing runner::This owned Mac's glaeda hook (GLAEDA_HOOK_SHA256='${hook_sha}') is not one $HOOKS_FILE lists. Only a listed hook waits out an earlier job's leftover processes; this run's Blacksmith job signs instead." exit 1 fi diff --git a/tests/test_ci_macos_xcode_selection.py b/tests/test_ci_macos_xcode_selection.py index b639a8462b1f..3f5b53c36f01 100644 --- a/tests/test_ci_macos_xcode_selection.py +++ b/tests/test_ci_macos_xcode_selection.py @@ -160,7 +160,7 @@ def steps(job: str) -> list[tuple[str, str]]: """(name, text) for each step of a job.""" lines = job.splitlines() try: - start = next(i for i, line in enumerate(lines) if re.match(r"^ steps:\s*$", line)) + start = next(i for i, line in enumerate(lines) if re.match(r"^ steps:(?:\s+&[\w-]+)?\s*$", line)) except StopIteration: return [] found = [] @@ -196,7 +196,15 @@ def check_workflow(name: str, text: str) -> list[str]: # A path such as scripts/build-x.sh keeps its name even when a job shares it. job_ids = [job_id for job_id in workflow_jobs if "-" in job_id] job_id_token = re.compile(r"(?-blacksmith, with the same steps, env, + outputs and permissions, that signs in the same run when the owned attempt + never passed its claim gate (glaeda refused the job, or an unlisted hook + admitted it), and that every job reading the signing job's result or + outputs also needs and reads, so no run fails or goes stale on a refused + owned Mac; +4. make its keychain with scripts/ci/signing-keychain.sh create (the certificate is decoded to a private temporary file that create itself removes, and delete removes again) and delete it in an `if: always()` step. @@ -37,6 +46,7 @@ import sys import tempfile import unittest +import urllib.request from pathlib import Path from typing import Any @@ -61,15 +71,35 @@ BLACKSMITH_PICKER = "scripts/ci/nightly_sign_runner.py" CLAIM = "scripts/ci/signing-runner-claim.sh" CLAIM_GATE_NAME = "Hold the whole Mac before any credential" +CLAIM_GATE_ID = "claim-gate" +# The glaeda hooks whose claim the signing jobs accept: one sha256 per line, then a note. +SIGNING_HOOKS_FILE = ROOT / "scripts/ci/glaeda-signing-hooks.txt" +SIGNING_HOOKS = [line.split()[0] for line in SIGNING_HOOKS_FILE.read_text().splitlines() + if line.strip() and not line.startswith("#")] +# glaeda's drain allowlist, every hook sha it ever pinned, oldest first; FIRST_SIGNING_HOOK is the first hook that waits +# out an earlier job's leftover processes (glaeda #1738), so nothing older may be listed. +GLAEDA_DISK_URL = "https://raw.githubusercontent.com/teamleaderleo/glaeda/main/scripts/glaeda-disk" +FIRST_SIGNING_HOOK = "2695209c13db5fe7514daf76d3e8e547b9569ad9a545225e52f7e07dd83d69e4" # The first step of an owned-routed signing job, word for word: it needs no checkout. -CLAIM_GATE = """owned="" -case "${RUNNER_NAME:-}" in *-glaeda|*-glaeda-*) owned=1 ;; esac +CLAIM_GATE = f"""owned="" +case "${{RUNNER_NAME:-}}" in *-glaeda|*-glaeda-*) owned=1 ;; esac if [ -d /Users/Shared/cmux-build-fleet ]; then owned=1; fi -if [ -n "$owned" ] && [ "${GLAEDA_JOB_CLAIM:-}" != exclusive ]; then - echo "::error title=Signing runner::This owned Mac did not give the job the whole mini; its re-run takes Blacksmith." - exit 1 +if [ -n "$owned" ]; then + if [ "${{GLAEDA_JOB_CLAIM:-}}" != exclusive ]; then + echo "::error title=Signing runner::This owned Mac did not give the job the whole mini; this run's Blacksmith job signs instead." + exit 1 + fi + case "${{GLAEDA_HOOK_SHA256:-}}" in + {"|".join(SIGNING_HOOKS)}) ;; + *) echo "::error title=Signing runner::glaeda hook ${{GLAEDA_HOOK_SHA256:-unnamed}} is not one scripts/ci/glaeda-signing-hooks.txt lists; this run's Blacksmith job signs instead." + exit 1 ;; + esac + echo "claim=owned" >> "$GITHUB_OUTPUT" fi """ +TWIN_SUFFIX = "-blacksmith" +# The hook from before the leftover-process check (glaeda before #1738): the gate refuses it. +OLD_HOOK = "fb0442d31e23f01326118b7d6be2effd0ba91bff9a0b59c62016c39bc23116da" NEEDS_OUTPUT = re.compile(r"needs\.([\w-]+)\.outputs\.([\w-]+)") STEP_OUTPUT = re.compile(r"steps\.([\w-]+)\.outputs\.") KEYCHAIN_DELETE = re.compile(r"signing-keychain\.sh\s+delete\b") @@ -147,6 +177,54 @@ def picker_scripts(jobs: dict, needed: str, output: str) -> list[str]: return found or [f"needs.{needed}.outputs.{output} (no step output)"] +def twin_failures(jobs: dict, name: str, owned_reads: list[str]) -> list[str]: + """Why the owned-routed signing job `name` lacks a same-run Blacksmith twin that every reader of it also reads.""" + twin_name = name + TWIN_SUFFIX + job, twin = jobs[name], jobs.get(twin_name) + if str((job.get("outputs") or {}).get("claim", "")).strip() != f"${{{{ steps.{CLAIM_GATE_ID}.outputs.claim }}}}": + return [f"its outputs must carry claim: ${{{{ steps.{CLAIM_GATE_ID}.outputs.claim }}}}, which its twin reads"] + if not isinstance(twin, dict): + return [f"no Blacksmith twin {twin_name}, so a refused owned attempt would leave the run unsigned"] + failures = [] + for key in ("steps", "env", "outputs", "permissions", "environment", "concurrency", "continue-on-error"): + if twin.get(key) != job.get(key): + failures.append(f"{twin_name} must have the same {key} as {name} (a YAML alias of it)") + runs_on = str(twin.get("runs-on", "")) + if any(read.split(".outputs.")[0] + ".outputs." + read.split(".outputs.")[1] in runs_on for read in owned_reads): + failures.append(f"{twin_name} runs-on reads the owned pick; it must take Blacksmith only") + needs = twin.get("needs") or [] + needs = [needs] if isinstance(needs, str) else list(needs) + own = needs[:-1] if needs and needs[-1] == name else None + first = job.get("needs") or [] + if own != ([first] if isinstance(first, str) else list(first)): + failures.append(f"{twin_name} needs must be {name}'s needs, then {name}") + condition = re.sub(r"\s+", " ", str(twin.get("if", ""))) + for read in owned_reads: + want = (f"!cancelled() && github.run_attempt == 1 && {read} != '' && needs.{name}.result != 'skipped' && " + f"needs.{name}.result != 'cancelled' && needs.{name}.outputs.claim != 'owned'") + if condition != want: + failures.append(f"{twin_name} if must be exactly: {want}") + for other, reader in jobs.items(): + if other in (name, twin_name) or not isinstance(reader, dict): + continue + text = json.dumps(reader) + reader_needs = reader.get("needs") or [] + reader_needs = [reader_needs] if isinstance(reader_needs, str) else list(reader_needs) + if name not in reader_needs and f"needs.{name}." not in text: + continue + if twin_name not in reader_needs: + failures.append(f"{other} needs {name} but not {twin_name}, so it never sees the twin's sign") + for ref in re.findall(rf"needs\.{re.escape(name)}\.(outputs\.[\w-]+|result)", text): + pair = (f"(needs.{twin_name}.{ref} || needs.{name}.{ref})" if ref.startswith("outputs.") + else f"(needs.{twin_name}.result == 'skipped' && needs.{name}.result || needs.{twin_name}.result)") + if text.count(f"needs.{name}.{ref}") != text.count(pair): + failures.append(f"{other} reads needs.{name}.{ref} without the twin first: write it as {pair}") + condition = str(reader.get("if", "")) + if not re.search(r"\b(always|cancelled|failure|success)\s*\(", condition): + failures.append(f"{other} if has no status check, so the twin it needs, skipped, skips it too") + return failures + + def check(documents: dict[str, dict]) -> tuple[list[str], int]: failures: list[str] = [] owned_jobs = 0 @@ -187,10 +265,12 @@ def check(documents: dict[str, dict]) -> tuple[list[str], int]: steps = [step for step in keychain_guard.inline_step_scripts(job).get("steps") or [] if isinstance(step, dict)] gate = steps[0] if steps else {} - if (str(gate.get("run", "")).strip() != CLAIM_GATE.strip() or set(gate) - {"name", "run", "shell"} - or str(gate.get("shell", "bash")) != "bash"): + if (str(gate.get("run", "")).strip() != CLAIM_GATE.strip() or set(gate) - {"name", "run", "shell", "id"} + or str(gate.get("shell", "bash")) != "bash" or gate.get("id") != CLAIM_GATE_ID): failures.append(f"{label}: the first step must be the claim gate ({CLAIM_GATE_NAME}): CLAIM_GATE word " - "for word, with no if, uses, env or continue-on-error, before the checkout's token") + f"for word, id {CLAIM_GATE_ID}, with no if, uses, env or continue-on-error, before the " + "checkout's token") + failures.extend(f"{label}: {why}" for why in twin_failures(jobs, name, owned_reads)) claim = next((index for index, step in enumerate(steps) if CLAIM in str(step.get("run", ""))), None) first_secret = next((index for index, step in enumerate(steps) if "secrets." in json.dumps(step)), None) if claim is None: @@ -220,6 +300,14 @@ def check(documents: dict[str, dict]) -> tuple[list[str], int]: return failures, owned_jobs +def paired(picker: dict, name: str, job: dict) -> dict: + """The jobs of a workflow with picker `pick`, the owned-routed signing job `name` and its Blacksmith twin.""" + twin = {**job, "runs-on": "blacksmith-6vcpu-macos-26", "needs": [*job.get("needs", []), name], + "if": f"!cancelled() && github.run_attempt == 1 && needs.pick.outputs.owned != '' && needs.{name}.result != " + f"'skipped' && needs.{name}.result != 'cancelled' && needs.{name}.outputs.claim != 'owned'"} + return {"pick": picker, name: job, name + TWIN_SUFFIX: twin} + + def runner(name: str, labels: list[str], busy: bool = False, status: str = "online") -> dict: return {"name": name, "status": status, "busy": busy, "labels": [{"name": label} for label in labels]} @@ -238,11 +326,12 @@ def test_the_guard_catches_a_missing_claim_and_cleanup(self) -> None: "steps": [{"id": "pick", "run": f"python3 {OWNED_PICKER} owned"}]} good = {"runs-on": "${{ github.run_attempt == 1 && needs.pick.outputs.owned && " "fromJSON(needs.pick.outputs.owned) || 'blacksmith-6vcpu-macos-26' }}", - "steps": [{"name": CLAIM_GATE_NAME, "run": CLAIM_GATE}, {"run": CLAIM}, + "needs": ["pick"], "outputs": {"claim": "${{ steps.claim-gate.outputs.claim }}"}, + "steps": [{"name": CLAIM_GATE_NAME, "id": CLAIM_GATE_ID, "run": CLAIM_GATE}, {"run": CLAIM}, {"env": {"APPLE_CERTIFICATE_BASE64": "${{ secrets.APPLE_CERTIFICATE_BASE64 }}"}, "run": "scripts/ci/signing-keychain.sh create"}, {"if": "always()", "run": "scripts/ci/signing-keychain.sh delete"}]} - self.assertEqual(check({"release.yml": {"jobs": {"pick": picker, "build-sign-notarize": good}}}), ([], 1)) + self.assertEqual(check({"release.yml": {"jobs": paired(picker, "build-sign-notarize", good)}}), ([], 1)) cases = { "not in SIGNING_JOBS": ("other.yml", "sign", good), "without a step that runs": ("release.yml", "build-sign-notarize", @@ -268,37 +357,60 @@ def test_the_guard_catches_a_missing_claim_and_cleanup(self) -> None: } for needle, (workflow, name, job) in cases.items(): with self.subTest(needle): - failures, _ = check({workflow: {"jobs": {"pick": picker, name: job}}}) + failures, _ = check({workflow: {"jobs": paired(picker, name, job)}}) + self.assertTrue(any(needle in failure for failure in failures), failures) + twin_name = "build-sign-notarize" + TWIN_SUFFIX + both = paired(picker, "build-sign-notarize", good) + reader = {"needs": ["build-sign-notarize", twin_name], + "if": "!cancelled() && (needs.build-sign-notarize-blacksmith.outputs.claim || " + "needs.build-sign-notarize.outputs.claim) != ''"} + self.assertEqual(check({"release.yml": {"jobs": {**both, "publish": reader}}}), ([], 1)) + twin_cases = { + "no Blacksmith twin": {k: v for k, v in both.items() if k != twin_name}, + "must carry claim": {**both, "build-sign-notarize": {**good, "outputs": {}}}, + "must have the same steps": {**both, twin_name: {**both[twin_name], "steps": good["steps"][1:]}}, + "must take Blacksmith only": {**both, twin_name: {**both[twin_name], "runs-on": good["runs-on"]}}, + "if must be exactly": {**both, twin_name: {**both[twin_name], "if": "always()"}}, + "needs must be": {**both, twin_name: {**both[twin_name], "needs": ["pick"]}}, + "but not build-sign-notarize-blacksmith": {**both, "publish": {**reader, "needs": ["build-sign-notarize"]}}, + "without the twin first": {**both, "publish": {**reader, "if": "!cancelled() && " + "needs.build-sign-notarize.outputs.claim != ''"}}, + "has no status check": {**both, "publish": {**reader, "if": "(needs.build-sign-notarize-blacksmith.outputs." + "claim || needs.build-sign-notarize.outputs.claim) != ''"}}, + } + for needle, jobs in twin_cases.items(): + with self.subTest(needle): + failures, _ = check({"release.yml": {"jobs": jobs}}) self.assertTrue(any(needle in failure for failure in failures), failures) failures, _ = check({"release.yml": {"env": {"K": "${{ secrets.K }}"}, - "jobs": {"pick": picker, "build-sign-notarize": good}}}) + "jobs": paired(picker, "build-sign-notarize", good)}}) self.assertTrue(any("workflow-level env names a secret" in failure for failure in failures), failures) brackets = {**good, "runs-on": "${{ github.run_attempt == 1 && needs['pick'].outputs['owned'] && " "fromJSON(needs[\"pick\"].outputs.owned) || 'blacksmith-6vcpu-macos-26' }}", "steps": [good["steps"][0], {"env": {"K": "${{ secrets['APPLE_CERTIFICATE_BASE64'] }}"}, "run": "true"}, *good["steps"][1:]]} - failures, owned = check({"release.yml": {"jobs": {"pick": picker, "build-sign-notarize": brackets}}}) + failures, owned = check({"release.yml": {"jobs": paired(picker, "build-sign-notarize", brackets)}}) self.assertEqual(owned, 1, "bracket notation still reads as the owned pick") self.assertTrue(any("names a secret before the claim" in failure for failure in failures), failures) dynamic = {**good, "steps": [*good["steps"][:2], {"env": {"K": "${{ secrets[format('A{0}', 'X')] }}"}, "run": "true"}, *good["steps"][2:]]} - failures, _ = check({"release.yml": {"jobs": {"pick": picker, "build-sign-notarize": dynamic}}}) + failures, _ = check({"release.yml": {"jobs": paired(picker, "build-sign-notarize", dynamic)}}) self.assertTrue(any("cannot read" in failure for failure in failures), failures) - failures, _ = check({"release.yml": {"jobs": {"pick": picker, "build-sign-notarize": { - **good, "runs-on": "${{ fromJSON(needs[format('pi{0}', 'ck')].outputs.owned) }}"}}}}) + failures, _ = check({"release.yml": {"jobs": paired(picker, "build-sign-notarize", { + **good, "runs-on": "${{ fromJSON(needs[format('pi{0}', 'ck')].outputs.owned) }}"})}}) self.assertTrue(any("cannot read" in failure for failure in failures), failures) open_picker = {**picker, "steps": [{"id": "mint", "if": "vars.X == '1'", "with": {"k": "${{ secrets.K }}"}}, *picker["steps"]]} - failures, _ = check({"release.yml": {"jobs": {"pick": open_picker, "build-sign-notarize": good}}}) + failures, _ = check({"release.yml": {"jobs": paired(open_picker, "build-sign-notarize", good)}}) self.assertTrue(any("names a secret on any ref" in failure for failure in failures), failures) pinned = {**picker, "steps": [{"id": "mint", "if": "vars.X == '1' && github.ref == 'refs/heads/main'", "with": {"k": "${{ secrets.K }}"}}, {"id": "again", "if": "steps.mint.outcome == 'failure'", "with": {"k": "${{ secrets.K }}"}}, *picker["steps"]]} - self.assertEqual(check({"release.yml": {"jobs": {"pick": pinned, "build-sign-notarize": good}}}), ([], 1)) + self.assertEqual(check({"release.yml": {"jobs": paired(pinned, "build-sign-notarize", good)}}), ([], 1)) other = {"outputs": {"owned": "${{ steps.pick.outputs.owned }}"}, "steps": [{"id": "pick", "run": "python3 scripts/ci/nightly_build_runner.py verify"}]} - failures, _ = check({"release.yml": {"jobs": {"pick": other, "build-sign-notarize": good}}}) + failures, _ = check({"release.yml": {"jobs": paired(other, "build-sign-notarize", good)}}) self.assertTrue(any("signing job's runner comes from" in failure for failure in failures), failures) def test_drained_runner_takes_only_a_mini_running_nothing(self) -> None: @@ -449,8 +561,15 @@ def test_claim(self) -> None: refused = self.run_claim(tmp, **env) self.assertEqual(refused.returncode, 1, refused.stdout) self.assertIn("did not give the job the whole mini", refused.stdout) + for sha in ("", OLD_HOOK, SIGNING_HOOKS[0][:12], "x" * 64): + with self.subTest(sha=sha): + refused = self.run_claim(tmp, RUNNER_NAME="cmux12-glaeda-3", GLAEDA_JOB_CLAIM="exclusive", + GLAEDA_HOOK_SHA256=sha) + self.assertEqual(refused.returncode, 1, refused.stdout) + self.assertIn("is not one", refused.stdout) (tmp / "keychains" / "build.keychain-db").write_text("left by a killed job") - claimed = self.run_claim(tmp, RUNNER_NAME="cmux12-glaeda-3", GLAEDA_JOB_CLAIM="exclusive") + claimed = self.run_claim(tmp, RUNNER_NAME="cmux12-glaeda-3", GLAEDA_JOB_CLAIM="exclusive", + GLAEDA_HOOK_SHA256=FIRST_SIGNING_HOOK) self.assertEqual(claimed.returncode, 0, claimed.stdout + claimed.stderr) self.assertIn("holds the whole mini", claimed.stdout) self.assertIn("removed the signing keychain build.keychain", claimed.stdout) @@ -498,19 +617,73 @@ def test_search_list(self) -> None: class ClaimGateTests(unittest.TestCase): def test_gate(self) -> None: - def gate(**env: str) -> subprocess.CompletedProcess: - return subprocess.run(["bash", "-e", "-c", CLAIM_GATE], env={"PATH": "/usr/bin:/bin", **env}, - capture_output=True, text=True, timeout=30, check=False) - - fleet = Path("/Users/Shared/cmux-build-fleet").is_dir() - if not fleet: - self.assertEqual(gate(RUNNER_NAME="blacksmith-6vcpu-macos-26-abc").returncode, 0) - for env in ({"RUNNER_NAME": "cmux12-glaeda-3"}, {"RUNNER_NAME": "cmux12-glaeda", "GLAEDA_JOB_CLAIM": "light"}): - with self.subTest(env): - refused = gate(**env) - self.assertEqual(refused.returncode, 1, refused.stdout) - self.assertIn("did not give the job the whole mini", refused.stdout) - self.assertEqual(gate(RUNNER_NAME="cmux12-glaeda-3", GLAEDA_JOB_CLAIM="exclusive").returncode, 0) + with tempfile.TemporaryDirectory() as raw: + output = Path(raw) / "output" + + def gate(**env: str) -> subprocess.CompletedProcess: + output.write_text("") + return subprocess.run(["bash", "-e", "-c", CLAIM_GATE], + env={"PATH": "/usr/bin:/bin", "GITHUB_OUTPUT": os.fspath(output), **env}, + capture_output=True, text=True, timeout=30, check=False) + + fleet = Path("/Users/Shared/cmux-build-fleet").is_dir() + if not fleet: + self.assertEqual(gate(RUNNER_NAME="blacksmith-6vcpu-macos-26-abc").returncode, 0) + self.assertEqual(output.read_text(), "", "an ephemeral runner claims nothing: its twin never runs") + for env in ({"RUNNER_NAME": "cmux12-glaeda-3"}, + {"RUNNER_NAME": "cmux12-glaeda", "GLAEDA_JOB_CLAIM": "light", "GLAEDA_HOOK_SHA256": FIRST_SIGNING_HOOK}): + with self.subTest(env): + refused = gate(**env) + self.assertEqual(refused.returncode, 1, refused.stdout) + self.assertIn("did not give the job the whole mini", refused.stdout) + self.assertEqual(output.read_text(), "") + owned = {"RUNNER_NAME": "cmux12-glaeda-3", "GLAEDA_JOB_CLAIM": "exclusive"} + for sha in (None, "", OLD_HOOK, FIRST_SIGNING_HOOK[:12], FIRST_SIGNING_HOOK + "0", "x" * 64): + with self.subTest(sha=sha): + refused = gate(**owned, **({} if sha is None else {"GLAEDA_HOOK_SHA256": sha})) + self.assertEqual(refused.returncode, 1, refused.stdout) + self.assertIn("is not one scripts/ci/glaeda-signing-hooks.txt lists", refused.stdout) + self.assertEqual(output.read_text(), "", "a refused owned attempt leaves claim empty: the twin signs") + for sha in SIGNING_HOOKS: + with self.subTest(sha=sha): + passed = gate(**owned, GLAEDA_HOOK_SHA256=sha) + self.assertEqual(passed.returncode, 0, passed.stdout + passed.stderr) + self.assertEqual(output.read_text(), "claim=owned\n") + self.assertIn(FIRST_SIGNING_HOOK, SIGNING_HOOKS, "the hook with the leftover-process check passes") + self.assertNotIn(OLD_HOOK, SIGNING_HOOKS, "the hook from before it is refused") + + +class SigningHooksInStepWithGlaedaTests(unittest.TestCase): + """scripts/ci/glaeda-signing-hooks.txt names only hooks glaeda released with the leftover-process check: each sha + is one glaeda-disk pins in CMUX_NEXT_DRAIN_HOOK_SHA256, at or after FIRST_SIGNING_HOOK in its order.""" + + def glaeda_pins(self) -> list[str]: + last: Exception | None = None + for _ in range(3): + try: + with urllib.request.urlopen(GLAEDA_DISK_URL, timeout=20) as response: + text = response.read().decode() + break + except OSError as error: + last = error + else: + self.fail(f"could not read glaeda's pins from {GLAEDA_DISK_URL}: {last}") + block = re.search(r"CMUX_NEXT_DRAIN_HOOK_SHA256 = frozenset\(\{(.*?)\}\)", text, re.S) + self.assertIsNotNone(block, "glaeda-disk no longer declares CMUX_NEXT_DRAIN_HOOK_SHA256") + return re.findall(r'"([0-9a-f]{64})"', block.group(1)) + + def test_every_listed_hook_is_one_glaeda_released_with_the_check(self) -> None: + self.assertEqual(len(SIGNING_HOOKS), len(set(SIGNING_HOOKS)), "a sha is listed twice") + for sha in SIGNING_HOOKS: + self.assertRegex(sha, r"^[0-9a-f]{64}$") + pins = self.glaeda_pins() + self.assertIn(FIRST_SIGNING_HOOK, pins) + released = pins[pins.index(FIRST_SIGNING_HOOK):] + for sha in SIGNING_HOOKS: + self.assertIn(sha, released, f"{sha} is not a hook glaeda pinned at or after {FIRST_SIGNING_HOOK[:8]}") + if pins[-1] not in SIGNING_HOOKS: + print(f"warning: glaeda's newest hook {pins[-1]} is not in {SIGNING_HOOKS_FILE.name}; minis on it sign " + "on Blacksmith until it is added", file=sys.stderr) if __name__ == "__main__": diff --git a/tests/test_cmux_next_dev_daily_dispatch.py b/tests/test_cmux_next_dev_daily_dispatch.py index 04d4ed1881f0..04f4bc7724f8 100755 --- a/tests/test_cmux_next_dev_daily_dispatch.py +++ b/tests/test_cmux_next_dev_daily_dispatch.py @@ -85,7 +85,7 @@ def check(ok, message): publish_text = PUBLISH_WORKFLOW.read_text() verify = re.search(r"^ verify:\n(.*?)(?=^ publish:\n)", publish_text, re.MULTILINE | re.DOTALL) verify_text = verify.group(1) if verify else "" -check("needs: [sign, runner]" in verify_text, +check("needs: [sign, runner, sign-blacksmith]" in verify_text, "DEV verify must wait for the owned-Mac placement job") check("needs.runner.outputs.label" in verify_text and "blacksmith-6vcpu-macos-26" in verify_text, "DEV verify must prefer the picked owned Mac and retain a Blacksmith fallback") diff --git a/tests/upload_artifact_retention_allowlist.json b/tests/upload_artifact_retention_allowlist.json index 645a150ff258..6521b5662686 100644 --- a/tests/upload_artifact_retention_allowlist.json +++ b/tests/upload_artifact_retention_allowlist.json @@ -36,6 +36,7 @@ {"step": ".github/workflows/cmux-next-car-guards.yml::linux::car-guards-${{ needs.plan.outputs.car }}-portable", "reason": "a small receipt, report or log a person or a later run reads; baseline 2026-10-25 (public repo, not billed), lower to 1 day when touched"}, {"step": ".github/workflows/cmux-next-car-guards.yml::mac::car-guards-${{ needs.plan.outputs.car }}-mac", "reason": "a small receipt, report or log a person or a later run reads; baseline 2026-10-25 (public repo, not billed), lower to 1 day when touched"}, {"step": ".github/workflows/cmux-next-dev-daily.yml::sign::cmux-next-dev-daily-signed", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, + {"step": ".github/workflows/cmux-next-dev-daily.yml::sign-blacksmith::cmux-next-dev-daily-signed", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, {"step": ".github/workflows/cmux-next-frame-pacing.yml::bench::frame-pacing-${{ github.run_id }}", "reason": "a small receipt, report or log a person or a later run reads; baseline 2026-10-25 (public repo, not billed), lower to 1 day when touched"}, {"step": ".github/workflows/cmux-next-frame-pacing.yml::skipped-receipt::frame-pacing-${{ github.run_id }}", "reason": "a small receipt, report or log a person or a later run reads; baseline 2026-10-25 (public repo, not billed), lower to 1 day when touched"}, {"step": ".github/workflows/cmux-next-media.yml::tour::${{ steps.build_base.outputs.artifact }}", "reason": "a large kind read by a later job or run; baseline 2026-10-25 until it moves to R2 or the fleet SSD store"}, @@ -129,14 +130,18 @@ {"step": ".github/workflows/nightly.yml::publish-nightly::cmux-nightly-${{ steps.source.outputs.build_short_sha }}", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, {"step": ".github/workflows/nightly.yml::recover-nightly-next-notarization::cmux-nightly-recovered-variant-arm64", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, {"step": ".github/workflows/nightly.yml::sign-nightly-identity-legacy::cmux-nightly-identity-pending-bundle", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, + {"step": ".github/workflows/nightly.yml::sign-nightly-identity-legacy-blacksmith::cmux-nightly-identity-pending-bundle", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, {"step": ".github/workflows/nightly.yml::sign-nightly-identity-legacy::cmux-nightly-identity-signed", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, + {"step": ".github/workflows/nightly.yml::sign-nightly-identity-legacy-blacksmith::cmux-nightly-identity-signed", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, {"step": ".github/workflows/nightly.yml::sign-release-notes::release-notes", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics"}, {"step": ".github/workflows/r2-storage-census.yml::binaries::r2-census-binaries", "reason": "a small receipt, report or log a person or a later run reads; baseline 2026-10-25 (public repo, not billed), lower to 1 day when touched"}, {"step": ".github/workflows/r2-storage-census.yml::cef::r2-census-cef", "reason": "a small receipt, report or log a person or a later run reads; baseline 2026-10-25 (public repo, not billed), lower to 1 day when touched"}, {"step": ".github/workflows/r2-storage-census.yml::ci-cache::r2-census-ci-cache", "reason": "a small receipt, report or log a person or a later run reads; baseline 2026-10-25 (public repo, not billed), lower to 1 day when touched"}, {"step": ".github/workflows/relay-tls.yml::system-keychain::relay-tls-results", "reason": "a small receipt, report or log a person or a later run reads; baseline 2026-10-25 (public repo, not billed), lower to 1 day when touched"}, {"step": ".github/workflows/release.yml::build-sign-notarize::cmux-release-dry-run", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, + {"step": ".github/workflows/release.yml::build-sign-notarize-blacksmith::cmux-release-dry-run", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, {"step": ".github/workflows/repair-v0-64-25-helper-rpaths.yml::repair::cmux-v0.64.25-rpath-repair-dry-run", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, + {"step": ".github/workflows/repair-v0-64-25-helper-rpaths.yml::repair-blacksmith::cmux-v0.64.25-rpath-repair-dry-run", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics; flagged as a large kind by name or path"}, {"step": ".github/workflows/resume-nightly-notarization.yml::resume::cmux-${{ inputs.channel }}-notarization-verified-${{ inputs.variant }}-${{ github.run_id }}", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics"}, {"step": ".github/workflows/resume-nightly-notarization.yml::resume::cmux-${{ inputs.channel }}-outer-notarization-recovery-${{ inputs.variant }}-${{ github.run_id }}", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics"}, {"step": ".github/workflows/sdk-bootstrap-crates.yml::build::cmux-sdk-bootstrap-crate", "reason": "a release, sign, publish or package handoff: only its storage may change, in its own reviewed PR that keeps publish semantics"}, From 66df9934a2b022eb13b43d78c7a123b4d76ef9fc Mon Sep 17 00:00:00 2001 From: Leo Li Date: Thu, 29 Oct 2026 02:29:27 -0700 Subject: [PATCH 2/4] ci: the twins join the step registry, runner table and nightly guards; the notary receipt step moves to a script for room Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- .github/nightly-step-coverage.json | 21 ++++++++++ .github/runners.json | 4 ++ .github/workflows/nightly.yml | 41 ++----------------- .../ci/nightly-steps/notary-test-receipt.sh | 37 +++++++++++++++++ tests/test_nightly_sign_runner.py | 2 + tests/test_nightly_universal_build.sh | 6 +-- 6 files changed, 70 insertions(+), 41 deletions(-) create mode 100755 scripts/ci/nightly-steps/notary-test-receipt.sh diff --git a/.github/nightly-step-coverage.json b/.github/nightly-step-coverage.json index 049dd7e7e0de..f235e0535d0f 100644 --- a/.github/nightly-step-coverage.json +++ b/.github/nightly-step-coverage.json @@ -316,6 +316,27 @@ "Hand the stapled cmux NIGHTLY DMG to the verify job": {"pr": "ci-guards.yml:workflow-guard-tests:Validate Sparkle appcast macOS floor"}, "Record the attempt": {"plumbing": true}, "Cleanup keychain": {"nightly_only": "deletes the signing keychain"}}, + "sign-nightly-identity-legacy-blacksmith": { + "Hold the whole Mac before any credential": {"pr": "ci-guards.yml:workflow-guard-tests:Validate a signing job on an owned Mac holds the whole Mac and deletes its keychain"}, + "Claim the whole Mac for signing": {"pr": "ci-guards.yml:workflow-guard-tests:Validate a signing job on an owned Mac holds the whole Mac and deletes its keychain"}, + "Mint the release App token for this job's gh calls": {"nightly_only": "mints release GitHub App token"}, + "Send this job's gh calls as the release App, github.token as the fallback": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the nightly publish jobs send gh calls as the release App first"}, + "Wait through GitHub API rate limits in this job's gh calls": {"pr": "ci-guards.yml:workflow-guard-tests:Validate nightly publish-path gh calls wait through a rate limit"}, + "Prepare current NEXT DMG for the parallel identity": {"nightly_only": "rewrites the prepared Release cmux.app's Info.plist for the cmux NEXT identity before Developer ID signing; needs the Release app only the nightly builds"}, + "Skip a build the old feed already has": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the nightly-next release-ring tags"}, + "Initialize cmux-next notices submodule": {"plumbing": true}, + "Parse signing and notarize scripts with /bin/bash 3.2": {"pr": "signing-keychain-macos.yml:bash32-syntax:Parse every scripts/ci shell script with bash 3.2"}, + "Select Xcode": {"plumbing": true}, "Install create-dmg": {"plumbing": true}, + "Download the published cmux NEXT DMG": {"plumbing": true}, + "Import signing cert": {"nightly_only": "imports Developer ID signing certificate"}, + "Sign the build as cmux NIGHTLY": {"nightly_only": "signs with Developer ID and profile"}, + "Download the pending cmux NIGHTLY-identity submission": {"plumbing": true}, + "Check out the build's own source for the bundle checks": {"plumbing": true}, + "Notarize the cmux NIGHTLY DMG": {"nightly_only": "notarizes cmux NIGHTLY DMG (Apple)"}, + "Keep the pending cmux NIGHTLY-identity submission": {"plumbing": true}, + "Hand the stapled cmux NIGHTLY DMG to the verify job": {"pr": "ci-guards.yml:workflow-guard-tests:Validate Sparkle appcast macOS floor"}, + "Record the attempt": {"plumbing": true}, + "Cleanup keychain": {"nightly_only": "deletes the signing keychain"}}, "verify-nightly-identity-legacy": { "Select Xcode": {"plumbing": true}, "Check out the build's own source for the bundle checks": {"plumbing": true}, diff --git a/.github/runners.json b/.github/runners.json index 4a5ffce9cfdc..cf1df53d3554 100644 --- a/.github/runners.json +++ b/.github/runners.json @@ -152,6 +152,7 @@ "cmux-next-cua-helper-identity.yml#helper-identity": "macos-runtime-tests", "cmux-next-daemon-macos-shadow.yml#cmux-next-daemon-macos-shadow": "macos-runtime-tests", "cmux-next-dev-daily.yml#sign": "apple-signing-release", + "cmux-next-dev-daily.yml#sign-blacksmith": "apple-signing-release", "cmux-next-dev-daily.yml#verify": "apple-signing-release", "cmux-next-fast-tier-seed.yml#seed": "apple-app-build", "cmux-next-frame-pacing.yml#bench": { @@ -224,6 +225,7 @@ "nightly.yml#refresh-compilation-cache": "apple-app-build", "nightly.yml#refresh-test-compilation-cache": "apple-app-build", "nightly.yml#sign-nightly-identity-legacy": "apple-signing-release", + "nightly.yml#sign-nightly-identity-legacy-blacksmith": "apple-signing-release", "nightly.yml#signer-dmg-image": "apple-signing-release", "nightly.yml#verify-final-dmg-nightly": "apple-signing-release", "nightly.yml#verify-nightly-identity-legacy": "apple-signing-release", @@ -237,9 +239,11 @@ "relay-tls.yml#system-keychain": "macos-runtime-tests", "release.yml#build-ghostty-cli-helper": "apple-app-build", "release.yml#build-sign-notarize": "apple-signing-release", + "release.yml#build-sign-notarize-blacksmith": "apple-signing-release", "reload-build.yml#build": "apple-app-build", "remote-daemon.yml#remote-daemon-macos-tests": "macos-runtime-tests", "repair-v0-64-25-helper-rpaths.yml#repair": "apple-signing-release", + "repair-v0-64-25-helper-rpaths.yml#repair-blacksmith": "apple-signing-release", "repair-v0-64-25-helper-rpaths.yml#verify-shipped": "apple-signing-release", "resume-nightly-notarization.yml#resume": "apple-signing-release", "resume-nightly-notarization.yml#verify-stapled": "apple-signing-release", diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c97b4b88327d..911344412d3f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -4983,41 +4983,8 @@ jobs: EMPTY_BROWSER_SOURCES: ${{ needs.decide.outputs.empty_browser_sources }} NOTARY_PROBE: ${{ needs.decide.outputs.notary_probe }} NOTARIZE_OUTCOME: ${{ steps.notarize-nightly.outcome }} - run: | - set -uo pipefail - out="notary-receipt" - mkdir -p "$out" - { - echo "run: $GITHUB_RUN_ID attempt ${GITHUB_RUN_ATTEMPT:-1}" - echo "commit: ${{ needs.resolve-nightly-cmux-tui-client.outputs.build_sha }}" - echo "bundle id: $CHANNEL_BUNDLE_ID" - echo "production probe: $NOTARY_PROBE" - echo "import variant: $IMPORT_VARIANT" - echo "swift flags: ${SWIFT_IMPORT_FLAGS:-none}" - echo "empty browser-sources.json: $EMPTY_BROWSER_SOURCES" - echo "notarize step: $NOTARIZE_OUTCOME" - if [ -f "$NIGHTLY_DMG_RELEASE" ]; then echo "dmg sha256: $(shasum -a 256 "$NIGHTLY_DMG_RELEASE" | cut -d' ' -f1)"; fi - if [ -f "$NIGHTLY_DMG_RELEASE.notarization.state" ]; then grep -E '^(submission_id|status)=' "$NIGHTLY_DMG_RELEASE.notarization.state" || true; fi - } > "$out/receipt.txt" - [ -f "$NIGHTLY_DMG_RELEASE.notarization.log" ] && cp "$NIGHTLY_DMG_RELEASE.notarization.log" "$out/notarization.log" - /usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$CHANNEL_APP_PATH/Contents/Info.plist" > "$out/bundle-id.txt" 2>&1 - if /usr/libexec/PlistBuddy -c "Print :SUPublicEDKey" "$CHANNEL_APP_PATH/Contents/Info.plist" >/dev/null 2>&1; then - echo "SUPublicEDKey present" > "$out/sparkle.txt" - else - echo "SUPublicEDKey absent (Sparkle disabled)" > "$out/sparkle.txt" - fi - codesign -dvv "$CHANNEL_APP_PATH" > "$out/codesign.txt" 2>&1 - xcrun stapler validate "$CHANNEL_APP_PATH" > "$out/stapler-app.txt" 2>&1 - [ -f "$NIGHTLY_DMG_RELEASE" ] && xcrun stapler validate "$NIGHTLY_DMG_RELEASE" > "$out/stapler-dmg.txt" 2>&1 - spctl -a -vv -t exec "$CHANNEL_APP_PATH" > "$out/spctl-app.txt" 2>&1 - [ -f "$NIGHTLY_DMG_RELEASE" ] && spctl -a -vv -t open --context context:primary-signature "$NIGHTLY_DMG_RELEASE" > "$out/spctl-dmg.txt" 2>&1 - syspolicy_check distribution "$CHANNEL_APP_PATH" > "$out/syspolicy-check.txt" 2>&1 - cat "$out/receipt.txt" "$out/sparkle.txt" "$out/stapler-app.txt" - { echo "### cx-f58x notary test receipt"; echo '```'; cat "$out/receipt.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" - # Receipts are text only: refuse to upload anything else. - if find "$out" -type f ! -name '*.txt' ! -name '*.log' | grep -q .; then echo "non-text receipt file" >&2; exit 1; fi - exit 0 - + BUILD_SHA: ${{ needs.resolve-nightly-cmux-tui-client.outputs.build_sha }} + run: bash scripts/ci/nightly-steps/notary-test-receipt.sh - name: Upload notary test receipt if: always() && steps.notary-receipt.outcome == 'success' @@ -7187,15 +7154,13 @@ jobs: sign-nightly-identity-legacy-blacksmith: needs: [decide, build-nightly-app, legacy-identity-target, pick-nightly-sign-runner, sign-nightly-identity-legacy] - # The same run signs on Blacksmith when the owned Mac took sign-nightly-identity-legacy but its claim gate never passed - # (glaeda refused the job, or a hook scripts/ci/glaeda-signing-hooks.txt does not list admitted it). + # Signs in this run when the owned attempt never passed its claim gate. if: "!cancelled() && github.run_attempt == 1 && needs.pick-nightly-sign-runner.outputs.sign_owned != '' && needs.sign-nightly-identity-legacy.result != 'skipped' && needs.sign-nightly-identity-legacy.result != 'cancelled' && needs.sign-nightly-identity-legacy.outputs.claim != 'owned'" continue-on-error: true environment: release-next concurrency: *sign-nightly-identity-legacy-concurrency permissions: *sign-nightly-identity-legacy-permissions outputs: *sign-nightly-identity-legacy-outputs - # Blacksmith only: this run's sign when the owned Mac refused the job (its claim gate). runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.run_attempt > 1 && needs.legacy-identity-target.outputs.sign_alternate || needs.legacy-identity-target.outputs.sign_runner || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26' }} timeout-minutes: 160 env: *sign-nightly-identity-legacy-env diff --git a/scripts/ci/nightly-steps/notary-test-receipt.sh b/scripts/ci/nightly-steps/notary-test-receipt.sh new file mode 100755 index 000000000000..842df89c94bf --- /dev/null +++ b/scripts/ci/nightly-steps/notary-test-receipt.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# nightly.yml build-sign-notarize-nightly: "Write notary test receipt". Moved out of the workflow to keep +# nightly.yml under GitHub's workflow file size limit; the step's env: still sets every input. +set -uo pipefail +out="notary-receipt" +mkdir -p "$out" +{ + echo "run: $GITHUB_RUN_ID attempt ${GITHUB_RUN_ATTEMPT:-1}" + echo "commit: $BUILD_SHA" + echo "bundle id: $CHANNEL_BUNDLE_ID" + echo "production probe: $NOTARY_PROBE" + echo "import variant: $IMPORT_VARIANT" + echo "swift flags: ${SWIFT_IMPORT_FLAGS:-none}" + echo "empty browser-sources.json: $EMPTY_BROWSER_SOURCES" + echo "notarize step: $NOTARIZE_OUTCOME" + if [ -f "$NIGHTLY_DMG_RELEASE" ]; then echo "dmg sha256: $(shasum -a 256 "$NIGHTLY_DMG_RELEASE" | cut -d' ' -f1)"; fi + if [ -f "$NIGHTLY_DMG_RELEASE.notarization.state" ]; then grep -E '^(submission_id|status)=' "$NIGHTLY_DMG_RELEASE.notarization.state" || true; fi +} > "$out/receipt.txt" +[ -f "$NIGHTLY_DMG_RELEASE.notarization.log" ] && cp "$NIGHTLY_DMG_RELEASE.notarization.log" "$out/notarization.log" +/usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$CHANNEL_APP_PATH/Contents/Info.plist" > "$out/bundle-id.txt" 2>&1 +if /usr/libexec/PlistBuddy -c "Print :SUPublicEDKey" "$CHANNEL_APP_PATH/Contents/Info.plist" >/dev/null 2>&1; then + echo "SUPublicEDKey present" > "$out/sparkle.txt" +else + echo "SUPublicEDKey absent (Sparkle disabled)" > "$out/sparkle.txt" +fi +codesign -dvv "$CHANNEL_APP_PATH" > "$out/codesign.txt" 2>&1 +xcrun stapler validate "$CHANNEL_APP_PATH" > "$out/stapler-app.txt" 2>&1 +[ -f "$NIGHTLY_DMG_RELEASE" ] && xcrun stapler validate "$NIGHTLY_DMG_RELEASE" > "$out/stapler-dmg.txt" 2>&1 +spctl -a -vv -t exec "$CHANNEL_APP_PATH" > "$out/spctl-app.txt" 2>&1 +[ -f "$NIGHTLY_DMG_RELEASE" ] && spctl -a -vv -t open --context context:primary-signature "$NIGHTLY_DMG_RELEASE" > "$out/spctl-dmg.txt" 2>&1 +syspolicy_check distribution "$CHANNEL_APP_PATH" > "$out/syspolicy-check.txt" 2>&1 +cat "$out/receipt.txt" "$out/sparkle.txt" "$out/stapler-app.txt" +{ echo "### cx-f58x notary test receipt"; echo '```'; cat "$out/receipt.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" +# Receipts are text only: refuse to upload anything else. +if find "$out" -type f ! -name '*.txt' ! -name '*.log' | grep -q .; then echo "non-text receipt file" >&2; exit 1; fi +exit 0 + diff --git a/tests/test_nightly_sign_runner.py b/tests/test_nightly_sign_runner.py index 3c19f3eb61f8..a1f1b9d59504 100644 --- a/tests/test_nightly_sign_runner.py +++ b/tests/test_nightly_sign_runner.py @@ -37,6 +37,8 @@ "recover-nightly-next-notarization": "needs.find-nightly-next-recovery.outputs.sign_runner", "verify-recovered-nightly-next": "needs.find-nightly-next-recovery.outputs.sign_runner", "sign-nightly-identity-legacy": "needs.legacy-identity-target.outputs.sign_runner", + # Its same-run Blacksmith twin (tests/test_ci_signing_owned_mac.py). + "sign-nightly-identity-legacy-blacksmith": "needs.legacy-identity-target.outputs.sign_runner", "verify-nightly-identity-legacy": "needs.legacy-identity-target.outputs.sign_runner", # Signs the Sparkle deltas (the release environment), after the feed is published. "generate-nightly-deltas": "needs.pick-nightly-sign-runner.outputs.runner", diff --git a/tests/test_nightly_universal_build.sh b/tests/test_nightly_universal_build.sh index b450e963f132..98c38d732398 100644 --- a/tests/test_nightly_universal_build.sh +++ b/tests/test_nightly_universal_build.sh @@ -1465,7 +1465,7 @@ for expected in \ grep -Fq -- "$expected" <<<"$publish_job$sign_job" || { echo "FAIL: parallel NEXT identity preparation must contain: $expected"; exit 1; } done for expected in \ - "needs: [legacy-identity-target, sign-nightly-identity-legacy, pick-nightly-sign-runner]" \ + "needs: [legacy-identity-target, sign-nightly-identity-legacy, pick-nightly-sign-runner, sign-nightly-identity-legacy-blacksmith]" \ "runs-on: \${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || needs.pick-nightly-sign-runner.outputs.attempt == github.run_attempt && needs.pick-nightly-sign-runner.outputs.owned || github.run_attempt > 1 && needs.legacy-identity-target.outputs.sign_alternate || needs.legacy-identity-target.outputs.sign_runner || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26' }}" \ 'stapled_flag=(--require-stapled-app)' \ './scripts/ci/verify-dmg-artifact.sh "$dmg" nightly --expect-sha256 "$SIGNED_SHA256" "${stapled_flag[@]}"' \ @@ -1477,11 +1477,11 @@ if grep -Eq 'environment:|secrets\.' <<<"$verify_job" || ! grep -Fq 'vars.MACOS_ exit 1 fi for expected in \ - "needs: [decide, publish-nightly, legacy-identity-target, sign-nightly-identity-legacy, verify-nightly-identity-legacy]" \ + "needs: [decide, publish-nightly, legacy-identity-target, sign-nightly-identity-legacy, verify-nightly-identity-legacy, sign-nightly-identity-legacy-blacksmith]" \ "TESTED_SHA256: \${{ needs.verify-nightly-identity-legacy.outputs.tested_sha256 }}" \ './scripts/ci/nightly-next-legacy-tested-digest.sh' \ "group: nightly-next-legacy-identity-publish" \ - "needs.sign-nightly-identity-legacy.result != 'cancelled'" \ + "(needs.sign-nightly-identity-legacy-blacksmith.result == 'skipped' && needs.sign-nightly-identity-legacy.result || needs.sign-nightly-identity-legacy-blacksmith.result) != 'cancelled'" \ "needs.publish-nightly.outputs.refused != 'true'" \ "needs.publish-nightly.outputs.build == needs.legacy-identity-target.outputs.build" \ "needs.publish-nightly.result == 'skipped' && needs.legacy-identity-target.outputs.action == 'continue'" \ From 5cd808cc99d0bc769e4838384123ed89c619c1b1 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Thu, 29 Oct 2026 02:46:08 -0700 Subject: [PATCH 3/4] ci: the claim script matches a listed hook exactly; the notary receipt script keeps the step's errexit Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/ci/nightly-steps/notary-test-receipt.sh | 3 ++- scripts/ci/signing-runner-claim.sh | 8 +++++++- tests/test_ci_signing_owned_mac.py | 8 +++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/ci/nightly-steps/notary-test-receipt.sh b/scripts/ci/nightly-steps/notary-test-receipt.sh index 842df89c94bf..1b8566e2bc8e 100755 --- a/scripts/ci/nightly-steps/notary-test-receipt.sh +++ b/scripts/ci/nightly-steps/notary-test-receipt.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # nightly.yml build-sign-notarize-nightly: "Write notary test receipt". Moved out of the workflow to keep # nightly.yml under GitHub's workflow file size limit; the step's env: still sets every input. -set -uo pipefail +# The inline step ran under Actions' `bash -e`, then `set -uo pipefail`: the same options here. +set -euo pipefail out="notary-receipt" mkdir -p "$out" { diff --git a/scripts/ci/signing-runner-claim.sh b/scripts/ci/signing-runner-claim.sh index 9d530af4c75c..4498e95c7671 100755 --- a/scripts/ci/signing-runner-claim.sh +++ b/scripts/ci/signing-runner-claim.sh @@ -44,7 +44,13 @@ if [ "${GLAEDA_JOB_CLAIM:-}" != "exclusive" ]; then exit 1 fi hook_sha="${GLAEDA_HOOK_SHA256:-}" -if [ -z "$hook_sha" ] || ! grep -Eq "^${hook_sha}( |\$)" "$HOOKS_FILE" 2>/dev/null; then +listed="" +if [[ "$hook_sha" =~ ^[0-9a-f]{64}$ ]] && [ -r "$HOOKS_FILE" ]; then + while read -r sha _; do + if [ "$sha" = "$hook_sha" ]; then listed=1; fi + done < "$HOOKS_FILE" +fi +if [ -z "$listed" ]; then echo "::error title=Signing runner::This owned Mac's glaeda hook (GLAEDA_HOOK_SHA256='${hook_sha}') is not one $HOOKS_FILE lists. Only a listed hook waits out an earlier job's leftover processes; this run's Blacksmith job signs instead." exit 1 fi diff --git a/tests/test_ci_signing_owned_mac.py b/tests/test_ci_signing_owned_mac.py index 62f71b6e83b3..5b5941c26a38 100644 --- a/tests/test_ci_signing_owned_mac.py +++ b/tests/test_ci_signing_owned_mac.py @@ -561,12 +561,18 @@ def test_claim(self) -> None: refused = self.run_claim(tmp, **env) self.assertEqual(refused.returncode, 1, refused.stdout) self.assertIn("did not give the job the whole mini", refused.stdout) - for sha in ("", OLD_HOOK, SIGNING_HOOKS[0][:12], "x" * 64): + for sha in ("", OLD_HOOK, SIGNING_HOOKS[0][:12], "x" * 64, ".*", ".*.*", "[0-9a-f]*", + SIGNING_HOOKS[0] + " ", "#", SIGNING_HOOKS[0][:63] + "."): with self.subTest(sha=sha): refused = self.run_claim(tmp, RUNNER_NAME="cmux12-glaeda-3", GLAEDA_JOB_CLAIM="exclusive", GLAEDA_HOOK_SHA256=sha) self.assertEqual(refused.returncode, 1, refused.stdout) self.assertIn("is not one", refused.stdout) + for sha in SIGNING_HOOKS: + with self.subTest(listed=sha): + passed = self.run_claim(tmp, RUNNER_NAME="cmux12-glaeda-3", GLAEDA_JOB_CLAIM="exclusive", + GLAEDA_HOOK_SHA256=sha) + self.assertEqual(passed.returncode, 0, passed.stdout + passed.stderr) (tmp / "keychains" / "build.keychain-db").write_text("left by a killed job") claimed = self.run_claim(tmp, RUNNER_NAME="cmux12-glaeda-3", GLAEDA_JOB_CLAIM="exclusive", GLAEDA_HOOK_SHA256=FIRST_SIGNING_HOOK) From 650560b8eb3a5ce6072f3596d9a6db0c20de523f Mon Sep 17 00:00:00 2001 From: Leo Li Date: Thu, 29 Oct 2026 02:58:23 -0700 Subject: [PATCH 4/4] ci: keep nightly.yml under its size margin by moving the publication marker outcome step to a script (the notary receipt step stays inline) The build-sign-notarize-nightly checkout is the build's commit, so a new script there would be missing for older builds; the publication marker step runs from .publish-tooling, the workflow commit. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- .github/workflows/nightly.yml | 93 +++++++------------ .../ci/nightly-steps/notary-test-receipt.sh | 38 -------- .../ci/nightly-steps/publish-tag-outcome.sh | 57 ++++++++++++ 3 files changed, 93 insertions(+), 95 deletions(-) delete mode 100755 scripts/ci/nightly-steps/notary-test-receipt.sh create mode 100755 scripts/ci/nightly-steps/publish-tag-outcome.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 911344412d3f..1b4866034723 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -4983,8 +4983,41 @@ jobs: EMPTY_BROWSER_SOURCES: ${{ needs.decide.outputs.empty_browser_sources }} NOTARY_PROBE: ${{ needs.decide.outputs.notary_probe }} NOTARIZE_OUTCOME: ${{ steps.notarize-nightly.outcome }} - BUILD_SHA: ${{ needs.resolve-nightly-cmux-tui-client.outputs.build_sha }} - run: bash scripts/ci/nightly-steps/notary-test-receipt.sh + run: | + set -uo pipefail + out="notary-receipt" + mkdir -p "$out" + { + echo "run: $GITHUB_RUN_ID attempt ${GITHUB_RUN_ATTEMPT:-1}" + echo "commit: ${{ needs.resolve-nightly-cmux-tui-client.outputs.build_sha }}" + echo "bundle id: $CHANNEL_BUNDLE_ID" + echo "production probe: $NOTARY_PROBE" + echo "import variant: $IMPORT_VARIANT" + echo "swift flags: ${SWIFT_IMPORT_FLAGS:-none}" + echo "empty browser-sources.json: $EMPTY_BROWSER_SOURCES" + echo "notarize step: $NOTARIZE_OUTCOME" + if [ -f "$NIGHTLY_DMG_RELEASE" ]; then echo "dmg sha256: $(shasum -a 256 "$NIGHTLY_DMG_RELEASE" | cut -d' ' -f1)"; fi + if [ -f "$NIGHTLY_DMG_RELEASE.notarization.state" ]; then grep -E '^(submission_id|status)=' "$NIGHTLY_DMG_RELEASE.notarization.state" || true; fi + } > "$out/receipt.txt" + [ -f "$NIGHTLY_DMG_RELEASE.notarization.log" ] && cp "$NIGHTLY_DMG_RELEASE.notarization.log" "$out/notarization.log" + /usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$CHANNEL_APP_PATH/Contents/Info.plist" > "$out/bundle-id.txt" 2>&1 + if /usr/libexec/PlistBuddy -c "Print :SUPublicEDKey" "$CHANNEL_APP_PATH/Contents/Info.plist" >/dev/null 2>&1; then + echo "SUPublicEDKey present" > "$out/sparkle.txt" + else + echo "SUPublicEDKey absent (Sparkle disabled)" > "$out/sparkle.txt" + fi + codesign -dvv "$CHANNEL_APP_PATH" > "$out/codesign.txt" 2>&1 + xcrun stapler validate "$CHANNEL_APP_PATH" > "$out/stapler-app.txt" 2>&1 + [ -f "$NIGHTLY_DMG_RELEASE" ] && xcrun stapler validate "$NIGHTLY_DMG_RELEASE" > "$out/stapler-dmg.txt" 2>&1 + spctl -a -vv -t exec "$CHANNEL_APP_PATH" > "$out/spctl-app.txt" 2>&1 + [ -f "$NIGHTLY_DMG_RELEASE" ] && spctl -a -vv -t open --context context:primary-signature "$NIGHTLY_DMG_RELEASE" > "$out/spctl-dmg.txt" 2>&1 + syspolicy_check distribution "$CHANNEL_APP_PATH" > "$out/syspolicy-check.txt" 2>&1 + cat "$out/receipt.txt" "$out/sparkle.txt" "$out/stapler-app.txt" + { echo "### cx-f58x notary test receipt"; echo '```'; cat "$out/receipt.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + # Receipts are text only: refuse to upload anything else. + if find "$out" -type f ! -name '*.txt' ! -name '*.log' | grep -q .; then echo "non-text receipt file" >&2; exit 1; fi + exit 0 + - name: Upload notary test receipt if: always() && steps.notary-receipt.outcome == 'success' @@ -6345,61 +6378,7 @@ jobs: NEXT_APPCASTS_OUTCOME: ${{ steps.upload_next_appcasts.outcome }} CHANNEL_RELEASE_TAG: ${{ needs.decide.outputs.release_tag }} HEAD_SHA: ${{ steps.source.outputs.build_sha }} - run: | - set -euo pipefail - if [ "$TAG_MOVE_OUTCOME" = "success" ] && [ "$CHANNEL_RELEASE_TAG" = nightly-next ]; then - echo "tag_status=updated" >> "$GITHUB_OUTPUT" - { - echo "### Feed updated" - echo "" - echo "\`$CHANNEL_FEED_BASE\` serves \`$HEAD_SHA\` from R2; publish-nightly-next-github copies it to the release and moves the tag." - } >> "$GITHUB_STEP_SUMMARY" - elif [ "$TAG_MOVE_OUTCOME" = "success" ]; then - echo "tag_status=updated" >> "$GITHUB_OUTPUT" - { - echo "### Channel release tag updated" - echo "" - echo "\`$CHANNEL_RELEASE_TAG\` now points to \`$HEAD_SHA\`." - } >> "$GITHUB_STEP_SUMMARY" - elif [ "$TAG_MOVE_OUTCOME" != "failure" ]; then - # The tag move runs only after every publication step succeeded, so - # a skipped or cancelled move means the publication itself stopped - # (run 40076543798 said "are live" when no asset had uploaded). - # Name what each part did instead of claiming any of it is live. - appcasts="$APPCASTS_OUTCOME" - if [ -z "$appcasts" ] || [ "$appcasts" = "skipped" ]; then - appcasts="$NEXT_APPCASTS_OUTCOME" - fi - case "$appcasts" in - success) appcast_state="updated" ;; - skipped|"") appcast_state="not uploaded, still serving the previous build" ;; - *) appcast_state="$appcasts, may be partly updated" ;; - esac - parts="release assets: ${ASSETS_OUTCOME:-not run}; release metadata: ${METADATA_OUTCOME:-not run}; appcasts: $appcast_state; $CHANNEL_RELEASE_TAG tag: not moved" - if [ "$ASSETS_OUTCOME" != "success" ] && [ "${METADATA_OUTCOME:-skipped}" = "skipped" ] && [ "$appcast_state" = "not uploaded, still serving the previous build" ]; then - headline="Nothing is live for $HEAD_SHA" - else - headline="Publication of $HEAD_SHA did not complete" - fi - echo "tag_status=unpublished" >> "$GITHUB_OUTPUT" - echo "::warning::$headline ($parts). See the first failed step above." - { - echo "### Not published" - echo "" - echo "$headline ($parts)." - } >> "$GITHUB_STEP_SUMMARY" - else - echo "tag_status=stale" >> "$GITHUB_OUTPUT" - echo "::warning::Published assets, release metadata, and appcasts are live; the $CHANNEL_RELEASE_TAG tag marker is stale because its final update failed." - { - echo "### Published, tag marker stale" - echo "" - echo "Assets, release metadata, and appcasts for \`$HEAD_SHA\` are published, but \`$CHANNEL_RELEASE_TAG\` was not advanced. The next eligible publication will retry the marker without republishing this commit." - } >> "$GITHUB_STEP_SUMMARY" - fi - - - # Never fails the job (later jobs need it); report-nightly-failure reads feed_check. + run: bash .publish-tooling/scripts/ci/nightly-steps/publish-tag-outcome.sh - name: Read the live nightly-next feed back id: feedcheck if: always() && steps.recheck.outputs.already_published != 'true' && steps.upload_next_appcasts.outcome == 'success' diff --git a/scripts/ci/nightly-steps/notary-test-receipt.sh b/scripts/ci/nightly-steps/notary-test-receipt.sh deleted file mode 100755 index 1b8566e2bc8e..000000000000 --- a/scripts/ci/nightly-steps/notary-test-receipt.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# nightly.yml build-sign-notarize-nightly: "Write notary test receipt". Moved out of the workflow to keep -# nightly.yml under GitHub's workflow file size limit; the step's env: still sets every input. -# The inline step ran under Actions' `bash -e`, then `set -uo pipefail`: the same options here. -set -euo pipefail -out="notary-receipt" -mkdir -p "$out" -{ - echo "run: $GITHUB_RUN_ID attempt ${GITHUB_RUN_ATTEMPT:-1}" - echo "commit: $BUILD_SHA" - echo "bundle id: $CHANNEL_BUNDLE_ID" - echo "production probe: $NOTARY_PROBE" - echo "import variant: $IMPORT_VARIANT" - echo "swift flags: ${SWIFT_IMPORT_FLAGS:-none}" - echo "empty browser-sources.json: $EMPTY_BROWSER_SOURCES" - echo "notarize step: $NOTARIZE_OUTCOME" - if [ -f "$NIGHTLY_DMG_RELEASE" ]; then echo "dmg sha256: $(shasum -a 256 "$NIGHTLY_DMG_RELEASE" | cut -d' ' -f1)"; fi - if [ -f "$NIGHTLY_DMG_RELEASE.notarization.state" ]; then grep -E '^(submission_id|status)=' "$NIGHTLY_DMG_RELEASE.notarization.state" || true; fi -} > "$out/receipt.txt" -[ -f "$NIGHTLY_DMG_RELEASE.notarization.log" ] && cp "$NIGHTLY_DMG_RELEASE.notarization.log" "$out/notarization.log" -/usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$CHANNEL_APP_PATH/Contents/Info.plist" > "$out/bundle-id.txt" 2>&1 -if /usr/libexec/PlistBuddy -c "Print :SUPublicEDKey" "$CHANNEL_APP_PATH/Contents/Info.plist" >/dev/null 2>&1; then - echo "SUPublicEDKey present" > "$out/sparkle.txt" -else - echo "SUPublicEDKey absent (Sparkle disabled)" > "$out/sparkle.txt" -fi -codesign -dvv "$CHANNEL_APP_PATH" > "$out/codesign.txt" 2>&1 -xcrun stapler validate "$CHANNEL_APP_PATH" > "$out/stapler-app.txt" 2>&1 -[ -f "$NIGHTLY_DMG_RELEASE" ] && xcrun stapler validate "$NIGHTLY_DMG_RELEASE" > "$out/stapler-dmg.txt" 2>&1 -spctl -a -vv -t exec "$CHANNEL_APP_PATH" > "$out/spctl-app.txt" 2>&1 -[ -f "$NIGHTLY_DMG_RELEASE" ] && spctl -a -vv -t open --context context:primary-signature "$NIGHTLY_DMG_RELEASE" > "$out/spctl-dmg.txt" 2>&1 -syspolicy_check distribution "$CHANNEL_APP_PATH" > "$out/syspolicy-check.txt" 2>&1 -cat "$out/receipt.txt" "$out/sparkle.txt" "$out/stapler-app.txt" -{ echo "### cx-f58x notary test receipt"; echo '```'; cat "$out/receipt.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" -# Receipts are text only: refuse to upload anything else. -if find "$out" -type f ! -name '*.txt' ! -name '*.log' | grep -q .; then echo "non-text receipt file" >&2; exit 1; fi -exit 0 - diff --git a/scripts/ci/nightly-steps/publish-tag-outcome.sh b/scripts/ci/nightly-steps/publish-tag-outcome.sh new file mode 100755 index 000000000000..a3251777433f --- /dev/null +++ b/scripts/ci/nightly-steps/publish-tag-outcome.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# nightly.yml publish-nightly: "Record nightly publication marker outcome". Moved out of the workflow to keep +# nightly.yml under GitHub's workflow file size limit; the step's env: still sets every input. +set -euo pipefail +if [ "$TAG_MOVE_OUTCOME" = "success" ] && [ "$CHANNEL_RELEASE_TAG" = nightly-next ]; then + echo "tag_status=updated" >> "$GITHUB_OUTPUT" + { + echo "### Feed updated" + echo "" + echo "\`$CHANNEL_FEED_BASE\` serves \`$HEAD_SHA\` from R2; publish-nightly-next-github copies it to the release and moves the tag." + } >> "$GITHUB_STEP_SUMMARY" +elif [ "$TAG_MOVE_OUTCOME" = "success" ]; then + echo "tag_status=updated" >> "$GITHUB_OUTPUT" + { + echo "### Channel release tag updated" + echo "" + echo "\`$CHANNEL_RELEASE_TAG\` now points to \`$HEAD_SHA\`." + } >> "$GITHUB_STEP_SUMMARY" +elif [ "$TAG_MOVE_OUTCOME" != "failure" ]; then + # The tag move runs only after every publication step succeeded, so + # a skipped or cancelled move means the publication itself stopped + # (run 40076543798 said "are live" when no asset had uploaded). + # Name what each part did instead of claiming any of it is live. + appcasts="$APPCASTS_OUTCOME" + if [ -z "$appcasts" ] || [ "$appcasts" = "skipped" ]; then + appcasts="$NEXT_APPCASTS_OUTCOME" + fi + case "$appcasts" in + success) appcast_state="updated" ;; + skipped|"") appcast_state="not uploaded, still serving the previous build" ;; + *) appcast_state="$appcasts, may be partly updated" ;; + esac + parts="release assets: ${ASSETS_OUTCOME:-not run}; release metadata: ${METADATA_OUTCOME:-not run}; appcasts: $appcast_state; $CHANNEL_RELEASE_TAG tag: not moved" + if [ "$ASSETS_OUTCOME" != "success" ] && [ "${METADATA_OUTCOME:-skipped}" = "skipped" ] && [ "$appcast_state" = "not uploaded, still serving the previous build" ]; then + headline="Nothing is live for $HEAD_SHA" + else + headline="Publication of $HEAD_SHA did not complete" + fi + echo "tag_status=unpublished" >> "$GITHUB_OUTPUT" + echo "::warning::$headline ($parts). See the first failed step above." + { + echo "### Not published" + echo "" + echo "$headline ($parts)." + } >> "$GITHUB_STEP_SUMMARY" +else + echo "tag_status=stale" >> "$GITHUB_OUTPUT" + echo "::warning::Published assets, release metadata, and appcasts are live; the $CHANNEL_RELEASE_TAG tag marker is stale because its final update failed." + { + echo "### Published, tag marker stale" + echo "" + echo "Assets, release metadata, and appcasts for \`$HEAD_SHA\` are published, but \`$CHANNEL_RELEASE_TAG\` was not advanced. The next eligible publication will retry the marker without republishing this commit." + } >> "$GITHUB_STEP_SUMMARY" +fi + + + # Never fails the job (later jobs need it); report-nightly-failure reads feed_check.