fix(ci): resume timed-out nightly notarization

.github/workflows/ci-guards.yml

@@ -1153,9 +1153,9 @@ jobs:
         if: ${{ matrix.group == 'release-notary' }}
         run: ./tests/test_notarize_nightly_dmg.sh
 
-      - name: Validate automatic nightly notarization recovery
+      - name: Validate nightly notarization recovery manifest
         if: ${{ matrix.group == 'release-notary' }}
-        run: ./tests/test_nightly_notarization_recovery.sh
+        run: python3 tests/test_notarization_recovery_manifest.py
 
       - name: Validate Sparkle delta finalization
         if: ${{ matrix.group == 'release-notary' }}

.github/workflows/nightly.yml

@@ -1416,7 +1416,7 @@ jobs:
           name: cmux-nightly-remote-daemon
           path: remote-daemon-assets/cmuxd-remote-*
           if-no-files-found: error
-          retention-days: 3
+          retention-days: 14
 
       - name: Strip unsigned nightly app before transfer
         run: |
@@ -2036,11 +2036,16 @@ jobs:
       - name: Notarize app ticket through final DMG
         id: notarize-nightly
         if: needs.decide.outputs.fast_build != 'true'
+        # Computer Use helper propagation can still take about 20 minutes;
+        # this budget covers that work and DMG validation, not Apple's outer
+        # notarization queue, which is asynchronous now.
+        timeout-minutes: 50
         env:
           ASC_API_KEY_ID: ${{ secrets.ASC_API_KEY_ID }}
           ASC_API_ISSUER_ID: ${{ secrets.ASC_API_ISSUER_ID }}
           ASC_API_KEY_P8_BASE64: ${{ secrets.ASC_API_KEY_P8_BASE64 }}
           APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }}
+          CMUX_NOTARY_SUBMIT_ONLY: ${{ needs.decide.outputs.should_publish }}
           CMUX_COMPUTER_USE_NOTARY_SUBMISSION_FILE: ${{ runner.temp }}/cmux-computer-use-notarization.state
           CMUX_CHANNEL: ${{ needs.decide.outputs.channel }}
         run: |
@@ -2049,21 +2054,108 @@ jobs:
             "$NIGHTLY_DMG_RELEASE" \
             "$NIGHTLY_DMG_IMMUTABLE"
 
-      # A timed-out Apple submission remains usable, but only with the exact
-      # signed DMG that produced its ID. Retain that image, app, and sidecars
-      # for a follow-up workflow; this artifact is never part of publication.
+      # Submit the exact signed DMG and hand Apple processing to the trusted
+      # workflow_run continuation. The build lane never waits on Apple's queue.
+      - name: Prepare pending notarization recovery artifact
+        id: prepare-notarization-recovery
+        if: always() && (steps.notarize-nightly.outputs.submission_pending == 'true' || steps.notarize-nightly.outcome == 'failure')
+        run: |
+          set -euo pipefail
+          for required in \
+            "$NIGHTLY_DMG_RELEASE" \
+            "$NIGHTLY_DMG_RELEASE.notarization.state" \
+            "$NIGHTLY_DMG_RELEASE.notarization.log" \
+            "$CHANNEL_APP_PATH"; do
+            if [ ! -e "$required" ]; then
+              echo "Missing notarization recovery input: $required" >&2
+              exit 1
+            fi
+          done
+          recovery_app_archive="cmux-nightly-notarization-recovery-app.tar.gz"
+          rm -f "$recovery_app_archive"
+          tar -C "$(dirname "$CHANNEL_APP_PATH")" -czf "$recovery_app_archive" "$(basename "$CHANNEL_APP_PATH")"
+          dmg_sha256="$(shasum -a 256 "$NIGHTLY_DMG_RELEASE" | awk '{print $1}')"
+          [[ "$dmg_sha256" =~ ^[[:xdigit:]]{64}$ ]] || {
+            echo "Could not record a SHA-256 for $NIGHTLY_DMG_RELEASE" >&2
+            exit 1
+          }
+          python3 - \
+            "$NIGHTLY_DMG_RELEASE.notarization.state" \
+            "$dmg_sha256" \
+            "$NIGHTLY_DMG_RELEASE" \
+            "$CHANNEL_APP_PATH" \
+            "$CHANNEL" \
+            "$NIGHTLY_VARIANT" \
+            "$CHANNEL_RELEASE_TAG" \
+            "$NIGHTLY_BUILD" \
+            "${{ needs.decide.outputs.head_sha }}" \
+            "${{ needs.decide.outputs.short_sha }}" <<'PY'
+          import json
+          import pathlib
+          import sys
+
+          state_path, actual_sha, dmg_path, app_path, channel, variant, release_tag, build, head_sha, short_sha = sys.argv[1:]
+          state = {}
+          for line in pathlib.Path(state_path).read_text(encoding="utf-8").splitlines():
+              if not line:
+                  continue
+              key, sep, value = line.partition("=")
+              if not sep or key in state:
+                  raise SystemExit(f"invalid notarization state line: {line!r}")
+              state[key] = value
+          if state.get("dmg_sha256") != actual_sha:
+              raise SystemExit("notarization state SHA-256 does not match the signed DMG")
+          if state.get("submit_exit", "0") != "0":
+              raise SystemExit("notarization submit exited nonzero; refusing automatic continuation")
+          submission_id = state.get("submission_id", "")
+          if not submission_id or not all(character.isalnum() or character in "._-" for character in submission_id):
+              raise SystemExit("notarization state has no valid Apple submission id")
+          for key in ("immutable_path", "release_tag", "dmg_prefix", "variant", "channel"):
+              if not state.get(key):
+                  raise SystemExit(f"notarization state is missing {key}")
+          if state["channel"] != channel or state["variant"] != variant or state["release_tag"] != release_tag:
+              raise SystemExit("notarization state metadata does not match the build")
+          if state.get("dmg_path", "").split("/")[-1] != pathlib.Path(dmg_path).name:
+              raise SystemExit("notarization state points at a different DMG")
+          pathlib.Path("cmux-nightly-notarization-recovery.json").write_text(
+              json.dumps({
+                  "schema": 1,
+                  "source_run_id": "${{ github.run_id }}",
+                  "source_run_attempt": "${{ github.run_attempt }}",
+                  "should_publish": "${{ needs.decide.outputs.should_publish }}" == "true",
+                  "channel": channel,
+                  "variant": variant,
+                  "release_tag": release_tag,
+                  "build": build,
+                  "head_sha": head_sha,
+                  "short_sha": short_sha,
+                  "dmg_path": pathlib.Path(dmg_path).name,
+                  "app_path": f"cmux-nightly-notarization-recovery-app/{pathlib.Path(app_path).name}",
+                  "app_archive_path": "cmux-nightly-notarization-recovery-app.tar.gz",
+                  "state_path": pathlib.Path(state_path).name,
+                  "log_path": f"{pathlib.Path(dmg_path).name}.notarization.log",
+                  "immutable_path": state.get("immutable_path", ""),
+                  "dmg_prefix": state.get("dmg_prefix", ""),
+                  "dmg_sha256": actual_sha,
+                  "submission_id": submission_id,
+              }, indent=2) + "\n",
+              encoding="utf-8",
+          )
+          PY
+
       - name: Upload pending notarization recovery artifact
-        if: failure() && needs.decide.outputs.fast_build != 'true' && steps.notarize-nightly.outcome == 'failure'
+        if: always() && steps.prepare-notarization-recovery.outcome == 'success'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: cmux-nightly-notarization-recovery-${{ matrix.variant }}-${{ needs.decide.outputs.short_sha }}
+          name: cmux-${{ needs.decide.outputs.channel }}-notarization-recovery-${{ matrix.variant }}-${{ needs.decide.outputs.short_sha }}
           path: |
             ${{ env.NIGHTLY_DMG_RELEASE }}
             ${{ env.NIGHTLY_DMG_RELEASE }}.notarization.state
             ${{ env.NIGHTLY_DMG_RELEASE }}.notarization.log
-            ${{ env.CHANNEL_APP_PATH }}
-          if-no-files-found: warn
-          retention-days: 3
+            cmux-nightly-notarization-recovery-app.tar.gz
+            cmux-nightly-notarization-recovery.json
+          if-no-files-found: error
+          retention-days: 14
 
       - name: Package signed fast dogfood DMG
         if: needs.decide.outputs.fast_build == 'true'
@@ -2078,11 +2170,11 @@ jobs:
             "$NIGHTLY_DMG_IMMUTABLE"
 
       - name: Gate distribution with syspolicy_check
-        if: needs.decide.outputs.fast_build != 'true'
+        if: needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.should_publish != 'true'
         run: syspolicy_check distribution "$CHANNEL_APP_PATH"
 
       - name: Generate Sparkle appcasts (nightly)
-        if: needs.decide.outputs.fast_build != 'true'
+        if: needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.should_publish != 'true'
         env:
           NIGHTLY_SPARKLE_KEY: ${{ vars.NIGHTLY_SPARKLE_KEY }}
           CHANNEL: ${{ needs.decide.outputs.channel }}
@@ -2101,7 +2193,7 @@ jobs:
           fi
 
       - name: Upload nightly variant artifacts
-        if: needs.decide.outputs.fast_build != 'true'
+        if: needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.should_publish != 'true'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: cmux-nightly-variant-${{ matrix.variant }}
@@ -2147,7 +2239,10 @@ jobs:
     needs: [decide, build-nightly-app, build-sign-notarize-nightly]
     outputs:
       tag_status: ${{ steps.tag-outcome.outputs.tag_status }}
-    if: needs.decide.outputs.should_build == 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.build_only != 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *')
+    # Published paths are completed by auto-resume-nightly-notarization after
+    # Apple accepts every exact DMG. This job remains only for private branch
+    # artifacts, where no Apple ticket or release publication is involved.
+    if: needs.decide.outputs.should_build == 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.build_only != 'true' && needs.decide.outputs.should_publish != 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *')
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
     timeout-minutes: 30
     env: