ci(nightly): republished nightly-next builds follow their source's CEF pack and source archive; report a failed fetch

.github/workflows/nightly.yml

@@ -4691,6 +4691,8 @@ jobs:
       tip_sha: ${{ steps.fetch.outputs.tip_sha }}
       behind: ${{ steps.fetch.outputs.behind }}
       behind_hours: ${{ steps.fetch.outputs.behind_hours }}
+      cef: ${{ steps.fetch.outputs.cef }}
+      source_archive: ${{ steps.fetch.outputs.source_archive }}
     steps:
       - *checkout-default
 
@@ -4719,7 +4721,8 @@ jobs:
           if-no-files-found: error
           retention-days: 3
 
-      # publish-nightly reads the pack by this run's name when CMUX_NEXT_CEF_ON_DEMAND is 1.
+      # publish-nightly reads the pack by this run's name when the source build has one (its cef
+      # output), whatever CMUX_NEXT_CEF_ON_DEMAND says now: the app's pin names that pack.
       - name: Upload the republished build's Chromium runtime pack
         if: steps.fetch.outputs.cef == 'true'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -5035,6 +5038,7 @@ jobs:
       # feed_written is false when a newer build was already on the feed.
       variant_pattern: ${{ steps.source.outputs.variant_pattern }}
       tested_pattern: ${{ steps.source.outputs.tested_pattern }}
+      source_archive: ${{ steps.source.outputs.source_archive }}
       tip_sha: ${{ steps.source.outputs.tip_sha }}
       behind: ${{ steps.source.outputs.behind }}
       behind_hours: ${{ steps.source.outputs.behind_hours }}
@@ -5098,9 +5102,19 @@ jobs:
           REPUBLISHED_TIP_SHA: ${{ needs.republish-nightly-next.outputs.tip_sha }}
           REPUBLISHED_BEHIND: ${{ needs.republish-nightly-next.outputs.behind }}
           REPUBLISHED_BEHIND_HOURS: ${{ needs.republish-nightly-next.outputs.behind_hours }}
+          # Chromium on demand and the source offer: the switches as they are now, except that a
+          # republished build publishes the pack its source signed with (the app's pin names it), and
+          # no source archive when its source made none.
+          CEF_ON_DEMAND: ${{ vars.CMUX_NEXT_CEF_ON_DEMAND }}
+          PUBLISH_SOURCE_ARCHIVE: ${{ vars.CMUX_NEXT_PUBLISH_SOURCE_ARCHIVE }}
+          REPUBLISHED_CEF: ${{ needs.republish-nightly-next.outputs.cef }}
+          REPUBLISHED_SOURCE_ARCHIVE: ${{ needs.republish-nightly-next.outputs.source_archive }}
         run: |
           set -euo pipefail
           run_id=
+          cef=false source_archive=false
+          [ "${CEF_ON_DEMAND:-}" = 1 ] && cef=true
+          [ "${PUBLISH_SOURCE_ARCHIVE:-}" = 1 ] && source_archive=true
           if [ "$OWN_RESULT" = success ] && [ "$OWN_PENDING" != true ] && [ "$OWN_VERIFIED" = success ] && [ "$OWN_DMG_VERIFIED" = success ] && [ "$OWN_TREE" = ok ]; then
             source=own build="$OWN_BUILD" build_sha="$OWN_BUILD_SHA" tip_sha="$OWN_TIP_SHA"
             behind="$OWN_BEHIND" behind_hours="$OWN_BEHIND_HOURS" pattern="cmux-nightly-variant-*"
@@ -5115,6 +5129,8 @@ jobs:
             source=recovered build="$REPUBLISHED_BUILD" build_sha="$REPUBLISHED_BUILD_SHA" tip_sha="$REPUBLISHED_TIP_SHA"
             behind="$REPUBLISHED_BEHIND" behind_hours="$REPUBLISHED_BEHIND_HOURS" pattern="cmux-nightly-republished-variant-*"
             tested_pattern="cmux-nightly-republished-tested-*" run_id="$REPUBLISHED_RUN_ID"
+            [ "${REPUBLISHED_CEF:-}" = true ] && cef=true || cef=false
+            [ "${REPUBLISHED_SOURCE_ARCHIVE:-}" = true ] || source_archive=false
             echo "::notice title=Republishing a verified nightly-next build::build $build from run $REPUBLISHED_RUN_ID, with this run's publication tooling"
           else
             echo "nothing accepted to publish" >&2
@@ -5132,6 +5148,8 @@ jobs:
             echo "variant_pattern=$pattern"
             echo "tested_pattern=$tested_pattern"
             echo "run_id=$run_id"
+            echo "cef=$cef"
+            echo "source_archive=$source_archive"
           } >> "$GITHUB_OUTPUT"
           echo "NIGHTLY_BUILD=$build" >> "$GITHUB_ENV"
 
@@ -5419,16 +5437,17 @@ jobs:
       # appcast enclosure and object key. It never receives CF_R2_* .
       # Chromium on demand: the pack the app's pin names goes up before the
       # appcast that offers the app. A recovered build from another run has no
-      # pack artifact here, so this fails and nothing is offered.
+      # pack artifact here, so this fails and nothing is offered. A republished
+      # build follows its source (steps.source.outputs.cef), not the switch.
       - name: Download the Chromium runtime pack
-        if: needs.decide.outputs.should_publish == 'true' && needs.decide.outputs.track == 'nightly-next' && vars.CMUX_NEXT_CEF_ON_DEMAND == '1' && steps.recheck.outputs.already_published != 'true' && steps.gate.outputs.refused != 'true'
+        if: needs.decide.outputs.should_publish == 'true' && needs.decide.outputs.track == 'nightly-next' && steps.source.outputs.cef == 'true' && steps.recheck.outputs.already_published != 'true' && steps.gate.outputs.refused != 'true'
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: cef-runtime-pack
           path: cef-runtime
 
       - name: Publish the Chromium runtime pack
-        if: needs.decide.outputs.should_publish == 'true' && needs.decide.outputs.track == 'nightly-next' && vars.CMUX_NEXT_CEF_ON_DEMAND == '1' && steps.recheck.outputs.already_published != 'true' && steps.gate.outputs.refused != 'true'
+        if: needs.decide.outputs.should_publish == 'true' && needs.decide.outputs.track == 'nightly-next' && steps.source.outputs.cef == 'true' && steps.recheck.outputs.already_published != 'true' && steps.gate.outputs.refused != 'true'
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.NIGHTLY_NEXT_R2_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.NIGHTLY_NEXT_R2_SECRET_ACCESS_KEY }}
@@ -5622,24 +5641,26 @@ jobs:
         run: bash .publish-tooling/scripts/ci/nightly-steps/github-publish-release-assets.sh
 
       - name: Download the cmux-next source archive
-        if: vars.CMUX_NEXT_PUBLISH_SOURCE_ARCHIVE == '1' && needs.publish-nightly.outputs.source == 'own'
+        if: needs.publish-nightly.outputs.source_archive == 'true' && needs.publish-nightly.outputs.source == 'own'
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
         with:
           name: cmux-next-source-archive
           path: source-archive
 
-      # Gated by vars.CMUX_NEXT_PUBLISH_SOURCE_ARCHIVE (the coordinator sets it).
+      # Gated by vars.CMUX_NEXT_PUBLISH_SOURCE_ARCHIVE (the coordinator sets it), through
+      # publish-nightly's source_archive, which is also false for a republished build whose
+      # source made no archive (republish-verified-nightly.py).
       # Tags the build commit cmux-next-src-<11 characters> and attaches
       # cmux-next-source-<build>.tar.gz to the nightly-next release. The prune
       # step removes it only together with the app assets of <build>, whose
       # source offer names it (prune_nightly_release_assets.py), never by age.
       # A recovered build carries its source archive in the recovered variant.
       - name: Take the recovered build's source archive
-        if: vars.CMUX_NEXT_PUBLISH_SOURCE_ARCHIVE == '1' && needs.publish-nightly.outputs.source == 'recovered'
+        if: needs.publish-nightly.outputs.source_archive == 'true' && needs.publish-nightly.outputs.source == 'recovered'
         run: mv nightly-out/source-archive source-archive
 
       - name: Publish the cmux-next source archive
-        if: vars.CMUX_NEXT_PUBLISH_SOURCE_ARCHIVE == '1'
+        if: needs.publish-nightly.outputs.source_archive == 'true'
         env:
           GH_TOKEN: ${{ github.token }}
           GH_APP_TOKEN: ${{ steps.gh-app-token.outputs.token }} # App first, GH_TOKEN as fallback
@@ -7062,7 +7083,7 @@ jobs:
   # published build whose notes jobs did not both succeed is reported too
   # (run 37526646874 published with its notes job cancelled and no alarm).
   report-nightly-failure:
-    needs: [decide, build-nightly-ghostty-cli-helper, build-nightly-app, build-nightly-remote-daemon, resolve-nightly-cmux-tui-client, prepare-nightly-variant, build-sign-notarize-nightly, verify-signed-nightly, verify-final-dmg-nightly, verify-recovered-nightly-next, publish-nightly, generate-nightly-deltas, republish-nightly-deltas, sign-release-notes, publish-release-notes]
+    needs: [decide, build-nightly-ghostty-cli-helper, build-nightly-app, build-nightly-remote-daemon, resolve-nightly-cmux-tui-client, prepare-nightly-variant, build-sign-notarize-nightly, verify-signed-nightly, verify-final-dmg-nightly, verify-recovered-nightly-next, republish-nightly-next, publish-nightly, generate-nightly-deltas, republish-nightly-deltas, sign-release-notes, publish-release-notes]
     if: always() && needs.decide.outputs.should_publish == 'true' && needs.decide.outputs.notary_test != 'true' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || needs.publish-nightly.outputs.feed_check == 'failure' || needs.publish-nightly.outputs.tag_status == 'stale' || needs.publish-nightly.outputs.refusal_reason != '' || (needs.decide.outputs.track == 'nightly-next' && needs.publish-nightly.result == 'success' && needs.publish-nightly.outputs.already_published != 'true' && needs.publish-nightly.outputs.refused != 'true' && needs.publish-nightly.outputs.source != 'recovered' && (needs.sign-release-notes.result != 'success' || needs.publish-release-notes.result != 'success')))
     # The declared nightly-only mini is enabled after live admission is verified.
     # Keep the existing fallback; never reuse the stale builder-only placement.

scripts/ci/republish-verified-nightly.py

@@ -18,7 +18,9 @@
 Usage: republish-verified-nightly.py --run-id N --attempt N [--out DIR]
 Env: GITHUB_REPOSITORY, GITHUB_SHA, GITHUB_REF, GITHUB_EVENT_NAME, GH_TOKEN (actions: read),
 CHANNEL_DMG_PREFIX; GITHUB_OUTPUT receives run_id, attempt, build, build_sha, tip_sha, behind,
-behind_hours and cef.
+behind_hours, cef (the source signed a Chromium runtime pack, so its app's pin names one) and
+source_archive (the source made a cmux-next source tarball; its artifact always carries the
+license trees, and the tarball only while CMUX_NEXT_PUBLISH_SOURCE_ARCHIVE was 1).
 """
 from __future__ import annotations
 
@@ -186,6 +188,11 @@ def restore(receipt, root: Path, prefix: str):
     return check_bytes(root, receipt["build"], prefix)
 
 
+def has_source_tarball(root: Path) -> bool:
+    """github-source-archive.sh publishes exactly one cmux-next-source-*.tar.gz from source-archive/."""
+    return len(list((root / ARTIFACTS["cmux-next-source-archive"][1]).glob("cmux-next-source-*.tar.gz"))) == 1
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     parser.add_argument("--run-id", required=True)
@@ -198,6 +205,7 @@ def main():
     count = restore(receipt, args.out, os.environ.get("CHANNEL_DMG_PREFIX", ""))
     outputs = {key: receipt[key] for key in ("run_id", "attempt", "build", "build_sha", "tip_sha", "behind", "behind_hours")}
     outputs["cef"] = "true" if "cef-runtime-pack" in receipt["artifacts"] else "false"
+    outputs["source_archive"] = "true" if has_source_tarball(args.out) else "false"
     (args.out / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n")
     with open(os.environ["GITHUB_OUTPUT"], "a") as output:
         for key, value in outputs.items():

tests/test_nightly_next_recovery.py

@@ -467,9 +467,10 @@ class FailureIssueTests(unittest.TestCase):
     def publication(self, source, **results):
         jobs = ("decide", "build-nightly-ghostty-cli-helper", "build-nightly-app", "build-nightly-remote-daemon", "resolve-nightly-cmux-tui-client",
                 "prepare-nightly-variant", "build-sign-notarize-nightly", "verify-signed-nightly", "verify-final-dmg-nightly",
-                "verify-recovered-nightly-next", "publish-nightly", "generate-nightly-deltas", "republish-nightly-deltas",
-                "sign-release-notes", "publish-release-notes")
+                "verify-recovered-nightly-next", "republish-nightly-next", "publish-nightly", "generate-nightly-deltas",
+                "republish-nightly-deltas", "sign-release-notes", "publish-release-notes")
         outcome = dict.fromkeys(jobs, "success")
+        outcome["republish-nightly-next"] = "skipped"
         if source == "own":
             outcome["verify-recovered-nightly-next"] = "skipped"
         if source == "recovered":
@@ -494,6 +495,12 @@ def test_a_recovered_publication_after_this_run_s_build_failed_reports(self):
         self.assertEqual(self.publication("recovered", **{"build-sign-notarize-nightly": "failure"}), (True, False))
         self.assertEqual(self.publication("recovered", **{"build-nightly-app": "failure", "build-sign-notarize-nightly": "skipped"}), (True, False))
 
+    def test_a_failed_republish_fetch_reports(self):
+        # publish-nightly is skipped then, and nothing else failed.
+        self.assertIn("republish-nightly-next", workflow()["jobs"]["report-nightly-failure"]["needs"])
+        self.assertEqual(self.publication("recovered", **{"republish-nightly-next": "failure", "publish-nightly": "skipped"})[0],
+                         True)
+
     def test_a_recovered_publication_that_failed_its_delta_republish_reports(self):
         self.assertEqual(self.publication("recovered", **{"republish-nightly-deltas": "failure"}), (True, False))
 
@@ -1063,6 +1070,17 @@ def test_optional_artifacts_may_be_absent_but_never_ambiguous(self):
         with self.assertRaises(SystemExit):
             self.receipt()
 
+    def test_source_archive_output_is_the_tarball_not_the_artifact(self):
+        # The artifact always carries the license trees; the tarball only while the source offered one.
+        with tempfile.TemporaryDirectory() as tmp:
+            archive = Path(tmp) / "variant/source-archive"
+            (archive / "ghostty-licenses").mkdir(parents=True)
+            self.assertFalse(self.republish.has_source_tarball(Path(tmp)))
+            (archive / "cmux-next-source-abc.tar.gz").write_bytes(b"tar")
+            self.assertTrue(self.republish.has_source_tarball(Path(tmp)))
+        with tempfile.TemporaryDirectory() as tmp:
+            self.assertFalse(self.republish.has_source_tarball(Path(tmp)))
+
     def test_no_compiled_source_receipt_refuses(self):
         self.artifacts = [a for a in self.artifacts if not a["name"].startswith("nightly-next-dsyms-")]
         with self.assertRaises(SystemExit):
@@ -1195,11 +1213,52 @@ def test_publish_takes_the_republished_build_through_every_guard(self):
         self.assertEqual(step(job, "Final publication gate")["env"]["RECOVERED_RUN_ID"], "${{ steps.source.outputs.run_id }}")
         self.assertEqual(run_select(**dict(OWN, **RECOVERED, OWN_RESULT="failure"))["run_id"], "498")
         self.assertEqual(run_select(**OWN)["run_id"], "")
+        self.assertEqual(job["outputs"]["source_archive"], "${{ steps.source.outputs.source_archive }}")
+
         # Every guard stays as it is for any other build: none of them names the republish.
         for kept in ("Take the nightly-next publish lease", "Recheck the recovered build against both feeds",
                      "Require every DMG to match a tested digest", "Final publication gate"):
             self.assertNotIn("republish", json.dumps(step(job, kept)), kept)
 
 
+    def test_a_republished_build_publishes_the_pack_its_source_signed(self):
+        republished = dict(OWN, OWN_RESULT="skipped", REPUBLISHED="success", REPUBLISHED_RUN_ID="41815641855",
+                           REPUBLISHED_BUILD="4181564185501", REPUBLISHED_BUILD_SHA="d" * 40,
+                           REPUBLISHED_TIP_SHA="e" * 40, REPUBLISHED_BEHIND="3", REPUBLISHED_BEHIND_HOURS="4.5")
+        for switch, source, want in (("", "true", "true"), ("1", "false", "false"), ("1", "true", "true"), ("", "false", "false")):
+            with self.subTest(switch=switch, source=source):
+                selected = run_select(**dict(republished, CEF_ON_DEMAND=switch, REPUBLISHED_CEF=source))
+                self.assertEqual(selected["cef"], want)
+        # Any other build follows the switch as before.
+        self.assertEqual(run_select(**dict(OWN, CEF_ON_DEMAND="1"))["cef"], "true")
+        self.assertEqual(run_select(**dict(OWN, CEF_ON_DEMAND=""))["cef"], "false")
+        self.assertEqual(run_select(**dict(OWN, **RECOVERED, OWN_RESULT="failure", CEF_ON_DEMAND="1"))["cef"], "true")
+        job = workflow()["jobs"]["publish-nightly"]
+        for name in ("Download the Chromium runtime pack", "Publish the Chromium runtime pack"):
+            self.assertIn("steps.source.outputs.cef == 'true'", step(job, name)["if"])
+            self.assertNotIn("CMUX_NEXT_CEF_ON_DEMAND", step(job, name)["if"])
+        self.assertEqual(workflow()["jobs"]["republish-nightly-next"]["outputs"]["cef"], "${{ steps.fetch.outputs.cef }}")
+
+    def test_a_republished_build_without_a_source_tarball_publishes_none(self):
+        republished = dict(OWN, OWN_RESULT="skipped", REPUBLISHED="success", REPUBLISHED_RUN_ID="41815641855",
+                           REPUBLISHED_BUILD="4181564185501", REPUBLISHED_BUILD_SHA="d" * 40,
+                           REPUBLISHED_TIP_SHA="e" * 40, REPUBLISHED_BEHIND="3", REPUBLISHED_BEHIND_HOURS="4.5")
+        # The switch stays the switch; a republish only drops an archive its source never made.
+        for switch, source, want in (("1", "true", "true"), ("1", "false", "false"), ("", "true", "false"), ("", "false", "false")):
+            with self.subTest(switch=switch, source=source):
+                selected = run_select(**dict(republished, PUBLISH_SOURCE_ARCHIVE=switch, REPUBLISHED_SOURCE_ARCHIVE=source))
+                self.assertEqual(selected["source_archive"], want)
+        self.assertEqual(run_select(**dict(OWN, PUBLISH_SOURCE_ARCHIVE="1"))["source_archive"], "true")
+        self.assertEqual(run_select(**dict(OWN, **RECOVERED, OWN_RESULT="failure", PUBLISH_SOURCE_ARCHIVE="1"))["source_archive"],
+                         "true")
+        github = workflow()["jobs"]["publish-nightly-next-github"]
+        for name in ("Download the cmux-next source archive", "Take the recovered build's source archive",
+                     "Publish the cmux-next source archive"):
+            self.assertIn("needs.publish-nightly.outputs.source_archive == 'true'", step(github, name)["if"])
+            self.assertNotIn("CMUX_NEXT_PUBLISH_SOURCE_ARCHIVE", step(github, name)["if"])
+        self.assertEqual(workflow()["jobs"]["republish-nightly-next"]["outputs"]["source_archive"],
+                         "${{ steps.fetch.outputs.source_archive }}")
+
+
 if __name__ == "__main__":
     unittest.main()