ci: seed each Swift job width admission compiles at

.github/workflows/nightly.yml

@@ -742,9 +742,19 @@ jobs:
         continue-on-error: true
         run: python3 scripts/ci/seed_derived_data.py prune "$CMUX_COMPILE_ADMISSION_DERIVED_DATA"
 
-      - name: Save DerivedData seed
+      # Seeds serve only the Swift job width they were built at; see
+      # scripts/ci/seed_derived_data.py.
+      - name: Scope DerivedData seed key
+        id: derived-data-seed-key
         if: steps.derived-data-bound.outputs.save == 'true'
         continue-on-error: true
+        env:
+          FINGERPRINT: ${{ steps.compilation-cache-key.outputs.fingerprint }}
+        run: echo "scoped=$(python3 scripts/ci/seed_derived_data.py scope "admission-derived-data-v1-${RUNNER_OS}-${RUNNER_ARCH}-${FINGERPRINT}-")" >> "$GITHUB_OUTPUT"
+
+      - name: Save DerivedData seed
+        if: steps.derived-data-bound.outputs.save == 'true' && steps.derived-data-seed-key.outputs.scoped != ''
+        continue-on-error: true
         uses: ./.github/actions/cache-save
         env:
           # Only an R2 save sees the bucket credentials.
@@ -755,7 +765,7 @@ jobs:
         with:
           backend: r2
           path: ${{ env.CMUX_COMPILE_ADMISSION_DERIVED_DATA }}
-          key: admission-derived-data-v1-${{ runner.os }}-${{ runner.arch }}-${{ steps.compilation-cache-key.outputs.fingerprint }}-${{ needs.decide.outputs.head_sha }}
+          key: ${{ steps.derived-data-seed-key.outputs.scoped }}${{ needs.decide.outputs.head_sha }}
 
   build-nightly-ghostty-cli-helper:
     needs: decide

.github/workflows/seed-derived-data.yml

@@ -34,16 +34,6 @@ permissions:
 env:
   CI_CACHE_R2_PUBLIC_URL: ${{ vars.CI_CACHE_R2_PUBLIC_URL || 'https://ci-cache.cmux.com' }}
 
-# One seed at a time per runner pool, and never cancel-in-progress. main
-# takes several merges an hour and a cold seed takes 15-20 minutes, so
-# cancelling the running seed for each newer push would starve publishing for
-# as long as merges keep arriving. With `false`, the running seed always
-# finishes and publishes, GitHub keeps at most one pending run, and a newer
-# push replaces that pending run, so the published seed is never more than
-# about one build behind main. tests/test_seed_derived_data.py pins this.
-concurrency:
-  group: seed-derived-data-${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || vars.MACOS_RUNNER_PR == 'blacksmith-6vcpu-macos-26' && 'blacksmith-12vcpu-macos-26' || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15' }}
-  cancel-in-progress: false
 
 jobs:
   decide:
@@ -84,11 +74,35 @@ jobs:
     # Must match pull-request and main full-suite compile admission: the seed
     # key carries the toolchain and the build paths, so a mismatch is a miss,
     # not a wrong hit. tests/test_seed_derived_data.py evaluates both routes.
-    # The 12 vCPU macOS 26 pool runs the same image and Xcode as the 6 vCPU
-    # one admission uses, but pull requests do not queue on it: on 2026-09-24
-    # seeds waited 19-25 minutes for a 6 vCPU runner while 12 vCPU jobs
-    # started in 15 s. Neither key names the runner size.
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || vars.MACOS_RUNNER_PR == 'blacksmith-6vcpu-macos-26' && 'blacksmith-12vcpu-macos-26' || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15' }}
+    #
+    # A seed serves only the width it was built at: Swift Build passes the
+    # runner's CPU count to swift-driver as -j<n>, so a 12 vCPU seed reran
+    # every SwiftDriver task and re-emitted 62 modules on a 6 vCPU admission
+    # (run 36043267820). Admission takes the 12 vCPU pool when it is free and
+    # the 6 vCPU one otherwise, so both seed, and seed_derived_data.py keys
+    # each seed by its width. The first entry is the 12 vCPU pool of the same
+    # image and Xcode, which starts in seconds and alone publishes the
+    # product; the second is admission's own pool, the same pool again under
+    # any other MACOS_RUNNER_PR. GitHub names these jobs "seed (<pool>)";
+    # seed_decide.py and reuse_app_host_products.py match that.
+    strategy:
+      fail-fast: false
+      matrix:
+        pool:
+          - ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || vars.MACOS_RUNNER_PR == 'blacksmith-6vcpu-macos-26' && 'blacksmith-12vcpu-macos-26' || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15' }}
+          - ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15' }}
+    runs-on: ${{ matrix.pool }}
+    # One seed at a time per pool, and never cancel-in-progress. main takes
+    # several merges an hour and a cold seed takes 15-20 minutes, so
+    # cancelling the running seed for each newer push would starve publishing
+    # for as long as merges keep arriving. With `false`, the running seed
+    # always finishes and publishes, GitHub keeps at most one pending job, and
+    # a newer push replaces it, so the published seed is never more than about
+    # one build behind main. Per pool, so the 6 vCPU queue never holds back
+    # the 12 vCPU seed. tests/test_seed_derived_data.py pins this.
+    concurrency:
+      group: seed-derived-data-${{ matrix.pool }}
+      cancel-in-progress: false
     environment: ${{ github.ref == 'refs/heads/main' && 'ci-cache-writer' || '' }}
     timeout-minutes: 60
     env:
@@ -128,7 +142,12 @@ jobs:
         run: |
           set -euo pipefail
           fingerprint="$(scripts/ci/compile-app-host-test-product.sh canonical-fingerprint "$CMUX_COMPILE_ADMISSION_DERIVED_DATA")"
-          echo "prefix=admission-derived-data-v1-${RUNNER_OS}-${RUNNER_ARCH}-${fingerprint}-" >> "$GITHUB_OUTPUT"
+          prefix="admission-derived-data-v1-${RUNNER_OS}-${RUNNER_ARCH}-${fingerprint}-"
+          # Adoption scopes the prefix itself; the save names this width.
+          scoped="$(python3 scripts/ci/seed_derived_data.py scope "$prefix")"
+          test -n "$scoped"
+          echo "prefix=$prefix" >> "$GITHUB_OUTPUT"
+          echo "scoped=$scoped" >> "$GITHUB_OUTPUT"
 
       - name: Install compilation dependencies
         run: |
@@ -277,17 +296,18 @@ jobs:
         with:
           backend: r2
           path: ${{ env.CMUX_COMPILE_ADMISSION_DERIVED_DATA }}
-          key: ${{ steps.key.outputs.prefix }}${{ github.sha }}
+          key: ${{ steps.key.outputs.scoped }}${{ github.sha }}
 
       # Publish the product exactly as ci-macos.yml compile admission does
       # (tests/test_seed_derived_data.py pins the copy). Staging and relocation
       # rewrite Build/Products, so these follow "Save seed". Only a main push
       # is a producer pull requests trust; a dispatch would publish a product
       # nothing adopts. A failure here costs pull requests the adoption, never
-      # the seed.
+      # the seed. The product does not depend on the width, so only the first
+      # pool publishes it, under the one artifact name admission looks up.
       - name: Stage compiled package frameworks
         id: stage-products
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && strategy.job-index == 0
         continue-on-error: true
         run: |
           set -euo pipefail

scripts/ci/reuse_app_host_products.py

@@ -103,6 +103,13 @@
 }
 
 
+def names_compile_job(name: object, compile_name: str) -> bool:
+    """Whether a listed job is COMPILE_JOBS' job, through a reusable workflow
+    ("<caller> / <name>") or a matrix ("<name> (<values>)")."""
+    last = str(name or "").rsplit(" / ", 1)[-1]
+    return last == compile_name or last.startswith(f"{compile_name} (")
+
+
 # Build controls the product contract hashes. Only non-secret values belong
 # here, because the contract is published in the artifact receipt.
 #
@@ -566,9 +573,11 @@ def select(api, value, current_run, current_attempt, consumer, reasons):
             # A reusable workflow reports "<caller job> / <job name>", so this
             # is "macos / macOS compile admission" when ci.yml reaches the job
             # through ci-macos.yml. Match the final segment.
+            # A matrix job adds " (<values>)", as seed-derived-data.yml's
+            # "seed (<pool>)" does.
             compile_name, compile_step = COMPILE_JOBS[run["path"]]
             compile_job = next((job for job in jobs
-                                if str(job.get("name") or "").rsplit(" / ", 1)[-1] == compile_name
+                                if names_compile_job(job.get("name"), compile_name)
                                 and job.get("status") == "completed"
                                 and job.get("conclusion") == "success"), None)
             if compile_job is None:

scripts/ci/seed_decide.py

@@ -54,26 +54,36 @@ def fingerprint(revision: str, xcode: str) -> str:
     ).strip()
 
 
+def is_seed_job(name: str | None) -> bool:
+    """The seed job is a matrix over pools, which GitHub names "seed (<pool>)"."""
+    return name == SEED_JOB or str(name or "").startswith(f"{SEED_JOB} (")
+
+
 def seed_state(api: Api, repository: str, run: dict) -> str:
     """'seeded', 'skipped' or 'none' for one seeder run.
 
-    Concurrency is workflow-wide, so this run's decide starts only after every
-    earlier seeder run finished: no ancestor's seed is still being built.
+    Each pool seeds its own Swift job width, so a commit is seeded only when
+    every pool's job saved. Concurrency is per pool, so an earlier run may
+    still be building; it reads as 'none' and the walk moves past it, which
+    can only make a push build, never skip wrongly.
     """
     status, conclusion = run.get("status"), run.get("conclusion")
     # A pending run can still be replaced by a newer push.
     if status != "completed" or conclusion != "success":
         return "none"
     jobs = api(f"repos/{repository}/actions/runs/{run['id']}/jobs?per_page=100").get("jobs", [])
-    job = next((j for j in jobs if j.get("name") == SEED_JOB), None)
-    if job is None:
+    seeds = [j for j in jobs if is_seed_job(j.get("name"))]
+    if not seeds:
         return "none"
-    if job.get("conclusion") == "skipped":
+    if all(j.get("conclusion") == "skipped" for j in seeds):
         return "skipped"
-    if job.get("conclusion") != "success":
-        return "none"
-    save = next((s for s in job.get("steps", []) if s.get("name") == SAVE_STEP), None)
-    return "seeded" if save is not None and save.get("conclusion") == "success" else "none"
+    for job in seeds:
+        if job.get("conclusion") != "success":
+            return "none"
+        save = next((s for s in job.get("steps", []) if s.get("name") == SAVE_STEP), None)
+        if save is None or save.get("conclusion") != "success":
+            return "none"
+    return "seeded"
 
 
 def nearest_seed(api: Api, repository: str, ancestors: Iterable[str]) -> str | None:

scripts/ci/seed_derived_data.py

@@ -5,6 +5,7 @@
     seed_derived_data.py prune DERIVED_DATA
     seed_derived_data.py start DERIVED_DATA PREFIX REVISION
     seed_derived_data.py adopt SOURCE DERIVED_DATA PREFIX REVISION
+    seed_derived_data.py scope PREFIX
 
 nightly.yml `refresh-test-compilation-cache` already compiles main cold on the
 runner, Xcode and canonical paths that ci-macos.yml compile admission uses.
@@ -34,6 +35,15 @@
 the download instead of downloading again; without a matching `start` it picks
 and downloads itself.
 
+A seed also serves only the machine width it was built at. Swift Build
+passes the runner's CPU count to every swift-driver invocation as -j<n>, so
+a seed built on 12 vCPU reran all 94 SwiftDriver tasks and re-emitted 62
+modules on a 6 vCPU admission at seed distance 0 (run 36043267820). Keys
+therefore carry the width, `<PREFIX>j<n>-<revision>`: `scope` prints the
+prefix a saver writes under, and `start` and `adopt` take the unscoped
+PREFIX, prefer their own width, and fall back to another width's seed,
+which still beats a cold build.
+
 Only jobs holding the bucket credentials can write R2 objects or pointers, and
 only the main-branch seeder is given them, so a pull request can read the seed
 but never replace it.
@@ -67,6 +77,8 @@
 # main seeds about one commit in ten, so fifty ancestors reach back several
 # seeds; past that the newest pointer is as good as anything.
 ANCESTOR_LIMIT = 50
+# The widths seed-derived-data.yml seeds at: the 12 and 6 vCPU macOS 26 pools.
+SEEDED_JOB_WIDTHS = (12, 6)
 USER_AGENT = "cmux-ci-seed-derived-data"
 # Shorter than the adopt step's 8-minute timeout, so adopt stops the detached
 # download itself rather than leaving it pulling a seed through the compile.
@@ -159,10 +171,30 @@ def nearest(prefix: str, revisions: list[str], exists=None) -> tuple[str, int] |
     return None
 
 
+def swift_jobs() -> int:
+    """The -j Swift Build gives swift-driver here: the active CPU count."""
+    override = os.environ.get("CMUX_SEED_SWIFT_JOBS")
+    return int(override) if override else os.sysconf("SC_NPROCESSORS_ONLN")
+
+
+def scoped(prefix: str, jobs: int | None = None) -> str:
+    return f"{prefix}j{jobs or swift_jobs()}-"
+
+
 def locate(prefix: str, revision: str) -> tuple[str, int | None]:
-    """The exact key to restore for REVISION, and its distance if a seed has it."""
-    found = nearest(prefix, lineage(revision))
-    return found if found else (prefix + revision, None)
+    """The exact key to restore for REVISION, and its distance if a seed has it.
+
+    PREFIX is unscoped. The nearest seed of this width wins; failing that,
+    the nearest of another width, whose extra module work is still far less
+    than a cold build.
+    """
+    revisions = lineage(revision)
+    own = swift_jobs()
+    for jobs in (own, *(width for width in SEEDED_JOB_WIDTHS if width != own)):
+        found = nearest(scoped(prefix, jobs), revisions)
+        if found:
+            return found
+    return scoped(prefix, own) + revision, None
 
 
 def beside(derived: Path, suffix: str) -> Path:
@@ -346,10 +378,14 @@ def main(argv: list[str]) -> int:
     if len(argv) == 3 and argv[1] == "prune":
         write_outputs(prune(Path(argv[2])))
         return 0
+    if len(argv) == 3 and argv[1] == "scope":
+        print(scoped(argv[2]))
+        return 0
     if len(argv) == 5 and argv[1] == "start":
         prefix, revision = argv[3], argv[4]
         exact, distance = locate(prefix, revision)
-        start(Path(argv[2]), exact, prefix, revision, distance)
+        # The newest-pointer fallback stays within this width.
+        start(Path(argv[2]), exact, scoped(prefix), revision, distance)
         return 0
     if len(argv) == 5 and argv[1] == "fetch":
         fetch_detached(Path(argv[2]), argv[3], argv[4])
@@ -358,8 +394,8 @@ def main(argv: list[str]) -> int:
         source, derived = Path(argv[2]).resolve(), Path(argv[3])
         prefix, revision = argv[4], argv[5]
         try:
-            exact, distance = picked(derived, prefix, revision) or locate(prefix, revision)
-            result = adopt(source, derived, exact, prefix)
+            exact, distance = picked(derived, scoped(prefix), revision) or locate(prefix, revision)
+            result = adopt(source, derived, exact, scoped(prefix))
             if result.get("hit") == "true":
                 # Commits between the seed and REVISION; empty means the
                 # newest pointer supplied it.

tests/test_ci_change_areas.py

@@ -5696,14 +5696,20 @@ def test_reuse_lookups_match_the_job_name_github_actually_reports() -> None:
     assert not admission_job_name("macos / some other job")
     assert not admission_job_name(None)
 
-    reuse = (ROOT / "scripts/ci/reuse_app_host_products.py").read_text(encoding="utf-8")
-    assert '.rsplit(" / ", 1)[-1] == compile_name' in reuse, (
-        "reuse_app_host_products.py must match the final segment of the job name"
-    )
     # Each trusted producer workflow names the job that has to have compiled.
     sys.path.insert(0, str(ROOT / "scripts/ci"))
     import reuse_app_host_products
 
+    # Match the final segment of the job name, and a matrix job's
+    # "<name> (<values>)", as seed-derived-data.yml's "seed (<pool>)".
+    names = reuse_app_host_products.names_compile_job
+    assert names(composed, ADMISSION_JOB), "reuse_app_host_products.py must match the final segment of the job name"
+    assert names(inner, ADMISSION_JOB)
+    assert names("seed (blacksmith-12vcpu-macos-26)", "seed")
+    assert not names("macos / some other job", ADMISSION_JOB)
+    assert not names("seeder", "seed")
+    assert not names(None, "seed")
+
     assert reuse_app_host_products.COMPILE_JOBS[".github/workflows/ci.yml"][0] == ADMISSION_JOB, (
         "reuse_app_host_products.py must look for ci.yml's admission job by its real name"
     )

tests/test_reuse_app_host_products.py

@@ -1239,7 +1239,8 @@ def test_pull_requests_accept_a_main_push_producer_in_one_direction(self):
     def use_main_push_producer(self):
         self.api.run.update(self.main_push_run(), head_sha="abc123")
         self.api.job = {
-            "name": "seed",
+            # A matrix over pools: GitHub names it "seed (<pool>)".
+            "name": "seed (blacksmith-12vcpu-macos-26)",
             "conclusion": "success",
             "status": "completed",
             "steps": [{

tests/test_seed_decide.py

@@ -17,9 +17,10 @@ def run(run_id, sha, status="completed", conclusion="success"):
     return {"id": run_id, "head_sha": sha, "status": status, "conclusion": conclusion}
 
 
-def seed_job(conclusion="success", status="completed", saved=True):
+def seed_job(conclusion="success", status="completed", saved=True, pool="blacksmith-12vcpu-macos-26"):
+    # The seed job is a matrix over pools, so GitHub names it "seed (<pool>)".
     steps = [{"name": "Save seed", "conclusion": "success" if saved else "skipped"}]
-    return {"name": "seed", "status": status, "conclusion": conclusion, "steps": steps}
+    return {"name": f"seed ({pool})", "status": status, "conclusion": conclusion, "steps": steps}
 
 
 class Api:
@@ -50,6 +51,15 @@ def test_a_replaced_parent_with_other_inputs_does_not_let_a_docs_push_skip(self)
         self.assertTrue(build, reason)
         self.assertIn("p2", reason)
 
+    def test_a_commit_is_seeded_only_when_every_pool_saved_its_width(self):
+        # A push may skip only if both widths already have the seed it would
+        # build; one missing width would go stale.
+        narrow = "blacksmith-6vcpu-macos-26"
+        both = Api([run(2, "p1")], {2: [seed_job(), seed_job(pool=narrow)]})
+        self.assertFalse(decide(both, ["p1"], {"HEAD": "v1", "p1": "v1"})[0])
+        one = Api([run(2, "p1")], {2: [seed_job(), seed_job(saved=False, pool=narrow)]})
+        self.assertTrue(decide(one, ["p1"], {"HEAD": "v1", "p1": "v1"})[0])
+
     def test_inputs_equal_to_the_nearest_seed_skip(self):
         api = Api([run(2, "p1")], {2: [seed_job()]})
         build, reason = decide(api, ["p1"], {"HEAD": "app-v1", "p1": "app-v1"})