From 353b5bf2e2cd8db7079e9fe2d7e8db34356d1b0a Mon Sep 17 00:00:00 2001 From: Leo Li Date: Thu, 24 Sep 2026 10:13:52 -0400 Subject: [PATCH 1/2] ci: take the build-fleet host lock for nightly mini builds A std mini that runs the nightly producer is also a build-fleet controller worker. Build under /Users/Shared/cmux-build-fleet/bin/with-host-lock when it exists, so a nightly and a dev build take turns instead of splitting one 48 GB machine. Hosted and fork runs have no lock tool and build as before. A lock refusal (exit 75, free-disk floor) fails the producer with a clear error and the router falls back to Blacksmith. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly-mini-build.yml | 20 +++++++++++++++++++- docs/ci/mac-fleet.md | 7 +++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly-mini-build.yml b/.github/workflows/nightly-mini-build.yml index ba426a9509af..0b03dce362fc 100644 --- a/.github/workflows/nightly-mini-build.yml +++ b/.github/workflows/nightly-mini-build.yml @@ -129,7 +129,16 @@ jobs: # products: an incremental build never deletes a resource, framework # or dSYM the source has since removed, and those would ship. rm -rf build-universal/Build/Products - python3 scripts/ci/run-with-helper-prebuild.py \ + # On a Manaflow mini, take the host lock the build-fleet controller + # holds while it builds, so a nightly and a dev build take turns + # instead of splitting the machine. Hosted and fork runs have no + # lock tool and build as before. Use bin/, not recipes/: recipes is a + # symlink and its lock path does not resolve. + lock=() + host_lock=/Users/Shared/cmux-build-fleet/bin/with-host-lock + if [ -x "$host_lock" ]; then lock=("$host_lock"); fi + set +e + ${lock[@]+"${lock[@]}"} python3 scripts/ci/run-with-helper-prebuild.py \ --derived-data build-universal --configuration Release --archs "$archs" \ --log "$RUNNER_TEMP/helper-prebuild.log" -- \ ./scripts/ci/run-xcodebuild-with-diagnostics.sh -- \ @@ -144,6 +153,15 @@ jobs: COMPILATION_CACHE_LIMIT_SIZE=3221225472 \ COMPILER_INDEX_STORE_ENABLE=NO \ CODE_SIGNING_ALLOWED=NO ASSETCATALOG_COMPILER_APPICON_NAME="$CHANNEL_ICON_NAME" build + status=$? + set -e + if [ "$status" -eq 75 ] && [ ${#lock[@]} -gt 0 ]; then + # with-host-lock refuses a job below its free-disk floor. The router + # reads the failed producer as producer_failure and the nightly + # compiles on Blacksmith. + echo "::error::the host lock refused this build (free-disk floor); the nightly falls back to Blacksmith" + fi + [ "$status" -eq 0 ] || exit "$status" echo "seconds=$(( $(date +%s) - started ))" >> "$GITHUB_OUTPUT" - name: Prune dead compilation cache generations diff --git a/docs/ci/mac-fleet.md b/docs/ci/mac-fleet.md index 28797eb7c0af..9b14564b5598 100644 --- a/docs/ci/mac-fleet.md +++ b/docs/ci/mac-fleet.md @@ -691,6 +691,13 @@ its own work directory. `nightly.yml` already runs one full nightly per branch at a time and throttles pushes, so a second mini does not double publishing nightlies. It takes a `build_only` measurement run while the first mini builds a full nightly, and it covers for a mini that is offline or busy. + +On a Manaflow mini the producer builds under +`/Users/Shared/cmux-build-fleet/bin/with-host-lock`, the same lock the +build-fleet controller holds for dev builds, so the two take turns. Time spent +waiting for the lock counts against `NIGHTLY_MAC_MINI_EXECUTION_SECONDS`, and a +lock that refuses admission (exit 75, below its free-disk floor) fails the +producer, so either way the nightly falls back to Blacksmith. The label avoids the bare word `nightly`, which the HQ build-fleet controller reserves as a tag. It is a separate registration from the compile lane's runner. From 8622dd2d54e51b1b8e6a3579d86dd8f082930100 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Thu, 24 Sep 2026 10:18:04 -0400 Subject: [PATCH 2/2] ci: address review notes on the nightly host lock Say that the build seconds include any lock wait, word the exit 75 error for what it is, and say a lock wait falls back only when it overruns the execution budget. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly-mini-build.yml | 4 ++-- docs/ci/mac-fleet.md | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly-mini-build.yml b/.github/workflows/nightly-mini-build.yml index 0b03dce362fc..7a1064089862 100644 --- a/.github/workflows/nightly-mini-build.yml +++ b/.github/workflows/nightly-mini-build.yml @@ -159,7 +159,7 @@ jobs: # with-host-lock refuses a job below its free-disk floor. The router # reads the failed producer as producer_failure and the nightly # compiles on Blacksmith. - echo "::error::the host lock refused this build (free-disk floor); the nightly falls back to Blacksmith" + echo "::error::exit 75: the host lock refuses a build below its free-disk floor, so this nightly compiles on Blacksmith" fi [ "$status" -eq 0 ] || exit "$status" echo "seconds=$(( $(date +%s) - started ))" >> "$GITHUB_OUTPUT" @@ -230,5 +230,5 @@ jobs: echo echo "- source: \`$SOURCE_SHA\`" echo "- runner: \`$RUNNER_NAME\`" - echo "- xcodebuild seconds: \`${BUILD_SECONDS:-n/a}\`" + echo "- build seconds, including any wait for the host lock: \`${BUILD_SECONDS:-n/a}\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/ci/mac-fleet.md b/docs/ci/mac-fleet.md index 9b14564b5598..ed5fa210cb8c 100644 --- a/docs/ci/mac-fleet.md +++ b/docs/ci/mac-fleet.md @@ -695,9 +695,11 @@ a full nightly, and it covers for a mini that is offline or busy. On a Manaflow mini the producer builds under `/Users/Shared/cmux-build-fleet/bin/with-host-lock`, the same lock the build-fleet controller holds for dev builds, so the two take turns. Time spent -waiting for the lock counts against `NIGHTLY_MAC_MINI_EXECUTION_SECONDS`, and a -lock that refuses admission (exit 75, below its free-disk floor) fails the -producer, so either way the nightly falls back to Blacksmith. +waiting for the lock counts against `NIGHTLY_MAC_MINI_EXECUTION_SECONDS`, so a +long wait plus the build can overrun it and fall back to Blacksmith. A lock that +refuses admission (exit 75, below its free-disk floor) fails the producer, which +also falls back. + The label avoids the bare word `nightly`, which the HQ build-fleet controller reserves as a tag. It is a separate registration from the compile lane's runner.