diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index cf0a4535a4d7..637e9ce5c529 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -662,6 +662,42 @@ jobs: echo "seconds=$seconds" >> "$GITHUB_OUTPUT" exit "$status" + # CANARY (do not merge): what checksum-only costs and absorbs at cmux scale. + - name: Canary probe checksum-only rebuilds + if: always() && steps.hosted-compile.outcome == 'success' + continue-on-error: true + timeout-minutes: 45 + run: | + set +e -u + dd="$CMUX_COMPILE_ADMISSION_DERIVED_DATA"; src=/private/tmp/cmux-ci/src + echo "manifest modes: $(grep -rho '"file-system":"[a-z-]*"' "$dd"/Build/Intermediates.noindex/XCBuildData/*.xcbuilddata/manifest.json 2>/dev/null | sort | uniq -c | tr '\n' ' ')" + for f in SwiftDriver SwiftEmitModule SwiftCompile CompileC ScanDependencies Copy CpResource Ld; do + printf '%s=%s ' "$f" "$(grep -cE "^$f " "$RUNNER_TEMP/cmux-compile-admission.txt")" + done; echo "<- admission compile" + probe() { + local name="$1" log="$RUNNER_TEMP/probe-$1.txt" started=$SECONDS + scripts/ci/compile-app-host-test-product.sh canonical-build "$dd" "$PWD/.ci-source-packages" \ + "$CMUX_COMPILE_ADMISSION_CAS" "$log" > /dev/null 2>&1 || echo "probe $name build failed" + local line="probe $name: $((SECONDS - started))s" + for f in SwiftDriver SwiftEmitModule SwiftCompile CompileC Copy CpResource Ld; do + line+=" $f=$(grep -cE "^$f " "$log")" + done + echo "$line"; echo "- $line" >> "$GITHUB_STEP_SUMMARY" + } + probe null + # A fresh runner: every checkout file, directory and package checkout + # gets a new mtime with the same bytes, and so do the generated + # module maps Xcode rewrites after adoption. + find "$src" -name .git -prune -o -print0 | xargs -0 touch -h + find "$dd/Build/Intermediates.noindex/GeneratedModuleMaps" -type f -print0 | xargs -0 touch -h + xcode_sdk="$(xcrun --show-sdk-path)" + touch -h "$xcode_sdk/usr/include/stdio.h" 2>&1 | head -1; stat -f 'sdk stdio.h mtime %Sm' "$xcode_sdk/usr/include/stdio.h" + sudo -n find "$xcode_sdk/usr/include" -maxdepth 1 -type f -exec touch -h {} + 2>&1 | head -2 + stat -f 'after sudo touch %Sm' "$xcode_sdk/usr/include/stdio.h" + probe touch-everything + printf '\n// canary\n' >> "$src/Sources/Sidebar/GPUSpinner.swift" + probe app-file-edit + # Adoption sets a per-user Xcode default. Only a build whose inputs were # just replayed should ignore inode changes, so a reused runner does not # carry it into an unrelated job. diff --git a/scripts/ci/compile-app-host-test-product.sh b/scripts/ci/compile-app-host-test-product.sh index 2b9011f9c19a..2dea48fafbfd 100755 --- a/scripts/ci/compile-app-host-test-product.sh +++ b/scripts/ci/compile-app-host-test-product.sh @@ -39,6 +39,21 @@ cache_limit_bytes=3221225472 # Keep in sync with scripts/ci/canonical-build-root.sh. CANONICAL_BUILD_ROOT="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}" +# How Swift Build's llbuild decides a file changed. The default, +# device-agnostic, compares modification times, which cannot survive a move to +# another runner: Blacksmith images install Xcode at different times, so every +# SDK header and prebuilt module a task discovered has another mtime there, and +# Xcode rewrites the generated package module maps with identical bytes on the +# first build after a DerivedData seed is adopted. Run 36022099083 adopted a +# seed of its own base commit and still reran 94 SwiftDriver and 64 +# SwiftEmitModule tasks, every third-party package included, for 1,616 Xcode +# files whose only difference was the mtime. checksum-only compares contents, +# so an input that did not change is not rebuilt wherever it came from. Swift +# Build reads the setting from the environment of the xcodebuild that launches +# it, so it reaches only these builds. A build database written in one mode +# reruns every task in the other, so the mode is part of the fingerprint. +XCBUILD_FILE_SYSTEM_MODE=checksum-only + fingerprint() { local derived_data="$1" # Canonical only when both paths are fixed: the source at the canonical @@ -52,6 +67,7 @@ fingerprint() { echo "canonical-v1" xcodebuild -version printf 'derived-data=%s\n' "${derived_data##*/}" + printf 'file-system=%s\n' "$XCBUILD_FILE_SYSTEM_MODE" } | shasum -a 256 | cut -c1-32 return fi @@ -59,6 +75,7 @@ fingerprint() { xcodebuild -version printf 'workspace=%s\n' "$PWD" printf 'derived-data=%s\n' "$derived_data" + printf 'file-system=%s\n' "$XCBUILD_FILE_SYSTEM_MODE" } | shasum -a 256 | cut -c1-32 } @@ -126,7 +143,7 @@ build() { # job warning-budget scope; subsequent schemes reuse the same app objects. # shellcheck disable=SC2016 # Xcode expands $(inherited), not the shell for scheme in cmux cmux-unit cmux-numeric-locale cmux-cli-tests; do - xcodebuild -project cmux.xcodeproj -scheme "$scheme" -configuration Debug \ + FileSystemMode="$XCBUILD_FILE_SYSTEM_MODE" xcodebuild -project cmux.xcodeproj -scheme "$scheme" -configuration Debug \ -derivedDataPath "$derived_data" \ -clonedSourcePackagesDirPath "$source_packages" \ -disableAutomaticPackageResolution \ diff --git a/tests/test_ci_canonical_build_root.py b/tests/test_ci_canonical_build_root.py index 87596232c733..ea83fa383487 100644 --- a/tests/test_ci_canonical_build_root.py +++ b/tests/test_ci_canonical_build_root.py @@ -315,5 +315,63 @@ def test_build_alone_refuses_a_stale_runtime_alias(self): for cwd, _args in records: self.assertEqual(cwd, str(root / "src")) +class SeededBuildFileSystemModeTests(unittest.TestCase): + """A seeded build must compare inputs by content, not by stat. + + Blacksmith images install Xcode at different times, so every SDK header + and prebuilt module carries a different mtime on each image, and Xcode + rewrites the generated package module maps with identical bytes on the + first build after adoption. Under the default device-agnostic mode each of + those invalidates the seed: run 36022099083 adopted a seed at distance 0 + and still reran 94 SwiftDriver and 64 SwiftEmitModule tasks. + """ + + def run_recipe(self, base: Path) -> tuple[list[dict], str]: + bin_dir = base / "bin" + bin_dir.mkdir() + calls = base / "calls.jsonl" + (bin_dir / "xcodebuild").write_text( + "#!/usr/bin/env python3\n" + "import os,sys,json\n" + "with open(os.environ['CALLS'], 'a') as f:\n" + " f.write(json.dumps({'args': sys.argv[1:], 'mode': os.environ.get('FileSystemMode')})+'\\n')\n" + "if '-version' in sys.argv: print('Xcode 26.3')\n") + (bin_dir / "xcodebuild").chmod(0o755) + workspace = base / "checkout" + (workspace / ".git").mkdir(parents=True) + root = base / "canonical" + root.mkdir() + env = dict(os.environ, PATH=f"{bin_dir}:" + os.environ["PATH"], CALLS=str(calls), + CMUX_CI_CANONICAL_ROOT=str(root)) + env.pop("FileSystemMode", None) + derived = str(root / "derived-data-compile-admission") + fingerprints = [] + for args in [("canonical-fingerprint", derived), + ("canonical-build", derived, str(workspace / ".ci-source-packages"), str(root / "cas"))]: + result = subprocess.run([str(SCRIPT), *args], cwd=workspace, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + fingerprints.append(result.stdout.strip()) + return [json.loads(line) for line in calls.read_text().splitlines()], fingerprints[0] + + def test_every_scheme_builds_in_checksum_only_mode(self): + with tempfile.TemporaryDirectory() as tmp: + records, _ = self.run_recipe(Path(tmp)) + builds = [r for r in records if "build-for-testing" in r["args"]] + self.assertEqual(len(builds), 4) + for record in builds: + self.assertEqual(record["mode"], "checksum-only", record["args"]) + + def test_the_seed_fingerprint_names_the_mode(self): + # A seed recorded under another mode reruns every task when adopted, + # so it must not share a key with a checksum-only one. + with tempfile.TemporaryDirectory() as tmp: + _, fingerprint = self.run_recipe(Path(tmp)) + old = subprocess.run( + ["shasum", "-a", "256"], input="canonical-v1\nXcode 26.3\nderived-data=derived-data-compile-admission\n", + capture_output=True, text=True, check=True, + ).stdout[:32] + self.assertNotEqual(fingerprint, old) + + if __name__ == "__main__": unittest.main(verbosity=2)