canary: measure checksum-only rebuilds at cmux scale (do not merge)

.github/workflows/ci-macos.yml

@@ -662,6 +662,42 @@ jobs:
           echo "seconds=$seconds" >> "$GITHUB_OUTPUT"
           exit "$status"
 
+      # CANARY (do not merge): what checksum-only costs and absorbs at cmux scale.
+      - name: Canary probe checksum-only rebuilds
+        if: always() && steps.hosted-compile.outcome == 'success'
+        continue-on-error: true
+        timeout-minutes: 45
+        run: |
+          set +e -u
+          dd="$CMUX_COMPILE_ADMISSION_DERIVED_DATA"; src=/private/tmp/cmux-ci/src
+          echo "manifest modes: $(grep -rho '"file-system":"[a-z-]*"' "$dd"/Build/Intermediates.noindex/XCBuildData/*.xcbuilddata/manifest.json 2>/dev/null | sort | uniq -c | tr '\n' ' ')"
+          for f in SwiftDriver SwiftEmitModule SwiftCompile CompileC ScanDependencies Copy CpResource Ld; do
+            printf '%s=%s ' "$f" "$(grep -cE "^$f " "$RUNNER_TEMP/cmux-compile-admission.txt")"
+          done; echo "<- admission compile"
+          probe() {
+            local name="$1" log="$RUNNER_TEMP/probe-$1.txt" started=$SECONDS
+            scripts/ci/compile-app-host-test-product.sh canonical-build "$dd" "$PWD/.ci-source-packages" \
+              "$CMUX_COMPILE_ADMISSION_CAS" "$log" > /dev/null 2>&1 || echo "probe $name build failed"
+            local line="probe $name: $((SECONDS - started))s"
+            for f in SwiftDriver SwiftEmitModule SwiftCompile CompileC Copy CpResource Ld; do
+              line+=" $f=$(grep -cE "^$f " "$log")"
+            done
+            echo "$line"; echo "- $line" >> "$GITHUB_STEP_SUMMARY"
+          }
+          probe null
+          # A fresh runner: every checkout file, directory and package checkout
+          # gets a new mtime with the same bytes, and so do the generated
+          # module maps Xcode rewrites after adoption.
+          find "$src" -name .git -prune -o -print0 | xargs -0 touch -h
+          find "$dd/Build/Intermediates.noindex/GeneratedModuleMaps" -type f -print0 | xargs -0 touch -h
+          xcode_sdk="$(xcrun --show-sdk-path)"
+          touch -h "$xcode_sdk/usr/include/stdio.h" 2>&1 | head -1; stat -f 'sdk stdio.h mtime %Sm' "$xcode_sdk/usr/include/stdio.h"
+          sudo -n find "$xcode_sdk/usr/include" -maxdepth 1 -type f -exec touch -h {} + 2>&1 | head -2
+          stat -f 'after sudo touch %Sm' "$xcode_sdk/usr/include/stdio.h"
+          probe touch-everything
+          printf '\n// canary\n' >> "$src/Sources/Sidebar/GPUSpinner.swift"
+          probe app-file-edit
+
       # Adoption sets a per-user Xcode default. Only a build whose inputs were
       # just replayed should ignore inode changes, so a reused runner does not
       # carry it into an unrelated job.

scripts/ci/compile-app-host-test-product.sh

@@ -39,6 +39,21 @@ cache_limit_bytes=3221225472
 # Keep in sync with scripts/ci/canonical-build-root.sh.
 CANONICAL_BUILD_ROOT="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}"
 
+# How Swift Build's llbuild decides a file changed. The default,
+# device-agnostic, compares modification times, which cannot survive a move to
+# another runner: Blacksmith images install Xcode at different times, so every
+# SDK header and prebuilt module a task discovered has another mtime there, and
+# Xcode rewrites the generated package module maps with identical bytes on the
+# first build after a DerivedData seed is adopted. Run 36022099083 adopted a
+# seed of its own base commit and still reran 94 SwiftDriver and 64
+# SwiftEmitModule tasks, every third-party package included, for 1,616 Xcode
+# files whose only difference was the mtime. checksum-only compares contents,
+# so an input that did not change is not rebuilt wherever it came from. Swift
+# Build reads the setting from the environment of the xcodebuild that launches
+# it, so it reaches only these builds. A build database written in one mode
+# reruns every task in the other, so the mode is part of the fingerprint.
+XCBUILD_FILE_SYSTEM_MODE=checksum-only
+
 fingerprint() {
   local derived_data="$1"
   # Canonical only when both paths are fixed: the source at the canonical
@@ -52,13 +67,15 @@ fingerprint() {
       echo "canonical-v1"
       xcodebuild -version
       printf 'derived-data=%s\n' "${derived_data##*/}"
+      printf 'file-system=%s\n' "$XCBUILD_FILE_SYSTEM_MODE"
     } | shasum -a 256 | cut -c1-32
     return
   fi
   {
     xcodebuild -version
     printf 'workspace=%s\n' "$PWD"
     printf 'derived-data=%s\n' "$derived_data"
+    printf 'file-system=%s\n' "$XCBUILD_FILE_SYSTEM_MODE"
   } | shasum -a 256 | cut -c1-32
 }
 
@@ -126,7 +143,7 @@ build() {
   # job warning-budget scope; subsequent schemes reuse the same app objects.
   # shellcheck disable=SC2016 # Xcode expands $(inherited), not the shell
   for scheme in cmux cmux-unit cmux-numeric-locale cmux-cli-tests; do
-    xcodebuild -project cmux.xcodeproj -scheme "$scheme" -configuration Debug \
+    FileSystemMode="$XCBUILD_FILE_SYSTEM_MODE" xcodebuild -project cmux.xcodeproj -scheme "$scheme" -configuration Debug \
       -derivedDataPath "$derived_data" \
       -clonedSourcePackagesDirPath "$source_packages" \
       -disableAutomaticPackageResolution \

tests/test_ci_canonical_build_root.py

@@ -315,5 +315,63 @@ def test_build_alone_refuses_a_stale_runtime_alias(self):
             for cwd, _args in records:
                 self.assertEqual(cwd, str(root / "src"))
 
+class SeededBuildFileSystemModeTests(unittest.TestCase):
+    """A seeded build must compare inputs by content, not by stat.
+
+    Blacksmith images install Xcode at different times, so every SDK header
+    and prebuilt module carries a different mtime on each image, and Xcode
+    rewrites the generated package module maps with identical bytes on the
+    first build after adoption. Under the default device-agnostic mode each of
+    those invalidates the seed: run 36022099083 adopted a seed at distance 0
+    and still reran 94 SwiftDriver and 64 SwiftEmitModule tasks.
+    """
+
+    def run_recipe(self, base: Path) -> tuple[list[dict], str]:
+        bin_dir = base / "bin"
+        bin_dir.mkdir()
+        calls = base / "calls.jsonl"
+        (bin_dir / "xcodebuild").write_text(
+            "#!/usr/bin/env python3\n"
+            "import os,sys,json\n"
+            "with open(os.environ['CALLS'], 'a') as f:\n"
+            " f.write(json.dumps({'args': sys.argv[1:], 'mode': os.environ.get('FileSystemMode')})+'\\n')\n"
+            "if '-version' in sys.argv: print('Xcode 26.3')\n")
+        (bin_dir / "xcodebuild").chmod(0o755)
+        workspace = base / "checkout"
+        (workspace / ".git").mkdir(parents=True)
+        root = base / "canonical"
+        root.mkdir()
+        env = dict(os.environ, PATH=f"{bin_dir}:" + os.environ["PATH"], CALLS=str(calls),
+                   CMUX_CI_CANONICAL_ROOT=str(root))
+        env.pop("FileSystemMode", None)
+        derived = str(root / "derived-data-compile-admission")
+        fingerprints = []
+        for args in [("canonical-fingerprint", derived),
+                     ("canonical-build", derived, str(workspace / ".ci-source-packages"), str(root / "cas"))]:
+            result = subprocess.run([str(SCRIPT), *args], cwd=workspace, env=env, capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            fingerprints.append(result.stdout.strip())
+        return [json.loads(line) for line in calls.read_text().splitlines()], fingerprints[0]
+
+    def test_every_scheme_builds_in_checksum_only_mode(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            records, _ = self.run_recipe(Path(tmp))
+        builds = [r for r in records if "build-for-testing" in r["args"]]
+        self.assertEqual(len(builds), 4)
+        for record in builds:
+            self.assertEqual(record["mode"], "checksum-only", record["args"])
+
+    def test_the_seed_fingerprint_names_the_mode(self):
+        # A seed recorded under another mode reruns every task when adopted,
+        # so it must not share a key with a checksum-only one.
+        with tempfile.TemporaryDirectory() as tmp:
+            _, fingerprint = self.run_recipe(Path(tmp))
+        old = subprocess.run(
+            ["shasum", "-a", "256"], input="canonical-v1\nXcode 26.3\nderived-data=derived-data-compile-admission\n",
+            capture_output=True, text=True, check=True,
+        ).stdout[:32]
+        self.assertNotEqual(fingerprint, old)
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)