diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 500b4b3b1e02..1e93c7d445c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -418,6 +418,7 @@ jobs: run: | python3 tests/test_ci_change_areas.py python3 tests/test_app_host_test_products.py + python3 tests/test_reuse_app_host_products.py - name: Validate Python R2 appcast upload guard run: ./tests/test_ci_r2_upload_python.sh @@ -2349,6 +2350,9 @@ jobs: if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' && needs.changes.outputs.macos == 'true' }} runs-on: ${{ vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }} timeout-minutes: 75 + permissions: + contents: read + actions: read outputs: artifact_id: ${{ steps.upload-products.outputs.artifact-id }} sha256: ${{ steps.package-products.outputs.sha256 }} @@ -2356,6 +2360,7 @@ jobs: CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" CMUX_SKIP_ZIG_BUILD: "1" + CMUX_PRODUCT_RUNNER: ${{ vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }} steps: - name: Clear stale git locks (self-hosted reused workspace) shell: bash @@ -2383,11 +2388,26 @@ jobs: - name: Select Xcode run: ./scripts/select-ci-xcode.sh + - name: Install Rust + run: ./scripts/install-rust-ci.sh + + - name: Identify reusable compiled products + id: product-key + run: python3 scripts/ci/reuse_app_host_products.py key "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" + + - name: Reuse compatible compiled products in merge queue + id: reuse-products + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/ci/reuse_app_host_products.py restore "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" + - name: Capture Ghostty revision + if: steps.reuse-products.outputs.hit != 'true' id: ghostty-revision run: echo "sha=$(git -C ghostty rev-parse HEAD)" >> "$GITHUB_OUTPUT" - name: Cache GhosttyKit.xcframework + if: steps.reuse-products.outputs.hit != 'true' id: cache-ghosttykit-admission uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -2395,13 +2415,11 @@ jobs: key: ghosttykit-sentry-off-v1-${{ steps.ghostty-revision.outputs.sha }} - name: Download pre-built GhosttyKit.xcframework - if: steps.cache-ghosttykit-admission.outputs.cache-hit != 'true' + if: steps.reuse-products.outputs.hit != 'true' && steps.cache-ghosttykit-admission.outputs.cache-hit != 'true' run: ./scripts/download-prebuilt-ghosttykit.sh - - name: Install Rust - run: ./scripts/install-rust-ci.sh - - name: Cache Swift packages + if: steps.reuse-products.outputs.hit != 'true' uses: ./.github/actions/cache-restore with: backend: ${{ inputs.cache_backend != 'default' && inputs.cache_backend || vars.CI_CACHE_BACKEND }} @@ -2410,9 +2428,11 @@ jobs: restore-keys: spm- - name: Sanitize Swift package cache + if: steps.reuse-products.outputs.hit != 'true' run: python3 scripts/ci/sanitize-xcode-source-packages-cache.py .ci-source-packages - name: Resolve Swift packages + if: steps.reuse-products.outputs.hit != 'true' run: | set -euo pipefail rm -rf "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" @@ -2420,6 +2440,7 @@ jobs: "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" "$PWD/.ci-source-packages" - name: Compute test compilation cache key + if: steps.reuse-products.outputs.hit != 'true' id: compilation-cache-key run: | set -euo pipefail @@ -2429,6 +2450,7 @@ jobs: # only writer. A cache saved here would be scoped to this pull request and # would spend the cache budget that keeps the main seed alive. - name: Restore test compilation cache + if: steps.reuse-products.outputs.hit != 'true' uses: ./.github/actions/cache-restore with: backend: ${{ inputs.cache_backend != 'default' && inputs.cache_backend || vars.CI_CACHE_BACKEND }} @@ -2438,6 +2460,7 @@ jobs: xcode-compilation-test-${{ runner.os }}-${{ runner.arch }}-${{ steps.compilation-cache-key.outputs.fingerprint }}- - name: Compile app-host test product + if: steps.reuse-products.outputs.hit != 'true' run: | set -euo pipefail scripts/ci/compile-app-host-test-product.sh build \ @@ -2464,6 +2487,7 @@ jobs: rsync -aL "$framework_root/" "$products/PackageFrameworks/" test -f "$products/PackageFrameworks/CmuxAgentJournal_27B6EF8727F6C277_PackageProduct.framework/Versions/A/CmuxAgentJournal_27B6EF8727F6C277_PackageProduct" python3 scripts/ci/app_host_test_products.py stamp "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" + python3 scripts/ci/reuse_app_host_products.py seal "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" archive="$RUNNER_TEMP/app-host-products.tar.gz" tar -chzf "$archive" -C "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" Build/Products echo "sha256=$(shasum -a 256 "$archive" | awk '{print $1}')" >> "$GITHUB_OUTPUT" @@ -2472,7 +2496,7 @@ jobs: id: upload-products uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: app-host-products-${{ github.run_attempt }} + name: app-host-products-v1-${{ steps.product-key.outputs.key }}-${{ github.run_attempt }} path: ${{ runner.temp }}/app-host-products.tar.gz if-no-files-found: error retention-days: 3 diff --git a/scripts/ci/reuse_app_host_products.py b/scripts/ci/reuse_app_host_products.py new file mode 100644 index 000000000000..ad67d6b0e50b --- /dev/null +++ b/scripts/ci/reuse_app_host_products.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Reuse compiled products, never test outcomes, across trusted CI runs. + +A conservative first version: the entire git tree and build environment must +match. Missing provenance, old artifacts, API errors and corrupt downloads are +cache misses. The original CMUXCommit embedded in the app is retained. +""" +from __future__ import annotations + +import hashlib +import gzip +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from pathlib import Path + +import app_host_test_products as products + +RECEIPT = "cmux-product-reuse.json" +PREFIX = "app-host-products-v1-" +# Current product archives are ~0.8 GiB compressed. Bound every expansion layer +# independently, including hardlink copies, with room for the UI product set. +MAX_ARCHIVE_BYTES = 2 * 1024**3 +MAX_MEMBER_BYTES = 4 * 1024**3 +MAX_EXPANDED_BYTES = 16 * 1024**3 +MAX_TAR_BYTES = 20 * 1024**3 +MAX_MEMBERS = 200_000 + + +def read(*args): + return subprocess.check_output(args, text=True, timeout=30).strip() + + +def contract(): + versions = {} + for command in ("rustc", "cargo", "go", "zig", "node", "bun"): + executable = shutil.which(command) + versions[command] = read(executable, "version" if command in {"go", "zig"} else "--version") if executable else "absent" + return { + "tree": read("git", "rev-parse", "HEAD^{tree}"), + "xcode": read("xcodebuild", "-version"), + "sdk": read("xcrun", "--sdk", "macosx", "--show-sdk-build-version"), + "os": read("sw_vers", "-buildVersion"), + "architecture": platform.machine(), + "tools": versions, + # Only non-secret build controls belong in the public artifact receipt. + "environment": {k: os.environ.get(k, "") for k in ( + "CMUX_CI_XCODE_APP", "CMUX_CI_REQUIRED_MACOS_SDK_MAJOR", "CMUX_SKIP_ZIG_BUILD", + "SDKROOT", "MACOSX_DEPLOYMENT_TARGET", "SWIFT_ACTIVE_COMPILATION_CONDITIONS", + "OTHER_SWIFT_FLAGS", "OTHER_CFLAGS", "OTHER_CPLUSPLUSFLAGS", "OTHER_LDFLAGS", + "RUSTFLAGS", "CFLAGS", "CXXFLAGS", "LDFLAGS", "ImageOS", "ImageVersion")}, + "runner": os.environ.get("CMUX_PRODUCT_RUNNER", ""), + } + + +def key(value): + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +class GitHub: + def __init__(self, repository): + self.repository = repository + + def get(self, path): + return json.loads(read("gh", "api", f"repos/{self.repository}/{path}")) + + def download(self, artifact_id, target): + with target.open("wb") as out: + subprocess.run(["gh", "api", f"repos/{self.repository}/actions/artifacts/{artifact_id}/zip"], + stdout=out, check=True, timeout=120) + + +def select(api, value, current_run): + """Inspect at most 300 recent artifacts and six matching producers.""" + prefix = PREFIX + key(value) + "-" + candidates = [] + for page in range(1, 4): + batch = api.get(f"actions/artifacts?per_page=100&page={page}")["artifacts"] + candidates.extend(a for a in batch if a.get("name", "").startswith(prefix)) + if len(candidates) >= 6 or len(batch) < 100: + break + for artifact in candidates[:6]: + suffix = artifact["name"][len(prefix):] + if artifact.get("expired") or not suffix.isdecimal(): + continue + if artifact.get("size_in_bytes", MAX_ARCHIVE_BYTES + 1) > MAX_ARCHIVE_BYTES: + continue + run_id = artifact.get("workflow_run", {}).get("id") + if not run_id or str(run_id) == str(current_run): + continue + run = api.get(f"actions/runs/{run_id}") + if (run.get("path") != ".github/workflows/ci.yml" + or run.get("event") not in {"pull_request", "merge_group"} + or run.get("head_repository", {}).get("full_name") != api.repository + or suffix != str(run["run_attempt"])): + continue + # GitHub's run head, not a candidate-authored receipt, establishes the + # source identity before downloading. The whole tree includes the CI + # workflow and every build/packaging script; different producer code + # cannot vouch for this checkout. This inherits CI's existing trust in + # the candidate workflow, not an independent base-controlled attestation. + head = run.get("head_sha", "") + if not re.fullmatch(r"[0-9a-f]{6,40}", head): + continue + if api.get(f"git/commits/{head}")["tree"]["sha"] != value["tree"]: + continue + attempt = run["run_attempt"] + jobs = [] + for page in range(1, 4): + batch = api.get(f"actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100&page={page}")["jobs"] + jobs.extend(batch) + if len(batch) < 100: + break + # The compile job must finish; unrelated tests in the producer run may + # still be running. No test result is being reused here. + if not any(j.get("name") == "macOS compile admission" and j.get("status") == "completed" + and j.get("conclusion") == "success" for j in jobs): + continue + if not artifact.get("digest", "").startswith("sha256:"): + continue + yield artifact, run + + +def bounded_copy(source, output, limit): + copied = 0 + while True: + chunk = source.read(min(1024 * 1024, limit - copied + 1)) + if not chunk: + return copied + copied += len(chunk) + if copied > limit: + raise ValueError("archive expansion limit exceeded") + output.write(chunk) + + +class BoundedReader: + def __init__(self, source, limit): + self.source, self.remaining = source, limit + + def read(self, size=-1): + size = self.remaining + 1 if size < 0 else min(size, self.remaining + 1) + chunk = self.source.read(size) + self.remaining -= len(chunk) + if self.remaining < 0: + raise ValueError("tar stream expansion limit exceeded") + return chunk + + +class BoundedTarInfo(tarfile.TarInfo): + @classmethod + def frombuf(cls, buf, encoding, errors): + info = super().frombuf(buf, encoding, errors) + # PAX/GNU extension bodies are read into memory by tarfile before it + # yields a member, so their limits must be checked at header parsing. + if info.size > MAX_MEMBER_BYTES or (info.type in {tarfile.XHDTYPE, tarfile.XGLTYPE, + tarfile.GNUTYPE_LONGNAME, tarfile.GNUTYPE_LONGLINK} and info.size > 1024 * 1024): + raise ValueError("tar header size limit exceeded") + return info + + +def unpack(archive, staging, digest): + if archive.stat().st_size > MAX_ARCHIVE_BYTES: + raise ValueError("archive size limit exceeded") + h = hashlib.sha256() + with archive.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + h.update(chunk) + if "sha256:" + h.hexdigest() != digest: + raise ValueError("artifact digest mismatch") + compressed = staging / "app-host-products.tar.gz" + with zipfile.ZipFile(archive) as z: + if z.namelist() != ["app-host-products.tar.gz"]: + raise ValueError("unexpected artifact contents") + info = z.infolist()[0] + if info.file_size > MAX_ARCHIVE_BYTES: + raise ValueError("zip expansion limit exceeded") + with z.open(info) as source, compressed.open("wb") as output: + bounded_copy(source, output, MAX_ARCHIVE_BYTES) + expanded = 0 + hardlinks = [] + # Limit the decompressed stream too: tar metadata/PAX headers must not + # bypass the per-file limits or force getmembers() to allocate unboundedly. + with gzip.open(compressed, "rb") as gz: + try: + with tarfile.open(fileobj=BoundedReader(gz, MAX_TAR_BYTES), mode="r|", tarinfo=BoundedTarInfo) as tar: + for count, member in enumerate(tar, 1): + if count > MAX_MEMBERS: + raise ValueError("archive member count limit exceeded") + parts = Path(member.name).parts + if parts[:2] != ("Build", "Products") or ".." in parts: + raise tarfile.ExtractError("unscoped product path") + if not (member.isdir() or member.isfile() or member.islnk()): + raise tarfile.ExtractError("unsupported product entry") + if member.size > MAX_MEMBER_BYTES or expanded + member.size > MAX_EXPANDED_BYTES: + raise ValueError("archive member size limit exceeded") + target = staging / member.name + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + elif member.isfile(): + target.parent.mkdir(parents=True, exist_ok=True) + with tar.extractfile(member) as source, target.open("wb") as output: + copied = bounded_copy(source, output, min(MAX_MEMBER_BYTES, MAX_EXPANDED_BYTES - expanded)) + if copied != member.size: + raise ValueError("truncated archive member") + expanded += copied + target.chmod(member.mode & 0o777) + else: + target_parts = Path(member.linkname).parts + if target_parts[:2] != ("Build", "Products") or ".." in target_parts: + raise tarfile.ExtractError("unscoped product hardlink") + hardlinks.append(member) + except (gzip.BadGzipFile, EOFError) as error: + raise tarfile.ReadError("invalid compressed product archive") from error + for member in hardlinks: + source_path = staging / member.linkname + target = staging / member.name + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + raise tarfile.ExtractError("duplicate product hardlink") + with source_path.open("rb") as source, target.open("wb") as output: + expanded += bounded_copy(source, output, min(MAX_MEMBER_BYTES, MAX_EXPANDED_BYTES - expanded)) + target.chmod(source_path.stat().st_mode & 0o777) + + + +def restore(api, value, derived, current_run, current_identity): + """Restore in staging; a miss never leaves partial products in DerivedData.""" + for artifact, run in select(api, value, current_run): + with tempfile.TemporaryDirectory(prefix="cmux-reuse-") as tmp: + staging = Path(tmp) + archive = staging / "artifact.zip" + try: + api.download(artifact["id"], archive) + unpack(archive, staging, artifact["digest"]) + root = staging / "Build/Products" + receipt = json.loads((root / RECEIPT).read_text()) + if receipt["contract"] != value or receipt["run_id"] != str(run["id"]) or receipt["run_attempt"] != str(run["run_attempt"]): + raise ValueError("artifact producer contract mismatch") + # Verify the actual checkout commit against GitHub, independent of + # the artifact name. Internal PR head trees must also match; when a + # PR merge includes additional base changes, conservatively rebuild. + for revision in (receipt["revision"], run["head_sha"]): + if not re.fullmatch(r"[0-9a-f]{6,40}", revision): + raise ValueError("invalid producer revision") + if api.get(f"git/commits/{revision}")["tree"]["sha"] != value["tree"]: + raise ValueError("producer source tree mismatch") + original = json.loads((root / products.RECEIPT).read_text()) + if original["revision"] != receipt["revision"]: + raise ValueError("producer revision mismatch") + products.restore(staging, {**current_identity, "revision": original["revision"]}) + # Relocate once more from staging into the actual consumer location. + products.stamp(staging, current_identity) + except (ValueError, KeyError, OSError, subprocess.SubprocessError, + tarfile.TarError, zipfile.BadZipFile) as error: + print(f"Skipping build artifact {artifact['id']} ({type(error).__name__}).") + continue + # After relocation starts, any failure must abort to main's cleanup. + destination = derived / "Build/Products" + if destination.exists(): + raise ValueError("reuse destination must be empty") + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(root), destination) + products.restore(derived, current_identity) + provenance = destination / "cmux-original-producer.json" + upstream = json.loads(provenance.read_text()) if provenance.exists() else None + provenance.write_text(json.dumps({ + "run_url": run["html_url"], "revision": receipt["revision"], + "artifact_id": artifact["id"], "consumer_revision": current_identity["revision"], + "upstream": upstream, + }, indent=2)) + print(f"Reused compiled products from {run['html_url']} (producer {receipt['revision']}); tests still run here.") + return True + return False + + +def main(): + mode, derived_raw = sys.argv[1:] + derived = Path(derived_raw) + try: + value = contract() + except (OSError, subprocess.SubprocessError): + value = None + print("Build environment cannot be fingerprinted; compiling normally.") + if mode == "key": + with open(os.environ["GITHUB_OUTPUT"], "a") as out: + fingerprint = key(value) if value is not None else "unavailable-" + os.environ["GITHUB_RUN_ID"] + out.write(f"key={fingerprint}\n") + elif mode == "seal": + if value is None: + return + root = derived / "Build/Products" + (root / RECEIPT).write_text(json.dumps({"contract": value, + "revision": read("git", "rev-parse", "HEAD"), + "run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": os.environ["GITHUB_RUN_ATTEMPT"]})) + elif mode == "restore": + hit = False + try: + if value is not None and os.environ.get("GITHUB_EVENT_NAME") == "merge_group": + hit = restore(GitHub(os.environ["GITHUB_REPOSITORY"]), value, derived, + os.environ["GITHUB_RUN_ID"], products.identity()) + except (ValueError, KeyError, OSError, subprocess.SubprocessError, tarfile.TarError, zipfile.BadZipFile) as error: + print(f"Build product reuse unavailable ({type(error).__name__}); compiling normally.") + shutil.rmtree(derived, ignore_errors=True) + with open(os.environ["GITHUB_OUTPUT"], "a") as out: + out.write(f"hit={'true' if hit else 'false'}\n") + else: + raise ValueError("expected key, seal or restore") + + +if __name__ == "__main__": + main() diff --git a/tests/test_reuse_app_host_products.py b/tests/test_reuse_app_host_products.py new file mode 100644 index 000000000000..503f0f3fda18 --- /dev/null +++ b/tests/test_reuse_app_host_products.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Exercise cross-run artifact reuse through real archives and product relocation.""" +import hashlib +import io +import json +import os +from unittest import mock +import shutil +import sys +import tarfile +import unittest +import zipfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts/ci")) +import reuse_app_host_products as reuse +from test_app_host_test_products import TestProductHandoff + + +class ReuseProducts(TestProductHandoff): + def setUp(self): + super().setUp() + self.contract = {"tree": "same-tree", "xcode": "same-xcode"} + self.api = FakeGitHub(self.contract) + self.api.archive = self.producer.parent / "artifact.zip" + self.seal() + + def seal(self): + reuse.products.stamp(self.producer, self.identity) + root = self.producer / "Build/Products" + (root / reuse.RECEIPT).write_text(json.dumps({"contract": self.contract, + "revision": self.identity["revision"], "run_id": "12", "run_attempt": str(self.api.run["run_attempt"])})) + archive = self.producer.parent / "app-host-products.tar.gz" + with tarfile.open(archive, "w:gz", dereference=True) as tar: + tar.add(root, arcname="Build/Products") + with zipfile.ZipFile(self.api.archive, "w") as z: + z.write(archive, "app-host-products.tar.gz") + self.api.artifact["digest"] = "sha256:" + hashlib.sha256(self.api.archive.read_bytes()).hexdigest() + + def restore_reuse(self): + current = {**self.identity, "revision": "def456", "checkout": "/queue/work/cmux"} + return reuse.restore(self.api, self.contract, self.consumer, "13", current) + + def test_other_commit_same_tree_reuses_and_relocates_without_test_result(self): + # The full run failed tests, while compilation itself succeeded. + self.api.run["conclusion"] = "failure" + self.assertTrue(self.restore_reuse()) + receipt = json.loads((self.consumer / "Build/Products" / reuse.products.RECEIPT).read_text()) + self.assertEqual(receipt["revision"], "def456") + provenance = json.loads((self.consumer / "Build/Products/cmux-original-producer.json").read_text()) + self.assertEqual(provenance["revision"], "abc123") + value = __import__('plistlib').loads(next((self.consumer / "Build/Products").glob('cmux-unit_*.xctestrun')).read_bytes()) + target = list(reuse.products.targets(value))[0] + self.assertEqual(target['EnvironmentVariables']['SOURCE'], '/queue/work/cmux/fixtures') + self.assertTrue(Path(target['DependentProductPaths'][0]).exists()) + + def test_fork_wrong_workflow_and_failed_compile_are_misses(self): + for field, value in [('event', 'workflow_dispatch'), ('path', '.github/workflows/untrusted.yml'), + ('head_repository', {'full_name': 'fork/cmux'})]: + with self.subTest(field=field): + old = self.api.run[field] + self.api.run[field] = value + self.assertFalse(self.restore_reuse()) + self.api.run[field] = old + self.api.job['conclusion'] = 'failure' + self.assertFalse(self.restore_reuse()) + + def test_expired_missing_digest_current_run_are_misses(self): + self.api.artifact['expired'] = True + self.assertFalse(self.restore_reuse()) + self.api.artifact['expired'] = False + self.api.artifact['workflow_run']['id'] = 13 + self.assertFalse(self.restore_reuse()) + self.api.artifact['workflow_run']['id'] = 12 + self.api.artifact.pop('digest') + self.assertFalse(self.restore_reuse()) + + def test_environment_changes_do_not_reuse(self): + self.contract = {**self.contract, 'xcode': 'different-xcode'} + self.assertFalse(self.restore_reuse()) + + def test_actual_source_and_attempt_must_match(self): + self.api.tree = 'different-tree' + self.assertFalse(self.restore_reuse()) + self.assertFalse(self.consumer.exists()) + self.api.tree = 'same-tree' + self.api.run['run_attempt'] = 2 + self.assertFalse(self.restore_reuse()) + + def test_corrupt_archive_never_populates_consumer(self): + self.api.archive.write_bytes(b'corrupt') + self.assertFalse(self.restore_reuse()) + self.assertFalse(self.consumer.exists()) + + def test_invalid_candidate_does_not_hide_later_valid_archive(self): + original_download = self.api.download + for failure in ('download', 'archive', 'receipt'): + with self.subTest(failure=failure): + bad = {**self.api.artifact, 'id': 41} + if failure == 'archive': + bad['digest'] = 'sha256:' + hashlib.sha256(b'corrupt').hexdigest() + bad_run = {**self.api.run, 'id': 99} if failure == 'receipt' else self.api.run + def download(artifact_id, target): + if artifact_id == 41 and failure == 'download': + raise OSError('candidate unavailable') + if artifact_id == 41 and failure == 'archive': + target.write_bytes(b'corrupt') + else: + original_download(42, target) + with mock.patch.object(reuse, 'select', return_value=[ + (bad, bad_run), (self.api.artifact, self.api.run)]), \ + mock.patch.object(self.api, 'download', side_effect=download) as calls: + self.assertTrue(self.restore_reuse()) + self.assertEqual([call.args[0] for call in calls.call_args_list], [41, 42]) + provenance = json.loads((self.consumer / 'Build/Products/cmux-original-producer.json').read_text()) + self.assertEqual(provenance['artifact_id'], 42) + shutil.rmtree(self.consumer) + + def test_failure_after_relocation_aborts_without_trying_another_candidate(self): + original_restore = reuse.products.restore + def restore(derived, identity): + if derived == self.consumer: + raise ValueError('consumer relocation failed') + return original_restore(derived, identity) + with mock.patch.object(reuse, 'select', return_value=[ + (self.api.artifact, self.api.run), (self.api.artifact, self.api.run)]), \ + mock.patch.object(reuse.products, 'restore', side_effect=restore), \ + mock.patch.object(self.api, 'download', wraps=self.api.download) as download: + with self.assertRaisesRegex(ValueError, 'consumer relocation failed'): + self.restore_reuse() + self.assertEqual(download.call_count, 1) + + def test_attempt_suffixed_artifact_remains_discoverable(self): + self.assertTrue(self.api.artifact['name'].endswith('-1')) + self.assertTrue(self.restore_reuse()) + + def test_later_attempt_gets_its_own_artifact_and_receipt(self): + self.api.run['run_attempt'] = 2 + self.assertFalse(self.restore_reuse()) + self.api.artifact['name'] = reuse.PREFIX + reuse.key(self.contract) + '-2' + self.seal() + self.assertTrue(self.restore_reuse()) + + def test_oversize_compressed_artifact_is_rejected_without_download(self): + with mock.patch.object(reuse, 'MAX_ARCHIVE_BYTES', 1), \ + mock.patch.object(self.api, 'download') as download: + self.assertFalse(self.restore_reuse()) + download.assert_not_called() + + def test_valid_digest_with_corrupt_tar_is_rejected(self): + with zipfile.ZipFile(self.api.archive, 'w') as z: + z.writestr('app-host-products.tar.gz', b'corrupt') + self.api.artifact['digest'] = 'sha256:' + hashlib.sha256(self.api.archive.read_bytes()).hexdigest() + self.assertFalse(self.restore_reuse()) + self.assertFalse(self.consumer.exists()) + + def test_archive_expansion_is_bounded(self): + for limit in ('MAX_MEMBER_BYTES', 'MAX_EXPANDED_BYTES', 'MAX_MEMBERS', 'MAX_TAR_BYTES'): + with self.subTest(limit=limit), mock.patch.object(reuse, limit, 1, create=True): + self.assertFalse(self.restore_reuse()) + self.assertFalse(self.consumer.exists()) + + def test_unrelated_producer_tree_rejected_before_download(self): + self.api.tree = 'different-tree' + with mock.patch.object(self.api, 'download', wraps=self.api.download) as download: + try: + self.restore_reuse() + except ValueError: + pass + download.assert_not_called() + + def test_completed_compile_can_be_used_while_other_tests_run(self): + self.api.run['status'] = 'in_progress' + self.assertTrue(self.restore_reuse()) + + def test_api_failure_cli_falls_back_to_compile(self): + output = self.producer.parent / 'github-output' + env = {'GITHUB_OUTPUT': str(output), 'GITHUB_EVENT_NAME': 'merge_group', + 'GITHUB_REPOSITORY': self.api.repository, 'GITHUB_RUN_ID': '13'} + with mock.patch.dict(os.environ, env), mock.patch.object(sys, 'argv', + ['reuse', 'restore', str(self.consumer)]), \ + mock.patch.object(reuse, 'contract', return_value=self.contract), \ + mock.patch.object(reuse.products, 'identity', return_value=self.identity), \ + mock.patch.object(reuse.GitHub, 'get', side_effect=OSError('API unavailable')): + reuse.main() + self.assertEqual(output.read_text(), 'hit=false\n') + self.assertFalse(self.consumer.exists()) + + + def test_tar_cannot_escape_staging(self): + tarbytes = io.BytesIO() + with tarfile.open(fileobj=tarbytes, mode='w:gz') as tar: + member = tarfile.TarInfo('../escape') + member.size = 1 + tar.addfile(member, io.BytesIO(b'x')) + with zipfile.ZipFile(self.api.archive, 'w') as z: + z.writestr('app-host-products.tar.gz', tarbytes.getvalue()) + self.api.artifact['digest'] = 'sha256:' + hashlib.sha256(self.api.archive.read_bytes()).hexdigest() + self.assertFalse(self.restore_reuse()) + self.assertFalse((self.producer.parent / 'escape').exists()) + + +class FakeGitHub: + repository = 'manaflow-ai/cmux' + def __init__(self, contract): + self.tree = contract['tree'] + self.artifact = {'id': 42, 'name': reuse.PREFIX + reuse.key(contract) + '-1', 'size_in_bytes': 100, + 'expired': False, 'workflow_run': {'id': 12}} + self.run = {'id': 12, 'path': '.github/workflows/ci.yml', 'event': 'pull_request', + 'head_repository': {'full_name': self.repository}, 'run_attempt': 1, + 'head_sha': 'abc123', 'html_url': 'https://github.com/manaflow-ai/cmux/actions/runs/12'} + self.job = {'name': 'macOS compile admission', 'conclusion': 'success', 'status': 'completed'} + def get(self, path): + if path.startswith('actions/artifacts?'): + return {'artifacts': [self.artifact]} + if '/jobs?' in path: + return {'jobs': [self.job]} + if path.startswith('actions/runs/'): + return self.run + if path.startswith('git/commits/'): + return {'tree': {'sha': self.tree}} + raise AssertionError(path) + def download(self, artifact_id, target): + assert artifact_id == 42 + shutil.copyfile(self.archive, target) + + +if __name__ == '__main__': + unittest.main()