diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index 482a4ee9f9b8..65a58cccf544 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -79,6 +79,10 @@ jobs: if: ${{ matrix.group == 'ci' }} run: python3 tests/test_ci_canonical_build_root.py + - name: Validate seeded macOS checkout + if: ${{ matrix.group == 'ci' }} + run: python3 tests/test_ci_git_seed.py + - name: Run canonical CMUX CI guard profile if: ${{ matrix.group == 'ci' }} run: | diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 352091ce85dc..eb426d0fe8f5 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -195,6 +195,21 @@ jobs: echo "CMUX_HOSTED_SOURCE_PREP_STARTED=$now" } >> "$GITHUB_ENV" + # Start from main's git objects, so checkout fetches only what changed + # since then instead of the whole tree and its submodules + # (scripts/ci/git-seed.sh). The script is read at this commit over HTTPS + # because nothing is checked out yet. A miss leaves the workspace empty + # and checkout clones as before. + - name: Restore git object seed + continue-on-error: true + timeout-minutes: 3 + run: | + set -euo pipefail + script="$RUNNER_TEMP/git-seed.sh" + curl --fail --silent --show-error --location --connect-timeout 10 --max-time 30 \ + -o "$script" "https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$GITHUB_SHA/scripts/ci/git-seed.sh" + bash "$script" restore "$GITHUB_WORKSPACE" + - name: Clear stale git locks (self-hosted reused workspace) shell: bash run: | @@ -213,6 +228,12 @@ jobs: submodules: recursive persist-credentials: false + # A seeded repository that checkout cannot use (a submodule moved to + # another URL) must not fail the retry the same way. + - name: Discard the git object seed after a failed checkout + if: steps.checkout.outcome == 'failure' + run: rm -rf "$GITHUB_WORKSPACE/.git" + # WarpBuild can lose GitHub DNS on an otherwise healthy runner. Retry the # checkout once only after that action fails; healthy jobs still perform # one checkout, and a second failure remains a hard failure with evidence. @@ -1171,6 +1192,21 @@ jobs: ;; esac + # Start from main's git objects, so checkout fetches only what changed + # since then instead of the whole tree and its submodules + # (scripts/ci/git-seed.sh). The script is read at this commit over HTTPS + # because nothing is checked out yet. A miss leaves the workspace empty + # and checkout clones as before. + - name: Restore git object seed + continue-on-error: true + timeout-minutes: 3 + run: | + set -euo pipefail + script="$RUNNER_TEMP/git-seed.sh" + curl --fail --silent --show-error --location --connect-timeout 10 --max-time 30 \ + -o "$script" "https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$GITHUB_SHA/scripts/ci/git-seed.sh" + bash "$script" restore "$GITHUB_WORKSPACE" + - name: Clear stale git locks (self-hosted reused workspace) shell: bash run: | @@ -1190,6 +1226,12 @@ jobs: with: persist-credentials: false + # A seeded repository that checkout cannot use (a submodule moved to + # another URL) must not fail the retry the same way. + - name: Discard the git object seed after a failed checkout + if: steps.checkout.outcome == 'failure' + run: rm -rf "$GITHUB_WORKSPACE/.git" + # WarpBuild can lose GitHub DNS on an otherwise healthy runner. Retry the # checkout once only after that action fails; healthy jobs still perform # one checkout, and a second failure remains a hard failure with evidence. @@ -2079,6 +2121,21 @@ jobs: ;; esac + # Start from main's git objects, so checkout fetches only what changed + # since then instead of the whole tree and its submodules + # (scripts/ci/git-seed.sh). The script is read at this commit over HTTPS + # because nothing is checked out yet. A miss leaves the workspace empty + # and checkout clones as before. + - name: Restore git object seed + continue-on-error: true + timeout-minutes: 3 + run: | + set -euo pipefail + script="$RUNNER_TEMP/git-seed.sh" + curl --fail --silent --show-error --location --connect-timeout 10 --max-time 30 \ + -o "$script" "https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$GITHUB_SHA/scripts/ci/git-seed.sh" + bash "$script" restore "$GITHUB_WORKSPACE" + - name: Clear stale git locks (self-hosted reused workspace) shell: bash run: | @@ -2310,6 +2367,21 @@ jobs: CMUX_CI_HELPER_XCODE_APP: ${{ vars.CMUX_CI_HELPER_XCODE_APP_MACOS_15 }} CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" steps: + # Start from main's git objects, so checkout fetches only what changed + # since then instead of the whole tree and its submodules + # (scripts/ci/git-seed.sh). The script is read at this commit over HTTPS + # because nothing is checked out yet. A miss leaves the workspace empty + # and checkout clones as before. + - name: Restore git object seed + continue-on-error: true + timeout-minutes: 3 + run: | + set -euo pipefail + script="$RUNNER_TEMP/git-seed.sh" + curl --fail --silent --show-error --location --connect-timeout 10 --max-time 30 \ + -o "$script" "https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$GITHUB_SHA/scripts/ci/git-seed.sh" + bash "$script" restore "$GITHUB_WORKSPACE" + - name: Clear stale git locks (self-hosted reused workspace) shell: bash run: | @@ -2323,6 +2395,8 @@ jobs: fi - name: Checkout + id: checkout + continue-on-error: true uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive @@ -2330,6 +2404,19 @@ jobs: # "Select package tests" diffs against. fetch-depth: 2 + # A seeded repository that checkout cannot use (a submodule moved to + # another URL) must not fail the retry the same way. + - name: Discard the git object seed after a failed checkout + if: steps.checkout.outcome == 'failure' + run: rm -rf "$GITHUB_WORKSPACE/.git" + + - name: Retry checkout without the git object seed + if: steps.checkout.outcome == 'failure' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: recursive + fetch-depth: 2 + - name: Select package tests id: select env: @@ -2808,6 +2895,21 @@ jobs: ;; esac + # Start from main's git objects, so checkout fetches only what changed + # since then instead of the whole tree and its submodules + # (scripts/ci/git-seed.sh). The script is read at this commit over HTTPS + # because nothing is checked out yet. A miss leaves the workspace empty + # and checkout clones as before. + - name: Restore git object seed + continue-on-error: true + timeout-minutes: 3 + run: | + set -euo pipefail + script="$RUNNER_TEMP/git-seed.sh" + curl --fail --silent --show-error --location --connect-timeout 10 --max-time 30 \ + -o "$script" "https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$GITHUB_SHA/scripts/ci/git-seed.sh" + bash "$script" restore "$GITHUB_WORKSPACE" + - name: Clear stale git locks (self-hosted reused workspace) shell: bash run: | diff --git a/.github/workflows/cli-pipe-regressions.yml b/.github/workflows/cli-pipe-regressions.yml index f05dced9c4cf..2b2d810bb938 100644 --- a/.github/workflows/cli-pipe-regressions.yml +++ b/.github/workflows/cli-pipe-regressions.yml @@ -34,12 +34,27 @@ jobs: # restore needs this set here. CI_CACHE_R2_PUBLIC_URL: ${{ vars.CI_CACHE_R2_PUBLIC_URL || 'https://ci-cache.cmux.com' }} steps: + # Start from main's git objects, so checkout fetches only what changed + # since then instead of the whole tree and its submodules + # (scripts/ci/git-seed.sh). The script is read at this commit over HTTPS + # because nothing is checked out yet. A miss leaves the workspace empty + # and checkout clones as before. + - name: Restore git object seed + continue-on-error: true + timeout-minutes: 3 + run: | + set -euo pipefail + script="$RUNNER_TEMP/git-seed.sh" + curl --fail --silent --show-error --location --connect-timeout 10 --max-time 30 \ + -o "$script" "https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$GITHUB_SHA/scripts/ci/git-seed.sh" + bash "$script" restore "$GITHUB_WORKSPACE" + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Initialize local Swift package dependency - run: git submodule update --init --depth 1 vendor/bonsplit + run: scripts/ci/git-seed.sh update-submodules "$GITHUB_WORKSPACE" vendor/bonsplit - name: Select Xcode run: ./scripts/select-ci-xcode.sh @@ -47,7 +62,7 @@ jobs: - name: Initialize Ghostty and download its binary framework run: | set -euo pipefail - git submodule update --init --depth 1 ghostty + scripts/ci/git-seed.sh update-submodules "$GITHUB_WORKSPACE" ghostty ./scripts/download-prebuilt-ghosttykit.sh - name: Install Rust toolchain diff --git a/.github/workflows/seed-derived-data.yml b/.github/workflows/seed-derived-data.yml index 511ae99a6dc3..c6498e0b1389 100644 --- a/.github/workflows/seed-derived-data.yml +++ b/.github/workflows/seed-derived-data.yml @@ -336,6 +336,22 @@ jobs: retention-days: 3 compression-level: 0 + # macOS jobs start their checkout from these objects, so a pull request + # fetches only what changed since this main commit instead of the whole + # tree and its submodules (scripts/ci/git-seed.sh). A failure costs them + # the head start, never this seed. + - name: Save git object seed + id: git-seed + if: github.ref == 'refs/heads/main' + continue-on-error: true + timeout-minutes: 5 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CI_CACHE_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_CACHE_R2_SECRET_ACCESS_KEY }} + CI_CACHE_R2_ENDPOINT: ${{ format('https://{0}.r2.cloudflarestorage.com', secrets.CI_CACHE_R2_ACCOUNT_ID) }} + CI_CACHE_R2_BUCKET: ${{ vars.CI_CACHE_R2_BUCKET }} + run: scripts/ci/git-seed.sh save "$GITHUB_WORKSPACE" + - name: Summarize if: always() env: diff --git a/scripts/ci/detect_ci_change_areas.py b/scripts/ci/detect_ci_change_areas.py index 678dd06df1cb..359302dc5ee7 100644 --- a/scripts/ci/detect_ci_change_areas.py +++ b/scripts/ci/detect_ci_change_areas.py @@ -630,6 +630,8 @@ def is_guard_only_test(path: str, references: Optional[tuple[frozenset[str], fro # What restore-app-host-test-product.sh itself runs. "scripts/ci/app_host_test_products.py", "scripts/ci/canonical-build-root.sh", + # Seeds the checkout of both CLI lanes and initializes cli-pipe's submodules. + "scripts/ci/git-seed.sh", }) CLI_LANE_INPUT_PREFIXES = ( diff --git a/scripts/ci/git-seed.sh b/scripts/ci/git-seed.sh new file mode 100755 index 000000000000..a8ca128ba4d4 --- /dev/null +++ b/scripts/ci/git-seed.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# git-seed.sh save WORKSPACE +# git-seed.sh restore WORKSPACE +# git-seed.sh update-submodules WORKSPACE PATH... +# +# Start a macOS checkout from main's git objects instead of an empty directory. +# +# actions/checkout fetches the tested commit with --depth=1 into an empty +# repository, so every macOS job downloads the whole tree: 129 MB in 29 s for +# the main repository, then 17 s for its submodules, most of it ghostty, whose +# depth-1 clone fetches its default branch before the pinned commit (job +# 107695138963). A fetch into a repository that already holds a recent main +# sends only what changed since: 92 KB to 1.2 MB in under a second for pull +# request merge commits measured on 2026-09-24, and a submodule whose pinned +# commit is already present is not fetched at all. +# +# `save` runs in a main-branch job after its checkout and publishes the +# repository's objects to R2 under git-seed-v1-. Only objects, the +# shallow list and commit ids travel: no config, hooks, index or credentials. +# +# `restore` runs before actions/checkout on a runner with no repository yet. +# It builds a repository whose HEAD is the seed commit and whose origin is the +# URL actions/checkout expects, so checkout keeps it: it resets to HEAD, then +# fetches the tested commit against those objects. Submodule git directories +# go where `git submodule update` looks for them, and it reuses them. The +# repository is assembled aside and renamed into place last, so any failure +# leaves the workspace empty and checkout clones from scratch, as it always +# has. Correctness never depends on the seed: checkout fetches and +# checks out the exact commit it was asked for. A pull request that moves a +# submodule to another URL would fetch its new pin from the seed's URL and +# fail; each job discards the seeded repository and retries cold then. +# +# Seeds are written only by main-branch jobs holding the R2 credentials, the +# same trust as the Swift package and DerivedData seeds these jobs restore. +set -euo pipefail + +mode="${1:-}" +workspace="${2:-${GITHUB_WORKSPACE:-$PWD}}" +PREFIX="git-seed-v1-" +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +usage() { + echo "usage: git-seed.sh save|restore WORKSPACE | update-submodules WORKSPACE PATH..." >&2 + exit 2 +} + +is_commit_id() { [[ "$1" =~ ^[0-9a-f]{40}$ ]]; } + +# Submodule names and paths from a .gitmodules file on stdin, as "namepath". +submodules() { + git config --file - --get-regexp '^submodule\..*\.path$' 2>/dev/null \ + | sed -E 's/^submodule\.(.*)\.path (.*)$/\1\t\2/' || true +} + +stage_seed() { # ; copies the seed out of WORKSPACE's repository + local stage="$1" head + cd "$workspace" + head="$(git rev-parse HEAD)" + is_commit_id "$head" || { echo "git-seed: HEAD is not a commit" >&2; exit 1; } + mkdir -p "$stage" + cp -cR .git/objects "$stage/objects" 2>/dev/null || cp -R .git/objects "$stage/objects" + [ ! -f .git/shallow ] || cp .git/shallow "$stage/shallow" + echo "$head" > "$stage/HEAD" + local name path gitdir commit + while IFS=$'\t' read -r name path; do + [ -n "$name" ] || continue + gitdir=".git/modules/$name" + [ -d "$gitdir/objects" ] || continue + commit="$(git -C "$path" rev-parse HEAD 2>/dev/null)" || continue + is_commit_id "$commit" || continue + mkdir -p "$stage/modules/$name" + cp -cR "$gitdir/objects" "$stage/modules/$name/objects" 2>/dev/null \ + || cp -R "$gitdir/objects" "$stage/modules/$name/objects" + [ ! -f "$gitdir/shallow" ] || cp "$gitdir/shallow" "$stage/modules/$name/shallow" + echo "$commit" > "$stage/modules/$name/HEAD" + done < <(submodules < .gitmodules) +} + +save() { + local scratch head + head="$(git -C "$workspace" rev-parse HEAD)" + scratch="$(mktemp -d)" + bash "${BASH_SOURCE[0]}" stage "$workspace" "$scratch/seed" + "$HERE/r2-cache.sh" save "$scratch/seed" "$PREFIX$head" + rm -rf "$scratch" +} + +fetch_seed() { # ; downloads and unpacks the newest seed + local stage="$1" base key + base="${CI_CACHE_R2_PUBLIC_URL:?CI_CACHE_R2_PUBLIC_URL is not set}" + base="${base%/}/v1/${RUNNER_OS:-$(uname -s)}-${RUNNER_ARCH:-$(uname -m)}" + key="$(curl --fail --silent --show-error --location --connect-timeout 10 --max-time 20 \ + "$base/latest/$PREFIX" | head -c 512 | tr -d '[:space:]')" + [[ "$key" =~ ^${PREFIX}[0-9a-f]{40}$ ]] || { echo "git-seed: no seed pointer"; return 1; } + mkdir -p "$stage" + # r2-cache.sh saves .tar.gz where the saver has no zstd. + local extension archive="$stage.archive" + for extension in tar.zst tar.gz; do + if curl --fail --silent --show-error --location --connect-timeout 10 --max-time 120 \ + -o "$archive" "$base/objects/$key.$extension" 2>/dev/null; then + if [ "$extension" = tar.zst ]; then + # Drain to EOF: bsdtar stops at the end-of-archive marker, and zstd + # would then die of SIGPIPE and fail a valid restore (r2-cache.sh). + zstd -dc "$archive" | { tar -xf - -C "$stage"; status=$?; cat > /dev/null; exit "$status"; } + else + tar -xzf "$archive" -C "$stage" + fi + rm -f "$archive" + echo "git-seed: unpacked $key.$extension" + return 0 + fi + done + echo "git-seed: $key has no archive" + return 1 +} + +install_seed() { # + local stage="$1" head url repo + head="$(cat "$stage/HEAD")" + is_commit_id "$head" || { echo "git-seed: seed HEAD is not a commit id" >&2; return 1; } + [ -d "$stage/objects/pack" ] || { echo "git-seed: seed has no packs" >&2; return 1; } + + # Built beside the stage and renamed into the workspace last, so a step + # killed partway leaves either no repository or a complete one. + repo="$(dirname "$stage")/repo" + git init -q "$repo" + cd "$repo" + rm -rf .git/objects + mv "$stage/objects" .git/objects + [ ! -f "$stage/shallow" ] || cp "$stage/shallow" .git/shallow + # The exact URL actions/checkout compares before it keeps a repository. + url="${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is not set}" + git remote add origin "$url" + git cat-file -e "$head^{tree}" + # A ref outside refs/remotes, so nothing mistakes it for today's main; the + # fetch still offers it to the server as a commit this repository has. + git update-ref refs/git-seed/main "$head" + git update-ref --no-deref HEAD "$head" + + # Module URLs come from the seed commit's own .gitmodules, never the archive. + local name path commit gitdir module_url + while IFS=$'\t' read -r name path; do + [ -n "$name" ] || continue + [ -f "$stage/modules/$name/HEAD" ] && [ -d "$stage/modules/$name/objects/pack" ] || continue + commit="$(cat "$stage/modules/$name/HEAD")" + is_commit_id "$commit" || continue + # Only a module at the commit main pins is worth keeping. + [ "$(git rev-parse -q --verify "$head:$path" 2>/dev/null)" = "$commit" ] || continue + module_url="$(git config --blob "$head:.gitmodules" "submodule.$name.url")" || continue + gitdir=".git/modules/$name" + mkdir -p "$gitdir/refs/heads" "$gitdir/refs/tags" + mv "$stage/modules/$name/objects" "$gitdir/objects" + [ ! -f "$stage/modules/$name/shallow" ] || cp "$stage/modules/$name/shallow" "$gitdir/shallow" + echo "$commit" > "$gitdir/HEAD" + git config --file "$gitdir/config" core.repositoryformatversion 0 + git config --file "$gitdir/config" core.bare false + git config --file "$gitdir/config" remote.origin.url "$module_url" + git config --file "$gitdir/config" remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*' + git --git-dir="$gitdir" cat-file -e "$commit^{tree}" || { rm -rf "$gitdir"; continue; } + echo "git-seed: module $name at $commit" + done < <(git show "$head:.gitmodules" 2>/dev/null | submodules) + mkdir -p "$workspace" + [ ! -e "$workspace/.git" ] || { echo "git-seed: $workspace/.git appeared meanwhile" >&2; return 1; } + mv "$repo/.git" "$workspace/.git" + echo "git-seed: repository at $head" +} + +restore() { + if [ -e "$workspace/.git" ]; then + echo "git-seed: $workspace already has a repository; leaving it to actions/checkout" + return 0 + fi + local scratch + scratch="$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/git-seed.XXXXXX")" + # Separate processes: errexit does not apply inside a function whose status + # is tested, so a failed step would otherwise be installed as if it passed. + if ! bash "${BASH_SOURCE[0]}" fetch "$workspace" "$scratch/seed" \ + || ! bash "${BASH_SOURCE[0]}" install "$workspace" "$scratch/seed"; then + echo "git-seed: no seed installed; actions/checkout clones from scratch" + fi + rm -rf "$scratch" +} + +# `git submodule update --init --depth 1 PATH...` for jobs that initialize +# submodules after checkout. A seeded module fetches a new pin from the URL the +# seed recorded, so when that fails, drop the seeded git directories and clone +# those modules cold. +update_submodules() { + cd "$workspace" + git submodule update --init --depth 1 -- "$@" && return 0 + echo "git-seed: submodule update failed; retrying without seeded modules" >&2 + local name path wanted + while IFS=$'\t' read -r name path; do + for wanted in "$@"; do + [ "$wanted" = "$path" ] || continue + rm -rf ".git/modules/$name" "$path" + done + done < <(submodules < .gitmodules) + git submodule update --init --depth 1 -- "$@" +} + +case "$mode" in + save) save ;; + restore) restore ;; + update-submodules) shift 2; update_submodules "$@" ;; + # Internal steps of `save` and `restore`. + stage) stage_seed "${3:?}" ;; + fetch) fetch_seed "${3:?}" ;; + install) install_seed "${3:?}" ;; + *) usage ;; +esac diff --git a/scripts/ci/r2_cache_prune.py b/scripts/ci/r2_cache_prune.py index 57bfe8c7350f..7324e67d3920 100644 --- a/scripts/ci/r2_cache_prune.py +++ b/scripts/ci/r2_cache_prune.py @@ -25,6 +25,8 @@ (#14015: 3 of 2,665 app jobs hit). On 2026-09-24 these were 141 GiB of a 143 GiB bucket after five days, about 35 GiB a day. + git-seed- 1 day. One main checkout's objects per seeded + commit; restores read only the newest pointer. everything else 30 days. Keyed by content (a Package.resolved or toolchain hash), so an old key stays exact for a pull request whose base still has that input. @@ -51,6 +53,7 @@ RETENTION_DAYS = ( ("admission-derived-data-", 1), ("xcode-compilation-", 1), + ("git-seed-", 1), ("", 30), ) ARCHIVE = re.compile(r"^(v1/[^/]+)/objects/(?P[A-Za-z0-9._-]+)\.(?:tar\.zst|tar\.gz)$") diff --git a/scripts/ci/select_package_tests.py b/scripts/ci/select_package_tests.py index cedc219772fa..97879859002e 100755 --- a/scripts/ci/select_package_tests.py +++ b/scripts/ci/select_package_tests.py @@ -29,6 +29,7 @@ ".github/workflows/ci.yml", ".github/workflows/ci-macos.yml", "scripts/build-ghostty-cli-helper.sh", + "scripts/ci/git-seed.sh", "scripts/ci/release-build-archs.sh", "scripts/ci/ci_process_tree.py", "scripts/ci/hung_test_watchdog.py", diff --git a/tests/test-execution.toml b/tests/test-execution.toml index 67554a491713..f921a9ee2053 100644 --- a/tests/test-execution.toml +++ b/tests/test-execution.toml @@ -1175,3 +1175,7 @@ lane = "macos-cli-no-socket" [[test]] path = "tests/test_e2e_sibling_build.py" lane = "linux-guard" + +[[test]] +path = "tests/test_ci_git_seed.py" +lane = "linux-guard" diff --git a/tests/test_ci_git_seed.py b/tests/test_ci_git_seed.py new file mode 100644 index 000000000000..2e9f0a2a265c --- /dev/null +++ b/tests/test_ci_git_seed.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""A checkout seeded from main's objects must end exactly where a cold one does.""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "ci" / "git-seed.sh" +WORKFLOWS = ROOT / ".github" / "workflows" + + +def git(*args: str, cwd: Path | None = None, env: dict | None = None) -> str: + return subprocess.run( + ["git", "-c", "protocol.file.allow=always", *args], cwd=cwd, env=env, + check=True, capture_output=True, text=True, + ).stdout.strip() + + +def commit(repo: Path, name: str, text: str) -> str: + (repo / name).write_text(text) + git("add", "-A", cwd=repo) + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", name, cwd=repo) + return git("rev-parse", "HEAD", cwd=repo) + + +class GitSeedTests(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.base = Path(os.path.realpath(self._tmp.name)) + # Upstream repositories at //, as GitHub lays them out. + self.server = self.base / "server" + self.module = self.server / "acme" / "module" + self.super = self.server / "acme" / "super" + for repo in (self.module, self.super): + repo.mkdir(parents=True) + git("init", "-q", "-b", "main", cwd=repo) + git("config", "uploadpack.allowAnySHA1InWant", "true", cwd=repo) + self.module_v1 = commit(self.module, "m.txt", "module v1") + git("submodule", "add", "-q", f"file://{self.module}", "vendor/module", cwd=self.super) + self.main = commit(self.super, "app.txt", "main") + self.env = dict( + os.environ, + GITHUB_SERVER_URL=f"file://{self.server}", + GITHUB_REPOSITORY="acme/super", + GIT_CONFIG_COUNT="1", GIT_CONFIG_KEY_0="protocol.file.allow", GIT_CONFIG_VALUE_0="always", + ) + + def seed_from_main(self) -> Path: + """What the main-branch seeder checks out and stages.""" + seeder = self.base / "seeder" + git("clone", "-q", "--depth=1", "--recurse-submodules", "--shallow-submodules", + f"file://{self.super}", str(seeder), env=self.env) + stage = self.base / "stage" / "seed" + self.run_seed("stage", seeder, stage) + return stage + + def run_seed(self, mode: str, workspace: Path, *extra: Path, check: bool = True): + result = subprocess.run( + ["bash", str(SCRIPT), mode, str(workspace), *map(str, extra)], + env=self.env, capture_output=True, text=True, + ) + if check: + self.assertEqual(result.returncode, 0, result.stderr) + return result + + def checkout(self, workspace: Path, sha: str) -> None: + """The git commands actions/checkout runs against an existing repository.""" + self.assertEqual(git("rev-parse", "--symbolic-full-name", "--verify", "--quiet", "HEAD", cwd=workspace), "HEAD") + self.assertEqual(git("config", "remote.origin.url", cwd=workspace), f"file://{self.server}/acme/super") + git("clean", "-ffdx", cwd=workspace) + git("reset", "--hard", "HEAD", cwd=workspace) + git("fetch", "--no-tags", "--prune", "--no-recurse-submodules", "--depth=1", "origin", + f"+{sha}:refs/remotes/pull/1/merge", cwd=workspace, env=self.env) + git("checkout", "--force", "refs/remotes/pull/1/merge", cwd=workspace) + git("submodule", "sync", "--recursive", cwd=workspace) + git("submodule", "update", "--init", "--force", "--depth=1", "--recursive", cwd=workspace, env=self.env) + + def test_the_seed_carries_objects_and_commit_ids_only(self): + stage = self.seed_from_main() + self.assertEqual((stage / "HEAD").read_text().strip(), self.main) + self.assertEqual((stage / "modules" / "vendor/module" / "HEAD").read_text().strip(), self.module_v1) + for path in stage.rglob("*"): + relative = path.relative_to(stage).as_posix() + self.assertTrue( + re.fullmatch(r"(modules/vendor/module/)?(objects(/.*)?|shallow|HEAD)|modules(/vendor(/module)?)?", relative), + relative, + ) + + def test_a_seeded_checkout_matches_the_tested_commit_and_its_submodule(self): + stage = self.seed_from_main() + # Main moves on after the seed: a new app file and a submodule bump. + module_v2 = commit(self.module, "m.txt", "module v2") + git("-C", "vendor/module", "fetch", "-q", "origin", cwd=self.super, env=self.env) + git("-C", "vendor/module", "checkout", "-q", module_v2, cwd=self.super) + tested = commit(self.super, "app.txt", "pull request") + + workspace = self.base / "ws" + workspace.mkdir() + self.run_seed("install", workspace, stage) + self.checkout(workspace, tested) + self.assertEqual(git("rev-parse", "HEAD", cwd=workspace), tested) + self.assertEqual((workspace / "app.txt").read_text(), "pull request") + self.assertEqual(git("rev-parse", "HEAD", cwd=workspace / "vendor/module"), module_v2) + self.assertEqual((workspace / "vendor/module/m.txt").read_text(), "module v2") + self.assertEqual(git("status", "--porcelain", cwd=workspace), "") + + def test_an_unchanged_submodule_checks_out_from_the_seed(self): + stage = self.seed_from_main() + tested = commit(self.super, "app.txt", "pull request") + workspace = self.base / "ws" + workspace.mkdir() + self.run_seed("install", workspace, stage) + # The module upstream disappears: only the seed can supply its commit. + self.module.rename(self.base / "gone") + self.checkout(workspace, tested) + self.assertEqual((workspace / "vendor/module/m.txt").read_text(), "module v1") + + def test_a_submodule_moved_to_another_url_falls_back_to_a_cold_clone(self): + stage = self.seed_from_main() + # The pull request points the module at a fork holding a new pin. + fork = self.server / "acme" / "fork" + git("clone", "-q", f"file://{self.module}", str(fork), env=self.env) + git("config", "uploadpack.allowAnySHA1InWant", "true", cwd=fork) + forked = commit(fork, "m.txt", "fork only") + git("config", "-f", ".gitmodules", "submodule.vendor/module.url", f"file://{fork}", cwd=self.super) + git("-C", "vendor/module", "fetch", "-q", f"file://{fork}", "main", cwd=self.super, env=self.env) + git("-C", "vendor/module", "checkout", "-q", forked, cwd=self.super) + tested = commit(self.super, ".gitmodules", (self.super / ".gitmodules").read_text()) + + workspace = self.base / "ws" + workspace.mkdir() + self.run_seed("install", workspace, stage) + with self.assertRaises(subprocess.CalledProcessError): + self.checkout(workspace, tested) + # Jobs that update submodules themselves retry without the seed. + result = self.run_seed("update-submodules", workspace, Path("vendor/module")) + self.assertIn("retrying without seeded modules", result.stderr) + self.assertEqual(git("rev-parse", "HEAD", cwd=workspace / "vendor/module"), forked) + + def test_the_seed_commit_is_not_published_as_main(self): + stage = self.seed_from_main() + workspace = self.base / "ws" + workspace.mkdir() + self.run_seed("install", workspace, stage) + refs = git("for-each-ref", "--format=%(refname)", cwd=workspace).splitlines() + self.assertEqual(refs, ["refs/git-seed/main"]) + + def test_restore_leaves_an_existing_repository_alone(self): + workspace = self.base / "ws" + workspace.mkdir() + git("init", "-q", cwd=workspace) + marker = workspace / ".git" / "marker" + marker.write_text("keep") + result = self.run_seed("restore", workspace) + self.assertIn("already has a repository", result.stdout) + self.assertEqual(marker.read_text(), "keep") + + def test_a_missing_seed_leaves_the_workspace_empty(self): + workspace = self.base / "ws" + workspace.mkdir() + self.env["CI_CACHE_R2_PUBLIC_URL"] = f"file://{self.base}/no-bucket" + result = self.run_seed("restore", workspace) + self.assertIn("clones from scratch", result.stdout) + self.assertEqual(list(workspace.iterdir()), []) + + @unittest.skipUnless(shutil.which("zstd"), "zstd is not installed") + def test_a_seed_served_by_the_bucket_is_installed(self): + stage = self.seed_from_main() + bucket = self.base / "bucket" / "v1" / "macOS-ARM64" + (bucket / "latest").mkdir(parents=True) + (bucket / "objects").mkdir() + key = f"git-seed-v1-{self.main}" + (bucket / "latest" / "git-seed-v1-").write_text(key + "\n") + subprocess.run( + f"tar -cf - -C {stage} . | zstd -q -o {bucket / 'objects' / (key + '.tar.zst')}", + shell=True, check=True, + ) + self.env.update(CI_CACHE_R2_PUBLIC_URL=f"file://{self.base}/bucket", RUNNER_OS="macOS", RUNNER_ARCH="ARM64") + workspace = self.base / "ws" + workspace.mkdir() + self.run_seed("restore", workspace) + self.assertEqual(git("rev-parse", "HEAD", cwd=workspace), self.main) + + def test_a_corrupt_seed_installs_nothing(self): + stage = self.seed_from_main() + (stage / "HEAD").write_text("0" * 40 + "\n") + workspace = self.base / "ws" + workspace.mkdir() + result = self.run_seed("install", workspace, stage, check=False) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(list(workspace.iterdir()), []) + + +class WorkflowWiringTests(unittest.TestCase): + def steps_before_checkout(self, text: str, job: str) -> list[str]: + body = re.search(rf"^ {re.escape(job)}:\n(.*?)(?=^ [A-Za-z0-9_-]+:\n|\Z)", text, re.S | re.M) + self.assertIsNotNone(body, job) + before = body.group(1).split("uses: actions/checkout@", 1)[0] + return re.findall(r"- name: (.+)", before) + + def test_macos_jobs_restore_the_seed_before_checkout(self): + text = (WORKFLOWS / "ci-macos.yml").read_text() + for job in ("macos-compile-admission", "app-host-unit-tests", "cli-product-tests", "swift-package-tests", "tests-build-and-lag"): + with self.subTest(job=job): + self.assertIn("Restore git object seed", self.steps_before_checkout(text, job)) + text = (WORKFLOWS / "cli-pipe-regressions.yml").read_text() + self.assertIn("Restore git object seed", self.steps_before_checkout(text, "cli-pipe-regressions")) + + def test_a_failed_seeded_checkout_retries_without_the_seed(self): + text = (WORKFLOWS / "ci-macos.yml").read_text() + for job in ("macos-compile-admission", "app-host-unit-tests", "swift-package-tests"): + with self.subTest(job=job): + body = re.search(rf"^ {job}:\n(.*?)(?=^ [A-Za-z0-9_-]+:\n)", text, re.S | re.M).group(1) + discard = body.index("Discard the git object seed after a failed checkout") + retry = body.index("- name: Retry checkout", discard) + self.assertIn('rm -rf "$GITHUB_WORKSPACE/.git"', body[discard:retry]) + self.assertIn("steps.checkout.outcome == 'failure'", body[retry:retry + 200]) + text = (WORKFLOWS / "cli-pipe-regressions.yml").read_text() + self.assertNotIn("git submodule update --init", text) + self.assertEqual(text.count("git-seed.sh update-submodules"), 2) + + def test_only_main_saves_the_seed(self): + text = (WORKFLOWS / "seed-derived-data.yml").read_text() + step = re.search(r"- name: Save git object seed\n(.*?)(?=\n - name:|\Z)", text, re.S) + self.assertIsNotNone(step) + self.assertIn("if: github.ref == 'refs/heads/main'", step.group(1)) + self.assertIn("continue-on-error: true", step.group(1)) + self.assertIn('git-seed.sh save "$GITHUB_WORKSPACE"', step.group(1)) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_r2_cache_prune.py b/tests/test_r2_cache_prune.py index 868e2d9072bf..83a8ea6423b0 100644 --- a/tests/test_r2_cache_prune.py +++ b/tests/test_r2_cache_prune.py @@ -57,6 +57,8 @@ def test_each_family_ages_out_on_its_own_retention(self): archive("admission-derived-data-v1-macOS-ARM64-fp-b", 0.5), archive("xcode-compilation-test-macOS-ARM64-fp-a", 2), archive("xcode-compilation-test-macOS-ARM64-fp-b", 0.5), + archive("git-seed-v1-" + "a" * 40, 2), + archive("git-seed-v1-" + "b" * 40, 0.5), archive("spm-a", 31), archive("spm-b", 29), archive("spm-c", 2), @@ -65,6 +67,7 @@ def test_each_family_ages_out_on_its_own_retention(self): self.assertEqual(sorted(bucket.deleted), sorted([ f"{NS}/objects/admission-derived-data-v1-macOS-ARM64-fp-a.tar.zst", f"{NS}/objects/xcode-compilation-test-macOS-ARM64-fp-a.tar.zst", + f"{NS}/objects/git-seed-v1-{'a' * 40}.tar.zst", f"{NS}/objects/spm-a.tar.zst", ]))