From 3d1f6c5f0477fc22f5c2f03f872590ba382c5e84 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Wed, 5 Aug 2026 17:43:48 -0700 Subject: [PATCH] ci(cache): save GitHub Actions caches only from main The repository is at 9.66 GB of its 10 GB cache limit, and most of it is copies no one can read. GitHub scopes a cache entry to the ref that wrote it, so a pull_request or merge_group run writes an entry only that ref can restore: used once, then holding quota until evicted. dependency-docs-bazel held 5.59 GB across four entries, three of them the same key written from main, a PR ref, and a merge-queue ref. That evicts the Bazel build caches, which are what actually make CI fast. Split every actions/cache use into restore + save and gate the save to pushes on main. Runs on any ref can still restore from the default branch, so no reuse is lost. This is the read/write split the remote cache already uses in tools/ci/bazel-cache-upload-mode. Also give the dependency-docs cache a restore-keys prefix; without one, any lock bump meant a fully cold build. Co-authored-by: Balaji Ganesan --- .github/workflows/bazel.yml | 49 +++++++++++++++++++--- .github/workflows/license-dependencies.yml | 30 ++++++++++++- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index ed44dd1ce..d9ede21ff 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -481,9 +481,17 @@ jobs: # # Concurrent saves to one key are safe: the first writer wins and the rest # no-op. Rows with their own module keep a per-row identity. - - name: Cache Bazel repository + disk caches + # Restore on every event, save only from main. GitHub scopes a cache + # entry to the ref that wrote it, so a pull_request or merge_group run + # writes a copy only that ref can read: used once, then holding quota + # until evicted. These entries are 1.5-1.9 GB against a 10 GB repository + # limit, so PR-scoped copies evict the main entry every other row is + # trying to restore. Runs on any ref can restore from the default branch, + # so main-only saves lose no reuse. Same read/write split as the remote + # cache in tools/ci/bazel-cache-upload-mode. + - name: Restore Bazel repository + disk caches if: steps.precheck.outputs.skip == 'false' - uses: actions/cache@v4 + uses: actions/cache/restore@v4 with: path: | ~/.cache/bazel/_bazel_${{ env.USER || 'root' }}/install @@ -498,9 +506,9 @@ jobs: # dependency graph and the Bazel cache above, so cache the genrule's Go # module + build cache separately, keyed on the collector go.sum, so cold # runners restore it instead of re-downloading. See NVIDIA/nvcf#373. - - name: Cache byoo-otel-collector Go module cache + - name: Restore byoo-otel-collector Go module cache if: matrix.subtree.id == 'byoo-otel-collector' && steps.precheck.outputs.skip == 'false' - uses: actions/cache@v4 + uses: actions/cache/restore@v4 with: path: | ${{ runner.temp }}/byoo-gomodcache @@ -864,6 +872,24 @@ jobs: if-no-files-found: error retention-days: 14 + - name: Save Bazel repository + disk caches + if: steps.precheck.outputs.skip == 'false' && github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/cache/save@v4 + with: + path: | + ~/.cache/bazel/_bazel_${{ env.USER || 'root' }}/install + ~/.cache/bazel/_bazel_${{ env.USER || 'root' }}/cache + key: bazel-${{ matrix.subtree.workdir == '.' && 'rootmodule' || matrix.subtree.id }}-${{ hashFiles(format('{0}/MODULE.bazel.lock', matrix.subtree.workdir), format('{0}/.bazelversion', matrix.subtree.workdir)) }} + + - name: Save byoo-otel-collector Go module cache + if: matrix.subtree.id == 'byoo-otel-collector' && steps.precheck.outputs.skip == 'false' && github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/cache/save@v4 + with: + path: | + ${{ runner.temp }}/byoo-gomodcache + ${{ runner.temp }}/byoo-gocache + key: byoo-gomod-${{ hashFiles('src/compute-plane-services/byoo-otel-collector/otelcol/go.sum') }} + # Docker-host lane. Each subtree that owns requires-docker # (Testcontainers) tests runs its FULL suite (unit + Testcontainers) here, in # its own visibly-named lane, with no tag filter -- so no test a subtree owns @@ -915,8 +941,10 @@ jobs: # Cache Bazel's install base + repository cache (external deps: Maven # artifacts, toolchains) across runs. Compiled action outputs come from # the remote cache below. - - name: Cache Bazel repository + disk caches - uses: actions/cache@v4 + # Restore on every event, save only from main; see the matching comment in + # the build-container lane above. + - name: Restore Bazel repository + disk caches + uses: actions/cache/restore@v4 with: path: | ~/.cache/bazel/*/install @@ -1027,6 +1055,15 @@ jobs: if-no-files-found: error retention-days: 14 + - name: Save Bazel repository + disk caches + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/cache/save@v4 + with: + path: | + ~/.cache/bazel/*/install + ~/.cache/bazel/*/cache + key: bazel-docker-${{ matrix.subtree.workdir == '.' && 'rootmodule' || matrix.subtree.id }}-${{ hashFiles(format('{0}/MODULE.bazel.lock', matrix.subtree.workdir), format('{0}/.bazelversion', matrix.subtree.workdir)) }} + bazel-verification: name: bazel required checks needs: [detect, bazel, bazel-docker] diff --git a/.github/workflows/license-dependencies.yml b/.github/workflows/license-dependencies.yml index 2faca9d9c..60ae9e777 100644 --- a/.github/workflows/license-dependencies.yml +++ b/.github/workflows/license-dependencies.yml @@ -79,13 +79,39 @@ jobs: chmod +x "${RUNNER_TEMP}/bin/bazel" echo "${RUNNER_TEMP}/bin" >> "${GITHUB_PATH}" - - name: Cache Bazel repository and disk caches - uses: actions/cache@v4 + # Restore on every event, but save only from main. + # + # GitHub scopes a cache entry to the ref that wrote it. A pull_request or + # merge_group run writes an entry only that one ref can ever read, so the + # copy is used once and then occupies quota until it is evicted. This + # cache is ~1.4 GB, and three such copies (main, a PR, a merge-queue ref) + # were holding 4.2 GB of the repository's 10 GB limit, evicting the Bazel + # build caches that actually make CI fast. Runs on any ref can restore + # from the default branch, so saving only on main loses no reuse. + # + # This mirrors the remote-cache policy in tools/ci/bazel-cache-upload-mode: + # untrusted refs read, they do not write. + - name: Restore Bazel repository and disk caches + uses: actions/cache/restore@v4 with: path: | ~/.cache/bazel/*/install ~/.cache/bazel/*/cache key: dependency-docs-bazel-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock', 'maven_install.json') }} + # Without a prefix fallback, any lock bump means a cold build. A + # partially stale Bazel disk cache is safe: entries are content + # addressed, so Bazel revalidates rather than trusting them blindly. + restore-keys: | + dependency-docs-bazel- - name: Run dependency docs freshness check run: ./tools/ci/check-dependency-docs + + - name: Save Bazel repository and disk caches + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/cache/save@v4 + with: + path: | + ~/.cache/bazel/*/install + ~/.cache/bazel/*/cache + key: dependency-docs-bazel-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock', 'maven_install.json') }}