ci: bound compiler and local build caches

.agents/skills/manage-ci/SKILL.md

@@ -228,6 +228,17 @@ owning source, and update the inventory and topology in the same change.
   selected-workflow restriction.
 - Provider changes must not alter source checkout, commands, profile,
   artifacts, tests, required checks, or plan membership.
+- Shared Linux Clippy/test/host/runtime compiler-cache publication has one
+  trusted GitHub-hosted warmer. It
+  publishes one bounded, exact-key sccache seed after successful main Quality;
+  GitHub-hosted PR jobs may restore it and write only to their job-local copy.
+  Depot selections must never restore this seed through Depot's repository-
+  scoped Actions-cache proxy. Do not restore per-shard Cargo target archives
+  or enable per-object GHA publication for Linux Clippy, Rust tests, host, or
+  runtime jobs.
+- Every seeded compiler job records whether the exact seed was warm or cold.
+  Enforce a measured minimum hit rate only for an exact warm restore; an
+  intentional cache miss is classified cold and must not fail the build.
 
 ### Bounded Depot PR cache-risk exception
 

.agents/skills/manage-ci/references/current-inventory.md

@@ -27,6 +27,7 @@ Read it with `../SKILL.md` and `ci/ci.md` before editing CI.
 | `website-pages.yml` | main website paths, dispatch | Public website deployment |
 | `pr_cleanup.yml` | PR close, dispatch | Positively matched cleanup only |
 | `pr_auto_assign.yml` | PR lifecycle | Metadata only |
+| `cache-warm-sccache.yml` (`Cache · Trusted sccache seed`) | successful Main Quality, dispatch | Sole bounded Linux compiler-seed publisher on GitHub-hosted infrastructure |
 
 Other scheduled, deployment, Docker, package, canary and cache-warming
 workflows are independent of required PR readiness.
@@ -129,6 +130,9 @@ from that same catalog.
   cache remains disabled. Hosted PR, release, and cache-warmer selections
   retain native GitHub cache behavior.
 - `configure-sccache-gha`: event/provider-derived compiler-cache setup.
+- `restore-sccache-seed`: exact-key restore of the trusted 2 GiB Linux seed;
+  central runner policy permits it only for GitHub-hosted selections, and
+  runtime rows must match the seed's container image and toolchain epoch.
 - `capture-sccache-stats`: machine-readable cache evidence.
 
 `scripts/collect-ci-metrics.py` is the read-only timing evidence collector. Its
@@ -145,8 +149,13 @@ PR artifacts generally retain for one day. Fork lanes cannot publish shared
 trusted-main caches. Same-repository PRs normally use GitHub's ref-scoped cache;
 an exact approved revision may temporarily use Depot's shared cross-branch
 namespace under `ci/DEPOT_PR_RISK_EXCEPTION.md`. That namespace is treated as
-untrusted input, not an authority or correctness boundary. Large Cargo target caches restore trusted-main entries but remain
-restore-only on PRs. Exact Linux static ABI, Swift ABI, macOS Metal unit ABI,
+untrusted input, not an authority or correctness boundary. Linux Clippy,
+Rust-test, host, and runtime jobs restore one bounded trusted sccache seed
+instead of per-row Cargo target archives. Depot selections cannot restore that
+seed through their cross-trust cache proxy. Its exact key fingerprints the
+warmer image and toolchain epoch, so mismatched native-runtime rows are cold and
+do not restore it. These four high-fanout families disable per-object GHA
+publication on every provider. Exact Linux static ABI, Swift ABI, macOS Metal unit ABI,
 and Windows native ABI caches may publish into GitHub's isolated PR merge-ref
 scope for same-PR reruns. The Website slice is the sole publisher for the
 shared pnpm key and owns the website npm cache; platform UI producers restore

.github/actions/capture-sccache-stats/action.yml

@@ -5,6 +5,14 @@ inputs:
   artifact_name:
     description: Unique artifact name for this workflow job and matrix row.
     required: true
+  cache_expectation:
+    description: "Expected cache state: cold, warm, or opportunistic."
+    required: false
+    default: opportunistic
+  minimum_hit_rate:
+    description: Minimum hit ratio required when cache_expectation is warm.
+    required: false
+    default: "0"
 
 outputs:
   stats_file:
@@ -31,6 +39,15 @@ outputs:
   cache_write_errors:
     description: Number of cache write errors observed since sccache was configured.
     value: ${{ steps.capture.outputs.cache_write_errors }}
+  hit_rate:
+    description: Cache hits divided by cache hits plus misses.
+    value: ${{ steps.capture.outputs.hit_rate }}
+  cache_classification:
+    description: Machine-readable cold, warm-pass, warm-failure, or opportunistic classification.
+    value: ${{ steps.capture.outputs.cache_classification }}
+  cache_passed:
+    description: Whether the configured cache expectation passed.
+    value: ${{ steps.capture.outputs.cache_passed }}
 
 runs:
   using: composite
@@ -41,10 +58,14 @@ runs:
       env:
         SCCACHE_STATS_ARTIFACT_NAME: ${{ inputs.artifact_name }}
         SCCACHE_STATS_OUTPUT_DIR: ${{ runner.temp }}/mesh-llm-sccache-evidence
+        SCCACHE_CACHE_EXPECTATION: ${{ inputs.cache_expectation }}
+        SCCACHE_MINIMUM_HIT_RATE: ${{ inputs.minimum_hit_rate }}
       run: |
         set -euo pipefail
         python3 "$GITHUB_ACTION_PATH/capture.py" \
           --artifact-name "$SCCACHE_STATS_ARTIFACT_NAME" \
+          --cache-expectation "$SCCACHE_CACHE_EXPECTATION" \
+          --minimum-hit-rate "$SCCACHE_MINIMUM_HIT_RATE" \
           --output "$SCCACHE_STATS_OUTPUT_DIR/$SCCACHE_STATS_ARTIFACT_NAME/sccache-stats.json" \
           --github-output "$GITHUB_OUTPUT"
 
@@ -55,3 +76,9 @@ runs:
         path: ${{ steps.capture.outputs.stats_file }}
         if-no-files-found: error
         retention-days: 14
+    - name: Enforce warm-cache regression threshold
+      if: ${{ steps.capture.outputs.cache_passed != 'true' }}
+      shell: bash
+      run: |
+        echo "Warm sccache hit rate did not meet the configured minimum." >&2
+        exit 1

.github/actions/capture-sccache-stats/capture.py

@@ -15,7 +15,7 @@
 
 ARTIFACT_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
 EVIDENCE_SCHEMA = "mesh-llm.sccache-stats"
-EVIDENCE_SCHEMA_VERSION = 1
+EVIDENCE_SCHEMA_VERSION = 2
 REQUIRED_COUNTERS = (
     "compile_requests",
     "requests_executed",
@@ -36,9 +36,42 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--artifact-name", required=True)
     parser.add_argument("--output", required=True, type=Path)
     parser.add_argument("--github-output", type=Path)
+    parser.add_argument(
+        "--cache-expectation",
+        choices=("cold", "warm", "opportunistic"),
+        default="opportunistic",
+    )
+    parser.add_argument("--minimum-hit-rate", type=float, default=0.0)
     return parser.parse_args()
 
 
+def assess_cache(
+    expectation: str,
+    minimum_hit_rate: float,
+    counters: dict[str, int],
+) -> dict[str, Any]:
+    if not 0 <= minimum_hit_rate <= 1:
+        raise EvidenceError("minimum hit rate must be between 0 and 1")
+    requests = counters["cache_hits"] + counters["cache_misses"]
+    rate = counters["cache_hits"] / requests if requests else None
+    if expectation == "cold":
+        classification, passed = "cold", True
+    elif expectation == "opportunistic":
+        classification, passed = "opportunistic", True
+    elif rate is not None and rate >= minimum_hit_rate:
+        classification, passed = "warm-pass", True
+    else:
+        classification, passed = "warm-failure", False
+    return {
+        "expectation": expectation,
+        "classification": classification,
+        "minimum_hit_rate": minimum_hit_rate,
+        "hit_rate": rate,
+        "cache_requests": requests,
+        "passed": passed,
+    }
+
+
 def require_counter(stats: dict[str, Any], name: str) -> int:
     value = stats.get(name)
     if isinstance(value, bool) or not isinstance(value, int) or value < 0:
@@ -123,6 +156,7 @@ def write_github_outputs(
     destination: Path | None,
     stats_file: Path,
     counters: dict[str, int],
+    assessment: dict[str, Any],
 ) -> None:
     if destination is None:
         return
@@ -138,6 +172,9 @@ def write_github_outputs(
             "cache_write_errors",
         ):
             output.write(f"{name}={counters[name]}\n")
+        output.write(f"hit_rate={assessment['hit_rate'] if assessment['hit_rate'] is not None else ''}\n")
+        output.write(f"cache_classification={assessment['classification']}\n")
+        output.write(f"cache_passed={str(assessment['passed']).lower()}\n")
 
 
 def main() -> int:
@@ -159,14 +196,20 @@ def main() -> int:
         except json.JSONDecodeError as error:
             raise EvidenceError(f"sccache returned invalid JSON: {error}") from error
         evidence, counters = sanitize_stats(payload)
+        assessment = assess_cache(
+            arguments.cache_expectation,
+            arguments.minimum_hit_rate,
+            counters,
+        )
+        evidence["assessment"] = assessment
 
         arguments.output.parent.mkdir(parents=True, exist_ok=True)
         arguments.output.write_text(
             json.dumps(evidence, indent=2, sort_keys=True) + "\n",
             encoding="utf-8",
         )
         stats_file = arguments.output.resolve()
-        write_github_outputs(arguments.github_output, stats_file, counters)
+        write_github_outputs(arguments.github_output, stats_file, counters, assessment)
 
         print(
             "sccache evidence: "
@@ -176,6 +219,12 @@ def main() -> int:
             f"misses={counters['cache_misses']} "
             f"writes={counters['cache_writes']}",
         )
+        print(
+            "sccache assessment: "
+            f"expectation={assessment['expectation']} "
+            f"classification={assessment['classification']} "
+            f"hit_rate={assessment['hit_rate']}",
+        )
         if counters["compile_requests"] == 0:
             print(
                 "::warning title=sccache reported zero compile requests::"

.github/actions/compute-changes/action.yml

@@ -164,7 +164,7 @@ runs:
         if [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]; then
           RUNNER_CONTRACT_REQUIRED="true"
         elif [[ -n "$CHANGED_FILES" ]]; then
-          RUNNER_CONTRACT_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^\.github/cache-version\.txt$|^\.github/actionlint\.yaml$|^\.github/actions/(capture-sccache-stats|configure-sccache-gha|resolve-native-toolchain-epoch|select-ci-runners)/|^\.github/workflows/(ci|ci-control|ci-.*-(lane|slice)|depot-canary|main_[a-z]+|native-sdk-artifact|pr_[a-z]+|release|sdk-smoke|static-abi-artifact|swift-sdk-artifact)\.yml$)' || true)
+          RUNNER_CONTRACT_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^\.github/cache-version\.txt$|^\.github/actionlint\.yaml$|^\.github/actions/(capture-sccache-stats|configure-sccache-gha|restore-sccache-seed|resolve-native-toolchain-epoch|select-ci-runners)/|^\.github/workflows/(cache-warm-sccache|ci|ci-control|ci-.*-(lane|slice)|depot-canary|main_[a-z]+|native-sdk-artifact|pr_[a-z]+|release|sdk-smoke|static-abi-artifact|swift-sdk-artifact)\.yml$)' || true)
           if [[ -n "$RUNNER_CONTRACT_INPUTS" ]]; then
             RUNNER_CONTRACT_REQUIRED="true"
           fi

.github/actions/restore-sccache-seed/action.yml

@@ -0,0 +1,46 @@
+name: Restore trusted sccache seed
+description: Restore the bounded compiler seed published by trusted main on GitHub-hosted runners.
+
+inputs:
+  allow_trusted_seed:
+    description: Central runner-policy decision; Depot selections must pass false.
+    required: true
+  cache_key:
+    description: Exact compatibility key owned by the trusted cache warmer.
+    required: true
+
+outputs:
+  cache_hit:
+    description: Whether the exact trusted seed was restored.
+    value: ${{ steps.restore.outputs.cache-hit }}
+
+runs:
+  using: composite
+  steps:
+    - name: Validate trusted seed policy and configure local tier
+      shell: bash
+      env:
+        ALLOW_TRUSTED_SEED: ${{ inputs.allow_trusted_seed }}
+      run: |
+        set -euo pipefail
+        if [[ "$ALLOW_TRUSTED_SEED" != "true" && "$ALLOW_TRUSTED_SEED" != "false" ]]; then
+          echo "allow_trusted_seed must be true or false" >&2
+          exit 1
+        fi
+        cache_dir="$RUNNER_TEMP/mesh-llm-sccache"
+        mkdir -p "$cache_dir"
+        echo "SCCACHE_DIR=$cache_dir" >> "$GITHUB_ENV"
+        echo "SCCACHE_CACHE_SIZE=2G" >> "$GITHUB_ENV"
+        if [[ "$ALLOW_TRUSTED_SEED" == "true" ]]; then
+          # The restored archive is the persistent read-only tier. Jobs may
+          # populate the same directory locally, but only the warmer saves it.
+          echo "SCCACHE_GHA_ENABLED=false" >> "$GITHUB_ENV"
+        fi
+    - name: Restore exact trusted compiler seed
+      id: restore
+      if: ${{ inputs.allow_trusted_seed == 'true' }}
+      uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+      with:
+        path: ${{ runner.temp }}/mesh-llm-sccache
+        key: ${{ inputs.cache_key }}
+        fail-on-cache-miss: false

.github/actions/select-ci-runners/action.yml

@@ -60,6 +60,9 @@ outputs:
   allow_native_github_cache:
     description: Whether this provider/trust selection may use GitHub Actions cache APIs, including Depot's transparent proxy on Depot runners.
     value: ${{ steps.select.outputs.allow_native_github_cache }}
+  allow_trusted_sccache_seed:
+    description: Whether this GitHub-hosted selection may restore the trusted, main-published compiler seed.
+    value: ${{ steps.select.outputs.allow_trusted_sccache_seed }}
   runner:
     description: Default two-vCPU Linux runner label.
     value: ${{ steps.select.outputs.runner }}
@@ -167,6 +170,7 @@ runs:
         depot_enabled=false
         allow_depot_remote_cache=false
         allow_native_github_cache=true
+        allow_trusted_sccache_seed=true
         is_direct_pull_request=false
         risk_exception_selected=false
         is_dispatched_pull_request=false
@@ -249,6 +253,7 @@ runs:
         if [[ "$depot_enabled" == "true" ]]; then
           allow_depot_remote_cache=false
           allow_native_github_cache=false
+          allow_trusted_sccache_seed=false
           if [[ "$depot_pr_exception_active" == "true" &&
             ( "$risk_exception_selected" == "true" ||
               ( "$INPUT_EVENT_NAME" == "push" &&
@@ -288,6 +293,7 @@ runs:
           echo "depot_enabled=$depot_enabled"
           echo "allow_depot_remote_cache=$allow_depot_remote_cache"
           echo "allow_native_github_cache=$allow_native_github_cache"
+          echo "allow_trusted_sccache_seed=$allow_trusted_sccache_seed"
           echo "runner=$runner"
           echo "runner_4=$runner_4"
           echo "runner_8=$runner_8"

.github/workflows/cache-warm-sccache.yml

@@ -0,0 +1,79 @@
+name: Cache · Trusted sccache seed
+
+on:
+  workflow_run:
+    workflows: ["Main · Quality"]
+    types: [completed]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: read
+
+concurrency:
+  group: trusted-sccache-seed
+  cancel-in-progress: false
+
+jobs:
+  warm:
+    name: Publish bounded Linux compiler seed
+    if: ${{ (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main') }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    container:
+      image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d
+    env:
+      CARGO_INCREMENTAL: "0"
+      MESH_LLM_SKIP_UI: "1"
+      RUSTC_WRAPPER: sccache
+      RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
+      SCCACHE_GHA_ENABLED: "false"
+      SCCACHE_CACHE_SIZE: 2G
+    steps:
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
+        with:
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}
+          persist-credentials: false
+      - name: Verify trusted cache-warmer environment
+        run: verify-runner-image public cpu
+      - name: Resolve compiler seed key
+        id: seed
+        shell: bash
+        run: |
+          set -euo pipefail
+          key="mesh-llm-sccache-seed-linux-x86_64-img-8d93de6b-epoch-8d93de6b-v2-${{ hashFiles('Cargo.lock', '.github/cache-version.txt') }}"
+          echo "key=$key" >> "$GITHUB_OUTPUT"
+      - name: Check for an existing exact seed
+        id: restore
+        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        with:
+          path: ${{ runner.temp }}/mesh-llm-sccache
+          key: ${{ steps.seed.outputs.key }}
+          lookup-only: true
+      - name: Configure bounded local sccache
+        if: ${{ steps.restore.outputs.cache-hit != 'true' }}
+        uses: ./.github/actions/configure-sccache-gha
+        with:
+          allow_depot_remote_cache: "false"
+          allow_native_github_cache: "false"
+      - name: Prepare UI placeholder
+        if: ${{ steps.restore.outputs.cache-hit != 'true' }}
+        run: mkdir -p crates/mesh-llm-ui/dist && printf '<html></html>' > crates/mesh-llm-ui/dist/index.html
+      - name: Compile the dominant host dependency graph
+        if: ${{ steps.restore.outputs.cache-hit != 'true' }}
+        run: just ci-sccache-seed-build
+      - name: Capture intentionally cold warmer evidence
+        if: ${{ steps.restore.outputs.cache-hit != 'true' && !cancelled() }}
+        uses: ./.github/actions/capture-sccache-stats
+        with:
+          artifact_name: sccache-trusted-seed-${{ github.run_attempt }}
+          cache_expectation: cold
+      - name: Stop sccache before archiving
+        if: ${{ steps.restore.outputs.cache-hit != 'true' && !cancelled() }}
+        run: sccache --stop-server
+      - name: Publish exact trusted compiler seed
+        if: ${{ steps.restore.outputs.cache-hit != 'true' && success() }}
+        uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        with:
+          path: ${{ runner.temp }}/mesh-llm-sccache
+          key: ${{ steps.seed.outputs.key }}