diff --git a/.agents/skills/manage-ci/SKILL.md b/.agents/skills/manage-ci/SKILL.md index ef40f06c0f..8bae439ee1 100644 --- a/.agents/skills/manage-ci/SKILL.md +++ b/.agents/skills/manage-ci/SKILL.md @@ -228,17 +228,6 @@ owning source, and update the inventory and topology in the same change. selected-workflow restriction. - Provider changes must not alter source checkout, commands, profile, artifacts, tests, required checks, or plan membership. -- Shared Linux Clippy/test/host/runtime compiler-cache publication has one - trusted GitHub-hosted warmer. It - publishes one bounded, exact-key sccache seed after successful main Quality; - GitHub-hosted PR jobs may restore it and write only to their job-local copy. - Depot selections must never restore this seed through Depot's repository- - scoped Actions-cache proxy. Do not restore per-shard Cargo target archives - or enable per-object GHA publication for Linux Clippy, Rust tests, host, or - runtime jobs. -- Every seeded compiler job records whether the exact seed was warm or cold. - Enforce a measured minimum hit rate only for an exact warm restore; an - intentional cache miss is classified cold and must not fail the build. ### Bounded Depot PR cache-risk exception diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 0d0da7d10d..b2ac053229 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -27,7 +27,6 @@ Read it with `../SKILL.md` and `ci/ci.md` before editing CI. | `website-pages.yml` | main website paths, dispatch | Public website deployment | | `pr_cleanup.yml` | PR close, dispatch | Positively matched cleanup only | | `pr_auto_assign.yml` | PR lifecycle | Metadata only | -| `cache-warm-sccache.yml` (`Cache · Trusted sccache seed`) | successful Main Quality, dispatch | Sole bounded Linux compiler-seed publisher on GitHub-hosted infrastructure | Other scheduled, deployment, Docker, package, canary and cache-warming workflows are independent of required PR readiness. @@ -130,9 +129,6 @@ from that same catalog. cache remains disabled. Hosted PR, release, and cache-warmer selections retain native GitHub cache behavior. - `configure-sccache-gha`: event/provider-derived compiler-cache setup. -- `restore-sccache-seed`: exact-key restore of the trusted 2 GiB Linux seed; - central runner policy permits it only for GitHub-hosted selections, and - runtime rows must match the seed's container image and toolchain epoch. - `capture-sccache-stats`: machine-readable cache evidence. `scripts/collect-ci-metrics.py` is the read-only timing evidence collector. Its @@ -149,13 +145,8 @@ PR artifacts generally retain for one day. Fork lanes cannot publish shared trusted-main caches. Same-repository PRs normally use GitHub's ref-scoped cache; an exact approved revision may temporarily use Depot's shared cross-branch namespace under `ci/DEPOT_PR_RISK_EXCEPTION.md`. That namespace is treated as -untrusted input, not an authority or correctness boundary. Linux Clippy, -Rust-test, host, and runtime jobs restore one bounded trusted sccache seed -instead of per-row Cargo target archives. Depot selections cannot restore that -seed through their cross-trust cache proxy. Its exact key fingerprints the -warmer image and toolchain epoch, so mismatched native-runtime rows are cold and -do not restore it. These four high-fanout families disable per-object GHA -publication on every provider. Exact Linux static ABI, Swift ABI, macOS Metal unit ABI, +untrusted input, not an authority or correctness boundary. Large Cargo target caches restore trusted-main entries but remain +restore-only on PRs. Exact Linux static ABI, Swift ABI, macOS Metal unit ABI, and Windows native ABI caches may publish into GitHub's isolated PR merge-ref scope for same-PR reruns. The Website slice is the sole publisher for the shared pnpm key and owns the website npm cache; platform UI producers restore diff --git a/.github/actions/capture-sccache-stats/action.yml b/.github/actions/capture-sccache-stats/action.yml index 42c92f269c..5119d2702c 100644 --- a/.github/actions/capture-sccache-stats/action.yml +++ b/.github/actions/capture-sccache-stats/action.yml @@ -5,14 +5,6 @@ inputs: artifact_name: description: Unique artifact name for this workflow job and matrix row. required: true - cache_expectation: - description: "Expected cache state: cold, warm, or opportunistic." - required: false - default: opportunistic - minimum_hit_rate: - description: Minimum hit ratio required when cache_expectation is warm. - required: false - default: "0" outputs: stats_file: @@ -39,15 +31,6 @@ outputs: cache_write_errors: description: Number of cache write errors observed since sccache was configured. value: ${{ steps.capture.outputs.cache_write_errors }} - hit_rate: - description: Cache hits divided by cache hits plus misses. - value: ${{ steps.capture.outputs.hit_rate }} - cache_classification: - description: Machine-readable cold, warm-pass, warm-failure, or opportunistic classification. - value: ${{ steps.capture.outputs.cache_classification }} - cache_passed: - description: Whether the configured cache expectation passed. - value: ${{ steps.capture.outputs.cache_passed }} runs: using: composite @@ -58,14 +41,10 @@ runs: env: SCCACHE_STATS_ARTIFACT_NAME: ${{ inputs.artifact_name }} SCCACHE_STATS_OUTPUT_DIR: ${{ runner.temp }}/mesh-llm-sccache-evidence - SCCACHE_CACHE_EXPECTATION: ${{ inputs.cache_expectation }} - SCCACHE_MINIMUM_HIT_RATE: ${{ inputs.minimum_hit_rate }} run: | set -euo pipefail python3 "$GITHUB_ACTION_PATH/capture.py" \ --artifact-name "$SCCACHE_STATS_ARTIFACT_NAME" \ - --cache-expectation "$SCCACHE_CACHE_EXPECTATION" \ - --minimum-hit-rate "$SCCACHE_MINIMUM_HIT_RATE" \ --output "$SCCACHE_STATS_OUTPUT_DIR/$SCCACHE_STATS_ARTIFACT_NAME/sccache-stats.json" \ --github-output "$GITHUB_OUTPUT" @@ -76,9 +55,3 @@ runs: path: ${{ steps.capture.outputs.stats_file }} if-no-files-found: error retention-days: 14 - - name: Enforce warm-cache regression threshold - if: ${{ steps.capture.outputs.cache_passed != 'true' }} - shell: bash - run: | - echo "Warm sccache hit rate did not meet the configured minimum." >&2 - exit 1 diff --git a/.github/actions/capture-sccache-stats/capture.py b/.github/actions/capture-sccache-stats/capture.py index efa125d6ef..194d45d775 100644 --- a/.github/actions/capture-sccache-stats/capture.py +++ b/.github/actions/capture-sccache-stats/capture.py @@ -15,7 +15,7 @@ ARTIFACT_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") EVIDENCE_SCHEMA = "mesh-llm.sccache-stats" -EVIDENCE_SCHEMA_VERSION = 2 +EVIDENCE_SCHEMA_VERSION = 1 REQUIRED_COUNTERS = ( "compile_requests", "requests_executed", @@ -36,42 +36,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--artifact-name", required=True) parser.add_argument("--output", required=True, type=Path) parser.add_argument("--github-output", type=Path) - parser.add_argument( - "--cache-expectation", - choices=("cold", "warm", "opportunistic"), - default="opportunistic", - ) - parser.add_argument("--minimum-hit-rate", type=float, default=0.0) return parser.parse_args() -def assess_cache( - expectation: str, - minimum_hit_rate: float, - counters: dict[str, int], -) -> dict[str, Any]: - if not 0 <= minimum_hit_rate <= 1: - raise EvidenceError("minimum hit rate must be between 0 and 1") - requests = counters["cache_hits"] + counters["cache_misses"] - rate = counters["cache_hits"] / requests if requests else None - if expectation == "cold": - classification, passed = "cold", True - elif expectation == "opportunistic": - classification, passed = "opportunistic", True - elif rate is not None and rate >= minimum_hit_rate: - classification, passed = "warm-pass", True - else: - classification, passed = "warm-failure", False - return { - "expectation": expectation, - "classification": classification, - "minimum_hit_rate": minimum_hit_rate, - "hit_rate": rate, - "cache_requests": requests, - "passed": passed, - } - - def require_counter(stats: dict[str, Any], name: str) -> int: value = stats.get(name) if isinstance(value, bool) or not isinstance(value, int) or value < 0: @@ -156,7 +123,6 @@ def write_github_outputs( destination: Path | None, stats_file: Path, counters: dict[str, int], - assessment: dict[str, Any], ) -> None: if destination is None: return @@ -172,9 +138,6 @@ def write_github_outputs( "cache_write_errors", ): output.write(f"{name}={counters[name]}\n") - output.write(f"hit_rate={assessment['hit_rate'] if assessment['hit_rate'] is not None else ''}\n") - output.write(f"cache_classification={assessment['classification']}\n") - output.write(f"cache_passed={str(assessment['passed']).lower()}\n") def main() -> int: @@ -196,12 +159,6 @@ def main() -> int: except json.JSONDecodeError as error: raise EvidenceError(f"sccache returned invalid JSON: {error}") from error evidence, counters = sanitize_stats(payload) - assessment = assess_cache( - arguments.cache_expectation, - arguments.minimum_hit_rate, - counters, - ) - evidence["assessment"] = assessment arguments.output.parent.mkdir(parents=True, exist_ok=True) arguments.output.write_text( @@ -209,7 +166,7 @@ def main() -> int: encoding="utf-8", ) stats_file = arguments.output.resolve() - write_github_outputs(arguments.github_output, stats_file, counters, assessment) + write_github_outputs(arguments.github_output, stats_file, counters) print( "sccache evidence: " @@ -219,12 +176,6 @@ def main() -> int: f"misses={counters['cache_misses']} " f"writes={counters['cache_writes']}", ) - print( - "sccache assessment: " - f"expectation={assessment['expectation']} " - f"classification={assessment['classification']} " - f"hit_rate={assessment['hit_rate']}", - ) if counters["compile_requests"] == 0: print( "::warning title=sccache reported zero compile requests::" diff --git a/.github/actions/compute-changes/action.yml b/.github/actions/compute-changes/action.yml index ead31cf32a..36a93891f0 100644 --- a/.github/actions/compute-changes/action.yml +++ b/.github/actions/compute-changes/action.yml @@ -164,7 +164,7 @@ runs: if [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]; then RUNNER_CONTRACT_REQUIRED="true" elif [[ -n "$CHANGED_FILES" ]]; then - RUNNER_CONTRACT_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^\.github/cache-version\.txt$|^\.github/actionlint\.yaml$|^\.github/actions/(capture-sccache-stats|configure-sccache-gha|restore-sccache-seed|resolve-native-toolchain-epoch|select-ci-runners)/|^\.github/workflows/(cache-warm-sccache|ci|ci-control|ci-.*-(lane|slice)|depot-canary|main_[a-z]+|native-sdk-artifact|pr_[a-z]+|release|sdk-smoke|static-abi-artifact|swift-sdk-artifact)\.yml$)' || true) + RUNNER_CONTRACT_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^\.github/cache-version\.txt$|^\.github/actionlint\.yaml$|^\.github/actions/(capture-sccache-stats|configure-sccache-gha|resolve-native-toolchain-epoch|select-ci-runners)/|^\.github/workflows/(ci|ci-control|ci-.*-(lane|slice)|depot-canary|main_[a-z]+|native-sdk-artifact|pr_[a-z]+|release|sdk-smoke|static-abi-artifact|swift-sdk-artifact)\.yml$)' || true) if [[ -n "$RUNNER_CONTRACT_INPUTS" ]]; then RUNNER_CONTRACT_REQUIRED="true" fi diff --git a/.github/actions/restore-sccache-seed/action.yml b/.github/actions/restore-sccache-seed/action.yml deleted file mode 100644 index 4722ae5f8c..0000000000 --- a/.github/actions/restore-sccache-seed/action.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Restore trusted sccache seed -description: Restore the bounded compiler seed published by trusted main on GitHub-hosted runners. - -inputs: - allow_trusted_seed: - description: Central runner-policy decision; Depot selections must pass false. - required: true - cache_key: - description: Exact compatibility key owned by the trusted cache warmer. - required: true - -outputs: - cache_hit: - description: Whether the exact trusted seed was restored. - value: ${{ steps.restore.outputs.cache-hit }} - -runs: - using: composite - steps: - - name: Validate trusted seed policy and configure local tier - shell: bash - env: - ALLOW_TRUSTED_SEED: ${{ inputs.allow_trusted_seed }} - run: | - set -euo pipefail - if [[ "$ALLOW_TRUSTED_SEED" != "true" && "$ALLOW_TRUSTED_SEED" != "false" ]]; then - echo "allow_trusted_seed must be true or false" >&2 - exit 1 - fi - cache_dir="$RUNNER_TEMP/mesh-llm-sccache" - mkdir -p "$cache_dir" - echo "SCCACHE_DIR=$cache_dir" >> "$GITHUB_ENV" - echo "SCCACHE_CACHE_SIZE=2G" >> "$GITHUB_ENV" - if [[ "$ALLOW_TRUSTED_SEED" == "true" ]]; then - # The restored archive is the persistent read-only tier. Jobs may - # populate the same directory locally, but only the warmer saves it. - echo "SCCACHE_GHA_ENABLED=false" >> "$GITHUB_ENV" - fi - - name: Restore exact trusted compiler seed - id: restore - if: ${{ inputs.allow_trusted_seed == 'true' }} - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 - with: - path: ${{ runner.temp }}/mesh-llm-sccache - key: ${{ inputs.cache_key }} - fail-on-cache-miss: false diff --git a/.github/actions/select-ci-runners/action.yml b/.github/actions/select-ci-runners/action.yml index e27caf3b88..75d3a256d2 100644 --- a/.github/actions/select-ci-runners/action.yml +++ b/.github/actions/select-ci-runners/action.yml @@ -60,9 +60,6 @@ outputs: allow_native_github_cache: description: Whether this provider/trust selection may use GitHub Actions cache APIs, including Depot's transparent proxy on Depot runners. value: ${{ steps.select.outputs.allow_native_github_cache }} - allow_trusted_sccache_seed: - description: Whether this GitHub-hosted selection may restore the trusted, main-published compiler seed. - value: ${{ steps.select.outputs.allow_trusted_sccache_seed }} runner: description: Default two-vCPU Linux runner label. value: ${{ steps.select.outputs.runner }} @@ -170,7 +167,6 @@ runs: depot_enabled=false allow_depot_remote_cache=false allow_native_github_cache=true - allow_trusted_sccache_seed=true is_direct_pull_request=false risk_exception_selected=false is_dispatched_pull_request=false @@ -253,7 +249,6 @@ runs: if [[ "$depot_enabled" == "true" ]]; then allow_depot_remote_cache=false allow_native_github_cache=false - allow_trusted_sccache_seed=false if [[ "$depot_pr_exception_active" == "true" && ( "$risk_exception_selected" == "true" || ( "$INPUT_EVENT_NAME" == "push" && @@ -293,7 +288,6 @@ runs: echo "depot_enabled=$depot_enabled" echo "allow_depot_remote_cache=$allow_depot_remote_cache" echo "allow_native_github_cache=$allow_native_github_cache" - echo "allow_trusted_sccache_seed=$allow_trusted_sccache_seed" echo "runner=$runner" echo "runner_4=$runner_4" echo "runner_8=$runner_8" diff --git a/.github/workflows/cache-warm-sccache.yml b/.github/workflows/cache-warm-sccache.yml deleted file mode 100644 index c4d64ae190..0000000000 --- a/.github/workflows/cache-warm-sccache.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Cache · Trusted sccache seed - -on: - workflow_run: - workflows: ["Main · Quality"] - types: [completed] - workflow_dispatch: - -permissions: - contents: read - packages: read - -concurrency: - group: trusted-sccache-seed - cancel-in-progress: false - -jobs: - warm: - name: Publish bounded Linux compiler seed - if: ${{ (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main') }} - runs-on: ubuntu-24.04 - timeout-minutes: 60 - container: - image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - env: - CARGO_INCREMENTAL: "0" - MESH_LLM_SKIP_UI: "1" - RUSTC_WRAPPER: sccache - RUSTFLAGS: "-C link-arg=-fuse-ld=lld" - SCCACHE_GHA_ENABLED: "false" - SCCACHE_CACHE_SIZE: 2G - steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} - persist-credentials: false - - name: Verify trusted cache-warmer environment - run: verify-runner-image public cpu - - name: Resolve compiler seed key - id: seed - shell: bash - run: | - set -euo pipefail - key="mesh-llm-sccache-seed-linux-x86_64-img-8d93de6b-epoch-8d93de6b-v2-${{ hashFiles('Cargo.lock', '.github/cache-version.txt') }}" - echo "key=$key" >> "$GITHUB_OUTPUT" - - name: Check for an existing exact seed - id: restore - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 - with: - path: ${{ runner.temp }}/mesh-llm-sccache - key: ${{ steps.seed.outputs.key }} - lookup-only: true - - name: Configure bounded local sccache - if: ${{ steps.restore.outputs.cache-hit != 'true' }} - uses: ./.github/actions/configure-sccache-gha - with: - allow_depot_remote_cache: "false" - allow_native_github_cache: "false" - - name: Prepare UI placeholder - if: ${{ steps.restore.outputs.cache-hit != 'true' }} - run: mkdir -p crates/mesh-llm-ui/dist && printf '' > crates/mesh-llm-ui/dist/index.html - - name: Compile the dominant host dependency graph - if: ${{ steps.restore.outputs.cache-hit != 'true' }} - run: just ci-sccache-seed-build - - name: Capture intentionally cold warmer evidence - if: ${{ steps.restore.outputs.cache-hit != 'true' && !cancelled() }} - uses: ./.github/actions/capture-sccache-stats - with: - artifact_name: sccache-trusted-seed-${{ github.run_attempt }} - cache_expectation: cold - - name: Stop sccache before archiving - if: ${{ steps.restore.outputs.cache-hit != 'true' && !cancelled() }} - run: sccache --stop-server - - name: Publish exact trusted compiler seed - if: ${{ steps.restore.outputs.cache-hit != 'true' && success() }} - uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 - with: - path: ${{ runner.temp }}/mesh-llm-sccache - key: ${{ steps.seed.outputs.key }} diff --git a/.github/workflows/ci-linux-host-slice.yml b/.github/workflows/ci-linux-host-slice.yml index 1ce4f4dd40..31383c6952 100644 --- a/.github/workflows/ci-linux-host-slice.yml +++ b/.github/workflows/ci-linux-host-slice.yml @@ -52,7 +52,7 @@ permissions: env: CACHE_NAMESPACE: mesh-llm CARGO_INCREMENTAL: "0" - SCCACHE_GHA_ENABLED: "false" + SCCACHE_GHA_ENABLED: "true" jobs: runner_policy: @@ -64,7 +64,6 @@ jobs: runner_8: ${{ steps.policy.outputs.runner_8 }} allow_depot_remote_cache: ${{ steps.policy.outputs.allow_depot_remote_cache }} allow_native_github_cache: ${{ steps.policy.outputs.allow_native_github_cache }} - allow_trusted_sccache_seed: ${{ steps.policy.outputs.allow_trusted_sccache_seed }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -121,16 +120,20 @@ jobs: ref: ${{ inputs.source_sha || github.sha }} persist-credentials: false - name: Verify prebuilt host environment - run: verify-runner-image public cpu - - id: sccache_seed - uses: ./.github/actions/restore-sccache-seed - with: - allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed == 'true' && matrix.host.architecture == 'x86_64' && 'true' || 'false' }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-8d93de6b-epoch-8d93de6b-v2-${{ hashFiles('Cargo.lock', '.github/cache-version.txt') }} + run: verify-runner-image public - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} allow_native_github_cache: ${{ needs.runner_policy.outputs.allow_native_github_cache }} + - if: ${{ needs.runner_policy.outputs.allow_native_github_cache == 'true' }} + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + continue-on-error: true + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: ci-host-linux-${{ inputs.profile }} + save-if: ${{ github.ref == 'refs/heads/main' && github.event.inputs.original_event_name != 'pull_request' && github.event.inputs.original_event_name != 'pull_request_target' }} - name: Download immutable UI distribution uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -156,5 +159,3 @@ jobs: uses: ./.github/actions/capture-sccache-stats with: artifact_name: sccache-ci-host-linux-${{ matrix.host.architecture }}-${{ github.run_attempt }} - cache_expectation: ${{ steps.sccache_seed.outputs.cache_hit == 'true' && 'warm' || 'cold' }} - minimum_hit_rate: "0.20" diff --git a/.github/workflows/ci-linux-product-slice.yml b/.github/workflows/ci-linux-product-slice.yml index f72a52f4d5..e312ed97cd 100644 --- a/.github/workflows/ci-linux-product-slice.yml +++ b/.github/workflows/ci-linux-product-slice.yml @@ -98,7 +98,7 @@ jobs: ref: ${{ inputs.source_sha || github.sha }} persist-credentials: false - name: Verify product composition environment - run: verify-runner-image public cpu + run: verify-runner-image public - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: ci-host-linux-${{ matrix.runtime.architecture }} diff --git a/.github/workflows/ci-linux-runtime-slice.yml b/.github/workflows/ci-linux-runtime-slice.yml index e27ffc979b..acd5cf5c62 100644 --- a/.github/workflows/ci-linux-runtime-slice.yml +++ b/.github/workflows/ci-linux-runtime-slice.yml @@ -44,7 +44,7 @@ permissions: env: CACHE_NAMESPACE: mesh-llm CARGO_INCREMENTAL: "0" - SCCACHE_GHA_ENABLED: "false" + SCCACHE_GHA_ENABLED: "true" jobs: runner_policy: @@ -57,7 +57,6 @@ jobs: runner_16: ${{ steps.policy.outputs.runner_16 }} allow_depot_remote_cache: ${{ steps.policy.outputs.allow_depot_remote_cache }} allow_native_github_cache: ${{ steps.policy.outputs.allow_native_github_cache }} - allow_trusted_sccache_seed: ${{ steps.policy.outputs.allow_trusted_sccache_seed }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -129,15 +128,19 @@ jobs: uses: ./.github/actions/resolve-native-toolchain-epoch with: pinned_epoch: ${{ matrix.runtime.toolchain_epoch }} - - id: sccache_seed - uses: ./.github/actions/restore-sccache-seed - with: - allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed == 'true' && matrix.runtime.architecture == 'x86_64' && matrix.runtime.container_image == 'ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d' && matrix.runtime.toolchain_epoch == 'mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d' && 'true' || 'false' }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-8d93de6b-epoch-8d93de6b-v2-${{ hashFiles('Cargo.lock', '.github/cache-version.txt') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} allow_native_github_cache: ${{ needs.runner_policy.outputs.allow_native_github_cache }} + - if: ${{ needs.runner_policy.outputs.allow_native_github_cache == 'true' }} + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + continue-on-error: true + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: ci-runtime-linux-${{ matrix.runtime.architecture }}-${{ matrix.runtime.backend }} + save-if: ${{ github.ref == 'refs/heads/main' && github.event.inputs.original_event_name != 'pull_request' && github.event.inputs.original_event_name != 'pull_request_target' }} - name: Prepare immutable Linux native runtime uses: ./.github/actions/prepare-native-runtime-input with: @@ -158,5 +161,3 @@ jobs: uses: ./.github/actions/capture-sccache-stats with: artifact_name: sccache-ci-runtime-linux-${{ matrix.runtime.architecture }}-${{ matrix.runtime.backend }}-${{ github.run_attempt }} - cache_expectation: ${{ steps.sccache_seed.outputs.cache_hit == 'true' && 'warm' || 'cold' }} - minimum_hit_rate: "0.01" diff --git a/.github/workflows/ci-quality-slice.yml b/.github/workflows/ci-quality-slice.yml index 0b35945934..34a5ae2ed2 100644 --- a/.github/workflows/ci-quality-slice.yml +++ b/.github/workflows/ci-quality-slice.yml @@ -49,7 +49,7 @@ on: env: CACHE_NAMESPACE: mesh-llm CARGO_INCREMENTAL: "0" - SCCACHE_GHA_ENABLED: "false" + SCCACHE_GHA_ENABLED: "true" permissions: contents: read @@ -66,7 +66,6 @@ jobs: runner_8: ${{ steps.policy.outputs.runner_8 }} allow_depot_remote_cache: ${{ steps.policy.outputs.allow_depot_remote_cache }} allow_native_github_cache: ${{ steps.policy.outputs.allow_native_github_cache }} - allow_trusted_sccache_seed: ${{ steps.policy.outputs.allow_trusted_sccache_seed }} authority_sentinel_runner: ${{ steps.sentinel_policy.outputs.runner }} authority_sentinel_depot_enabled: ${{ steps.sentinel_policy.outputs.depot_enabled }} steps: @@ -205,16 +204,20 @@ jobs: ref: ${{ inputs.source_sha || github.sha }} persist-credentials: false - name: Verify prebuilt CI environment - run: verify-runner-image public cpu - - id: sccache_seed - uses: ./.github/actions/restore-sccache-seed - with: - allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-8d93de6b-epoch-8d93de6b-v2-${{ hashFiles('Cargo.lock', '.github/cache-version.txt') }} + run: verify-runner-image public - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} allow_native_github_cache: ${{ needs.runner_policy.outputs.allow_native_github_cache }} + - if: ${{ needs.runner_policy.outputs.allow_native_github_cache == 'true' }} + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + continue-on-error: true + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: ci-quality-clippy-${{ matrix.batch.id }} + save-if: ${{ github.ref == 'refs/heads/main' && github.event.inputs.original_event_name != 'pull_request' && github.event.inputs.original_event_name != 'pull_request_target' }} - name: Run one Clippy invocation for the batch env: CLIPPY_CRATES: ${{ toJson(matrix.batch.crates) }} @@ -236,8 +239,6 @@ jobs: uses: ./.github/actions/capture-sccache-stats with: artifact_name: sccache-ci-quality-clippy-${{ matrix.batch.id }}-${{ github.run_attempt }} - cache_expectation: ${{ steps.sccache_seed.outputs.cache_hit == 'true' && 'warm' || 'cold' }} - minimum_hit_rate: "0.20" cli_docs_sync: name: CLI documentation sync diff --git a/.github/workflows/ci-rust-tests-slice.yml b/.github/workflows/ci-rust-tests-slice.yml index 6e3f8b8df4..62f70f281d 100644 --- a/.github/workflows/ci-rust-tests-slice.yml +++ b/.github/workflows/ci-rust-tests-slice.yml @@ -48,7 +48,7 @@ permissions: env: CACHE_NAMESPACE: mesh-llm CARGO_INCREMENTAL: "0" - SCCACHE_GHA_ENABLED: "false" + SCCACHE_GHA_ENABLED: "true" jobs: runner_policy: @@ -61,7 +61,6 @@ jobs: runner: ${{ steps.policy.outputs.runner }} allow_depot_remote_cache: ${{ steps.policy.outputs.allow_depot_remote_cache }} allow_native_github_cache: ${{ steps.policy.outputs.allow_native_github_cache }} - allow_trusted_sccache_seed: ${{ steps.policy.outputs.allow_trusted_sccache_seed }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -120,20 +119,24 @@ jobs: ref: ${{ inputs.source_sha || github.sha }} persist-credentials: false - name: Verify prebuilt test environment - run: verify-runner-image public cpu + run: verify-runner-image public - name: Resolve static ABI toolchain epoch uses: ./.github/actions/resolve-native-toolchain-epoch with: pinned_epoch: ${{ inputs.static_abi_toolchain_epoch }} - - id: sccache_seed - uses: ./.github/actions/restore-sccache-seed - with: - allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-8d93de6b-epoch-8d93de6b-v2-${{ hashFiles('Cargo.lock', '.github/cache-version.txt') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} allow_native_github_cache: ${{ needs.runner_policy.outputs.allow_native_github_cache }} + - if: ${{ needs.runner_policy.outputs.allow_native_github_cache == 'true' }} + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + continue-on-error: true + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: ci-rust-tests-${{ matrix.batch.id }} + save-if: ${{ github.ref == 'refs/heads/main' && github.event.inputs.original_event_name != 'pull_request' && github.event.inputs.original_event_name != 'pull_request_target' }} - name: Prepare UI placeholder run: mkdir -p crates/mesh-llm-ui/dist && printf '' > crates/mesh-llm-ui/dist/index.html - name: Prepare patched llama.cpp when required @@ -170,5 +173,3 @@ jobs: uses: ./.github/actions/capture-sccache-stats with: artifact_name: sccache-ci-rust-tests-${{ matrix.batch.id }}-${{ github.run_attempt }} - cache_expectation: ${{ steps.sccache_seed.outputs.cache_hit == 'true' && 'warm' || 'cold' }} - minimum_hit_rate: "0.20" diff --git a/.github/workflows/native-sdk-artifact.yml b/.github/workflows/native-sdk-artifact.yml index 96f0f86b32..7cf674afa9 100644 --- a/.github/workflows/native-sdk-artifact.yml +++ b/.github/workflows/native-sdk-artifact.yml @@ -250,7 +250,7 @@ jobs: fi - name: Verify prebuilt native SDK environment - run: verify-runner-image public cpu + run: verify-runner-image public - name: Resolve static ABI toolchain epoch uses: ./.github/actions/resolve-native-toolchain-epoch diff --git a/.github/workflows/scripted-binary-smoke.yml b/.github/workflows/scripted-binary-smoke.yml index 3c171dbf57..240e367a96 100644 --- a/.github/workflows/scripted-binary-smoke.yml +++ b/.github/workflows/scripted-binary-smoke.yml @@ -55,21 +55,13 @@ env: permissions: contents: read - packages: read jobs: scripted_binary_smoke: name: Scripted Binary Smoke - # Model credentials stay on an isolated GitHub-hosted runner; the - # container just supplies prebuilt smoke utilities, it does not move - # execution to self-hosted/depot infra. + # Model credentials stay on an isolated GitHub-hosted runner. runs-on: ubuntu-24.04 timeout-minutes: ${{ inputs.timeout_minutes }} - container: - image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - defaults: - run: - shell: bash env: HF_TOKEN: ${{ secrets.HF_TOKEN }} HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} @@ -79,8 +71,18 @@ jobs: ref: ${{ inputs.source_sha || github.sha }} persist-credentials: false - - name: Verify prebuilt smoke environment - run: verify-runner-image public cpu + - name: Verify smoke runtime utilities + run: | + set -euo pipefail + for utility in \ + curl \ + jq \ + lsof; do + command -v "$utility" >/dev/null || { + echo "missing smoke runtime utility: $utility" >&2 + exit 1 + } + done - uses: ./.github/actions/restore-smoke-inputs with: diff --git a/.github/workflows/static-abi-artifact.yml b/.github/workflows/static-abi-artifact.yml index 4f3743c306..1e32ff7c5a 100644 --- a/.github/workflows/static-abi-artifact.yml +++ b/.github/workflows/static-abi-artifact.yml @@ -161,7 +161,7 @@ jobs: persist-credentials: false - name: Verify prebuilt static ABI environment - run: verify-runner-image public cpu + run: verify-runner-image public - name: Resolve static ABI toolchain epoch id: native_toolchain diff --git a/.omo/specs/pr-ci-optimization.md b/.omo/specs/pr-ci-optimization.md index e6b7c14524..1d738a5a7e 100644 --- a/.omo/specs/pr-ci-optimization.md +++ b/.omo/specs/pr-ci-optimization.md @@ -207,11 +207,8 @@ exact-SHA, same-epoch runs. Main/manual and unrelated workflows are never targets. The monitor is the only owner of Actions-write permission; checked-out PR code cannot invoke the cancellation API. -PR caching is selective. Linux Clippy, Rust tests, host, and runtime restore one -bounded trusted sccache seed on GitHub-hosted runners and no longer restore -per-row Cargo target archives. PR writes stay job-local; only the protected -post-Main-Quality warmer publishes the exact 2 GiB seed, and Depot selections -cannot restore it. Exact verified static, +PR caching is selective. Large Cargo target caches restore trusted main and do +not publish per-PR copies; sccache remains job-local. Exact verified static, Swift, Metal-unit and Windows ABI caches may publish into the PR merge-ref scope for same-PR reruns. Website owns the single pnpm publisher and its npm store cache, while platform UI producers are restore-only for the shared pnpm diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6da499b347..5c6921f05d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -134,27 +134,6 @@ just compat-smoke ~/.cache/huggingface/hub/.gguf # optional 2-node + 1- just --list # list all recipes ``` -On macOS and Linux, local Cargo artifacts are bounded independently from the 10 GiB sccache -compiler-object cache. Inspect and prune them with: - -```bash -just cache-status -just cache-prune-dry-run max_size=80GiB max_age=14 -just cache-prune max_size=80GiB max_age=14 -``` - -Pruning evicts the oldest incremental sessions first, then uses -`cargo clean -p` for old or size-dominant workspace packages. It is scoped to -this worktree's `target/` and reports before/after bytes. On macOS and Linux, -`just build` and `just build-dev` hold a shared lock for their full build while -cache status and dry-run pruning take the same lock in shared mode. Executed -pruning requires the corresponding exclusive lock before measuring or deleting -artifacts. Direct Cargo and lower-level build commands do not share that lock, -so pruning also refuses to run when it detects an active Cargo or Rust compiler -process as a best-effort safeguard. Cargo configurations that separate -`build.build-dir` from `target-dir` are rejected because the cache manager does -not report, lock, or clean a second artifact tree. - On native Windows, `just check-release` runs the host-safe Rust/doc invariant subset and skips the Bash-only `install.sh` / `package-release.sh` parity checks. Run it on macOS or Linux when you need full shell parity coverage. ## CI / GitHub Actions diff --git a/Justfile b/Justfile index 3d76f06476..5aea3bb9f0 100644 --- a/Justfile +++ b/Justfile @@ -94,21 +94,16 @@ with-lld *COMMAND: with-lld *COMMAND: @powershell -NoProfile -ExecutionPolicy Bypass -Command "$$ErrorActionPreference = 'Stop'; $$linker = $$null; try { $$sysroot = (& rustc --print sysroot).Trim(); foreach ($$target in @('x86_64-pc-windows-msvc', 'aarch64-pc-windows-msvc')) { $$candidate = Join-Path $$sysroot \"lib\rustlib\$$target\bin\rust-lld.exe\"; if (Test-Path $$candidate) { $$linker = $$candidate; break } } } catch {}; if (-not $$linker) { foreach ($$name in @('rust-lld.exe', 'lld-link.exe')) { $$command = Get-Command $$name -ErrorAction SilentlyContinue; if ($$command) { $$linker = $$command.Source; break } } }; if (-not $$linker) { Write-Error \"LLVM lld was not found for the Windows MSVC target.`n`nlld is required for faster Rust builds (measured up to 26% faster locally).`n`nInstall one of these, then rerun the just command:`n rustup component add llvm-tools-preview`n`nOr install LLVM lld-link:`n winget install LLVM.LLVM`n choco install llvm`n`nThe build requires lld. It looks for rust-lld.exe in the active Rust sysroot first, then falls back to rust-lld.exe or lld-link.exe on PATH.\"; exit 1 }; $$env:CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER = $$linker; $$env:CARGO_TARGET_AARCH64_PC_WINDOWS_MSVC_LINKER = $$linker; Invoke-Expression '{{ COMMAND }}'" -[private] -[unix] -with-build-cache-lock *COMMAND: - @python3 scripts/manage-build-cache.py build -- {{ COMMAND }} - # Build a local product for the current platform. This is always a # backend-neutral dynamic host plus an adjacent packaged runtime. [macos] build backend="" cuda_arch="" rocm_arch="": - @just with-build-cache-lock scripts/build-development-product.sh --backend "{{ backend }}" --cuda-arch "{{ cuda_arch }}" --rocm-arch "{{ rocm_arch }}" + @scripts/build-development-product.sh --backend "{{ backend }}" --cuda-arch "{{ cuda_arch }}" --rocm-arch "{{ rocm_arch }}" # Fast local iteration build: dynamic host + adjacent native runtime + UI. [macos] build-dev: - @just with-build-cache-lock env MESH_LLM_BUILD_PROFILE=dev scripts/build-development-product.sh --profile dev + @MESH_LLM_BUILD_PROFILE=dev scripts/build-development-product.sh --profile dev # Linux overrides: # just build backend=cpu @@ -117,12 +112,12 @@ build-dev: # just build backend=vulkan [linux] build backend="" cuda_arch="" rocm_arch="": - @just with-build-cache-lock scripts/build-development-product.sh --backend "{{ backend }}" --cuda-arch "{{ cuda_arch }}" --rocm-arch "{{ rocm_arch }}" + @scripts/build-development-product.sh --backend "{{ backend }}" --cuda-arch "{{ cuda_arch }}" --rocm-arch "{{ rocm_arch }}" # Fast local iteration build: dynamic host + adjacent native runtime + UI. [linux] build-dev backend="" cuda_arch="" rocm_arch="": - @just with-build-cache-lock env MESH_LLM_BUILD_PROFILE=dev scripts/build-development-product.sh --profile dev --backend "{{ backend }}" --cuda-arch "{{ cuda_arch }}" --rocm-arch "{{ rocm_arch }}" + @MESH_LLM_BUILD_PROFILE=dev scripts/build-development-product.sh --profile dev --backend "{{ backend }}" --cuda-arch "{{ cuda_arch }}" --rocm-arch "{{ rocm_arch }}" # Windows overrides: # just build backend=cpu @@ -469,11 +464,6 @@ ui-test: # ── Full Validation Gate ─────────────────────────────────────── -# Populate the trusted Linux compiler-object seed with the dominant host graph. -[linux] -ci-sccache-seed-build: - cargo clippy --locked -p mesh-llm --all-targets -- -D warnings - # Validate CI definitions, planner fixtures, and repository consistency. ci-validate: actionlint -config-file .github/actionlint.yaml @@ -618,31 +608,6 @@ auto: build # ── Utilities ────────────────────────────────────────────────── -# Measure repository-local Cargo build-cache usage (80 GiB / 14 day defaults). -[unix] -cache-status max_size="80GiB" max_age="14": - python3 scripts/manage-build-cache.py status --max-size "{{ max_size }}" --max-age "{{ max_age }}" - -# Emit Cargo's effective workspace and artifact-directory metadata as JSON. -[unix] -cache-cargo-metadata: - @cargo metadata --no-deps --format-version 1 - -# Remove one package from an explicitly validated Cargo target directory. -[unix] -cache-cargo-clean: - @cargo clean --target-dir "$MESH_LLM_CACHE_TARGET_DIR" -p "$MESH_LLM_CACHE_PACKAGE" - -# Preview bounded, oldest-first incremental and package-aware Cargo cleanup. -[unix] -cache-prune-dry-run max_size="80GiB" max_age="14": - python3 scripts/manage-build-cache.py prune --max-size "{{ max_size }}" --max-age "{{ max_age }}" - -# Prune only repository-local Cargo artifacts; refuses during compilation. -[unix] -cache-prune max_size="80GiB" max_age="14": - python3 scripts/manage-build-cache.py prune --execute --max-size "{{ max_size }}" --max-age "{{ max_age }}" - # Update both tracked llama.cpp pin files from the prepared checkout. llama-update-pin: scripts/update-llama-pin.sh diff --git a/ci/METRICS.md b/ci/METRICS.md index 9d1e0e3fff..5b05977e29 100644 --- a/ci/METRICS.md +++ b/ci/METRICS.md @@ -179,11 +179,9 @@ python3 scripts/summarize-sccache-stats.py \ /tmp/sccache-evidence ``` -The threshold is an acceptance gate only for comparable warm cohorts. CI's -capture action labels each observation `cold`, `opportunistic`, `warm-pass`, -or `warm-failure`; only an exact seed restore is held to its configured warm -threshold. A cache hit is not correctness evidence; native artifacts must -still pass their build stamp, manifest and checksum verification. +The threshold is an acceptance gate only for comparable warm cohorts. A cache +hit is not correctness evidence; native artifacts must still pass their build +stamp, manifest and checksum verification. ## Composition migration targets diff --git a/ci/ci.md b/ci/ci.md index 5188848c62..ad0f97e015 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -468,8 +468,8 @@ The implemented policy uses that isolation selectively: | Cache class | PR publication | Effective rerun behavior | | --- | --- | --- | -| Linux sccache compiler objects | Exact trusted 2 GiB seed plus job-local writes on GitHub-hosted jobs | Main Quality completion owns publication; PRs mutate only their ephemeral copy | -| Linux Cargo `target` directories | Disabled for Clippy, Rust tests, host, and runtime | Avoids sharded multi-GiB generations and their restore/upload latency | +| sccache compiler objects | Job-local disk only | Helps repeated compilation inside one job; no reuse by another job or rerun | +| Cargo `target` directories | Restore trusted main, never save from PR | A rerun reuses the latest compatible main cache, but not objects compiled by the earlier PR run | | Static Linux ABI and Swift native ABI | Exact PR-scoped cache on miss | Same-PR reruns reuse the verified native input when its full recipe/toolchain key is unchanged | | macOS Metal unit ABI and Windows native ABI | Exact PR-scoped cache on miss | Same-PR reruns avoid the native rebuild; no restore prefixes cross an ABI boundary | | Console pnpm store | Website is the sole publisher; platform UI jobs restore only | Avoids four platform workflows racing to upload the same entry; later same-PR runs reuse a lockfile-keyed store | @@ -495,16 +495,10 @@ sentinel evidence and rollback procedure, is documented in `ci/DEPOT_PR_RISK_EXCEPTION.md`; the exact-SHA canary, metrics, and hosted rollback evidence are recorded in `.omo/specs/depot-pr-rollout-evidence.md`. -This is intentionally not a universal PR write-through policy. One protected -GitHub-hosted warmer publishes an exact-key compiler seed capped at 2 GiB after -successful Main Quality. Central runner policy denies that seed to every Depot -selection because Depot's Actions-cache proxy crosses trust scopes. Seeded -jobs enforce measured hit-rate floors only after an exact warm restore; a -missing seed is explicitly cold and does not fail. The seed key fingerprints -the warmer container image and toolchain epoch; runtime rows whose image or -epoch differs from the warmer are explicitly cold and skip seed restoration. -These four high-fanout job families also disable the per-object GHA backend on -every provider. Small exact native +This is intentionally not a universal PR write-through policy. Cargo target +caches are commonly hundreds of megabytes to several gigabytes per row; making +every PR matrix row publish one would multiply storage, increase upload time, +and evict the trusted main caches available to every PR. Small exact native caches have substantially better reuse-to-storage value. Cache hits are always an optimization: native stamps/manifests/checksums are verified, and every job must still regenerate successfully after a miss. diff --git a/ci/ownership.yml b/ci/ownership.yml index 136091ff9b..c4ea867c88 100644 --- a/ci/ownership.yml +++ b/ci/ownership.yml @@ -46,9 +46,7 @@ ".github/actions/select-ci-runners/**", ".github/actions/configure-sccache-gha/**", ".github/actions/capture-sccache-stats/**", - ".github/actions/restore-sccache-seed/**", ".github/actions/resolve-native-toolchain-epoch/**", - ".github/workflows/cache-warm-sccache.yml", "runner-images/**" ] }, diff --git a/scripts/manage-build-cache.py b/scripts/manage-build-cache.py deleted file mode 100755 index 8dadc3b716..0000000000 --- a/scripts/manage-build-cache.py +++ /dev/null @@ -1,361 +0,0 @@ -#!/usr/bin/env python3 -"""Measure and safely prune repository-local Cargo build artifacts.""" - -from __future__ import annotations - -import argparse -from contextlib import contextmanager -import fcntl -import json -import os -from pathlib import Path -import re -import shutil -import subprocess -import sys -import time -from typing import Any, BinaryIO, Iterable, Iterator - - -DEFAULT_MAX_BYTES = 80 * 1024**3 -DEFAULT_MAX_AGE_DAYS = 14 -SIZE_PATTERN = re.compile(r"^(\d+(?:\.\d+)?)\s*([kmgt]?i?b)?$", re.I) -UNIT_BYTES = { - "": 1, "b": 1, "kb": 1000, "kib": 1024, "mb": 1000**2, - "mib": 1024**2, "gb": 1000**3, "gib": 1024**3, - "tb": 1000**4, "tib": 1024**4, -} - - -class CacheError(RuntimeError): - """Raised when cache inspection or pruning cannot proceed safely.""" - - -def parse_size(value: str) -> int: - value = value.removeprefix("max_size=") - match = SIZE_PATTERN.fullmatch(value.strip()) - if not match: - raise argparse.ArgumentTypeError(f"invalid size: {value}") - return int(float(match.group(1)) * UNIT_BYTES[(match.group(2) or "").lower()]) - - -def parse_age(value: str) -> int: - try: - return int(value.removeprefix("max_age=")) - except ValueError as error: - raise argparse.ArgumentTypeError(f"invalid age in days: {value}") from error - - -def human_size(value: int) -> str: - amount = float(value) - for unit in ("B", "KiB", "MiB", "GiB", "TiB"): - if amount < 1024 or unit == "TiB": - return f"{amount:.1f} {unit}" - amount /= 1024 - raise AssertionError("unreachable") - - -def tree_metrics(path: Path) -> tuple[int, float]: - if not path.exists(): - return 0, 0.0 - if path.is_file() or path.is_symlink(): - stat = path.lstat() - return stat.st_size, stat.st_mtime - total = 0 - newest = path.stat().st_mtime - for root, directories, files in os.walk(path, followlinks=False): - root_path = Path(root) - for name in directories: - candidate = root_path / name - if candidate.is_symlink(): - stat = candidate.lstat() - total += stat.st_size - newest = max(newest, stat.st_mtime) - for name in files: - stat = (root_path / name).lstat() - total += stat.st_size - newest = max(newest, stat.st_mtime) - return total, newest - - -def immediate_entries(path: Path) -> list[dict[str, Any]]: - entries = [] - if path.is_dir(): - for child in path.iterdir(): - size, newest = tree_metrics(child) - entries.append({"path": str(child), "bytes": size, "newest_mtime": newest}) - return sorted(entries, key=lambda entry: entry["bytes"], reverse=True) - - -def cargo_metadata(workspace: Path) -> dict[str, Any]: - result = subprocess.run( - ["just", "cache-cargo-metadata"], - cwd=workspace, check=False, capture_output=True, text=True, - ) - if result.returncode != 0: - raise CacheError("cargo metadata failed; refusing build-cache management") - return json.loads(result.stdout) - - -def cargo_packages(workspace: Path) -> list[str]: - return sorted({package["name"] for package in cargo_metadata(workspace)["packages"]}) - - -def reject_separate_build_directory(workspace: Path, managed_target: Path) -> None: - managed_target = managed_target.resolve() - if os.environ.get("CARGO_BUILD_BUILD_DIR"): - raise CacheError("CARGO_BUILD_BUILD_DIR is unsupported by build-cache management") - if not (workspace / "Cargo.toml").is_file(): - return - metadata = cargo_metadata(workspace) - target_directory = Path(metadata["target_directory"]).resolve() - build_directory = Path(metadata.get("build_directory") or target_directory).resolve() - if build_directory != target_directory: - raise CacheError( - "Cargo build.build-dir outside target-dir is unsupported by build-cache management: " - f"{build_directory}" - ) - if managed_target != target_directory: - raise CacheError( - "managed target directory does not match Cargo's effective target directory: " - f"{target_directory}" - ) - - -def artifact_roots(target: Path, leaf: str) -> list[Path]: - """Return host and cross-target Cargo profile artifact roots.""" - roots = [*target.glob(f"*/{leaf}"), *target.glob(f"*/*/{leaf}")] - return sorted(path for path in roots if path.is_dir() and not path.is_symlink()) - - -def package_metrics(target: Path, packages: Iterable[str]) -> list[dict[str, Any]]: - normalized = {package: package.replace("-", "_") for package in packages} - totals = {package: [0, 0.0] for package in normalized} - roots = [*artifact_roots(target, "deps"), *artifact_roots(target, "build")] - for root in roots: - stems = normalized if root.name == "deps" else {package: package for package in normalized} - for child in root.iterdir(): - for package, stem in stems.items(): - name = child.name - if name == stem or name.startswith(f"{stem}-") or name.startswith(f"lib{stem}-"): - size, newest = tree_metrics(child) - totals[package][0] += size - totals[package][1] = max(totals[package][1], newest) - break - return sorted( - ({"package": package, "bytes": values[0], "newest_mtime": values[1]} - for package, values in totals.items() if values[0]), - key=lambda item: (item["newest_mtime"], -item["bytes"]), - ) - - -def active_compilers() -> list[str]: - result = subprocess.run( - ["ps", "-axo", "pid=,comm=,args="], check=True, capture_output=True, text=True, - ) - active = [] - for line in result.stdout.splitlines(): - fields = line.strip().split(maxsplit=2) - if len(fields) >= 2 and int(fields[0]) != os.getpid(): - if Path(fields[1]).name in {"cargo", "rustc", "rustdoc", "clippy-driver"}: - active.append(line.strip()) - return active - - -@contextmanager -def cache_lock(target: Path, *, exclusive: bool, nonblocking: bool) -> Iterator[BinaryIO]: - target.mkdir(parents=True, exist_ok=True) - lock_file = (target / ".mesh-llm-cache-prune.lock").open("a+b") - operation = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH - if nonblocking: - operation |= fcntl.LOCK_NB - try: - fcntl.flock(lock_file, operation) - except BlockingIOError as error: - lock_file.close() - raise CacheError("build-cache cleanup or a local build is already running") from error - try: - yield lock_file - finally: - lock_file.close() - - -def remove_tree(path: Path, target: Path) -> None: - candidate = Path(os.path.abspath(path)) - target_absolute = Path(os.path.abspath(target)) - target_resolved = target.resolve() - if candidate == target_absolute or target_absolute not in candidate.parents: - raise CacheError(f"refusing to remove path outside target: {path}") - if path.is_symlink(): - path.unlink() - return - parent_resolved = path.parent.resolve() - if parent_resolved != target_resolved and target_resolved not in parent_resolved.parents: - raise CacheError(f"refusing to remove path through a parent outside target: {path}") - if not path.is_dir(): - raise CacheError(f"refusing to remove non-directory path: {path}") - shutil.rmtree(path) - - -def prune_incremental( - target: Path, cutoff: float, current_bytes: int, max_bytes: int, execute: bool, -) -> tuple[int, list[dict[str, Any]]]: - candidates = [] - for root in artifact_roots(target, "incremental"): - for child in root.iterdir(): - size, newest = tree_metrics(child) - if newest < cutoff or current_bytes > max_bytes: - candidates.append((newest, child, size)) - actions = [] - for newest, path, size in sorted(candidates): - if newest >= cutoff and current_bytes <= max_bytes: - break - actions.append({"kind": "incremental", "path": str(path), "bytes": size}) - if execute: - remove_tree(path, target) - current_bytes = max(0, current_bytes - size) - return current_bytes, actions - - -def prune_packages( - workspace: Path, target: Path, current_bytes: int, max_bytes: int, - cutoff: float, execute: bool, -) -> tuple[int, list[dict[str, Any]]]: - actions = [] - for metrics in package_metrics(target, cargo_packages(workspace)): - if current_bytes <= max_bytes and metrics["newest_mtime"] >= cutoff: - continue - actions.append({ - "kind": "cargo-package", "package": metrics["package"], - "estimated_bytes": metrics["bytes"], - }) - if execute: - environment = os.environ.copy() - environment.update({ - "MESH_LLM_CACHE_TARGET_DIR": str(target), - "MESH_LLM_CACHE_PACKAGE": metrics["package"], - }) - result = subprocess.run( - ["just", "cache-cargo-clean"], - cwd=workspace, env=environment, check=False, - ) - if result.returncode != 0: - raise CacheError(f"cargo clean failed for {metrics['package']}") - current_bytes, _ = tree_metrics(target) - else: - current_bytes = max(0, current_bytes - metrics["bytes"]) - if current_bytes <= max_bytes and metrics["newest_mtime"] >= cutoff: - break - return current_bytes, actions - - -def snapshot(workspace: Path, target: Path, max_bytes: int, max_age_days: int) -> dict[str, Any]: - total, newest = tree_metrics(target) - return { - "schema": "mesh-llm.local-build-cache", "schema_version": 1, - "workspace": str(workspace), "target": str(target), "target_bytes": total, - "target_limit_bytes": max_bytes, "target_over_limit_bytes": max(0, total - max_bytes), - "max_age_days": max_age_days, "newest_mtime": newest, - "entries": immediate_entries(target), - } - - -def render_status(report: dict[str, Any]) -> None: - print(f"Cargo target: {human_size(report['target_bytes'])}") - print(f"Configured limit: {human_size(report['target_limit_bytes'])}") - print(f"Configured maximum age: {report['max_age_days']} days") - if report["target_over_limit_bytes"]: - print(f"Over limit: {human_size(report['target_over_limit_bytes'])}") - print("Largest target entries:") - for entry in report["entries"][:10]: - print(f" {human_size(entry['bytes']):>10} {entry['path']}") - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - commands = parser.add_subparsers(dest="command", required=True) - for command in ("status", "prune"): - subparser = commands.add_parser(command) - subparser.add_argument("--workspace", type=Path, default=Path.cwd()) - subparser.add_argument("--target-dir", type=Path) - subparser.add_argument("--max-size", type=parse_size, default=DEFAULT_MAX_BYTES) - subparser.add_argument("--max-age", type=parse_age, default=DEFAULT_MAX_AGE_DAYS) - subparser.add_argument("--json", action="store_true") - if command == "prune": - subparser.add_argument("--execute", action="store_true") - build = commands.add_parser("build") - build.add_argument("--workspace", type=Path, default=Path.cwd()) - build.add_argument("--target-dir", type=Path) - build.add_argument("build_command", nargs=argparse.REMAINDER) - return parser.parse_args() - - -def main() -> int: - arguments = parse_args() - workspace = arguments.workspace.resolve() - target = (arguments.target_dir or workspace / "target").resolve() - if target == workspace or workspace not in target.parents: - raise CacheError("target directory must be a child of the workspace") - reject_separate_build_directory(workspace, target) - if arguments.command == "build": - build_command = arguments.build_command - if build_command[:1] == ["--"]: - build_command = build_command[1:] - if not build_command: - raise CacheError("build command is required") - with cache_lock(target, exclusive=False, nonblocking=False): - return subprocess.run(build_command, cwd=workspace, check=False).returncode - if arguments.max_age < 0: - raise CacheError("max age must be non-negative") - if arguments.command == "status": - with cache_lock(target, exclusive=False, nonblocking=True): - before = snapshot(workspace, target, arguments.max_size, arguments.max_age) - print(json.dumps(before, indent=2, sort_keys=True)) if arguments.json else render_status(before) - return 0 - if arguments.execute: - with cache_lock(target, exclusive=True, nonblocking=True): - if active_compilers(): - raise CacheError("active Cargo/Rust compiler processes detected; refusing cleanup") - return run_prune(arguments, workspace, target) - with cache_lock(target, exclusive=False, nonblocking=True): - return run_prune(arguments, workspace, target) - - -def run_prune(arguments: argparse.Namespace, workspace: Path, target: Path) -> int: - before = snapshot(workspace, target, arguments.max_size, arguments.max_age) - cutoff = time.time() - arguments.max_age * 86400 - current, incremental = prune_incremental( - target, cutoff, before["target_bytes"], arguments.max_size, arguments.execute, - ) - current, packages = prune_packages( - workspace, target, current, arguments.max_size, cutoff, arguments.execute, - ) - after = snapshot(workspace, target, arguments.max_size, arguments.max_age) - final_bytes = after["target_bytes"] if arguments.execute else current - report = { - "schema": "mesh-llm.local-build-cache-prune", "schema_version": 1, - "mode": "execute" if arguments.execute else "dry-run", - "before_bytes": before["target_bytes"], "after_bytes": final_bytes, - "reclaimed_bytes": before["target_bytes"] - final_bytes, - "actions": [*incremental, *packages], - } - if arguments.json: - print(json.dumps(report, indent=2, sort_keys=True)) - else: - print(f"Mode: {report['mode']}") - print(f"Before: {human_size(report['before_bytes'])}") - print(f"After: {human_size(report['after_bytes'])}") - print(f"Reclaimed: {human_size(report['reclaimed_bytes'])}") - for action in report["actions"]: - identity = action.get("package", action.get("path")) - size = action.get("estimated_bytes", action.get("bytes", 0)) - print(f" {action['kind']}: {identity} ({human_size(size)})") - return 0 - - -if __name__ == "__main__": - try: - raise SystemExit(main()) - except (CacheError, OSError, json.JSONDecodeError) as error: - print(f"ERROR: {error}", file=sys.stderr) - raise SystemExit(1) from error diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index 4c4200f776..eb5edb847b 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -2224,10 +2224,6 @@ def test_runner_selection_uses_event_repository_and_ref_policy(self) -> None: outputs["allow_native_github_cache"], expected_native_cache, ) - self.assertEqual( - outputs["allow_trusted_sccache_seed"], - "false" if enabled == "true" else "true", - ) self.assertEqual(outputs["runner"], runner) expected_arm = ( "depot-ubuntu-24.04-arm" @@ -2356,7 +2352,6 @@ def test_runner_selection_uses_event_repository_and_ref_policy(self) -> None: self.assertEqual(canary_pr["runner"], "depot-ubuntu-24.04") self.assertEqual(canary_pr["allow_depot_remote_cache"], "false") self.assertEqual(canary_pr["allow_native_github_cache"], "false") - self.assertEqual(canary_pr["allow_trusted_sccache_seed"], "false") unapproved_pr = self.run_runner_selector( event_name="pull_request", @@ -2424,10 +2419,6 @@ def test_runner_selection_uses_event_repository_and_ref_policy(self) -> None: trusted_main_cross_branch_cache["allow_native_github_cache"], "true", ) - self.assertEqual( - trusted_main_cross_branch_cache["allow_trusted_sccache_seed"], - "false", - ) for name, kwargs in ( ( @@ -2537,15 +2528,10 @@ def test_dispatched_pr_cache_writes_remain_blocked_with_depot( self.assertIn("CACHE_NAMESPACE: mesh-llm", workflow) self.assertNotIn("CACHE_NAMESPACE: mesh-llm-pr", workflow) self.assertNotIn("'mesh-llm-pr'", workflow) - if workflow_name in { - "ci-quality-slice.yml", - "ci-rust-tests-slice.yml", - "ci-linux-host-slice.yml", - "ci-linux-runtime-slice.yml", - }: - self.assertIn('SCCACHE_GHA_ENABLED: "false"', workflow) - else: - self.assertNotIn('SCCACHE_GHA_ENABLED: "false"', workflow) + self.assertNotIn( + 'SCCACHE_GHA_ENABLED: "false"', + workflow, + ) if "uses: Swatinem/rust-cache@" in workflow: self.assertIn( "save-if: ${{ github.ref == 'refs/heads/main' && ", @@ -2565,15 +2551,15 @@ def test_dispatched_pr_cache_writes_remain_blocked_with_depot( def test_depot_pr_native_cache_consumers_obey_central_policy(self) -> None: eligible_consumers = { - "ci-quality-slice.yml": ("uses: ./.github/actions/restore-sccache-seed",), + "ci-quality-slice.yml": ("Swatinem/rust-cache@",), "ci-web-slice.yml": ( "uses: actions/cache/restore@", "uses: actions/cache/save@", ), "ci-ui-artifact-slice.yml": ("uses: actions/cache/restore@",), - "ci-linux-host-slice.yml": ("uses: ./.github/actions/restore-sccache-seed",), - "ci-linux-runtime-slice.yml": ("uses: ./.github/actions/restore-sccache-seed",), - "ci-rust-tests-slice.yml": ("uses: ./.github/actions/restore-sccache-seed",), + "ci-linux-host-slice.yml": ("Swatinem/rust-cache@",), + "ci-linux-runtime-slice.yml": ("Swatinem/rust-cache@",), + "ci-rust-tests-slice.yml": ("Swatinem/rust-cache@",), "ci-macos-host-slice.yml": ("Swatinem/rust-cache@",), "ci-platform-checks-slice.yml": ( "uses: actions/cache/restore@", @@ -2650,10 +2636,7 @@ def step_block(workflow: str, marker: str) -> str: for marker in markers: block = step_block(workflow, marker) with self.subTest(consumer=marker): - if "restore-sccache-seed" in marker: - self.assertIn("allow_trusted_sccache_seed", block) - else: - self.assertIn("allow_native_github_cache", block) + self.assertIn("allow_native_github_cache", block) for filename, jobs in expected_jobs.items(): workflow = ( diff --git a/scripts/tests/test_manage_build_cache.py b/scripts/tests/test_manage_build_cache.py deleted file mode 100644 index 82c2117845..0000000000 --- a/scripts/tests/test_manage_build_cache.py +++ /dev/null @@ -1,325 +0,0 @@ -from __future__ import annotations - -import importlib.util -import json -import os -from pathlib import Path -import subprocess -import sys -import tempfile -import time -import unittest -from unittest import mock - - -ROOT = Path(__file__).resolve().parents[2] -SCRIPT = ROOT / "scripts" / "manage-build-cache.py" -SPEC = importlib.util.spec_from_file_location("manage_build_cache", SCRIPT) -assert SPEC and SPEC.loader -CACHE = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(CACHE) - - -class ManageBuildCacheTests(unittest.TestCase): - def test_size_parser_accepts_binary_units(self) -> None: - self.assertEqual(CACHE.parse_size("80GiB"), 80 * 1024**3) - self.assertEqual(CACHE.parse_size("max_size=80GiB"), 80 * 1024**3) - self.assertEqual(CACHE.parse_size("1.5 MiB"), int(1.5 * 1024**2)) - self.assertEqual(CACHE.parse_age("max_age=14"), 14) - - def test_status_emits_machine_readable_metrics(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - workspace = Path(temporary) - artifact = workspace / "target" / "debug" / "deps" / "item" - artifact.parent.mkdir(parents=True) - artifact.write_bytes(b"x" * 128) - result = subprocess.run( - [sys.executable, str(SCRIPT), "status", "--workspace", str(workspace), - "--max-size", "64B", "--json"], - check=False, capture_output=True, text=True, - ) - self.assertEqual(result.returncode, 0, result.stderr) - report = json.loads(result.stdout) - self.assertEqual(report["schema"], "mesh-llm.local-build-cache") - self.assertEqual(report["target_bytes"], 128) - self.assertEqual(report["target_over_limit_bytes"], 64) - - def test_package_metrics_count_direct_dependency_files(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - target = Path(temporary) / "target" - artifact = target / "debug" / "deps" / "mesh_llm-abc.rcgu.o" - artifact.parent.mkdir(parents=True) - artifact.write_bytes(b"x" * 256) - metrics = CACHE.package_metrics(target, ["mesh-llm"]) - self.assertEqual(metrics[0]["package"], "mesh-llm") - self.assertEqual(metrics[0]["bytes"], 256) - - def test_package_metrics_count_cross_target_debug_files(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - target = Path(temporary) / "target" - artifact = ( - target / "x86_64-unknown-linux-gnu" / "debug" / "deps" - / "mesh_llm-abc.rcgu.o" - ) - artifact.parent.mkdir(parents=True) - artifact.write_bytes(b"x" * 384) - metrics = CACHE.package_metrics(target, ["mesh-llm"]) - self.assertEqual(metrics[0]["package"], "mesh-llm") - self.assertEqual(metrics[0]["bytes"], 384) - - def test_package_metrics_count_hyphenated_build_root(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - target = Path(temporary) / "target" - artifact = target / "debug" / "build" / "mesh-llm-abc" / "output" - artifact.parent.mkdir(parents=True) - artifact.write_bytes(b"x" * 512) - metrics = CACHE.package_metrics(target, ["mesh-llm"]) - self.assertEqual(metrics[0]["package"], "mesh-llm") - self.assertEqual(metrics[0]["bytes"], 512) - - def test_separate_cargo_build_directory_is_rejected(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - workspace = Path(temporary) - (workspace / "Cargo.toml").write_text("[workspace]\n", encoding="utf-8") - metadata = { - "target_directory": str(workspace / "target"), - "build_directory": str(workspace / "build-artifacts"), - } - with mock.patch.object(CACHE, "cargo_metadata", return_value=metadata): - with self.assertRaisesRegex(CACHE.CacheError, "build.build-dir"): - CACHE.reject_separate_build_directory(workspace, workspace / "target") - - def test_cargo_build_directory_environment_is_rejected(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - with mock.patch.dict(os.environ, {"CARGO_BUILD_BUILD_DIR": "elsewhere"}): - with self.assertRaisesRegex(CACHE.CacheError, "CARGO_BUILD_BUILD_DIR"): - workspace = Path(temporary) - CACHE.reject_separate_build_directory(workspace, workspace / "target") - - def test_cargo_target_directory_environment_mismatch_is_rejected(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - workspace = Path(temporary) - configured_target = workspace / "configured-target" - (workspace / "Cargo.toml").write_text("[workspace]\n", encoding="utf-8") - metadata = { - "target_directory": str(configured_target), - "build_directory": str(configured_target), - } - with mock.patch.dict( - os.environ, {"CARGO_TARGET_DIR": str(configured_target)}, clear=False, - ): - with mock.patch.object(CACHE, "cargo_metadata", return_value=metadata): - with self.assertRaisesRegex(CACHE.CacheError, "effective target"): - CACHE.reject_separate_build_directory( - workspace, workspace / "target", - ) - - def test_explicit_target_directory_must_match_cargo_metadata(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - workspace = Path(temporary) - cargo_target = workspace / "target" - explicit_target = workspace / "other-target" - (workspace / "Cargo.toml").write_text("[workspace]\n", encoding="utf-8") - metadata = { - "target_directory": str(cargo_target), - "build_directory": str(cargo_target), - } - with mock.patch.object(CACHE, "cargo_metadata", return_value=metadata): - with mock.patch.object( - sys, - "argv", - [ - str(SCRIPT), "status", "--workspace", str(workspace), - "--target-dir", str(explicit_target), - ], - ): - with self.assertRaisesRegex(CACHE.CacheError, "effective target"): - CACHE.main() - - def test_explicit_target_directory_matching_cargo_metadata_is_valid(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - workspace = Path(temporary) - explicit_target = workspace / "configured-target" - (workspace / "Cargo.toml").write_text("[workspace]\n", encoding="utf-8") - metadata = { - "target_directory": str(explicit_target), - "build_directory": str(explicit_target), - } - with mock.patch.object(CACHE, "cargo_metadata", return_value=metadata): - CACHE.reject_separate_build_directory(workspace, explicit_target) - - def test_cargo_operations_use_just_recipes(self) -> None: - workspace = Path("/workspace") - metadata_result = subprocess.CompletedProcess( - ["just", "cache-cargo-metadata"], 0, stdout='{"packages": []}', stderr="", - ) - clean_result = subprocess.CompletedProcess( - ["just", "cache-cargo-clean"], 0, - ) - with mock.patch.object( - CACHE.subprocess, "run", side_effect=[metadata_result, clean_result], - ) as run: - self.assertEqual(CACHE.cargo_metadata(workspace), {"packages": []}) - with mock.patch.object( - CACHE, "package_metrics", - return_value=[{"package": "mesh-llm", "bytes": 1, "newest_mtime": 0}], - ): - with mock.patch.object(CACHE, "cargo_packages", return_value=["mesh-llm"]): - with mock.patch.object(CACHE, "tree_metrics", return_value=(0, 0)): - CACHE.prune_packages(workspace, workspace / "target", 1, 0, 0, True) - self.assertEqual(run.call_args_list[0].args[0], ["just", "cache-cargo-metadata"]) - self.assertEqual( - run.call_args_list[1].args[0], - ["just", "cache-cargo-clean"], - ) - clean_environment = run.call_args_list[1].kwargs["env"] - self.assertEqual( - clean_environment["MESH_LLM_CACHE_TARGET_DIR"], "/workspace/target", - ) - self.assertEqual(clean_environment["MESH_LLM_CACHE_PACKAGE"], "mesh-llm") - - def test_incremental_pruning_is_oldest_first_and_target_scoped(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - target = Path(temporary) / "target" - old = target / "debug" / "incremental" / "old" - fresh = target / "debug" / "incremental" / "fresh" - old.mkdir(parents=True) - fresh.mkdir() - (old / "artifact").write_bytes(b"x" * 100) - (fresh / "artifact").write_bytes(b"y" * 100) - old_time = time.time() - 30 * 86400 - os.utime(old / "artifact", (old_time, old_time)) - os.utime(old, (old_time, old_time)) - remaining, actions = CACHE.prune_incremental( - target, time.time() - 14 * 86400, 200, 150, True, - ) - self.assertFalse(old.exists()) - self.assertTrue(fresh.exists()) - self.assertEqual(actions[0]["path"], str(old)) - self.assertEqual(remaining, 100) - - def test_incremental_pruning_finds_cross_target_debug_profile(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - target = Path(temporary) / "target" - old = ( - target / "x86_64-unknown-linux-gnu" / "debug" / "incremental" / "old" - ) - old.mkdir(parents=True) - (old / "artifact").write_bytes(b"x" * 100) - old_time = time.time() - 30 * 86400 - os.utime(old / "artifact", (old_time, old_time)) - os.utime(old, (old_time, old_time)) - remaining, actions = CACHE.prune_incremental( - target, time.time() - 14 * 86400, 100, 100, True, - ) - self.assertFalse(old.exists()) - self.assertEqual(actions[0]["path"], str(old)) - self.assertEqual(remaining, 0) - - def test_remove_tree_unlinks_symlink_without_deleting_target(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - target = Path(temporary) / "target" - retained = target / "debug" / "incremental" / "retained" - retained.mkdir(parents=True) - (retained / "artifact").write_bytes(b"keep") - candidate = target / "debug" / "incremental" / "candidate" - candidate.symlink_to(retained, target_is_directory=True) - CACHE.remove_tree(candidate, target) - self.assertFalse(candidate.exists()) - self.assertTrue((retained / "artifact").exists()) - - def test_build_command_holds_shared_cache_lock(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - workspace = Path(temporary) - with mock.patch.object(CACHE, "cache_lock") as cache_lock: - cache_lock.return_value.__enter__.return_value = None - cache_lock.return_value.__exit__.return_value = None - with mock.patch.object( - sys, - "argv", - [str(SCRIPT), "build", "--workspace", str(workspace), "--", "true"], - ): - self.assertEqual(CACHE.main(), 0) - cache_lock.assert_called_once_with( - workspace.resolve() / "target", exclusive=False, nonblocking=False, - ) - - def test_shared_build_lock_excludes_pruning(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - target = Path(temporary) / "target" - with CACHE.cache_lock(target, exclusive=False, nonblocking=False): - with self.assertRaises(CACHE.CacheError): - with CACHE.cache_lock(target, exclusive=True, nonblocking=True): - self.fail("exclusive prune lock unexpectedly acquired") - - def test_status_holds_nonblocking_shared_cache_lock(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - workspace = Path(temporary) - with mock.patch.object(CACHE, "cache_lock") as cache_lock: - cache_lock.return_value.__enter__.return_value = None - cache_lock.return_value.__exit__.return_value = None - with mock.patch.object(CACHE, "render_status"): - with mock.patch.object( - sys, "argv", [str(SCRIPT), "status", "--workspace", str(workspace)], - ): - self.assertEqual(CACHE.main(), 0) - cache_lock.assert_called_once_with( - workspace.resolve() / "target", exclusive=False, nonblocking=True, - ) - - def test_dry_run_holds_nonblocking_shared_cache_lock(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - workspace = Path(temporary) - with mock.patch.object(CACHE, "cache_lock") as cache_lock: - cache_lock.return_value.__enter__.return_value = None - cache_lock.return_value.__exit__.return_value = None - with mock.patch.object(CACHE, "run_prune", return_value=0): - with mock.patch.object( - sys, "argv", [str(SCRIPT), "prune", "--workspace", str(workspace)], - ): - self.assertEqual(CACHE.main(), 0) - cache_lock.assert_called_once_with( - workspace.resolve() / "target", exclusive=False, nonblocking=True, - ) - - def test_execute_refuses_when_a_compiler_is_active(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - workspace = Path(temporary) - (workspace / "target").mkdir() - with mock.patch.object(CACHE, "active_compilers", return_value=["1 cargo"]): - with mock.patch.object( - sys, "argv", [str(SCRIPT), "prune", "--workspace", str(workspace), "--execute"], - ): - with self.assertRaises(CACHE.CacheError): - CACHE.main() - - def test_execute_acquires_exclusive_lock_before_compiler_check(self) -> None: - events = [] - - class RecordingLock: - def __enter__(self): - events.append("lock") - - def __exit__(self, *_args): - events.append("unlock") - - with tempfile.TemporaryDirectory() as temporary: - workspace = Path(temporary) - with mock.patch.object(CACHE, "cache_lock", return_value=RecordingLock()): - with mock.patch.object( - CACHE, - "active_compilers", - side_effect=lambda: events.append("compiler-check") or ["1 cargo"], - ): - with mock.patch.object( - sys, - "argv", - [str(SCRIPT), "prune", "--workspace", str(workspace), "--execute"], - ): - with self.assertRaises(CACHE.CacheError): - CACHE.main() - self.assertEqual(events, ["lock", "compiler-check", "unlock"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/tests/test_pr_workflow_artifacts.py b/scripts/tests/test_pr_workflow_artifacts.py index b5c42c8192..30694135fa 100644 --- a/scripts/tests/test_pr_workflow_artifacts.py +++ b/scripts/tests/test_pr_workflow_artifacts.py @@ -268,9 +268,8 @@ def test_pr_cache_publishers_are_exact_and_bounded(self): self.assertIn("key: ${{ steps.llama_cache.outputs.cache-primary-key }}", platform) rust_tests = self.workflow("ci-rust-tests-slice.yml") - self.assertNotIn("Swatinem/rust-cache@", rust_tests) - self.assertIn("uses: ./.github/actions/restore-sccache-seed", rust_tests) - self.assertIn("allow_trusted_sccache_seed", rust_tests) + self.assertIn("github.ref == 'refs/heads/main'", rust_tests) + self.assertIn("original_event_name != 'pull_request'", rust_tests) if __name__ == "__main__": diff --git a/scripts/tests/test_sccache_evidence.py b/scripts/tests/test_sccache_evidence.py index 360374eed1..bdeada3d96 100644 --- a/scripts/tests/test_sccache_evidence.py +++ b/scripts/tests/test_sccache_evidence.py @@ -3,7 +3,6 @@ import json import os from pathlib import Path -import re import stat import subprocess import sys @@ -34,18 +33,6 @@ NATIVE_SDK_WORKFLOW = ( ROOT / ".github" / "workflows" / "native-sdk-artifact.yml" ) -SEED_WARMER = ROOT / ".github" / "workflows" / "cache-warm-sccache.yml" -SEED_KEY_PATTERN = re.compile( - r"mesh-llm-sccache-seed-[^\n]+-\$\{\{ hashFiles\('[^'\n]+', '[^'\n]+'\) \}\}" -) -SEED_IMAGE = ( - "ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:" - "8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d" -) -SEED_EPOCH = ( - "mesh-llm-cuda-runner-sha256-" - "8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d" -) def valid_payload(*, compile_requests: int = 12) -> dict[str, object]: @@ -95,7 +82,6 @@ def test_configure_callers_declare_native_cache_policy(self) -> None: ("ci-rust-tests-slice.yml", "rust_tests"): policy, ("ci-windows-host-slice.yml", "windows_host"): policy, ("ci-windows-runtime-slice.yml", "windows_runtime"): policy, - ("cache-warm-sccache.yml", "warm"): "false", ("hf-download-smoke.yml", "hf_download_smoke"): "true", ("native-sdk-artifact.yml", "linux_native_sdk_artifact"): policy, ("native-sdk-artifact.yml", "macos_native_sdk_artifact"): policy, @@ -136,8 +122,6 @@ def run_capture( *, artifact_name: str = "sccache-test-1", sccache_error: str = "", - cache_expectation: str = "opportunistic", - minimum_hit_rate: str = "0", ) -> tuple[subprocess.CompletedProcess[str], Path, Path]: temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup) @@ -175,10 +159,6 @@ def run_capture( str(stats_file), "--github-output", str(github_output), - "--cache-expectation", - cache_expectation, - "--minimum-hit-rate", - minimum_hit_rate, ], env={ **os.environ, @@ -203,15 +183,7 @@ def test_capture_writes_only_sanitized_machine_readable_counters(self) -> None: json.loads(stats_file.read_text()), { "schema": "mesh-llm.sccache-stats", - "schema_version": 2, - "assessment": { - "expectation": "opportunistic", - "classification": "opportunistic", - "minimum_hit_rate": 0.0, - "hit_rate": 0.6, - "cache_requests": 10, - "passed": True, - }, + "schema_version": 1, "stats": { "compile_requests": 12, "requests_executed": 10, @@ -230,9 +202,6 @@ def test_capture_writes_only_sanitized_machine_readable_counters(self) -> None: self.assertIn("requests_executed=10", outputs) self.assertIn("cache_hits=6", outputs) self.assertIn("cache_misses=4", outputs) - self.assertIn("hit_rate=0.6", outputs) - self.assertIn("cache_classification=opportunistic", outputs) - self.assertIn("cache_passed=true", outputs) def test_raw_secrets_urls_and_paths_cannot_reach_logs_or_evidence(self) -> None: payload = valid_payload() @@ -310,26 +279,6 @@ def test_zero_compile_requests_warns_but_remains_valid_evidence(self) -> None: self.assertTrue(stats_file.is_file()) self.assertIn("::warning title=sccache reported zero compile requests", result.stdout) - def test_warm_and_cold_observations_are_classified_separately(self) -> None: - warm_result, warm_file, warm_output = self.run_capture( - valid_payload(), cache_expectation="warm", minimum_hit_rate="0.80", - ) - cold_result, cold_file, cold_output = self.run_capture( - valid_payload(), cache_expectation="cold", minimum_hit_rate="0.80", - ) - self.assertEqual(warm_result.returncode, 0, warm_result.stderr) - self.assertEqual(cold_result.returncode, 0, cold_result.stderr) - self.assertEqual( - json.loads(warm_file.read_text())["assessment"]["classification"], - "warm-failure", - ) - self.assertIn("cache_passed=false", warm_output.read_text()) - self.assertEqual( - json.loads(cold_file.read_text())["assessment"]["classification"], - "cold", - ) - self.assertIn("cache_passed=true", cold_output.read_text()) - def test_missing_or_invalid_counter_rejects_evidence(self) -> None: payload = valid_payload() stats = payload["stats"] @@ -447,62 +396,6 @@ def test_swift_uses_trusted_main_seeded_dependency_cache(self) -> None: swift, ) - def test_linux_seed_producer_and_consumers_share_compatible_key(self) -> None: - warmer_workflow = yaml.safe_load(SEED_WARMER.read_text(encoding="utf-8")) - warmer_steps = warmer_workflow["jobs"]["warm"]["steps"] - seed_steps = [step for step in warmer_steps if step.get("id") == "seed"] - self.assertEqual(len(seed_steps), 1) - seed_assignments = [ - match.group(1) - for line in seed_steps[0]["run"].splitlines() - if (match := re.fullmatch(r'\s*key="([^"\n]+)"\s*', line)) - ] - self.assertEqual(len(seed_assignments), 1) - expected_key = seed_assignments[0] - self.assertIsNotNone(SEED_KEY_PATTERN.fullmatch(expected_key)) - - cache_steps = [ - step for step in warmer_steps - if str(step.get("uses", "")).startswith("actions/cache/") - ] - self.assertEqual(len(cache_steps), 2) - for step in cache_steps: - self.assertEqual(step["with"]["key"], "${{ steps.seed.outputs.key }}") - - consumers = ( - WORKFLOWS["quality"], - WORKFLOWS["rust-tests"], - WORKFLOWS["host"], - WORKFLOWS["runtime"], - ) - for path in consumers: - with self.subTest(workflow=path.name): - workflow = yaml.safe_load(path.read_text(encoding="utf-8")) - keys = [ - step.get("with", {}).get("cache_key") - for job in workflow["jobs"].values() - for step in job.get("steps", []) - if step.get("uses") == "./.github/actions/restore-sccache-seed" - ] - self.assertEqual(len(keys), 1) - self.assertIsNotNone(SEED_KEY_PATTERN.fullmatch(keys[0])) - self.assertEqual(keys[0], expected_key) - warmer = SEED_WARMER.read_text(encoding="utf-8") - self.assertIn("run: just ci-sccache-seed-build", warmer) - self.assertNotIn( - "run: cargo clippy --locked -p mesh-llm --all-targets -- -D warnings", - warmer, - ) - restore = ( - ROOT / ".github" / "actions" / "restore-sccache-seed" / "action.yml" - ).read_text(encoding="utf-8") - self.assertIn('echo "SCCACHE_CACHE_SIZE=2G" >> "$GITHUB_ENV"', restore) - - def test_runtime_seed_restore_requires_matching_image_and_epoch(self) -> None: - runtime = WORKFLOWS["runtime"].read_text(encoding="utf-8") - self.assertIn(f"matrix.runtime.container_image == '{SEED_IMAGE}'", runtime) - self.assertIn(f"matrix.runtime.toolchain_epoch == '{SEED_EPOCH}'", runtime) - def test_instrumented_workflows_use_unique_evidence_artifacts(self) -> None: for workflow_name in INSTRUMENTED: path = WORKFLOWS[workflow_name]