From 9b3e4648d4d8fd6a4b260cdc733ab3cf487de0fe Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 14:58:40 -0400 Subject: [PATCH 01/23] ci: compose reusable products and add Depot routing --- .agents/skills/manage-ci/SKILL.md | 52 +- .../manage-ci/references/current-inventory.md | 116 +- .github/actionlint.yaml | 4 + .../actions/compose-product-input/action.yml | 74 ++ .github/actions/compute-changes/action.yml | 22 +- .../actions/configure-sccache-gha/action.yml | 151 ++- .github/actions/prepare-host-input/action.yml | 142 ++ .../prepare-native-runtime-input/action.yml | 77 ++ .../prepare-windows-host-input/action.yml | 180 +++ .../actions/restore-smoke-inputs/action.yml | 57 +- .github/actions/select-ci-runners/action.yml | 113 ++ .../actions/setup-windows-rocm-sdk/action.yml | 2 +- .github/workflows/ci.yml | 1138 ++++++++++++----- .github/workflows/depot-canary.yml | 61 + .github/workflows/docker-precheck.yml | 2 +- .github/workflows/docker.yml | 4 +- .github/workflows/fly-deploy-console.yml | 4 +- .github/workflows/hf-download-smoke.yml | 14 +- .github/workflows/llama-upstream-canary.yml | 12 +- .github/workflows/nightly-stability-run.yml | 6 +- .github/workflows/pr_auto_assign.yml | 2 +- .github/workflows/pr_builds.yml | 1068 ++++++++++------ .github/workflows/pr_cleanup.yml | 14 +- .github/workflows/pr_quality.yml | 109 +- .github/workflows/pr_website.yml | 6 +- .../queue-unsloth-layer-packages.yml | 6 +- .github/workflows/release.yml | 959 +++++++------- .github/workflows/reset-caches.yml | 2 +- .github/workflows/scripted-binary-smoke.yml | 2 +- .github/workflows/sdk-smoke.yml | 19 +- .github/workflows/smoke.yml | 6 +- .github/workflows/stale-prs.yml | 2 +- .github/workflows/website-pages.yml | 8 +- .github/workflows/windows-warm-caches.yml | 34 +- README.md | 14 + ci/DEPOT_MIGRATION.md | 365 ++++++ ci/METRICS.md | 78 ++ ci/ci.md | 172 ++- scripts/build-host.sh | 31 + scripts/build-release.sh | 144 +-- scripts/build-windows.ps1 | 12 +- scripts/ci-client-readiness-process.py | 10 +- scripts/ci-compose-product-input.sh | 237 ++++ scripts/ci-kotlin-sdk-smoke.sh | 8 +- scripts/ci-prepare-native-runtime.sh | 225 +++- scripts/ci-rust-sdk-smoke.sh | 7 +- scripts/ci-swift-sdk-smoke.sh | 8 +- scripts/collect-ci-metrics.py | 661 ++++++++++ scripts/compose-product-bundle.py | 21 +- scripts/package-release.ps1 | 216 +++- scripts/package-release.sh | 117 +- scripts/tests/test_build_release.py | 17 +- scripts/tests/test_build_windows.py | 43 +- scripts/tests/test_ci_artifact_actions.py | 514 ++++++++ .../tests/test_ci_client_readiness_process.py | 15 + .../tests/test_ci_prepare_native_runtime.py | 306 +++++ scripts/tests/test_ci_workflow_artifacts.py | 363 ++++++ scripts/tests/test_collect_ci_metrics.py | 312 +++++ scripts/tests/test_depot_canary_workflow.py | 44 + scripts/tests/test_package_release_ps1.py | 104 ++ scripts/tests/test_pr_workflow_artifacts.py | 467 +++++++ .../tests/test_release_workflow_artifacts.py | 332 ++++- .../tests/test_verify_host_dependencies.py | 27 +- tools/xtask/src/publish_consistency.rs | 4 +- tools/xtask/src/workflow_checks.rs | 277 +++- 65 files changed, 7814 insertions(+), 1805 deletions(-) create mode 100644 .github/actions/compose-product-input/action.yml create mode 100644 .github/actions/prepare-host-input/action.yml create mode 100644 .github/actions/prepare-native-runtime-input/action.yml create mode 100644 .github/actions/prepare-windows-host-input/action.yml create mode 100644 .github/actions/select-ci-runners/action.yml create mode 100644 .github/workflows/depot-canary.yml create mode 100644 ci/DEPOT_MIGRATION.md create mode 100644 ci/METRICS.md create mode 100755 scripts/ci-compose-product-input.sh create mode 100755 scripts/collect-ci-metrics.py create mode 100644 scripts/tests/test_ci_artifact_actions.py create mode 100644 scripts/tests/test_ci_prepare_native_runtime.py create mode 100644 scripts/tests/test_ci_workflow_artifacts.py create mode 100644 scripts/tests/test_collect_ci_metrics.py create mode 100644 scripts/tests/test_depot_canary_workflow.py create mode 100644 scripts/tests/test_pr_workflow_artifacts.py diff --git a/.agents/skills/manage-ci/SKILL.md b/.agents/skills/manage-ci/SKILL.md index 450afb5e00..e0ce80e130 100644 --- a/.agents/skills/manage-ci/SKILL.md +++ b/.agents/skills/manage-ci/SKILL.md @@ -81,12 +81,18 @@ update the skill resources in the same change. that ignore applicable `docs_only`, `rust_changed`, `backend_changed`, `inference_artifact_required`, `windows_*_build_required`, `sdk_smoke_required`, `ui_changed`, or `website_changed` outputs. -- Keep Linux, macOS, and Windows as top-level target matrices in - `pr_builds.yml`. Linux/macOS CPU rows produce downstream smoke artifacts. - Keep macOS CUDA, ROCm, and Vulkan rows as explicit unsupported skips. +- Model Linux, macOS, and Windows executable products as independent + backend-neutral host and native-runtime producers followed by + composition-only product jobs. A platform/backend matrix belongs on the + runtime and product layers, never on a host producer. Do not add no-op macOS + CUDA, ROCm, or Vulkan rows for unsupported combinations. - Gate native backend lanes on backend inputs, not on every Rust change. Workflow-only and docs-only changes must not fan out into build, GPU, benchmark, or SDK smoke lanes without a matching product input. +- Gate public/ARC runner-image contract jobs on runner workflow, cache + integration, or cache-version changes (plus manual dispatch). Do not make + ordinary source/docs PRs pay an infrastructure canary that validates no + changed contract. - Keep Clippy sharding driven by `scripts/plan-clippy-batches.sh`; do not add hand-maintained static batches. - Keep crate-test sharding driven by `scripts/plan-test-batches.sh`. It derives @@ -163,6 +169,23 @@ update the skill resources in the same change. names, node selectors, resource requests, or worker counts in only one side of the contract. Update workflow routing, runner/GitOps configuration, inventory, and verification together. +- Route trusted main/release Depot jobs through the repository-wide + `DEPOT_RUNNERS_ENABLED` exact-string gate, with a typed manual canary input + accepted only when `github.ref == 'refs/heads/main'` and a GitHub-hosted + fallback for tags and every other ref. Current + `pull_request` workflows must always select GitHub-hosted runners; + `DEPOT_PR_RUNNERS_ENABLED` is ignored. +- Treat a checked-out repository-local selector as defense in depth, never as + the PR runner trust boundary: pull requests can modify both their workflow and + local action code. Before any PR uses Depot, automatic cache authority must be + disabled and isolated, then a default-branch-pinned, narrowly typed reusable + workflow must be restricted to its exact `@refs/heads/main` workflow ref with + `restricted_to_workflows=true`. Do not use `pull_request_target` to build or + execute PR content. +- Confirm the Depot GitHub App, public-repository runner-group access, + selected-workflow restriction, and every selected runner label before + enabling Depot. Hardware-qualified GPU execution remains on a restricted + device runner. ## Dependencies and runner setup @@ -230,6 +253,14 @@ update the skill resources in the same change. MeshLLM version, platform, and backend compatibility. Product artifacts must record the exact host and runtime digests and preserve the `mesh-bundle/native-runtimes/` layout. +- Use `.github/actions/prepare-host-input`, + `.github/actions/prepare-windows-host-input`, + `.github/actions/prepare-native-runtime-input`, and + `.github/actions/compose-product-input` for PR/main/release producers. + `prepare-host-input` owns Unix hosts and `prepare-windows-host-input` owns + Windows hosts. Add release-only signing/publishing around those actions; do + not fork their build/package/compose commands into workflow-local shell + blocks. - Keep host, runtime, and product matrices separate in release workflows. Product jobs download producer artifacts and verify compatibility and digest metadata before composition. Never satisfy a missing producer by rebuilding @@ -252,6 +283,15 @@ update the skill resources in the same change. reuse unsafe: OS, architecture, backend/toolchain, relevant lockfiles, `.github/cache-version.txt`, and build inputs. Do not broaden restore keys across incompatible or untrusted contexts. +- GitHub-hosted PR jobs may share the normal key namespace with main because + GitHub scopes PR writes to the merge ref and trusted main does not restore + them. Do not assume that isolation applies to another cache provider. +- Depot's GitHub cache namespace is repository-scoped and has no branch + isolation. With automatic Depot Cache enabled, its authority is injected into + the whole runner job and cannot be contained by sccache disk-only mode or + cache-key conventions. Current PR workflows must not use Depot, including + through a trusted reusable caller, until automatic injection is disabled and + complete token/API isolation is proven. - Do not save large shared Rust caches from PR merge refs. Shared caches are written from trusted main/release/cache-warming paths. PR cleanup may delete positively matched PR caches/artifacts but must not delete workflow runs or @@ -261,8 +301,10 @@ update the skill resources in the same change. follows the release policy, not the PR default. - Restore producer artifacts through `.github/actions/restore-smoke-inputs`. Reuse `smoke.yml`, `scripted-binary-smoke.yml`, `sdk-smoke.yml`, and - `hf-download-smoke.yml`; do not rebuild MeshLLM or duplicate model/artifact - restore blocks in consumers. + `hf-download-smoke.yml`; do not rebuild MeshLLM, native runtimes, or duplicate + model/artifact restore blocks in consumers. SDK smokes consume the runtime + adjacent to their staged producer binary and must fail rather than silently + compiling a replacement in CI. - Never put credentials, local absolute paths, private endpoints, or secret material into cache/artifact content or workflow summaries. diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index bb657993bc..69f80789ba 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -54,22 +54,50 @@ The Windows host input also carries the checksum-protected `xtask` executable that performed producer-side attestation. Windows product composers invoke that prebuilt verifier for the immutable host instead of compiling workspace code. -`ci.yml` applies the same executable-product rule to trusted main validation: -Linux and macOS debug artifact producers upload both the backend-neutral host -and its adjacent runtime; their consumer reruns JSON client readiness from those -exact bytes. Linux and Windows backend rows build a release host separately, -package exactly one selected runtime, and require `runtime list` plus no-driver -client readiness. They never invoke a backend-linked host, inject a CUDA driver -stub, or skip client startup merely because a hosted runner has no GPU. +`ci.yml` applies the same executable-product rule to trusted main validation. +Linux and macOS build immutable release-profile hosts and separately packaged +CPU or Metal runtimes, then upload complete product-v2 trees from +composition-only jobs. Linux CUDA, ROCm, and Vulkan rows download the same +Linux host, build only one selected runtime, and compose a product through the +shared action. SDK consumers reuse the producer's adjacent runtime and fail if +CI would silently rebuild it. Windows likewise builds one immutable +release-profile host, independent CPU/CUDA/ROCm/Vulkan runtime inputs, and +composition-only products. Broad main Rust changes exercise the Windows CPU +product; Windows GPU products remain limited to GPU/backend inputs or manual +dispatch. Every composed backend product requires `runtime list` plus +no-driver client readiness; hosted GPU rows neither inject a driver stub nor +skip startup because no device is present. + +`pr_builds.yml` uses the same split producer/composer shape for Linux CPU and +macOS Metal products while retaining debug-profile hosts for lightweight PR +iteration. Windows broad-Rust validation stays at lightweight Cargo checks; +the debug host plus CPU or GPU runtime/product graph runs only for its +platform/backend input or manual dispatch. Unsupported macOS CUDA, ROCm, and +Vulkan combinations are omitted rather than emitted as no-op jobs. Local actions: - `.github/actions/compute-changes` owns path, crate, backend, SDK, UI, website, Windows, and docs-only routing outputs. +- `.github/actions/select-ci-runners` routes trusted push/dispatch jobs through + the Depot gate only for `refs/heads/main`, returns GitHub-hosted labels for + tags and every other ref, and unconditionally returns GitHub-hosted labels + for `pull_request` events. Repository ownership and the deprecated + `DEPOT_PR_RUNNERS_ENABLED` variable do not alter that decision. Its cache + permission is derived from the same typed trust decision. - `.github/actions/configure-sccache-gha` exports ephemeral Actions cache - credentials to the baked `sccache`, configures job-local disk storage ahead - of GHA in a fail-open multi-level cache, and restarts the server with disk-only - storage when the remote probe fails. + credentials to the baked `sccache`, permits Depot WebDAV only for an explicit + trusted call, and uses disk-only storage if a future pull-request trust + context is ever evaluated on Depot. +- `.github/actions/prepare-host-input` owns Unix neutral-host build, optional + release attestation, import-policy verification, and checksumming. +- `.github/actions/prepare-windows-host-input` owns the equivalent Windows + debug/release neutral-host build, optional release attestation, import-policy + verification, checksum, and verifier artifact. +- `.github/actions/prepare-native-runtime-input` owns runtime build/package + invocation and the release-grade artifact verifier. +- `.github/actions/compose-product-input` verifies producer inputs, creates one + product-v2 tree without compiling, and runs CLI/client readiness. - `.github/actions/restore-smoke-inputs` owns producer artifact staging and model restoration for smoke consumers. - `.github/actions/setup-windows-rocm-sdk` owns reusable Windows ROCm setup. @@ -94,6 +122,35 @@ GitHub-hosted labels currently used: - macOS: `macos-15` and legacy `macos-latest` - Windows: `windows-2022` +Depot labels referenced behind the rollout gate: + +- routing/summary: `depot-ubuntu-24.04` +- light build/planning: `depot-ubuntu-24.04-4` +- Rust/native build: `depot-ubuntu-24.04-8` +- measured high-parallelism native build: `depot-ubuntu-24.04-16` + +Current PR jobs and non-main refs never select these labels. They use the corresponding +GitHub-hosted label regardless of repository ownership or +`DEPOT_PR_RUNNERS_ENABLED`; that variable is ignored. Trusted main/release jobs +use `DEPOT_RUNNERS_ENABLED`; a trusted +main-ref manual dispatch can set `use_depot=true`. Hardware-qualified GPU +execution is not part of the gate. + +The Depot dashboard reports the `Mesh-LLM` GitHub connection active, and +GitHub lists both `depot-managed-runners` and `depot-code-access` installations +for all organization repositories. GitHub's organization settings show the +Depot-managed `Default` group currently allows all workflows but excludes +public repositories, while the separate `mesh-llm` group owns the two dedicated +GPU scale sets. Before a canary, restrict `Default` to this repository and exact +default-branch workflow refs, then enable public-repository access. + +The checked-out local selector is not the security boundary because PRs can +modify workflow and local-action files. The Depot runner group must use +`restricted_to_workflows=true` and exact default-branch selected-workflow refs. +Automatic Depot Cache still grants repository-scoped cache authority to the +whole job, so even a trusted reusable caller cannot safely execute untrusted PR +code while that injection is enabled. PRs remain GitHub-hosted. + Legacy/dedicated self-hosted label arrays currently referenced: - NVIDIA AMD64: `["self-hosted","Linux","X64","amd64","gpu-nvidia"]` @@ -104,12 +161,13 @@ ARC scale-set labels for the prebuilt runner rollout: - `mesh-llm-amd64` - `mesh-llm-arm64` -`pr_builds.yml` runs `public_runner_image_contract` in the public image and -`arc_runner_image_contract` on both ARC labels for every pull request. The ARC -job executes directly in each ephemeral runner pod, verifies the self-hosted -image contract and native architecture, and runs a small Rust check. It -intentionally has no hosted fallback because its purpose is to detect an ARC, -K3s scheduling, architecture, or runner-image regression before merge. +`pr_builds.yml` runs `public_runner_image_contract` in the public image when +the runner workflow, cache integration, or cache version changes, plus manual +dispatches. Trusted main `ci.yml` owns `arc_runner_image_contract` on both ARC +labels for the same change class. Untrusted PR-event jobs never request those +labels. The ARC job executes directly in each ephemeral runner pod, verifies +the self-hosted image contract and native architecture, and runs a small Rust +check. It intentionally has no hosted fallback. Runner images are published from [`Mesh-LLM/mesh-llm-runner-images`](https://github.com/Mesh-LLM/mesh-llm-runner-images) @@ -157,17 +215,19 @@ MeshLLM workflows pin the public digest. The Flux repository must independently roll the ARC HelmReleases to the paired self-hosted digest; that cross-repository change cannot be delivered by a MeshLLM pull request. -Public-image Rust jobs use the baked `sccache` binary and start with -`SCCACHE_GHA_ENABLED=false`. The local `configure-sccache-gha` action exports -the ephemeral Actions cache URL/token, enables a `disk,gha` cache chain, ignores -cache write errors, and probes it by starting the server. The disk tier uses a -job-local directory beside, rather than inside, the checkout. Cache read -failures degrade to misses, cache write failures are warnings, and compiler -invocations fall back locally if the server becomes unavailable. If the -initial remote probe fails, the action stops the remote-configured server and -restarts `sccache` with disk-only storage. Persistent Cargo target and ABI reuse -remains owned by `Swatinem/rust-cache` and `actions/cache`. Do not reintroduce -the sccache download action merely to export credentials. +Public-image Rust jobs use the baked `sccache` binary. Trusted calls to +`configure-sccache-gha` may use Depot's `SCCACHE_WEBDAV_ENDPOINT` plus +`SCCACHE_WEBDAV_TOKEN`/`DEPOT_CACHE_TOKEN` in a fail-open `disk,webdav` chain. +When that permission is false and Depot is detected, the action gives the +sccache server a credential-free environment and uses job-local disk only. +GitHub-hosted jobs retain `disk,gha` or explicit disk-only mode. Persistent +Cargo target and ABI reuse remains owned by `Swatinem/rust-cache` and +`actions/cache`. Current PR jobs use the normal `mesh-llm` key namespace and +GitHub's merge-ref scoping; trusted main does not restore PR-written entries. +A future Depot PR entrypoint must instead use keys that trusted main/release +jobs never restore because Depot cache entries are repository-scoped. Key +separation alone is not a security boundary while the job receives Depot cache +authority, so no such entrypoint is currently eligible. `USE_SELF_HOSTED` currently controls selected GPU/release routes. Unset or a value other than the exact string `true` selects the hosted fallback. Any new @@ -180,6 +240,8 @@ All GitHub Actions variables are strings. | Variable | Purpose and fallback | | --- | --- | | `USE_SELF_HOSTED` | Exact `true` selects supported self-hosted GPU/release lanes; otherwise hosted | +| `DEPOT_PR_RUNNERS_ENABLED` | Deprecated compatibility variable; the current selector ignores it and all PR jobs remain GitHub-hosted | +| `DEPOT_RUNNERS_ENABLED` | Exact `true` routes eligible trusted main/release Linux jobs to Depot; otherwise GitHub-hosted | | `CUDA_VERSION` | Windows CUDA toolkit selection; Linux CUDA lanes use digest-pinned backend images | | `VULKAN_SDK_VERSION` | Windows Vulkan SDK; fallback `1.4.328.1` | | `LLAMA_UPSTREAM_CANARY_SMOKE` | Enables canary smoke; fallback `1` | diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index feff309cec..31dd3e4cc5 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -12,3 +12,7 @@ self-hosted-runner: - blacksmith-6vcpu-macos-15 - blacksmith-4vcpu-ubuntu-2404-arm - blacksmith-8vcpu-windows-2025 + - depot-ubuntu-24.04 + - depot-ubuntu-24.04-4 + - depot-ubuntu-24.04-8 + - depot-ubuntu-24.04-16 diff --git a/.github/actions/compose-product-input/action.yml b/.github/actions/compose-product-input/action.yml new file mode 100644 index 0000000000..c1f5e8920d --- /dev/null +++ b/.github/actions/compose-product-input/action.yml @@ -0,0 +1,74 @@ +name: Compose verified product input +description: Compose one verified host and one verified native runtime without compiling either input. + +inputs: + host_input_dir: + description: Directory containing the immutable host, import report, and checksum. + required: true + runtime_input_dir: + description: Directory containing one runtime archive or one extracted runtime. + required: true + output_dir: + description: Directory that receives the composed product-v2 tree. + required: false + default: product-input + backend: + description: Product backend/flavor. + required: true + version: + description: Product version. Defaults to the runtime manifest mesh version. + required: false + default: "" + binary_name: + description: Host executable filename. + required: false + default: mesh-llm + readiness_smoke: + description: Run noninteractive client readiness against the composed bytes. + required: false + default: "true" + attestation_public_key_file: + description: Optional public key used with the producer's prebuilt attestation verifier. + required: false + default: "" + attestation_verifier: + description: Optional verifier path; defaults to release-attestation-verifier in the host input. + required: false + default: "" + +outputs: + product_dir: + description: Absolute path to the product-v2 tree. + value: ${{ steps.compose.outputs.product_dir }} + binary_path: + description: Absolute path to the composed host executable. + value: ${{ steps.compose.outputs.binary_path }} + runtime_root: + description: Absolute path to the composed native-runtimes directory. + value: ${{ steps.compose.outputs.runtime_root }} + runtime_dir: + description: Absolute path to the selected runtime. + value: ${{ steps.compose.outputs.runtime_dir }} + archive_path: + description: Tar archive containing the exact verified product tree, including executable modes. + value: ${{ steps.compose.outputs.archive_path }} + +runs: + using: composite + steps: + - name: Verify and compose producer inputs + id: compose + shell: bash + env: + INPUT_HOST_INPUT_DIR: ${{ inputs.host_input_dir }} + INPUT_RUNTIME_INPUT_DIR: ${{ inputs.runtime_input_dir }} + INPUT_OUTPUT_DIR: ${{ inputs.output_dir }} + INPUT_BACKEND: ${{ inputs.backend }} + INPUT_VERSION: ${{ inputs.version }} + INPUT_BINARY_NAME: ${{ inputs.binary_name }} + INPUT_READINESS_SMOKE: ${{ inputs.readiness_smoke }} + INPUT_ATTESTATION_PUBLIC_KEY_FILE: ${{ inputs.attestation_public_key_file }} + INPUT_ATTESTATION_VERIFIER: ${{ inputs.attestation_verifier }} + run: | + set -euo pipefail + scripts/ci-compose-product-input.sh diff --git a/.github/actions/compute-changes/action.yml b/.github/actions/compute-changes/action.yml index e8836aff37..e22d7d3412 100644 --- a/.github/actions/compute-changes/action.yml +++ b/.github/actions/compute-changes/action.yml @@ -72,6 +72,9 @@ outputs: sdk_smoke_required: description: 'Boolean: true if SDK smoke tests should run as soon as a binary is ready' value: ${{ steps.derive.outputs.sdk_smoke_required }} + runner_contract_required: + description: 'Boolean: true if runner-image or cache integration contracts changed' + value: ${{ steps.derive.outputs.runner_contract_required }} linux_test_groups_json: description: 'JSON array of Linux test groups to run' value: ${{ steps.derive.outputs.linux_test_groups_json }} @@ -142,6 +145,16 @@ runs: # Read changed files for docs_only and rust_changed logic CHANGED_FILES=$(cat /tmp/changed_files.txt | grep -v "^__force_all__$" || true) + RUNNER_CONTRACT_REQUIRED="false" + if [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]; then + RUNNER_CONTRACT_REQUIRED="true" + elif [[ -n "$CHANGED_FILES" ]]; then + RUNNER_CONTRACT_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^\.github/cache-version\.txt$|^\.github/actionlint\.yaml$|^\.github/actions/(configure-sccache-gha|select-ci-runners)/|^\.github/workflows/(ci|depot-canary|pr_builds|pr_quality|release)\.yml$)' || true) + if [[ -n "$RUNNER_CONTRACT_INPUTS" ]]; then + RUNNER_CONTRACT_REQUIRED="true" + fi + fi + # Determine docs_only: true if all_rust=false, UI/website are unchanged, # and all files match authored docs patterns. DOCS_ONLY="false" @@ -287,7 +300,7 @@ runs: if [[ "$ALL_RUST" == "true" ]]; then BACKEND_CHANGED="true" elif [[ -n "$CHANGED_FILES" ]]; then - BACKEND_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^third_party/llama\.cpp/|^crates/skippy-ffi/|^scripts/(build-llama|prepare-llama|build-linux|build-linux-rocm|build-mac|build-windows|install-windows-sdk)\.|^\.github/actions/setup-windows-rocm-sdk/|^\.github/cache-version\.txt$)' || true) + BACKEND_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^third_party/llama\.cpp/|^crates/skippy-ffi/|^scripts/(build-llama|prepare-llama|build-linux|build-linux-rocm|build-mac|build-windows|install-windows-sdk|build-host|build-release|package-native-runtime|verify-native-runtime-package|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^\.github/actions/(prepare-host-input|prepare-windows-host-input|prepare-native-runtime-input|compose-product-input|restore-smoke-inputs|setup-windows-rocm-sdk)/|^\.github/workflows/(ci|pr_builds|release|sdk-smoke|smoke)\.yml$|^\.github/cache-version\.txt$)' || true) if [[ -n "$BACKEND_INPUTS" ]] || [[ "$BACKEND_RECIPE_CHANGED" == "true" ]]; then BACKEND_CHANGED="true" fi @@ -296,8 +309,8 @@ runs: WINDOWS_CPU_BUILD_REQUIRED="false" WINDOWS_GPU_BUILD_REQUIRED="false" if [[ -n "$CHANGED_FILES" ]]; then - WINDOWS_CPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/mesh-llm-release-footer/|^crates/mesh-llm-nodejs/|^crates/skippy-ffi/|^scripts/build-windows\.ps1$|^third_party/llama\.cpp/|^Cargo\.toml$|^Cargo\.lock$|^\.github/cache-version\.txt$)' || true) - WINDOWS_GPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/skippy-ffi/|^scripts/(build-windows|install-windows-sdk)\.ps1$|^scripts/(package-native-runtime|verify-native-runtime-package)\.sh$|^scripts/windows-native-runtime-deps\.py$|^scripts/tests/test_windows_native_runtime_deps\.py$|^third_party/llama\.cpp/|^\.github/cache-version\.txt$|^\.github/actions/(compute-changes/action\.yml$|setup-windows-rocm-sdk/))' || true) + WINDOWS_CPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/mesh-llm-release-footer/|^crates/mesh-llm-nodejs/|^crates/skippy-ffi/|^scripts/build-windows\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^third_party/llama\.cpp/|^Cargo\.toml$|^Cargo\.lock$|^\.github/cache-version\.txt$|^\.github/actions/(compute-changes/action\.yml$|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/))' || true) + WINDOWS_GPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/skippy-ffi/|^scripts/(build-windows|install-windows-sdk)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^scripts/windows-native-runtime-deps\.py$|^scripts/tests/test_windows_native_runtime_deps\.py$|^third_party/llama\.cpp/|^\.github/cache-version\.txt$|^\.github/actions/(compute-changes/action\.yml$|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|setup-windows-rocm-sdk/))' || true) if [[ -n "$WINDOWS_CPU_INPUTS" ]] || [[ "$BACKEND_RECIPE_CHANGED" == "true" ]]; then WINDOWS_CPU_BUILD_REQUIRED="true" fi @@ -312,7 +325,7 @@ runs: if [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]; then SDK_SMOKE_REQUIRED="true" elif [[ -n "$CHANGED_FILES" ]]; then - DIRECT_SDK_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^sdk/|^Package\.swift$|^scripts/ci-(native|kotlin|swift)-sdk-smoke\.sh$|^scripts/ci-sdk-fixture\.sh$|^\.github/workflows/sdk-smoke\.yml$)' || true) + DIRECT_SDK_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^sdk/|^Package\.swift$|^scripts/ci-(rust|kotlin|swift)-sdk-smoke\.sh$|^scripts/ci-prepare-native-runtime\.sh$|^scripts/ci-sdk-fixture\.sh$|^\.github/workflows/sdk-smoke\.yml$)' || true) if [[ -n "$DIRECT_SDK_INPUTS" ]]; then SDK_SMOKE_REQUIRED="true" elif echo "$AFFECTED_CRATES" | jq -e 'index("mesh-llm-client") or index("mesh-llm-api-client") or index("mesh-llm-api-server") or index("mesh-llm-config") or index("mesh-llm-console-server") or index("mesh-llm-ffi") or index("mesh-llm-native-runtime") or index("mesh-llm-protocol") or index("mesh-llm-routing") or index("mesh-llm-types")' >/dev/null; then @@ -373,5 +386,6 @@ runs: echo "windows_cpu_build_required=$WINDOWS_CPU_BUILD_REQUIRED" echo "windows_gpu_build_required=$WINDOWS_GPU_BUILD_REQUIRED" echo "sdk_smoke_required=$SDK_SMOKE_REQUIRED" + echo "runner_contract_required=$RUNNER_CONTRACT_REQUIRED" echo "linux_test_groups_json=$LINUX_TEST_GROUPS_JSON" } >> "$GITHUB_OUTPUT" diff --git a/.github/actions/configure-sccache-gha/action.yml b/.github/actions/configure-sccache-gha/action.yml index 65ecc35344..4977c8db6b 100644 --- a/.github/actions/configure-sccache-gha/action.yml +++ b/.github/actions/configure-sccache-gha/action.yml @@ -1,13 +1,127 @@ -name: Configure baked sccache GHA backend -description: Use job-local disk plus best-effort Actions cache from the baked sccache binary. +name: Configure baked sccache remote backend +description: Prefer Depot WebDAV, then Actions cache, with a job-local disk fallback. + +inputs: + allow_depot_remote_cache: + description: Allow this trusted job to read and write Depot's repository-scoped WebDAV cache. + required: false + default: "false" runs: using: composite steps: - name: Export cache credentials and start sccache - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + INPUT_ALLOW_DEPOT_REMOTE_CACHE: ${{ inputs.allow_depot_remote_cache }} with: script: | + const allowDepotRemoteCache = + (process.env.INPUT_ALLOW_DEPOT_REMOTE_CACHE || '').toLowerCase(); + if ( + allowDepotRemoteCache !== 'true' && + allowDepotRemoteCache !== 'false' + ) { + core.setFailed( + 'allow_depot_remote_cache must be the string true or false.', + ); + return; + } + + const webdavEndpoint = process.env.SCCACHE_WEBDAV_ENDPOINT || ''; + const webdavToken = + process.env.SCCACHE_WEBDAV_TOKEN || + process.env.DEPOT_CACHE_TOKEN || + ''; + const diskOnlyEnvironment = () => ({ + ...process.env, + ACTIONS_CACHE_URL: '', + ACTIONS_RESULTS_URL: '', + ACTIONS_RUNTIME_TOKEN: '', + SCCACHE_WEBDAV_ENDPOINT: '', + SCCACHE_WEBDAV_TOKEN: '', + SCCACHE_WEBDAV_USERNAME: '', + SCCACHE_WEBDAV_PASSWORD: '', + }); + + if ( + allowDepotRemoteCache === 'true' && + webdavEndpoint && + webdavToken + ) { + core.setSecret(webdavToken); + core.exportVariable('SCCACHE_WEBDAV_ENDPOINT', webdavEndpoint); + core.exportVariable('SCCACHE_WEBDAV_TOKEN', webdavToken); + core.exportVariable('SCCACHE_GHA_ENABLED', 'false'); + core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk,webdav'); + core.exportVariable('SCCACHE_IGNORE_SERVER_IO_ERROR', '1'); + core.exportVariable('SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'ignore'); + await exec.exec('sccache', ['--stop-server'], { + ignoreReturnCode: true, + }); + const depotExitCode = await exec.exec('sccache', ['--start-server'], { + ignoreReturnCode: true, + }); + if (depotExitCode === 0) { + core.info( + 'Baked sccache is using job-local disk with Depot WebDAV cache.', + ); + return; + } + core.warning( + 'Depot WebDAV cache is unavailable; restarting baked sccache with job-local disk.', + ); + await exec.exec('sccache', ['--stop-server'], { + ignoreReturnCode: true, + }); + core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk'); + const depotFallbackExitCode = await exec.exec( + 'sccache', + ['--start-server'], + { + env: diskOnlyEnvironment(), + ignoreReturnCode: true, + }, + ); + if (depotFallbackExitCode !== 0) { + core.setFailed('Unable to start baked sccache with Depot or job-local storage.'); + } + return; + } + + if ( + allowDepotRemoteCache === 'false' && + (webdavEndpoint || webdavToken) + ) { + core.info( + 'Depot cache is present but disabled for this trust context; using job-local disk only.', + ); + core.exportVariable('SCCACHE_WEBDAV_ENDPOINT', ''); + core.exportVariable('SCCACHE_WEBDAV_TOKEN', ''); + core.exportVariable('SCCACHE_WEBDAV_USERNAME', ''); + core.exportVariable('SCCACHE_WEBDAV_PASSWORD', ''); + core.exportVariable('SCCACHE_GHA_ENABLED', 'false'); + core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk'); + core.exportVariable('SCCACHE_IGNORE_SERVER_IO_ERROR', '1'); + await exec.exec('sccache', ['--stop-server'], { + ignoreReturnCode: true, + }); + const untrustedDiskOnlyExitCode = await exec.exec( + 'sccache', + ['--start-server'], + { + env: diskOnlyEnvironment(), + ignoreReturnCode: true, + }, + ); + if (untrustedDiskOnlyExitCode !== 0) { + core.setFailed( + 'Unable to start baked sccache with its trust-isolated disk cache.', + ); + } + return; + } + // Honor an explicit job-level opt-out of the GitHub Actions cache // backend. Long-running lanes (CUDA/ROCm/Vulkan release builds that // exceed the ephemeral Actions cache token lifetime) set @@ -15,23 +129,27 @@ runs: // server started after the token expires probes the gha tier at // startup, and that storage read fails permanently (HTTP 400), // aborting the build before any fail-open policy applies. Clearing the - // cache URL/token as well prevents any residual gha configuration - // from re-enabling the remote tier. + // cache URL/token for the server process as well prevents any + // residual gha configuration from re-enabling the remote tier. Do + // not clear them job-wide: later actions/cache consumers still need + // their credentials. if ((process.env.SCCACHE_GHA_ENABLED || '').toLowerCase() === 'false') { core.info( 'SCCACHE_GHA_ENABLED=false is set; using baked sccache with disk-only cache.', ); core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk'); core.exportVariable('SCCACHE_IGNORE_SERVER_IO_ERROR', '1'); - core.exportVariable('ACTIONS_CACHE_URL', ''); - core.exportVariable('ACTIONS_RESULTS_URL', ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', ''); await exec.exec('sccache', ['--stop-server'], { ignoreReturnCode: true, }); - const diskOnlyExitCode = await exec.exec('sccache', ['--start-server'], { - ignoreReturnCode: true, - }); + const diskOnlyExitCode = await exec.exec( + 'sccache', + ['--start-server'], + { + env: diskOnlyEnvironment(), + ignoreReturnCode: true, + }, + ); if (diskOnlyExitCode !== 0) { core.setFailed('Unable to start baked sccache with its disk-only cache.'); } @@ -77,9 +195,14 @@ runs: core.exportVariable('SCCACHE_GHA_ENABLED', 'false'); core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk'); - const localExitCode = await exec.exec('sccache', ['--start-server'], { - ignoreReturnCode: true, - }); + const localExitCode = await exec.exec( + 'sccache', + ['--start-server'], + { + env: diskOnlyEnvironment(), + ignoreReturnCode: true, + }, + ); if (localExitCode !== 0) { core.setFailed('Unable to start baked sccache with either remote or job-local storage.'); } diff --git a/.github/actions/prepare-host-input/action.yml b/.github/actions/prepare-host-input/action.yml new file mode 100644 index 0000000000..fffd2dc372 --- /dev/null +++ b/.github/actions/prepare-host-input/action.yml @@ -0,0 +1,142 @@ +name: Prepare backend-neutral host input +description: Build, optionally attest, and verify one immutable MeshLLM host input. + +inputs: + profile: + description: Host build profile (debug, dev, or release). + required: false + default: release + output_dir: + description: Directory that receives the immutable host input. + required: false + default: host-input + binary_name: + description: Filename used inside the host input. + required: false + default: mesh-llm + skip_ui: + description: Skip rebuilding the React UI when a prepared dist tree is available. + required: false + default: "false" + build_version: + description: Optional build version override. + required: false + default: "" + attestation_signing_key_file: + description: Optional release-attestation signing key path. + required: false + default: "" + attestation_public_key_file: + description: Optional release-attestation public key path used to inspect the stamp. + required: false + default: "" + +outputs: + binary_path: + description: Absolute path to the prepared host executable. + value: ${{ steps.prepare.outputs.binary_path }} + imports_path: + description: Absolute path to the host import-policy report. + value: ${{ steps.prepare.outputs.imports_path }} + checksum_path: + description: Absolute path to the host checksum. + value: ${{ steps.prepare.outputs.checksum_path }} + attestation_verifier_path: + description: Optional prebuilt verifier copied beside an attested release host. + value: ${{ steps.prepare.outputs.attestation_verifier_path }} + +runs: + using: composite + steps: + - name: Build and prepare immutable host + id: prepare + shell: bash + env: + INPUT_PROFILE: ${{ inputs.profile }} + INPUT_OUTPUT_DIR: ${{ inputs.output_dir }} + INPUT_BINARY_NAME: ${{ inputs.binary_name }} + INPUT_SKIP_UI: ${{ inputs.skip_ui }} + INPUT_BUILD_VERSION: ${{ inputs.build_version }} + INPUT_ATTESTATION_SIGNING_KEY_FILE: ${{ inputs.attestation_signing_key_file }} + INPUT_ATTESTATION_PUBLIC_KEY_FILE: ${{ inputs.attestation_public_key_file }} + run: | + set -euo pipefail + + case "$INPUT_PROFILE" in + debug|dev) source_binary="target/debug/mesh-llm" ;; + release) source_binary="target/release/mesh-llm" ;; + *) + echo "unsupported host profile: $INPUT_PROFILE" >&2 + exit 1 + ;; + esac + + if [[ "$INPUT_SKIP_UI" == "true" ]]; then + export MESH_LLM_SKIP_UI=1 + fi + if [[ -n "$INPUT_BUILD_VERSION" ]]; then + export MESH_LLM_BUILD_VERSION="$INPUT_BUILD_VERSION" + fi + scripts/build-host.sh --profile "$INPUT_PROFILE" + + mkdir -p "$INPUT_OUTPUT_DIR" + binary_path="$INPUT_OUTPUT_DIR/$INPUT_BINARY_NAME" + imports_path="$INPUT_OUTPUT_DIR/host-imports.json" + checksum_path="$INPUT_OUTPUT_DIR/$INPUT_BINARY_NAME.sha256" + cp "$source_binary" "$binary_path" + chmod +x "$binary_path" + + if [[ -n "$INPUT_ATTESTATION_SIGNING_KEY_FILE" || -n "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" ]]; then + if [[ -z "$INPUT_ATTESTATION_SIGNING_KEY_FILE" || -z "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" ]]; then + echo "both attestation key files are required when release attestation is enabled" >&2 + exit 1 + fi + cargo build -q -p xtask --bin xtask + cargo_target_dir="$( + cargo metadata --no-deps --format-version 1 | + python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])' + )" + attestation_verifier="$cargo_target_dir/debug/xtask" + "$attestation_verifier" release-attestation stamp \ + --binary "$binary_path" \ + --signing-key-file "$INPUT_ATTESTATION_SIGNING_KEY_FILE" + "$attestation_verifier" release-attestation inspect \ + --binary "$binary_path" \ + --public-key-file "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" \ + --json + cp "$attestation_verifier" "$INPUT_OUTPUT_DIR/release-attestation-verifier" + chmod +x "$INPUT_OUTPUT_DIR/release-attestation-verifier" + fi + + python3 scripts/verify-host-dependencies.py \ + "$binary_path" \ + --report "$imports_path" + if command -v sha256sum >/dev/null 2>&1; then + checksum="$(sha256sum "$binary_path" | awk '{print $1}')" + else + checksum="$(shasum -a 256 "$binary_path" | awk '{print $1}')" + fi + printf '%s %s\n' "$checksum" "$INPUT_BINARY_NAME" > "$checksum_path" + attestation_verifier_path="" + if [[ -f "$INPUT_OUTPUT_DIR/release-attestation-verifier" ]]; then + attestation_verifier_path="$INPUT_OUTPUT_DIR/release-attestation-verifier" + if command -v sha256sum >/dev/null 2>&1; then + verifier_checksum="$(sha256sum "$attestation_verifier_path" | awk '{print $1}')" + else + verifier_checksum="$(shasum -a 256 "$attestation_verifier_path" | awk '{print $1}')" + fi + printf '%s release-attestation-verifier\n' "$verifier_checksum" \ + > "$INPUT_OUTPUT_DIR/release-attestation-verifier.sha256" + fi + + output_dir="$(cd "$INPUT_OUTPUT_DIR" && pwd)" + { + echo "binary_path=$output_dir/$INPUT_BINARY_NAME" + echo "imports_path=$output_dir/host-imports.json" + echo "checksum_path=$output_dir/$INPUT_BINARY_NAME.sha256" + if [[ -n "$attestation_verifier_path" ]]; then + echo "attestation_verifier_path=$output_dir/release-attestation-verifier" + else + echo "attestation_verifier_path=" + fi + } >> "$GITHUB_OUTPUT" diff --git a/.github/actions/prepare-native-runtime-input/action.yml b/.github/actions/prepare-native-runtime-input/action.yml new file mode 100644 index 0000000000..d2f51697ac --- /dev/null +++ b/.github/actions/prepare-native-runtime-input/action.yml @@ -0,0 +1,77 @@ +name: Prepare native runtime input +description: Build, package, and verify one immutable MeshLLM native runtime input. + +inputs: + backend: + description: Native runtime backend. + required: true + target: + description: Optional Rust target triple. + required: false + default: "" + output_dir: + description: Directory that receives the runtime directory and archive. + required: false + default: dist/native-runtimes + build: + description: Build the native libraries before packaging. + required: false + default: "true" + +outputs: + runtime_dir: + description: Absolute path to the verified runtime directory. + value: ${{ steps.prepare.outputs.runtime_dir }} + archive_path: + description: Absolute path to the verified runtime archive. + value: ${{ steps.prepare.outputs.archive_path }} + checksum_path: + description: Absolute path to the runtime archive checksum. + value: ${{ steps.prepare.outputs.checksum_path }} + +runs: + using: composite + steps: + - name: Package and verify native runtime + id: prepare + shell: bash + env: + INPUT_BACKEND: ${{ inputs.backend }} + INPUT_TARGET: ${{ inputs.target }} + INPUT_OUTPUT_DIR: ${{ inputs.output_dir }} + INPUT_BUILD: ${{ inputs.build }} + run: | + set -euo pipefail + + args=(--backend "$INPUT_BACKEND" --out "$INPUT_OUTPUT_DIR") + if [[ "$INPUT_BUILD" == "true" ]]; then + args=(--build "${args[@]}") + fi + if [[ -n "$INPUT_TARGET" ]]; then + args+=(--target "$INPUT_TARGET") + fi + scripts/package-native-runtime.sh "${args[@]}" + + runtime_dirs=() + while IFS= read -r manifest; do + runtime_dirs+=("$(dirname "$manifest")") + done < <(find "$INPUT_OUTPUT_DIR" -mindepth 2 -maxdepth 2 -type f -name manifest.json -print) + if [[ "${#runtime_dirs[@]}" -ne 1 ]]; then + echo "expected exactly one native runtime in $INPUT_OUTPUT_DIR; found ${#runtime_dirs[@]}" >&2 + exit 1 + fi + + runtime_dir="${runtime_dirs[0]}" + archive_path="$runtime_dir.tar.gz" + checksum_path="$archive_path.sha256" + test -s "$archive_path" + test -s "$checksum_path" + scripts/verify-native-runtime-package.sh "$runtime_dir" "$archive_path" + + runtime_parent="$(cd "$(dirname "$runtime_dir")" && pwd)" + runtime_dir="$runtime_parent/$(basename "$runtime_dir")" + { + echo "runtime_dir=$runtime_dir" + echo "archive_path=$runtime_dir.tar.gz" + echo "checksum_path=$runtime_dir.tar.gz.sha256" + } >> "$GITHUB_OUTPUT" diff --git a/.github/actions/prepare-windows-host-input/action.yml b/.github/actions/prepare-windows-host-input/action.yml new file mode 100644 index 0000000000..ea59a78403 --- /dev/null +++ b/.github/actions/prepare-windows-host-input/action.yml @@ -0,0 +1,180 @@ +name: Prepare backend-neutral Windows host input +description: Build, optionally attest, import-check, and checksum one immutable Windows host. + +inputs: + profile: + description: Windows host build profile (debug or release). + required: false + default: release + skip_ui: + description: Use a minimal prepared UI tree instead of rebuilding the React console. + required: false + default: "false" + attestation_signing_key_file: + description: Optional release-attestation signing key path. + required: false + default: "" + attestation_public_key_file: + description: Optional release-attestation public key path used to inspect the stamp. + required: false + default: "" + +outputs: + binary_path: + description: Absolute path to the prepared Windows host executable. + value: ${{ steps.prepare.outputs.binary_path }} + imports_path: + description: Absolute path to the host import-policy report. + value: ${{ steps.prepare.outputs.imports_path }} + checksum_path: + description: Absolute path to the host checksum. + value: ${{ steps.prepare.outputs.checksum_path }} + attestation_verifier_path: + description: Optional prebuilt verifier copied beside an attested release host. + value: ${{ steps.prepare.outputs.attestation_verifier_path }} + +runs: + using: composite + steps: + - name: Build and prepare immutable Windows host + id: prepare + shell: pwsh + env: + INPUT_PROFILE: ${{ inputs.profile }} + INPUT_SKIP_UI: ${{ inputs.skip_ui }} + INPUT_ATTESTATION_SIGNING_KEY_FILE: ${{ inputs.attestation_signing_key_file }} + INPUT_ATTESTATION_PUBLIC_KEY_FILE: ${{ inputs.attestation_public_key_file }} + run: | + $ErrorActionPreference = "Stop" + + $profile = "$env:INPUT_PROFILE".Trim().ToLowerInvariant() + if ($profile -notin @("debug", "release")) { + throw "unsupported Windows host profile: $profile" + } + if ($env:INPUT_SKIP_UI -notin @("true", "false")) { + throw "INPUT_SKIP_UI must be true or false" + } + + $signingKey = "$env:INPUT_ATTESTATION_SIGNING_KEY_FILE".Trim() + $publicKey = "$env:INPUT_ATTESTATION_PUBLIC_KEY_FILE".Trim() + if ([string]::IsNullOrWhiteSpace($signingKey) -ne [string]::IsNullOrWhiteSpace($publicKey)) { + throw "both attestation key files are required when release attestation is enabled" + } + + if ($profile -eq "debug") { + $pkgid = (& cargo pkgid -p mesh-llm).Trim() + if ($LASTEXITCODE -ne 0 -or -not $pkgid.Contains("#")) { + throw "failed to derive the debug host MeshLLM version" + } + $env:MESH_LLM_BUILD_VERSION = $pkgid.Substring($pkgid.LastIndexOf("#") + 1) + } + + if ($env:INPUT_SKIP_UI -eq "true") { + $env:MESH_LLM_SKIP_UI = "1" + New-Item -ItemType Directory -Force -Path crates/mesh-llm-ui/dist | Out-Null + "" | Set-Content -Path crates/mesh-llm-ui/dist/index.html -Encoding utf8 + } else { + Remove-Item Env:MESH_LLM_SKIP_UI -ErrorAction SilentlyContinue + } + + & .\scripts\build-windows.ps1 -BuildProfile $profile -HostOnly + if ($LASTEXITCODE -ne 0) { + throw "backend-neutral Windows host build failed" + } + + $sourceProfile = if ($profile -eq "release") { "release" } else { "debug" } + $sourceBinary = Join-Path "target" "$sourceProfile\mesh-llm.exe" + if (-not (Test-Path -LiteralPath $sourceBinary -PathType Leaf)) { + throw "Windows host build did not produce $sourceBinary" + } + + $outputDir = "host-input" + New-Item -ItemType Directory -Force -Path $outputDir | Out-Null + $binaryPath = Join-Path $outputDir "mesh-llm.exe" + $importsPath = Join-Path $outputDir "host-imports.json" + $checksumPath = Join-Path $outputDir "mesh-llm.exe.sha256" + Copy-Item -LiteralPath $sourceBinary -Destination $binaryPath -Force + + $attestationVerifierPath = "" + if ($signingKey -and $publicKey) { + foreach ($keyFile in @($signingKey, $publicKey)) { + if (-not (Test-Path -LiteralPath $keyFile -PathType Leaf) -or + (Get-Item -LiteralPath $keyFile).Length -eq 0) { + throw "release attestation key file is empty or missing: $keyFile" + } + } + + & cargo build -q -p xtask --bin xtask + if ($LASTEXITCODE -ne 0) { + throw "failed to build the release attestation verifier" + } + $cargoMetadata = & cargo metadata --no-deps --format-version 1 | ConvertFrom-Json + if ($LASTEXITCODE -ne 0) { + throw "failed to resolve the Cargo target directory" + } + $verifier = Join-Path $cargoMetadata.target_directory "debug\xtask.exe" + if (-not (Test-Path -LiteralPath $verifier -PathType Leaf)) { + throw "release attestation verifier was not built: $verifier" + } + + & $verifier release-attestation stamp ` + --binary $binaryPath ` + --signing-key-file $signingKey + if ($LASTEXITCODE -ne 0) { + throw "failed to stamp the Windows release host" + } + & $verifier release-attestation inspect ` + --binary $binaryPath ` + --public-key-file $publicKey ` + --json + if ($LASTEXITCODE -ne 0) { + throw "failed to verify the Windows release host attestation" + } + + $attestationVerifierPath = Join-Path $outputDir "release-attestation-verifier.exe" + Copy-Item -LiteralPath $verifier -Destination $attestationVerifierPath -Force + } + + & python scripts\verify-host-dependencies.py $binaryPath --report $importsPath + if ($LASTEXITCODE -ne 0) { + throw "backend-neutral host dependency verification failed" + } + + function Get-Sha256Hex { + param([string]$Path) + + $sha256 = [System.Security.Cryptography.SHA256]::Create() + try { + $stream = [System.IO.File]::OpenRead($Path) + try { + $bytes = $sha256.ComputeHash($stream) + } finally { + $stream.Dispose() + } + } finally { + $sha256.Dispose() + } + return [System.BitConverter]::ToString($bytes).Replace("-", "").ToLowerInvariant() + } + + $hostHash = Get-Sha256Hex $binaryPath + Set-Content -Path $checksumPath -Value "$hostHash mesh-llm.exe" -NoNewline + if ($attestationVerifierPath) { + $verifierHash = Get-Sha256Hex $attestationVerifierPath + Set-Content ` + -Path "$attestationVerifierPath.sha256" ` + -Value "$verifierHash release-attestation-verifier.exe" ` + -NoNewline + } + + $absoluteOutputDir = [System.IO.Path]::GetFullPath($outputDir) + @( + "binary_path=$absoluteOutputDir\mesh-llm.exe" + "imports_path=$absoluteOutputDir\host-imports.json" + "checksum_path=$absoluteOutputDir\mesh-llm.exe.sha256" + if ($attestationVerifierPath) { + "attestation_verifier_path=$absoluteOutputDir\release-attestation-verifier.exe" + } else { + "attestation_verifier_path=" + } + ) | Add-Content -Path $env:GITHUB_OUTPUT diff --git a/.github/actions/restore-smoke-inputs/action.yml b/.github/actions/restore-smoke-inputs/action.yml index 98abf552b6..8d815b0ef5 100644 --- a/.github/actions/restore-smoke-inputs/action.yml +++ b/.github/actions/restore-smoke-inputs/action.yml @@ -40,11 +40,56 @@ runs: using: composite steps: - name: Download built mesh-llm artifact - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: ${{ inputs.artifact_name }} path: ${{ inputs.artifact_path }} + - name: Extract and verify composed product + shell: bash + run: | + set -euo pipefail + artifact_path="${{ inputs.artifact_path }}" + archive="" + archive_count=0 + while IFS= read -r candidate; do + archive="$candidate" + archive_count=$((archive_count + 1)) + done < <(find "$artifact_path" -mindepth 1 -maxdepth 1 -type f -name '*.tar.gz' -print) + if [[ "$archive_count" -ne 1 ]]; then + echo "expected exactly one composed product archive in $artifact_path; found $archive_count" >&2 + exit 1 + fi + tar -xzf "$archive" -C "$artifact_path" + rm -f -- "$archive" + + product_manifest="$artifact_path/product-manifest.json" + test -s "$product_manifest" + IFS=$'\t' read -r version backend host_path runtime_path < <( + python3 - "$product_manifest" <<'PY' + import json + import sys + + with open(sys.argv[1], encoding="utf-8") as handle: + manifest = json.load(handle) + print( + manifest["mesh_version"], + manifest["backend"], + manifest["host"]["path"], + manifest["runtime"]["path"], + sep="\t", + ) + PY + ) + scripts/verify-native-runtime-package.sh "$artifact_path/$runtime_path" + python3 scripts/compose-product-bundle.py \ + --bundle "$artifact_path" \ + --host "$artifact_path/$host_path" \ + --runtime "$artifact_path/$runtime_path" \ + --version "$version" \ + --backend "$backend" \ + --check + - name: Stage mesh-llm binary shell: bash run: | @@ -72,8 +117,9 @@ runs: } - name: Restore integration model cache + if: ${{ inputs.model_url != '' && inputs.model_file != '' }} id: cache-model - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/.models/${{ inputs.model_file }} key: ${{ inputs.cache_key_prefix }}mesh-llm-${{ runner.os }}-${{ inputs.model_cache_scope }}-${{ inputs.model_file }}-${{ hashFiles('.github/cache-version.txt') }} @@ -81,6 +127,7 @@ runs: ${{ inputs.cache_key_prefix }}mesh-llm-${{ runner.os }}-${{ inputs.model_cache_scope }}-${{ inputs.model_file }}- - name: Check restored integration model + if: ${{ inputs.model_url != '' && inputs.model_file != '' }} id: model-file shell: bash run: | @@ -93,7 +140,7 @@ runs: fi - name: Download integration model - if: ${{ steps.model-file.outputs.present != 'true' }} + if: ${{ inputs.model_url != '' && inputs.model_file != '' && steps.model-file.outputs.present != 'true' }} shell: bash run: | set -euo pipefail @@ -122,8 +169,8 @@ runs: ls -lh "$output" - name: Save integration model cache - if: ${{ inputs.save_model_cache == 'true' && steps.cache-model.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v5 + if: ${{ inputs.model_url != '' && inputs.model_file != '' && inputs.save_model_cache == 'true' && steps.cache-model.outputs.cache-hit != 'true' }} + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/.models/${{ inputs.model_file }} key: ${{ steps.cache-model.outputs.cache-primary-key }} diff --git a/.github/actions/select-ci-runners/action.yml b/.github/actions/select-ci-runners/action.yml new file mode 100644 index 0000000000..57f9ba98ab --- /dev/null +++ b/.github/actions/select-ci-runners/action.yml @@ -0,0 +1,113 @@ +name: Select trusted CI runners +description: Select Depot or GitHub-hosted Linux runners without routing pull-request events to Depot. + +inputs: + event_name: + description: GitHub event name. + required: true + ref: + description: Fully qualified Git ref used to enforce the trusted-main boundary. + required: true + depot_main_enabled: + description: Repository-wide Depot gate for trusted push and dispatch events. + required: false + default: "false" + manual_use_depot: + description: Typed workflow-dispatch canary selection. + required: false + default: "false" + +outputs: + depot_enabled: + description: Whether the selected event may use Depot. + value: ${{ steps.select.outputs.depot_enabled }} + allow_depot_remote_cache: + description: Whether this trusted runner selection may use Depot's remote cache. + value: ${{ steps.select.outputs.allow_depot_remote_cache }} + runner: + description: Default two-vCPU Linux runner label. + value: ${{ steps.select.outputs.runner }} + runner_4: + description: Four-vCPU Linux runner label. + value: ${{ steps.select.outputs.runner_4 }} + runner_8: + description: Eight-vCPU Linux runner label. + value: ${{ steps.select.outputs.runner_8 }} + runner_16: + description: Sixteen-vCPU Linux runner label for measured high-parallelism builds. + value: ${{ steps.select.outputs.runner_16 }} + +runs: + using: composite + steps: + - name: Apply runner trust policy + id: select + shell: bash + env: + INPUT_EVENT_NAME: ${{ inputs.event_name }} + INPUT_REF: ${{ inputs.ref }} + INPUT_DEPOT_MAIN_ENABLED: ${{ inputs.depot_main_enabled }} + INPUT_MANUAL_USE_DEPOT: ${{ inputs.manual_use_depot }} + run: | + set -euo pipefail + + for value_name in \ + INPUT_DEPOT_MAIN_ENABLED \ + INPUT_MANUAL_USE_DEPOT + do + value="${!value_name}" + if [[ "$value" != "true" && "$value" != "false" ]]; then + echo "$value_name must be true or false, got: $value" >&2 + exit 1 + fi + done + + depot_enabled=false + case "$INPUT_EVENT_NAME" in + pull_request|pull_request_target) + # Current PR workflows and local actions are evaluated from + # PR-controlled code. A repository variable or same-repo check + # therefore cannot establish a runner trust boundary. + depot_enabled=false + ;; + workflow_dispatch) + if [[ "$INPUT_REF" == "refs/heads/main" && ( \ + "$INPUT_DEPOT_MAIN_ENABLED" == "true" || \ + "$INPUT_MANUAL_USE_DEPOT" == "true" \ + ) ]]; then + depot_enabled=true + fi + ;; + push) + if [[ \ + "$INPUT_REF" == "refs/heads/main" && \ + "$INPUT_DEPOT_MAIN_ENABLED" == "true" \ + ]]; then + depot_enabled=true + fi + ;; + *) + depot_enabled=false + ;; + esac + + if [[ "$depot_enabled" == "true" ]]; then + runner=depot-ubuntu-24.04 + runner_4=depot-ubuntu-24.04-4 + runner_8=depot-ubuntu-24.04-8 + runner_16=depot-ubuntu-24.04-16 + else + runner=ubuntu-24.04 + runner_4=ubuntu-24.04 + runner_8=ubuntu-24.04 + runner_16=ubuntu-24.04 + fi + + { + echo "depot_enabled=$depot_enabled" + echo "allow_depot_remote_cache=$depot_enabled" + echo "runner=$runner" + echo "runner_4=$runner_4" + echo "runner_8=$runner_8" + echo "runner_16=$runner_16" + } >> "$GITHUB_OUTPUT" diff --git a/.github/actions/setup-windows-rocm-sdk/action.yml b/.github/actions/setup-windows-rocm-sdk/action.yml index e62d64e3e6..2008800035 100644 --- a/.github/actions/setup-windows-rocm-sdk/action.yml +++ b/.github/actions/setup-windows-rocm-sdk/action.yml @@ -16,7 +16,7 @@ runs: steps: - name: Cache HIP SDK installers id: installer-cache - uses: actions/cache@v5 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ inputs.installer-cache-dir }}/rocm key: windows-ci-sdk-installers-rocm-${{ inputs.rocm-hip-sdk-filename }}-v1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2a1bd1cf2..b13855a80b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,12 @@ name: CI on: workflow_dispatch: + inputs: + use_depot: + description: Route eligible non-GPU jobs to Depot for rollout benchmarking. + required: false + default: false + type: boolean push: branches: [main] @@ -53,14 +59,29 @@ jobs: inference_artifact_required: ${{ steps.compute.outputs.inference_artifact_required }} backend_recipe_changed: ${{ steps.compute.outputs.backend_recipe_changed }} sdk_smoke_required: ${{ steps.compute.outputs.sdk_smoke_required }} + runner_contract_required: ${{ steps.compute.outputs.runner_contract_required }} ui_dist_cache_key: ${{ steps.ui_key.outputs.ui_dist_cache_key }} linux_inference_artifact_required: ${{ github.event_name == 'workflow_dispatch' || steps.compute.outputs.inference_artifact_required == 'true' }} + depot_enabled: ${{ steps.runners.outputs.depot_enabled }} + allow_depot_remote_cache: ${{ steps.runners.outputs.allow_depot_remote_cache }} + runner: ${{ steps.runners.outputs.runner }} + runner_4: ${{ steps.runners.outputs.runner_4 }} + runner_8: ${{ steps.runners.outputs.runner_8 }} + runner_16: ${{ steps.runners.outputs.runner_16 }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false fetch-depth: 0 - - uses: dorny/paths-filter@v4 + - name: Select trusted Linux runners + id: runners + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + ref: ${{ github.ref }} + depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + manual_use_depot: ${{ inputs.use_depot == true }} + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 id: filter with: filters: | @@ -76,6 +97,7 @@ jobs: - '.github/workflows/ci.yml' - '.github/workflows/smoke.yml' - '.github/actions/compute-changes/action.yml' + - '.github/actions/prepare-windows-host-input/**' ui: - 'crates/mesh-llm-ui/**' @@ -138,6 +160,7 @@ jobs: - 'Cargo.lock' - 'Justfile' - '.github/cache-version.txt' + - '.github/actions/prepare-windows-host-input/**' - '.github/workflows/ci.yml' - '.github/workflows/windows-warm-caches.yml' windows_gpu: @@ -151,6 +174,7 @@ jobs: - 'third_party/llama.cpp/**' - 'Justfile' - '.github/cache-version.txt' + - '.github/actions/prepare-windows-host-input/**' - '.github/actions/setup-windows-rocm-sdk/**' - '.github/actions/compute-changes/action.yml' - '.github/workflows/ci.yml' @@ -173,11 +197,41 @@ jobs: HASH=$(git ls-files -s crates/mesh-llm-ui .github/cache-version.txt | git hash-object --stdin) echo "ui_dist_cache_key=${CACHE_NAMESPACE}-ui-dist-${HASH}" >> "$GITHUB_OUTPUT" - linux_cpu_artifact: + arc_runner_image_contract: needs: changes - if: ${{ needs.changes.outputs.linux_inference_artifact_required == 'true' && needs.changes.outputs.docs_only != 'true' }} - name: Linux CPU - runs-on: ubuntu-24.04 + if: ${{ github.ref == 'refs/heads/main' && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.runner_contract_required == 'true') }} + name: Runner image / ARC ${{ matrix.architecture }} + runs-on: ${{ matrix.runner }} + permissions: + contents: read + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runner: mesh-llm-amd64 + machine: x86_64 + - architecture: arm64 + runner: mesh-llm-arm64 + machine: aarch64 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Verify prebuilt ARC environment + env: + EXPECTED_MACHINE: ${{ matrix.machine }} + run: | + test "$(uname -m)" = "$EXPECTED_MACHINE" + verify-runner-image self-hosted + cargo check -p mesh-llm-config + + linux_host_input: + needs: changes + if: ${{ needs.changes.outputs.docs_only != 'true' }} + name: Linux immutable release host + runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read packages: read @@ -190,18 +244,20 @@ jobs: run: shell: bash env: - LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-dynamic-cpu MESH_LLM_SKIP_UI: "1" MESH_LLM_REQUIRE_SCCACHE: "1" RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Verify prebuilt CI environment run: verify-runner-image public + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - name: Build UI if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} @@ -215,7 +271,7 @@ jobs: - name: Save UI dist cache if: ${{ github.ref == 'refs/heads/main' && needs.changes.outputs.ui == 'true' }} - uses: actions/cache/save@v4 + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: crates/mesh-llm-ui/dist key: ${{ needs.changes.outputs.ui_dist_cache_key }} @@ -228,13 +284,13 @@ jobs: rustflags = ["-C", "link-arg=-fuse-ld=lld"] EOF - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: workspaces: . -> target cache-bin: "false" prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} - shared-key: linux + shared-key: linux-release-host save-if: ${{ github.ref == 'refs/heads/main' }} - name: Check release-target repo consistency @@ -250,28 +306,24 @@ jobs: exit 1 fi - - name: Build backend-neutral debug host - run: scripts/build-host.sh --profile debug - - - name: Package CPU native runtime beside debug host - run: scripts/package-native-runtime.sh --build --backend cpu --out target/debug/native-runtimes - - - name: CLI smoke test + - name: Prepare UI placeholder run: | - target/debug/mesh-llm --log-format json --version - target/debug/mesh-llm --log-format json --help | head -5 - target/debug/mesh-llm --log-format json runtime list + mkdir -p crates/mesh-llm-ui/dist + test -f crates/mesh-llm-ui/dist/index.html || + printf '' > crates/mesh-llm-ui/dist/index.html - - name: Composed debug client readiness smoke - run: scripts/ci-client-readiness-smoke.sh target/debug/mesh-llm target/debug/native-runtimes + - name: Prepare backend-neutral release host + uses: ./.github/actions/prepare-host-input + with: + profile: release + output_dir: host-input + skip_ui: "true" - - name: Upload Linux composed debug product - uses: actions/upload-artifact@v6 + - name: Upload immutable host input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: - name: ci-linux-inference-binaries - path: | - target/debug/mesh-llm - target/debug/native-runtimes/** + name: ci-linux-host-input + path: host-input/ if-no-files-found: error retention-days: 1 @@ -289,11 +341,170 @@ jobs: echo "::warning::sccache reported zero compile requests; check RUSTC_WRAPPER wiring if this was not a fully reused target cache." fi - rust_crate_tests: + linux_cpu_runtime_input: needs: changes - if: ${{ needs.changes.outputs.rust_changed == 'true' && needs.changes.outputs.docs_only != 'true' }} + if: ${{ needs.changes.outputs.docs_only != 'true' }} + name: Linux CPU runtime + runs-on: ${{ needs.changes.outputs.runner_8 }} + permissions: + contents: read + packages: read + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + env: + LLAMA_STAGE_BACKEND: cpu + LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-dynamic-cpu + MESH_LLM_REQUIRE_SCCACHE: "1" + SCCACHE_GHA_ENABLED: "false" + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - name: Verify prebuilt CI environment + run: verify-runner-image public + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} + + - name: Prepare CPU native runtime + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: cpu + target: x86_64-unknown-linux-gnu + output_dir: runtime-input + + - name: Upload immutable CPU runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-linux-cpu-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + - name: Show sccache stats + if: ${{ always() }} + run: sccache --show-stats || true + + linux_cpu_artifact: + needs: [changes, linux_host_input, linux_cpu_runtime_input] + if: ${{ needs.linux_host_input.result == 'success' && needs.linux_cpu_runtime_input.result == 'success' }} + name: Linux CPU product + runs-on: ${{ needs.changes.outputs.runner_4 }} + permissions: + contents: read + packages: read + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Verify prebuilt CI environment + run: verify-runner-image public + - name: Download immutable host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-linux-host-input + path: host-input + - name: Download immutable CPU runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-linux-cpu-runtime-input + path: runtime-input + + - name: Compose CPU product from producer inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: ci-product + backend: cpu + + - name: Upload Linux composed product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-linux-inference-binaries + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 + + linux_static_abi_input: + needs: changes + if: ${{ needs.changes.outputs.docs_only != 'true' && (needs.changes.outputs.test_batches_json != '[]' || needs.changes.outputs.linux_test_groups_json != '[]') }} + name: Linux static ABI input + runs-on: ${{ needs.changes.outputs.runner_8 }} + permissions: + contents: read + packages: read + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + env: + LLAMA_STAGE_BACKEND: cpu + LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static + MESH_LLM_REQUIRE_SCCACHE: "1" + SCCACHE_GHA_ENABLED: "false" + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Verify prebuilt CI environment + run: verify-runner-image public + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} + - name: Cache patched llama.cpp ABI build + id: static_abi_cache + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: .deps/llama.cpp/build-stage-abi-static + key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-cpu-${{ hashFiles('scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} + - name: Prepare patched llama.cpp ABI checkout + if: ${{ steps.static_abi_cache.outputs.cache-hit != 'true' }} + run: scripts/prepare-llama.sh pinned + - name: Build patched llama.cpp ABI libraries + if: ${{ steps.static_abi_cache.outputs.cache-hit != 'true' }} + run: scripts/build-llama.sh + - name: Archive immutable static ABI input + run: | + test -d "$LLAMA_STAGE_BUILD_DIR" + tar -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" \ + -czf "$RUNNER_TEMP/mesh-llm-static-abi.tar.gz" \ + "$(basename "$LLAMA_STAGE_BUILD_DIR")" + - name: Upload immutable static ABI input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-linux-static-abi-input + path: ${{ runner.temp }}/mesh-llm-static-abi.tar.gz + if-no-files-found: error + retention-days: 1 + + rust_crate_tests: + needs: [changes, linux_static_abi_input] + if: ${{ needs.linux_static_abi_input.result == 'success' && needs.changes.outputs.test_batches_json != '[]' && needs.changes.outputs.docs_only != 'true' }} name: Rust crate tests (${{ matrix.batch.idx }}) - runs-on: ubuntu-24.04 + runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read packages: read @@ -318,12 +529,15 @@ jobs: RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Verify prebuilt CI environment run: verify-runner-image public - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: workspaces: . -> target @@ -336,20 +550,17 @@ jobs: - name: Prepare patched llama.cpp source checkout if: ${{ contains(matrix.batch.crates, 'skippy-quantize') }} run: scripts/prepare-llama.sh pinned - - name: Ensure ABI cache directory - run: mkdir -p "$LLAMA_STAGE_BUILD_DIR" - - name: Cache patched llama.cpp ABI build - id: rust_tests_llama_cache - uses: actions/cache@v5 + - name: Download immutable static ABI input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: - path: .deps/llama.cpp/build-stage-abi-static - key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-cpu-${{ hashFiles('scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - - name: Prepare patched llama.cpp ABI checkout - if: ${{ steps.rust_tests_llama_cache.outputs.cache-hit != 'true' }} - run: scripts/prepare-llama.sh pinned - - name: Build patched llama.cpp ABI libraries - if: ${{ steps.rust_tests_llama_cache.outputs.cache-hit != 'true' }} - run: scripts/build-llama.sh + name: ci-linux-static-abi-input + path: ${{ runner.temp }}/static-abi-input + - name: Restore immutable static ABI input + run: | + mkdir -p "$(dirname "$LLAMA_STAGE_BUILD_DIR")" + tar -xzf "$RUNNER_TEMP/static-abi-input/mesh-llm-static-abi.tar.gz" \ + -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" + test -d "$LLAMA_STAGE_BUILD_DIR" - name: Run crate tests env: TEST_CRATES: ${{ toJson(matrix.batch.crates) }} @@ -365,10 +576,10 @@ jobs: run: sccache --show-stats || true linux_test_groups: - needs: changes - if: ${{ needs.changes.outputs.docs_only != 'true' && needs.changes.outputs.linux_test_groups_json != '[]' }} + needs: [changes, linux_static_abi_input] + if: ${{ needs.linux_static_abi_input.result == 'success' && needs.changes.outputs.docs_only != 'true' && needs.changes.outputs.linux_test_groups_json != '[]' }} name: Linux tests (${{ matrix.group }}) - runs-on: ubuntu-24.04 + runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read packages: read @@ -394,12 +605,15 @@ jobs: RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Verify prebuilt CI environment run: verify-runner-image public + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - name: Configure Linux Rust linker run: | @@ -409,7 +623,7 @@ jobs: rustflags = ["-C", "link-arg=-fuse-ld=lld"] EOF - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: workspaces: . -> target @@ -418,28 +632,22 @@ jobs: shared-key: ${{ matrix.cache_key }} save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Ensure ABI cache directory - run: mkdir -p "$LLAMA_STAGE_BUILD_DIR" - - - name: Cache patched llama.cpp ABI build - id: llama_cache - uses: actions/cache@v5 + - name: Download immutable static ABI input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: - path: .deps/llama.cpp/build-stage-abi-static - key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-cpu-${{ hashFiles('scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - - - name: Prepare patched llama.cpp ABI checkout - if: ${{ steps.llama_cache.outputs.cache-hit != 'true' }} - run: scripts/prepare-llama.sh pinned - - - name: Build patched llama.cpp ABI libraries - if: ${{ steps.llama_cache.outputs.cache-hit != 'true' }} - run: scripts/build-llama.sh + name: ci-linux-static-abi-input + path: ${{ runner.temp }}/static-abi-input + - name: Restore immutable static ABI input + run: | + mkdir -p "$(dirname "$LLAMA_STAGE_BUILD_DIR")" + tar -xzf "$RUNNER_TEMP/static-abi-input/mesh-llm-static-abi.tar.gz" \ + -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" + test -d "$LLAMA_STAGE_BUILD_DIR" - name: Restore Skippy smoke model cache if: ${{ matrix.group == 'skippy-smoke' }} id: skippy_smoke_model_cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ runner.temp }}/skippy-ci-smoke-models key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-ci-smoke-models-SmolLM2-135M-Instruct.Q4_K_M.gguf-Falcon-H1-0.5B-Instruct-Q4_K_M.gguf-${{ hashFiles('.github/cache-version.txt') }} @@ -464,7 +672,7 @@ jobs: - name: Save Skippy smoke model cache if: ${{ matrix.group == 'skippy-smoke' && github.ref == 'refs/heads/main' && steps.skippy_smoke_model_cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v5 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ runner.temp }}/skippy-ci-smoke-models key: ${{ steps.skippy_smoke_model_cache.outputs.cache-primary-key }} @@ -483,26 +691,6 @@ jobs: echo "::warning::sccache reported zero compile requests; check RUSTC_WRAPPER wiring if this was not a fully reused target cache." fi - linux_client_readiness: - needs: [changes, linux_cpu_artifact] - if: ${{ needs.linux_cpu_artifact.result == 'success' && needs.changes.outputs.linux_inference_artifact_required == 'true' && needs.changes.outputs.docs_only != 'true' }} - name: Linux composed client readiness - runs-on: ubuntu-24.04 - permissions: - contents: read - steps: - - uses: actions/checkout@v5 - with: - persist-credentials: false - - uses: actions/download-artifact@v6 - with: - name: ci-linux-inference-binaries - path: target/debug - - name: Make mesh-llm executable - run: chmod +x target/debug/mesh-llm - - name: Verify composed client readiness - run: scripts/ci-client-readiness-smoke.sh target/debug/mesh-llm target/debug/native-runtimes - hf_download_smoke: needs: changes if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'model-artifact')) && needs.changes.outputs.docs_only != 'true' }} @@ -511,6 +699,7 @@ jobs: contents: read uses: ./.github/workflows/hf-download-smoke.yml with: + runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} timeout_minutes: 15 secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -523,15 +712,16 @@ jobs: uses: ./.github/workflows/smoke.yml with: artifact_name: ci-linux-inference-binaries - mesh_binary_target: target/debug/mesh-llm + mesh_binary_target: target/release/mesh-llm cache_key_prefix: '' + runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} agent_live_smokes: needs: [changes, linux_cpu_artifact] if: ${{ needs.linux_cpu_artifact.result == 'success' && (vars.MESH_AGENT_BASE_URL != '' || vars.MESH_OPENCODE_BASE_URL != '') && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'openai-frontend') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-client')) && needs.changes.outputs.docs_only != 'true' }} - runs-on: ubuntu-24.04 + runs-on: ${{ needs.changes.outputs.runner_4 }} permissions: contents: read timeout-minutes: 45 @@ -548,10 +738,10 @@ jobs: steps: # Keep live-agent smoke on an explicit Node version so CI does not depend # on GitHub-hosted image defaults. - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 24 - name: Install agent CLIs @@ -587,9 +777,10 @@ jobs: with: artifact_name: ci-linux-inference-binaries artifact_path: ci-artifacts/linux - staged_binary_path: target/debug/mesh-llm + staged_binary_path: target/release/mesh-llm model_cache_scope: two-node-smoke-model smoke_script: scripts/ci-two-node-client-serving-smoke.sh + runs_on: ${{ toJson(needs.changes.outputs.runner_4) }} timeout_minutes: 20 secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -604,9 +795,9 @@ jobs: sdk_kind: rust artifact_name: ci-linux-inference-binaries artifact_path: ci-artifacts/linux - staged_binary_path: target/debug/mesh-llm + staged_binary_path: target/release/mesh-llm model_cache_scope: sdk-smoke-model - runs_on: '"ubuntu-24.04"' + runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -620,15 +811,15 @@ jobs: sdk_kind: kotlin artifact_name: ci-linux-inference-binaries artifact_path: ci-artifacts/linux - staged_binary_path: target/debug/mesh-llm + staged_binary_path: target/release/mesh-llm model_cache_scope: sdk-smoke-model - runs_on: '"ubuntu-24.04"' + runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} swift_sdk_smoke: - needs: [changes, macos] - if: ${{ needs.macos.result == 'success' && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.sdk == 'true') && needs.changes.outputs.docs_only != 'true' }} + needs: [changes, macos_cpu_artifact, macos_unit_tests] + if: ${{ always() && needs.macos_cpu_artifact.result == 'success' && (needs.macos_unit_tests.result == 'success' || needs.macos_unit_tests.result == 'skipped') && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.sdk == 'true') && needs.changes.outputs.docs_only != 'true' }} permissions: contents: read uses: ./.github/workflows/sdk-smoke.yml @@ -636,31 +827,31 @@ jobs: sdk_kind: swift artifact_name: ci-macos-inference-binaries artifact_path: ci-artifacts/macos - staged_binary_path: target/debug/mesh-llm + staged_binary_path: target/release/mesh-llm model_cache_scope: sdk-smoke-model runs_on: '"macos-latest"' timeout_minutes: 40 secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} - macos: + macos_host_input: needs: changes if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.rust == 'true' || needs.changes.outputs.ui == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: macOS immutable release host runs-on: macos-latest permissions: contents: read env: - LLAMA_STAGE_BACKEND: metal MESH_LLM_SKIP_UI: "1" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} with: version: latest - - uses: actions/setup-node@v5 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} with: node-version: 24 @@ -671,7 +862,7 @@ jobs: - name: Restore UI dist cache id: ui-cache if: needs.changes.outputs.ui == 'true' || github.event_name == 'workflow_dispatch' - uses: actions/cache/restore@v4 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: crates/mesh-llm-ui/dist key: ${{ needs.changes.outputs.ui_dist_cache_key }} @@ -696,15 +887,144 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} working-directory: crates/mesh-llm-ui run: pnpm test - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 with: workspaces: . -> target cache-bin: "false" prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} shared-key: macos save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Install build dependencies + - name: Install host build dependencies + run: brew install jq lld + - name: Configure macOS Rust linker + run: | + mkdir -p .cargo + lld_prefix="$(brew --prefix lld)" + cat > .cargo/config.toml <' > crates/mesh-llm-ui/dist/index.html + + - name: Prepare backend-neutral release host + uses: ./.github/actions/prepare-host-input + with: + profile: release + output_dir: host-input + skip_ui: "true" + + - name: Upload immutable macOS host input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-macos-host-input + path: host-input/ + if-no-files-found: error + retention-days: 1 + + macos_metal_runtime_input: + needs: changes + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.rust == 'true' || needs.changes.outputs.ui == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: macOS Metal runtime + runs-on: macos-latest + permissions: + contents: read + env: + LLAMA_STAGE_BACKEND: metal + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-dynamic-metal + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Install native runtime dependencies + run: brew install cmake ninja jq + - name: Prepare Metal native runtime + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: metal + output_dir: runtime-input + + - name: Upload immutable Metal runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-macos-metal-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + macos_cpu_artifact: + needs: [changes, macos_host_input, macos_metal_runtime_input] + if: ${{ needs.macos_host_input.result == 'success' && needs.macos_metal_runtime_input.result == 'success' }} + name: macOS Metal release product + runs-on: macos-latest + permissions: + contents: read + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Download immutable macOS host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-macos-host-input + path: host-input + - name: Download immutable Metal runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-macos-metal-runtime-input + path: runtime-input + - name: Compose macOS Metal product + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: ci-product + backend: metal + + - name: Upload macOS composed release product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-macos-inference-binaries + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 + + macos_unit_tests: + needs: changes + if: ${{ needs.changes.outputs.docs_only != 'true' && (needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'model-artifact') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-host-runtime') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm')) }} + name: macOS unit tests + runs-on: macos-latest + permissions: + contents: read + env: + AFFECTED: ${{ needs.changes.outputs.affected_crates }} + ALL_RUST: ${{ needs.changes.outputs.all_rust }} + LLAMA_STAGE_BACKEND: metal + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-static-metal + MESH_LLM_SKIP_UI: "1" + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: macos-unit-tests + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Install unit-test dependencies run: brew install cmake ninja jq lld - name: Configure macOS Rust linker run: | @@ -717,15 +1037,21 @@ jobs: [target.x86_64-apple-darwin] rustflags = ["-C", "link-arg=-fuse-ld=$lld_prefix/bin/ld64.lld"] EOF - - name: Build backend-neutral debug host - run: scripts/build-host.sh --profile debug - - name: Package Metal native runtime beside debug host - run: scripts/package-native-runtime.sh --build --backend metal --out target/debug/native-runtimes + - name: Ensure static Metal ABI cache directory + run: mkdir -p "$LLAMA_STAGE_BUILD_DIR" + - name: Cache static Metal ABI build + id: llama_cache + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: ${{ env.LLAMA_STAGE_BUILD_DIR }} + key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-static-metal-${{ hashFiles('scripts/build-mac.sh', 'scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} + - name: Prepare patched llama.cpp ABI checkout + if: ${{ steps.llama_cache.outputs.cache-hit != 'true' }} + run: scripts/prepare-llama.sh pinned + - name: Build patched llama.cpp ABI libraries + if: ${{ steps.llama_cache.outputs.cache-hit != 'true' }} + run: scripts/build-llama.sh - name: Unit tests - if: ${{ needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'model-artifact') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-host-runtime') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm') }} - env: - AFFECTED: ${{ needs.changes.outputs.affected_crates }} - ALL_RUST: ${{ needs.changes.outputs.all_rust }} run: | should_test() { local crate="$1" @@ -739,28 +1065,12 @@ jobs: echo "Skipping $c on macOS (not affected)" fi done - - name: CLI smoke test - run: | - target/debug/mesh-llm --log-format json --version - target/debug/mesh-llm --log-format json --help | head -5 - target/debug/mesh-llm --log-format json runtime list - - name: Composed debug client readiness smoke - run: scripts/ci-client-readiness-smoke.sh target/debug/mesh-llm target/debug/native-runtimes - - name: Upload macOS composed debug product - uses: actions/upload-artifact@v6 - with: - name: ci-macos-inference-binaries - path: | - target/debug/mesh-llm - target/debug/native-runtimes/** - if-no-files-found: error - retention-days: 1 linux_cuda: - needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} - name: Linux CUDA slim - runs-on: ${{ fromJson(vars.USE_SELF_HOSTED == 'true' && '["self-hosted","Linux","X64","amd64","gpu-nvidia"]' || '["ubuntu-24.04"]') }} + needs: [changes, linux_host_input] + if: ${{ needs.changes.outputs.docs_only != 'true' }} + name: Linux CUDA compatibility + runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read packages: read @@ -770,55 +1080,62 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} env: - MESH_LLM_SKIP_UI: "1" + LLAMA_STAGE_BACKEND: cuda + LLAMA_STAGE_CUDA_ARCHITECTURES: "75;80;86;87;89;90" + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-dynamic-cuda-sm75_80_86_87_89_90 + MESH_CUDA_VERSION: "12.9.2" + MESH_LLM_CUDA_TOOLKIT_MAJOR: "12" # CI containers have no GPU driver (libcuda.so.1). Disable VMM and NCCL # to avoid linking libraries that require the CUDA driver at runtime. GGML_CUDA_NO_VMM: "1" LLAMA_STAGE_SKIP_NCCL: "1" MESH_LLM_REQUIRE_SCCACHE: "1" - RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Verify prebuilt backend environment run: verify-runner-image public cuda - - name: Configure Linux Rust linker - run: | - mkdir -p .cargo - cat > .cargo/config.toml <<'EOF' - [target.x86_64-unknown-linux-gnu] - rustflags = ["-C", "link-arg=-fuse-ld=lld"] - EOF - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/configure-sccache-gha with: - workspaces: . -> target - cache-bin: "false" - prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} - shared-key: linux-cuda-slim - save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Prepare UI placeholder - run: mkdir -p crates/mesh-llm-ui/dist && printf '' > crates/mesh-llm-ui/dist/index.html + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - - name: Build backend-neutral Linux release host - run: scripts/build-host.sh --profile release - - name: Package Linux CUDA native runtime - env: - MESH_CUDA_VERSION: "12.9.2" - MESH_LLM_CUDA_TOOLKIT_MAJOR: "12" - run: scripts/package-native-runtime.sh --build --backend cuda --target x86_64-unknown-linux-gnu --out dist/native-runtimes - - name: Composed CUDA CLI and client readiness smoke - run: | - target/release/mesh-llm --log-format json --version - MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR="$PWD/dist/native-runtimes" target/release/mesh-llm --log-format json runtime list - scripts/ci-client-readiness-smoke.sh target/release/mesh-llm dist/native-runtimes + - name: Download immutable host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-linux-host-input + path: host-input + + - name: Prepare Linux CUDA native runtime + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: cuda + target: x86_64-unknown-linux-gnu + output_dir: runtime-input + + - name: Compose CUDA product + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + backend: cuda + + - name: Upload composed CUDA product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-linux-cuda-product + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 linux_rocm: - needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + needs: [changes, linux_host_input] + if: ${{ needs.changes.outputs.docs_only != 'true' }} name: Linux ROCm slim - runs-on: ubuntu-24.04 + runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read packages: read @@ -828,53 +1145,57 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} env: - LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-rocm-gfx1100 + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-dynamic-rocm-gfx1100 LLAMA_STAGE_BACKEND: rocm LLAMA_STAGE_AMDGPU_TARGETS: gfx1100 - MESH_LLM_SKIP_UI: "1" + CMAKE_PREFIX_PATH: /opt/rocm MESH_LLM_REQUIRE_SCCACHE: "1" - RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Verify prebuilt backend environment run: verify-runner-image public rocm - - name: Configure Linux Rust linker - run: | - mkdir -p .cargo - cat > .cargo/config.toml <<'EOF' - [target.x86_64-unknown-linux-gnu] - rustflags = ["-C", "link-arg=-fuse-ld=lld"] - EOF - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/configure-sccache-gha with: - workspaces: . -> target - cache-bin: "false" - prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} - shared-key: linux-rocm-slim - save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Prepare UI placeholder - run: mkdir -p crates/mesh-llm-ui/dist && printf '' > crates/mesh-llm-ui/dist/index.html + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - - name: Build backend-neutral Linux release host - run: scripts/build-host.sh --profile release - - name: Package Linux ROCm native runtime - run: | - export CMAKE_PREFIX_PATH="/opt/rocm:${CMAKE_PREFIX_PATH:-}" - LLAMA_STAGE_AMDGPU_TARGETS=gfx1100 scripts/package-native-runtime.sh --build --backend rocm --target x86_64-unknown-linux-gnu --out dist/native-runtimes - - name: Composed ROCm CLI and client readiness smoke - run: | - target/release/mesh-llm --log-format json --version - MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR="$PWD/dist/native-runtimes" target/release/mesh-llm --log-format json runtime list - scripts/ci-client-readiness-smoke.sh target/release/mesh-llm dist/native-runtimes + - name: Download immutable host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-linux-host-input + path: host-input + + - name: Prepare Linux ROCm native runtime + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: rocm + target: x86_64-unknown-linux-gnu + output_dir: runtime-input + + - name: Compose ROCm product + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + backend: rocm + + - name: Upload composed ROCm product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-linux-rocm-product + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 linux_vulkan: - needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + needs: [changes, linux_host_input] + if: ${{ needs.changes.outputs.docs_only != 'true' }} name: Linux Vulkan - runs-on: ubuntu-24.04 + runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read packages: read @@ -885,122 +1206,90 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} env: LLAMA_STAGE_BACKEND: vulkan - LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-vulkan - MESH_LLM_SKIP_UI: "1" + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-dynamic-vulkan MESH_LLM_REQUIRE_SCCACHE: "1" - RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Verify prebuilt backend environment run: verify-runner-image public vulkan - - name: Configure Linux Rust linker - run: | - mkdir -p .cargo - cat > .cargo/config.toml <<'EOF' - [target.x86_64-unknown-linux-gnu] - rustflags = ["-C", "link-arg=-fuse-ld=lld"] - EOF - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/configure-sccache-gha with: - workspaces: . -> target - cache-bin: "false" - prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} - shared-key: linux-vulkan - save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Prepare UI placeholder - run: mkdir -p crates/mesh-llm-ui/dist && printf '' > crates/mesh-llm-ui/dist/index.html - - name: Build backend-neutral Linux release host - run: scripts/build-host.sh --profile release - - name: Package Linux Vulkan native runtime - run: scripts/package-native-runtime.sh --build --backend vulkan --target x86_64-unknown-linux-gnu --out dist/native-runtimes - - name: Composed Vulkan CLI and client readiness smoke - run: | - target/release/mesh-llm --log-format json --version - MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR="$PWD/dist/native-runtimes" target/release/mesh-llm --log-format json runtime list - scripts/ci-client-readiness-smoke.sh target/release/mesh-llm dist/native-runtimes - windows_cpu: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} + + - name: Download immutable host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-linux-host-input + path: host-input + + - name: Prepare Linux Vulkan native runtime + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: vulkan + target: x86_64-unknown-linux-gnu + output_dir: runtime-input + + - name: Compose Vulkan product + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + backend: vulkan + + - name: Upload composed Vulkan product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-linux-vulkan-product + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 + + windows_node_checks: needs: changes if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.rust == 'true') && needs.changes.outputs.docs_only != 'true' }} - name: Windows CPU + name: Windows Node SDK checks runs-on: windows-2022 permissions: contents: read env: - RUN_WINDOWS_CPU_FULL: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_cpu == 'true' }} - MESH_LLM_SKIP_UI: "1" - MESH_LLM_REQUIRE_SCCACHE: "1" + RUN_WINDOWS_NODE_RELEASE: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_cpu == 'true' }} RUSTC_WRAPPER: sccache steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - name: Use fast Windows CPU check - if: ${{ env.RUN_WINDOWS_CPU_FULL != 'true' }} - shell: pwsh - run: Write-Host "Running Windows CPU cargo check because this change does not touch Windows CPU build inputs." - - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v6 - - uses: mozilla-actions/sccache-action@v0.0.9 - - uses: Swatinem/rust-cache@v2 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 with: workspaces: . -> target cache-bin: "false" prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} - shared-key: windows-cpu + shared-key: windows-node save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Prepare UI placeholder - shell: pwsh - run: | - New-Item -ItemType Directory -Force -Path crates/mesh-llm-ui/dist | Out-Null - '' | Set-Content -Path crates/mesh-llm-ui/dist/index.html -Encoding utf8 - - name: Initialize MSVC for composed product - if: ${{ env.RUN_WINDOWS_CPU_FULL == 'true' }} - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 - with: - arch: x64 - - name: Build backend-neutral Windows release host - if: ${{ env.RUN_WINDOWS_CPU_FULL == 'true' }} - shell: pwsh - run: .\scripts\build-windows.ps1 -BuildProfile release -HostOnly - - name: Package Windows CPU native runtime - if: ${{ env.RUN_WINDOWS_CPU_FULL == 'true' }} - shell: bash - run: scripts/package-native-runtime.sh --build --backend cpu --target x86_64-pc-windows-msvc --out target/release/native-runtimes - - name: Check mesh-llm binary on Windows - if: ${{ env.RUN_WINDOWS_CPU_FULL != 'true' }} - shell: pwsh - run: cargo check --locked -p mesh-llm --bin mesh-llm --features dynamic-native-runtime - name: Check Node SDK addon on Windows shell: pwsh run: cargo check --locked -p mesh-llm-nodejs - name: Build Node SDK addon on Windows - if: ${{ env.RUN_WINDOWS_CPU_FULL == 'true' }} + if: ${{ env.RUN_WINDOWS_NODE_RELEASE == 'true' }} shell: pwsh run: cargo build --release --locked -p mesh-llm-nodejs - - name: Composed Windows CPU CLI and client readiness smoke - if: ${{ env.RUN_WINDOWS_CPU_FULL == 'true' }} - shell: pwsh - run: | - .\target\release\mesh-llm.exe --log-format json --version - .\target\release\mesh-llm.exe --log-format json --help | Select-Object -First 5 - $env:MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR = "$PWD\target\release\native-runtimes" - .\target\release\mesh-llm.exe --log-format json runtime list - bash scripts/ci-client-readiness-smoke.sh target/release/mesh-llm.exe target/release/native-runtimes - if ($LASTEXITCODE -ne 0) { throw "composed Windows CPU client readiness smoke failed" } - - windows_gpu: + + windows_gpu_runtime_inputs: needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.rust == 'true') && needs.changes.outputs.docs_only != 'true' }} - name: Windows ${{ matrix.name }} + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_gpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Windows ${{ matrix.name }} runtime input runs-on: windows-2022 permissions: contents: read env: - RUN_WINDOWS_GPU: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_gpu == 'true' }} - MESH_LLM_SKIP_UI: "1" + LLAMA_STAGE_BACKEND: ${{ matrix.backend }} + LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-${{ matrix.backend }} RUSTC_WRAPPER: sccache WINDOWS_CUDA_VERSION: ${{ vars.CUDA_VERSION || '12.6.3' }} WINDOWS_VULKAN_SDK_VERSION: ${{ vars.VULKAN_SDK_VERSION || '1.4.328.1' }} @@ -1018,43 +1307,28 @@ jobs: - name: Vulkan backend: vulkan steps: - - name: Skip Windows GPU build - if: ${{ env.RUN_WINDOWS_GPU != 'true' }} - shell: pwsh - run: Write-Host "Skipping Windows ${{ matrix.name }} because this change does not touch Windows GPU build inputs." - - uses: actions/checkout@v5 - if: ${{ env.RUN_WINDOWS_GPU == 'true' }} + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: actions/setup-python@v6 - if: ${{ env.RUN_WINDOWS_GPU == 'true' }} + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' - name: Test Windows native-runtime dependency resolver - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'vulkan' }} + if: ${{ matrix.backend == 'vulkan' }} shell: pwsh run: python -m unittest scripts.tests.test_windows_native_runtime_deps -v - - uses: dtolnay/rust-toolchain@stable - if: ${{ env.RUN_WINDOWS_GPU == 'true' }} - - uses: mozilla-actions/sccache-action@v0.0.9 - if: ${{ env.RUN_WINDOWS_GPU == 'true' }} - - uses: Swatinem/rust-cache@v2 - if: ${{ env.RUN_WINDOWS_GPU == 'true' }} + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 with: workspaces: . -> target cache-bin: "false" prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} - shared-key: windows-${{ matrix.backend }} + shared-key: windows-runtime-${{ matrix.backend }} save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Prepare UI placeholder - if: ${{ env.RUN_WINDOWS_GPU == 'true' }} - shell: pwsh - run: | - New-Item -ItemType Directory -Force -Path crates/mesh-llm-ui/dist | Out-Null - '' | Set-Content -Path crates/mesh-llm-ui/dist/index.html -Encoding utf8 - name: Install CUDA toolkit - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'cuda' }} - uses: Jimver/cuda-toolkit@v0.2.35 + if: ${{ matrix.backend == 'cuda' }} + uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 with: cuda: ${{ env.WINDOWS_CUDA_VERSION }} method: network @@ -1063,28 +1337,24 @@ jobs: use-local-cache: true log-file-suffix: windows-cuda - name: Verify CUDA toolkit - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'cuda' }} + if: ${{ matrix.backend == 'cuda' }} shell: pwsh run: | if (-not $env:CUDA_PATH -or -not (Test-Path $env:CUDA_PATH)) { throw "CUDA_PATH was not configured by Jimver/cuda-toolkit." } & nvcc --version - foreach ($library in @("cuda.lib", "cudart.lib", "cublas.lib", "cublasLt.lib")) { - $path = Join-Path $env:CUDA_PATH "lib\x64\$library" - if (-not (Test-Path $path)) { - throw "Expected CUDA import library was not found: $path" - } - } + $cudaMajor = ("$env:WINDOWS_CUDA_VERSION".Trim() -split '\.', 2)[0] + "MESH_LLM_CUDA_TOOLKIT_MAJOR=$cudaMajor" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Install Vulkan SDK - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'vulkan' }} - uses: jakoch/install-vulkan-sdk-action@v1.5.2 + if: ${{ matrix.backend == 'vulkan' }} + uses: jakoch/install-vulkan-sdk-action@3c53c378c9bfbb2ea122a1cc164a837d4004c871 # v1.5.2 with: vulkan_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} cache: true stripdown: true - name: Verify Vulkan SDK - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'vulkan' }} + if: ${{ matrix.backend == 'vulkan' }} shell: pwsh run: | if (-not $env:VULKAN_SDK -or -not (Test-Path $env:VULKAN_SDK)) { @@ -1095,44 +1365,206 @@ jobs: throw "glslc.exe was not found at $glslc" } & $glslc --version - $vulkanLib = Join-Path $env:VULKAN_SDK "Lib\vulkan-1.lib" - if (-not (Test-Path $vulkanLib)) { - throw "Expected Vulkan import library was not found: $vulkanLib" - } - name: Install ROCm HIP SDK - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'rocm' }} + if: ${{ matrix.backend == 'rocm' }} uses: ./.github/actions/setup-windows-rocm-sdk with: rocm-hip-sdk-filename: ${{ env.ROCM_HIP_SDK_FILENAME }} - - name: Initialize MSVC for composed product - if: ${{ env.RUN_WINDOWS_GPU == 'true' }} - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 + - name: Initialize MSVC for native runtime + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: x64 - - name: Build backend-neutral Windows release host - if: ${{ env.RUN_WINDOWS_GPU == 'true' }} - shell: pwsh - run: .\scripts\build-windows.ps1 -BuildProfile release -HostOnly - - name: Package Windows GPU native runtime - if: ${{ env.RUN_WINDOWS_GPU == 'true' }} - shell: bash + - name: Prepare immutable Windows GPU runtime + uses: ./.github/actions/prepare-native-runtime-input env: - LLAMA_STAGE_BACKEND: ${{ matrix.backend }} LLAMA_STAGE_CUDA_ARCHITECTURES: ${{ matrix.cuda_architectures }} LLAMA_STAGE_AMDGPU_TARGETS: ${{ matrix.rocm_architectures }} - MESH_LLM_CUDA_TOOLKIT_MAJOR: "12" - run: | - scripts/package-native-runtime.sh \ - --build \ - --backend "${{ matrix.backend }}" \ - --target x86_64-pc-windows-msvc \ - --out target/release/native-runtimes - - name: Composed Windows GPU CLI and client readiness smoke - if: ${{ env.RUN_WINDOWS_GPU == 'true' }} - shell: pwsh - run: | - .\target\release\mesh-llm.exe --log-format json --version - $env:MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR = "$PWD\target\release\native-runtimes" - .\target\release\mesh-llm.exe --log-format json runtime list - bash scripts/ci-client-readiness-smoke.sh target/release/mesh-llm.exe target/release/native-runtimes - if ($LASTEXITCODE -ne 0) { throw "composed Windows ${{ matrix.name }} client readiness smoke failed" } + with: + backend: ${{ matrix.backend }} + target: x86_64-pc-windows-msvc + output_dir: runtime-input + - name: Upload immutable Windows GPU runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-windows-${{ matrix.backend }}-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + windows_host_input: + needs: changes + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.rust == 'true' || needs.changes.outputs.windows_cpu == 'true' || needs.changes.outputs.windows_gpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Windows immutable release host + runs-on: windows-2022 + env: + MESH_LLM_REQUIRE_SCCACHE: "1" + RUSTC_WRAPPER: sccache + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.x' + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: windows-host-release + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Initialize MSVC for neutral host + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + with: + arch: x64 + - name: Prepare immutable Windows release host + uses: ./.github/actions/prepare-windows-host-input + with: + profile: release + skip_ui: "true" + - name: Upload immutable Windows host input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-windows-host-input + path: host-input/* + if-no-files-found: error + retention-days: 1 + + windows_cpu_runtime_input: + needs: changes + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.rust == 'true' || needs.changes.outputs.windows_cpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Windows CPU runtime input + runs-on: windows-2022 + env: + LLAMA_STAGE_BACKEND: cpu + LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-cpu + RUSTC_WRAPPER: sccache + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.x' + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: windows-runtime-cpu + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Initialize MSVC for CPU runtime + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + with: + arch: x64 + - name: Prepare immutable Windows CPU runtime + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: cpu + target: x86_64-pc-windows-msvc + output_dir: runtime-input + - name: Upload immutable Windows CPU runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-windows-cpu-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + windows_cpu_product: + needs: [changes, windows_host_input, windows_cpu_runtime_input] + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.rust == 'true' || needs.changes.outputs.windows_cpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Windows CPU + runs-on: windows-2022 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.x' + - name: Download immutable Windows host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-windows-host-input + path: host-input + - name: Download immutable Windows CPU runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-windows-cpu-runtime-input + path: runtime-input + - name: Compose Windows CPU product + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + backend: cpu + binary_name: mesh-llm.exe + readiness_smoke: "true" + - name: Upload composed Windows CPU product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-windows-cpu-product + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 + + windows_gpu_products: + needs: [changes, windows_host_input, windows_gpu_runtime_inputs] + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_gpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Windows ${{ matrix.name }} + runs-on: windows-2022 + strategy: + fail-fast: false + matrix: + include: + - name: CUDA + backend: cuda + - name: ROCm + backend: rocm + - name: Vulkan + backend: vulkan + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.x' + - name: Download immutable Windows host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-windows-host-input + path: host-input + - name: Download immutable Windows GPU runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-windows-${{ matrix.backend }}-runtime-input + path: runtime-input + - name: Compose Windows GPU product + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + backend: ${{ matrix.backend }} + binary_name: mesh-llm.exe + readiness_smoke: "true" + - name: Upload composed Windows GPU product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-windows-${{ matrix.backend }}-product + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/depot-canary.yml b/.github/workflows/depot-canary.yml new file mode 100644 index 0000000000..f7e75772dc --- /dev/null +++ b/.github/workflows/depot-canary.yml @@ -0,0 +1,61 @@ +name: Depot Runner Canary + +on: + workflow_dispatch: + +permissions: {} + +concurrency: + group: depot-runner-canary + cancel-in-progress: false + +jobs: + runner: + name: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + runner: + - depot-ubuntu-24.04 + - depot-ubuntu-24.04-4 + - depot-ubuntu-24.04-8 + - depot-ubuntu-24.04-16 + runs-on: ${{ matrix.runner }} + timeout-minutes: 10 + steps: + - name: Verify ephemeral runner resources + shell: bash + run: | + set -euo pipefail + + { + echo "### ${{ matrix.runner }}" + echo + echo "| Signal | Value |" + echo "| --- | --- |" + printf '| architecture | %s |\n' "$(uname -m)" + printf '| logical CPUs | %s |\n' "$(nproc)" + printf '| memory KiB | %s |\n' "$(awk '/MemTotal/ { print $2 }' /proc/meminfo)" + printf '| workspace free KiB | %s |\n' "$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR == 2 { print $4 }')" + printf '| Depot cache token injected | %s |\n' \ + "$([[ -n "${DEPOT_CACHE_TOKEN:-}" ]] && echo yes || echo no)" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Restore Depot cache probe + id: cache + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: .depot-canary-cache + key: depot-runner-canary-v1-${{ matrix.runner }} + + - name: Record cache probe + shell: bash + env: + CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} + run: | + set -euo pipefail + + mkdir -p .depot-canary-cache + printf 'Depot cache canary\n' > .depot-canary-cache/probe.txt + printf '| cache hit | %s |\n' "${CACHE_HIT:-false}" \ + >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/docker-precheck.yml b/.github/workflows/docker-precheck.yml index 456151c3b0..3d25c004a4 100644 --- a/.github/workflows/docker-precheck.yml +++ b/.github/workflows/docker-precheck.yml @@ -34,7 +34,7 @@ jobs: name: ${{ inputs.job_name }} runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Validate target Dockerfile exists run: test -f "${{ inputs.dockerfile_path }}" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a10cfee78b..48e85c61ad 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,9 +14,9 @@ jobs: docker-client: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: docker/setup-buildx-action@v4 + - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Validate client Dockerfile run: docker buildx build --check --file docker/Dockerfile.client . diff --git a/.github/workflows/fly-deploy-console.yml b/.github/workflows/fly-deploy-console.yml index 0f812c58ff..b805e9f2d6 100644 --- a/.github/workflows/fly-deploy-console.yml +++ b/.github/workflows/fly-deploy-console.yml @@ -31,12 +31,12 @@ jobs: environment: fly-console steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ inputs.ref || github.ref }} - name: Set up flyctl - uses: superfly/flyctl-actions/setup-flyctl@master + uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 # 1.6 - name: Deploy console env: diff --git a/.github/workflows/hf-download-smoke.yml b/.github/workflows/hf-download-smoke.yml index b94f8c5f9b..ee0f211819 100644 --- a/.github/workflows/hf-download-smoke.yml +++ b/.github/workflows/hf-download-smoke.yml @@ -27,11 +27,11 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v6 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev libdbus-1-dev lld - name: Configure Linux Rust linker @@ -41,8 +41,8 @@ jobs: [target.x86_64-unknown-linux-gnu] rustflags = ["-C", "link-arg=-fuse-ld=lld"] EOF - - uses: mozilla-actions/sccache-action@v0.0.9 - - uses: Swatinem/rust-cache@v2 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 with: workspaces: . -> target cache-bin: "false" @@ -50,7 +50,7 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Restore Hugging Face download test model cache id: hf_download_model_cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ runner.temp }}/mesh-llm-hf-download-cache key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-hf-download-smoke-models-${{ hashFiles('.github/cache-version.txt') }} @@ -62,7 +62,7 @@ jobs: run: scripts/ci-hf-download-smoke.sh - name: Save Hugging Face download test model cache if: ${{ github.ref == 'refs/heads/main' && steps.hf_download_model_cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v5 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ runner.temp }}/mesh-llm-hf-download-cache key: ${{ steps.hf_download_model_cache.outputs.cache-primary-key }} diff --git a/.github/workflows/llama-upstream-canary.yml b/.github/workflows/llama-upstream-canary.yml index 9eecfa58ec..283a3279f0 100644 --- a/.github/workflows/llama-upstream-canary.yml +++ b/.github/workflows/llama-upstream-canary.yml @@ -20,13 +20,13 @@ jobs: SCCACHE_GHA_ENABLED: "true" RUSTC_WRAPPER: "sccache" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - - uses: taiki-e/install-action@just + - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just - - uses: mozilla-actions/sccache-action@v0.0.9 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - name: Install Hugging Face CLI run: python -m pip install --upgrade "huggingface_hub[cli]" @@ -63,7 +63,7 @@ jobs: - name: Restore Skippy smoke model cache if: steps.sha.outputs.changed == 'true' && env.LLAMA_UPSTREAM_CANARY_SMOKE != '0' && env.LLAMA_UPSTREAM_CANARY_SMOKE != 'false' id: skippy_smoke_model_cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ runner.temp }}/skippy-ci-smoke-models key: mesh-llm-${{ runner.os }}-skippy-ci-smoke-models-SmolLM2-135M-Instruct.Q4_K_M.gguf-Falcon-H1-0.5B-Instruct-Q4_K_M.gguf-${{ hashFiles('.github/cache-version.txt') }} @@ -80,7 +80,7 @@ jobs: - name: Save Skippy smoke model cache if: steps.sha.outputs.changed == 'true' && env.LLAMA_UPSTREAM_CANARY_SMOKE != '0' && env.LLAMA_UPSTREAM_CANARY_SMOKE != 'false' && github.ref == 'refs/heads/main' && steps.skippy_smoke_model_cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v5 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ runner.temp }}/skippy-ci-smoke-models key: ${{ steps.skippy_smoke_model_cache.outputs.cache-primary-key }} diff --git a/.github/workflows/nightly-stability-run.yml b/.github/workflows/nightly-stability-run.yml index 22dd4ae429..d7f159d1b4 100644 --- a/.github/workflows/nightly-stability-run.yml +++ b/.github/workflows/nightly-stability-run.yml @@ -57,7 +57,7 @@ jobs: OPENCODE_DISABLE_LSP_DOWNLOAD: "true" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Preflight configuration id: preflight @@ -80,7 +80,7 @@ jobs: echo "run=true" >> "$GITHUB_OUTPUT" - - uses: actions/setup-node@v5 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 if: ${{ steps.preflight.outputs.run == 'true' && inputs.agent_smokes != '' }} with: node-version: 24 @@ -148,7 +148,7 @@ jobs: - name: Upload stability evidence if: ${{ always() && steps.preflight.outputs.run == 'true' }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: nightly-stability-${{ github.run_number }}-${{ github.sha }} path: ${{ inputs.output_dir }}/ diff --git a/.github/workflows/pr_auto_assign.yml b/.github/workflows/pr_auto_assign.yml index bff9091004..be34a71ec7 100644 --- a/.github/workflows/pr_auto_assign.yml +++ b/.github/workflows/pr_auto_assign.yml @@ -18,7 +18,7 @@ jobs: # This privileged workflow must not check out or run pull request code. - name: Comment on draft pull requests if: github.event.pull_request.draft == true - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: github-token: ${{ github.token }} script: | diff --git a/.github/workflows/pr_builds.yml b/.github/workflows/pr_builds.yml index d60803edd5..19f5944a1a 100644 --- a/.github/workflows/pr_builds.yml +++ b/.github/workflows/pr_builds.yml @@ -1,10 +1,13 @@ name: PR Builds -# Set USE_SELF_HOSTED=true to route CUDA matrix rows to the dedicated NVIDIA -# runner; unset or false uses GitHub-hosted runners. - on: workflow_dispatch: + inputs: + use_depot: + description: Route eligible non-GPU jobs to Depot for rollout benchmarking. + required: false + default: false + type: boolean pull_request: types: [opened, synchronize, reopened, ready_for_review] @@ -17,6 +20,10 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: + # GitHub scopes pull-request cache writes to refs/pull//merge while still + # allowing read-only reuse of the base branch. Depot cache access remains + # disabled for pull_request events because Depot's repository cache is not + # branch-isolated. CACHE_NAMESPACE: mesh-llm MODEL_URL: https://huggingface.co/unsloth/SmolLM2-135M-Instruct-GGUF/resolve/9e6855bc4be717fca1ef21360a1db4b29d5c559a/SmolLM2-135M-Instruct-Q8_0.gguf MODEL_FILE: SmolLM2-135M-Instruct-Q8_0.gguf @@ -45,6 +52,7 @@ jobs: test_crates: ${{ steps.compute.outputs.test_crates }} batches_json: ${{ steps.compute.outputs.batches_json }} test_batches_json: ${{ steps.compute.outputs.test_batches_json }} + linux_test_groups_json: ${{ steps.compute.outputs.linux_test_groups_json }} all_rust: ${{ steps.compute.outputs.all_rust }} docs_only: ${{ steps.compute.outputs.docs_only }} rust_changed: ${{ steps.compute.outputs.rust_changed }} @@ -53,14 +61,29 @@ jobs: inference_artifact_required: ${{ steps.compute.outputs.inference_artifact_required }} backend_recipe_changed: ${{ steps.compute.outputs.backend_recipe_changed }} sdk_smoke_required: ${{ steps.compute.outputs.sdk_smoke_required }} + runner_contract_required: ${{ steps.compute.outputs.runner_contract_required }} ui_dist_cache_key: ${{ steps.ui_key.outputs.ui_dist_cache_key }} linux_inference_artifact_required: ${{ github.event_name == 'workflow_dispatch' || steps.compute.outputs.inference_artifact_required == 'true' }} macos_inference_artifact_required: ${{ github.event_name == 'workflow_dispatch' || steps.compute.outputs.inference_artifact_required == 'true' || steps.filter.outputs.benchmarks == 'true' }} + depot_enabled: ${{ steps.runners.outputs.depot_enabled }} + allow_depot_remote_cache: ${{ steps.runners.outputs.allow_depot_remote_cache }} + runner: ${{ steps.runners.outputs.runner }} + runner_4: ${{ steps.runners.outputs.runner_4 }} + runner_8: ${{ steps.runners.outputs.runner_8 }} + runner_16: ${{ steps.runners.outputs.runner_16 }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: fetch-depth: 0 - - uses: dorny/paths-filter@v4 + - name: Select trusted Linux runners + id: runners + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + ref: ${{ github.ref }} + depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + manual_use_depot: ${{ inputs.use_depot == true }} + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 id: filter with: filters: | @@ -144,8 +167,9 @@ jobs: public_runner_image_contract: needs: changes + if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.runner_contract_required == 'true' }} name: Runner image / GitHub-hosted public - runs-on: ubuntu-24.04 + runs-on: ${{ needs.changes.outputs.runner_4 }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d credentials: @@ -156,7 +180,7 @@ jobs: shell: bash timeout-minutes: 15 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Verify prebuilt public environment @@ -165,40 +189,11 @@ jobs: hf --help >/dev/null cargo check -p mesh-llm-config - arc_runner_image_contract: - needs: changes - name: Runner image / ARC ${{ matrix.architecture }} - runs-on: ${{ matrix.runner }} - permissions: - contents: read - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - include: - - architecture: amd64 - runner: mesh-llm-amd64 - machine: x86_64 - - architecture: arm64 - runner: mesh-llm-arm64 - machine: aarch64 - steps: - - uses: actions/checkout@v5 - with: - persist-credentials: false - - name: Verify prebuilt ARC environment - env: - EXPECTED_MACHINE: ${{ matrix.machine }} - run: | - test "$(uname -m)" = "$EXPECTED_MACHINE" - verify-runner-image self-hosted - cargo check -p mesh-llm-config - - linux_cpu_artifact: + linux_host_input: needs: changes - if: ${{ needs.changes.outputs.linux_inference_artifact_required == 'true' && needs.changes.outputs.docs_only != 'true' }} - name: Linux CPU - runs-on: ubuntu-24.04 + if: ${{ (needs.changes.outputs.linux_inference_artifact_required == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Linux immutable host + runs-on: ${{ needs.changes.outputs.runner_8 }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d credentials: @@ -208,17 +203,18 @@ jobs: run: shell: bash env: - LLAMA_STAGE_BACKEND: cpu - LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-dynamic-cpu MESH_LLM_SKIP_UI: "1" MESH_LLM_REQUIRE_SCCACHE: "1" RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Verify prebuilt CI environment run: verify-runner-image public + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - name: Build UI if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} @@ -231,8 +227,8 @@ jobs: run: pnpm test - name: Save UI dist cache - if: ${{ github.ref == 'refs/heads/main' && needs.changes.outputs.ui == 'true' }} - uses: actions/cache/save@v4 + if: ${{ needs.changes.outputs.ui == 'true' }} + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: crates/mesh-llm-ui/dist key: ${{ needs.changes.outputs.ui_dist_cache_key }} @@ -245,7 +241,7 @@ jobs: rustflags = ["-C", "link-arg=-fuse-ld=lld"] EOF - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: workspaces: . -> target @@ -267,46 +263,130 @@ jobs: exit 1 fi - - name: Build backend-neutral debug host - env: - MESH_LLM_SKIP_UI: "1" - run: scripts/build-host.sh --profile debug + - name: Prepare UI placeholder + run: | + mkdir -p crates/mesh-llm-ui/dist + test -f crates/mesh-llm-ui/dist/index.html || + printf '' > crates/mesh-llm-ui/dist/index.html - - name: Package CPU native runtime beside debug host - run: scripts/package-native-runtime.sh --build --backend cpu --out target/debug/native-runtimes + - name: Prepare backend-neutral host + uses: ./.github/actions/prepare-host-input + with: + profile: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true') && 'release' || 'debug' }} + output_dir: host-input + skip_ui: "true" - - name: CLI smoke test - run: | - target/debug/mesh-llm --log-format json --version - target/debug/mesh-llm --log-format json --help | head -5 - target/debug/mesh-llm --log-format json runtime list - - name: Composed debug client readiness smoke - run: scripts/ci-client-readiness-smoke.sh target/debug/mesh-llm target/debug/native-runtimes + - name: Upload immutable host input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-linux-host-input + path: host-input/ + if-no-files-found: error + retention-days: 1 - - name: Upload Linux inference binary - uses: actions/upload-artifact@v6 + - name: Show sccache stats + if: ${{ always() }} + run: sccache --show-stats || true + + linux_cpu_runtime_input: + needs: changes + if: ${{ needs.changes.outputs.linux_inference_artifact_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + name: Linux CPU runtime + runs-on: ${{ needs.changes.outputs.runner_8 }} + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + env: + LLAMA_STAGE_BACKEND: cpu + LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-dynamic-cpu + MESH_LLM_REQUIRE_SCCACHE: "1" + SCCACHE_GHA_ENABLED: "false" + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: - name: ci-linux-inference-binaries + persist-credentials: false + - name: Verify prebuilt CI environment + run: verify-runner-image public + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} + - name: Prepare CPU native runtime + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: cpu + target: x86_64-unknown-linux-gnu + output_dir: runtime-input + + - name: Upload immutable CPU runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-linux-cpu-runtime-input path: | - target/debug/mesh-llm - target/debug/native-runtimes/** + runtime-input/*.tar.gz + runtime-input/*.sha256 if-no-files-found: error retention-days: 1 - name: Show sccache stats if: ${{ always() }} - run: | - sccache --show-stats || true - requests="$(sccache --show-stats 2>/dev/null | awk '/Compile requests/ { print $3; exit }')" - if [ "${requests:-0}" = "0" ]; then - echo "::warning::sccache reported zero compile requests; check RUSTC_WRAPPER wiring if this was not a fully reused target cache." - fi + run: sccache --show-stats || true + + linux_cpu_artifact: + needs: [changes, linux_host_input, linux_cpu_runtime_input] + if: ${{ needs.linux_host_input.result == 'success' && needs.linux_cpu_runtime_input.result == 'success' }} + name: Linux CPU product + runs-on: ${{ needs.changes.outputs.runner_4 }} + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Verify prebuilt CI environment + run: verify-runner-image public + - name: Download immutable host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-linux-host-input + path: host-input + - name: Download immutable CPU runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-linux-cpu-runtime-input + path: runtime-input + - name: Compose CPU product from producer inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: ci-product + backend: cpu + + - name: Upload Linux inference binary + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-linux-inference-binaries + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 linux_targets: - needs: changes + needs: [changes, linux_host_input] if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} name: Linux ${{ matrix.name }} - runs-on: ${{ fromJson(vars.USE_SELF_HOSTED == 'true' && matrix.self_hosted_runs_on || matrix.runs_on) }} + runs-on: ${{ needs.changes.outputs.runner_8 }} container: image: ${{ matrix.container }} credentials: @@ -321,133 +401,145 @@ jobs: include: - name: CUDA slim backend: cuda - container: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:295341c6c9f17c9eb69281fd454bda953799406d6915f472c914fb5f024a88ed - cache_key: linux-cuda-slim - runs_on: '["ubuntu-24.04"]' - self_hosted_runs_on: '["self-hosted","Linux","X64","amd64","gpu-nvidia"]' + container: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd + build_dir: .deps/llama-build/build-stage-abi-dynamic-cuda-sm86 + cuda_architectures: "86" + cuda_version: "12.9.2" + cuda_major: "12" - name: ROCm slim backend: rocm container: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:6b88ca9371ada2c507d6e36b71f0e0538fee378c6a5e2b39c17249b4b7e5088a - cache_key: linux-rocm-slim - build_dir: .deps/llama-build/build-stage-abi-rocm-gfx1100 + build_dir: .deps/llama-build/build-stage-abi-dynamic-rocm-gfx1100 amdgpu_targets: gfx1100 - runs_on: '"ubuntu-24.04"' + cmake_prefix_path: /opt/rocm - name: Vulkan backend: vulkan container: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:ce55fed5c680cd3184b5d4770d9a77c43a702687690906e5753efd2cea27ed80 - cache_key: linux-vulkan - build_dir: .deps/llama-build/build-stage-abi-vulkan - runs_on: '"ubuntu-24.04"' + build_dir: .deps/llama-build/build-stage-abi-dynamic-vulkan env: - RUN_LINUX_BACKEND: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true' }} LLAMA_STAGE_BACKEND: ${{ matrix.backend }} LLAMA_STAGE_BUILD_DIR: ${{ matrix.build_dir }} + LLAMA_STAGE_CUDA_ARCHITECTURES: ${{ matrix.cuda_architectures }} LLAMA_STAGE_AMDGPU_TARGETS: ${{ matrix.amdgpu_targets }} - MESH_LLM_SKIP_UI: "1" + MESH_CUDA_VERSION: ${{ matrix.cuda_version }} + MESH_LLM_CUDA_TOOLKIT_MAJOR: ${{ matrix.cuda_major }} + CMAKE_PREFIX_PATH: ${{ matrix.cmake_prefix_path }} MESH_LLM_REQUIRE_SCCACHE: "1" - RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" GGML_CUDA_NO_VMM: "1" LLAMA_STAGE_SKIP_NCCL: "1" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false + - name: Verify prebuilt backend environment - if: ${{ env.RUN_LINUX_BACKEND == 'true' }} run: verify-runner-image public "${{ matrix.backend }}" - - name: Skip Linux backend build - if: ${{ env.RUN_LINUX_BACKEND != 'true' }} - run: echo "Skipping Linux ${{ matrix.name }} backend build; backend inputs did not change." - - name: Configure Linux Rust linker - if: ${{ env.RUN_LINUX_BACKEND == 'true' }} - run: | - mkdir -p .cargo - cat > .cargo/config.toml <<'EOF' - [target.x86_64-unknown-linux-gnu] - rustflags = ["-C", "link-arg=-fuse-ld=lld"] - EOF - - uses: Swatinem/rust-cache@v2 - if: ${{ env.RUN_LINUX_BACKEND == 'true' }} - continue-on-error: true + - uses: ./.github/actions/configure-sccache-gha with: - workspaces: . -> target - cache-bin: "false" - prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} - shared-key: ${{ matrix.cache_key }} - save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Prepare UI placeholder - if: ${{ env.RUN_LINUX_BACKEND == 'true' }} - run: mkdir -p crates/mesh-llm-ui/dist && printf '' > crates/mesh-llm-ui/dist/index.html + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} + + - name: Download immutable host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-linux-host-input + path: host-input + - name: Ensure Vulkan ABI cache directory - if: ${{ env.RUN_LINUX_BACKEND == 'true' && matrix.backend == 'vulkan' }} + if: matrix.backend == 'vulkan' run: mkdir -p "$LLAMA_STAGE_BUILD_DIR" - name: Cache Vulkan ABI build - if: ${{ env.RUN_LINUX_BACKEND == 'true' && matrix.backend == 'vulkan' }} - id: backend_llama_cache - uses: actions/cache@v5 + if: matrix.backend == 'vulkan' + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: .deps/llama-build/build-stage-abi-vulkan + path: ${{ env.LLAMA_STAGE_BUILD_DIR }} key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-vulkan-${{ hashFiles('scripts/build-linux.sh', 'scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - - name: Build Linux CUDA backend - if: ${{ env.RUN_LINUX_BACKEND == 'true' && matrix.backend == 'cuda' }} - env: - MESH_CUDA_VERSION: "12.9.2" - run: just --shell bash --shell-arg -lc release-build-cuda - - name: Build Linux ROCm backend - if: ${{ env.RUN_LINUX_BACKEND == 'true' && matrix.backend == 'rocm' }} - run: | - export CMAKE_PREFIX_PATH="/opt/rocm:${CMAKE_PREFIX_PATH:-}" - just --shell bash --shell-arg -lc release-build-rocm gfx1100 - - name: Build Linux Vulkan backend - if: ${{ env.RUN_LINUX_BACKEND == 'true' && matrix.backend == 'vulkan' && steps.backend_llama_cache.outputs.cache-hit != 'true' }} - run: just --shell bash --shell-arg -lc release-build-vulkan - - name: Build Linux Vulkan binary only - if: ${{ env.RUN_LINUX_BACKEND == 'true' && matrix.backend == 'vulkan' && steps.backend_llama_cache.outputs.cache-hit == 'true' }} - run: | - scripts/build-release.sh - scripts/package-native-runtime.sh \ - --backend vulkan \ - --target x86_64-unknown-linux-gnu \ - --out dist/native-runtimes - - name: Linux CUDA CLI smoke test - if: ${{ env.RUN_LINUX_BACKEND == 'true' && matrix.backend == 'cuda' }} - run: | - # This must work without libcuda.so.1 or an injected driver stub. The - # CUDA runtime is selected dynamically from the packaged artifact; - # the backend-neutral host must remain executable on a CPU-only node. - export MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR="$PWD/dist/native-runtimes" - target/release/mesh-llm --log-format json --version - target/release/mesh-llm --log-format json runtime list - scripts/ci-client-readiness-smoke.sh target/release/mesh-llm dist/native-runtimes - - name: Linux ROCm CLI smoke test - if: ${{ env.RUN_LINUX_BACKEND == 'true' && matrix.backend == 'rocm' }} - run: | - export MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR="$PWD/dist/native-runtimes" - target/release/mesh-llm --log-format json --version - target/release/mesh-llm --log-format json runtime list - scripts/ci-client-readiness-smoke.sh target/release/mesh-llm dist/native-runtimes - - name: Linux Vulkan CLI smoke test - if: ${{ env.RUN_LINUX_BACKEND == 'true' && matrix.backend == 'vulkan' }} - run: | - export MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR="$PWD/dist/native-runtimes" - target/release/mesh-llm --log-format json --version - target/release/mesh-llm --log-format json runtime list - scripts/ci-client-readiness-smoke.sh target/release/mesh-llm dist/native-runtimes - - name: Show sccache stats - if: ${{ env.RUN_LINUX_BACKEND == 'true' && (always()) }} + + - name: Prepare native runtime + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: ${{ matrix.backend }} + target: x86_64-unknown-linux-gnu + output_dir: runtime-input + + - name: Compose backend product from immutable inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + backend: ${{ matrix.backend }} + + - name: Upload composed backend product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-linux-${{ matrix.backend }}-product + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 + + linux_static_abi_input: + needs: changes + if: ${{ needs.changes.outputs.docs_only != 'true' && (needs.changes.outputs.test_batches_json != '[]' || needs.changes.outputs.linux_test_groups_json != '[]') }} + name: Linux static ABI input + runs-on: ${{ needs.changes.outputs.runner_8 }} + permissions: + contents: read + packages: read + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + env: + LLAMA_STAGE_BACKEND: cpu + LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static + MESH_LLM_REQUIRE_SCCACHE: "1" + SCCACHE_GHA_ENABLED: "false" + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Verify prebuilt CI environment + run: verify-runner-image public + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} + - name: Cache patched llama.cpp ABI build + id: static_abi_cache + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: .deps/llama.cpp/build-stage-abi-static + key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-cpu-${{ hashFiles('scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} + - name: Prepare patched llama.cpp ABI checkout + if: ${{ steps.static_abi_cache.outputs.cache-hit != 'true' }} + run: scripts/prepare-llama.sh pinned + - name: Build patched llama.cpp ABI libraries + if: ${{ steps.static_abi_cache.outputs.cache-hit != 'true' }} + run: scripts/build-llama.sh + - name: Archive immutable static ABI input run: | - sccache --show-stats || true - requests="$(sccache --show-stats 2>/dev/null | awk '/Compile requests/ { print $3; exit }')" - if [ "${requests:-0}" = "0" ]; then - echo "::warning::sccache reported zero compile requests; check RUSTC_WRAPPER wiring if this was not a fully reused target cache." - fi + test -d "$LLAMA_STAGE_BUILD_DIR" + tar -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" \ + -czf "$RUNNER_TEMP/mesh-llm-static-abi.tar.gz" \ + "$(basename "$LLAMA_STAGE_BUILD_DIR")" + - name: Upload immutable static ABI input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-linux-static-abi-input + path: ${{ runner.temp }}/mesh-llm-static-abi.tar.gz + if-no-files-found: error + retention-days: 1 rust_crate_tests: - needs: changes - if: ${{ needs.changes.outputs.rust_changed == 'true' && needs.changes.outputs.docs_only != 'true' }} + needs: [changes, linux_static_abi_input] + if: ${{ needs.linux_static_abi_input.result == 'success' && needs.changes.outputs.test_batches_json != '[]' && needs.changes.outputs.docs_only != 'true' }} name: Rust crate tests (${{ matrix.batch.idx }}) - runs-on: ubuntu-24.04 + runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read packages: read @@ -472,12 +564,15 @@ jobs: RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Verify prebuilt CI environment run: verify-runner-image public - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: workspaces: . -> target @@ -490,20 +585,17 @@ jobs: - name: Prepare patched llama.cpp source checkout if: ${{ contains(matrix.batch.crates, 'skippy-quantize') }} run: scripts/prepare-llama.sh pinned - - name: Ensure ABI cache directory - run: mkdir -p "$LLAMA_STAGE_BUILD_DIR" - - name: Cache patched llama.cpp ABI build - id: rust_tests_llama_cache - uses: actions/cache@v5 + - name: Download immutable static ABI input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: - path: .deps/llama.cpp/build-stage-abi-static - key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-cpu-${{ hashFiles('scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - - name: Prepare patched llama.cpp ABI checkout - if: ${{ steps.rust_tests_llama_cache.outputs.cache-hit != 'true' }} - run: scripts/prepare-llama.sh pinned - - name: Build patched llama.cpp ABI libraries - if: ${{ steps.rust_tests_llama_cache.outputs.cache-hit != 'true' }} - run: scripts/build-llama.sh + name: pr-linux-static-abi-input + path: ${{ runner.temp }}/static-abi-input + - name: Restore immutable static ABI input + run: | + mkdir -p "$(dirname "$LLAMA_STAGE_BUILD_DIR")" + tar -xzf "$RUNNER_TEMP/static-abi-input/mesh-llm-static-abi.tar.gz" \ + -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" + test -d "$LLAMA_STAGE_BUILD_DIR" - name: Run crate tests env: TEST_CRATES: ${{ toJson(matrix.batch.crates) }} @@ -519,10 +611,10 @@ jobs: run: sccache --show-stats || true linux_test_groups: - needs: [changes, linux_cpu_artifact] - if: ${{ needs.linux_cpu_artifact.result == 'success' && needs.changes.outputs.docs_only != 'true' && needs.changes.outputs.linux_inference_artifact_required == 'true' }} + needs: [changes, linux_static_abi_input] + if: ${{ needs.linux_static_abi_input.result == 'success' && needs.changes.outputs.docs_only != 'true' && needs.changes.outputs.linux_test_groups_json != '[]' }} name: Linux tests (${{ matrix.group }}) - runs-on: ubuntu-24.04 + runs-on: ${{ needs.changes.outputs.runner_8 }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d credentials: @@ -534,11 +626,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - group: protocol - cache_key: linux-tests-protocol - - group: skippy-smoke - cache_key: linux-tests-skippy-smoke + include: ${{ fromJson(needs.changes.outputs.linux_test_groups_json) }} env: AFFECTED: ${{ needs.changes.outputs.affected_crates }} ALL_RUST: ${{ needs.changes.outputs.all_rust }} @@ -549,10 +637,13 @@ jobs: RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Verify prebuilt CI environment run: verify-runner-image public + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - name: Configure Linux Rust linker run: | @@ -562,7 +653,7 @@ jobs: rustflags = ["-C", "link-arg=-fuse-ld=lld"] EOF - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: workspaces: . -> target @@ -571,28 +662,22 @@ jobs: shared-key: ${{ matrix.cache_key }} save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Ensure ABI cache directory - run: mkdir -p "$LLAMA_STAGE_BUILD_DIR" - - - name: Cache patched llama.cpp ABI build - id: llama_cache - uses: actions/cache@v5 + - name: Download immutable static ABI input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: - path: .deps/llama.cpp/build-stage-abi-static - key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-cpu-${{ hashFiles('scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - - - name: Prepare patched llama.cpp ABI checkout - if: ${{ steps.llama_cache.outputs.cache-hit != 'true' }} - run: scripts/prepare-llama.sh pinned - - - name: Build patched llama.cpp ABI libraries - if: ${{ steps.llama_cache.outputs.cache-hit != 'true' }} - run: scripts/build-llama.sh + name: pr-linux-static-abi-input + path: ${{ runner.temp }}/static-abi-input + - name: Restore immutable static ABI input + run: | + mkdir -p "$(dirname "$LLAMA_STAGE_BUILD_DIR")" + tar -xzf "$RUNNER_TEMP/static-abi-input/mesh-llm-static-abi.tar.gz" \ + -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" + test -d "$LLAMA_STAGE_BUILD_DIR" - name: Restore Skippy smoke model cache if: ${{ matrix.group == 'skippy-smoke' }} id: skippy_smoke_model_cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ runner.temp }}/skippy-ci-smoke-models key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-ci-smoke-models-SmolLM2-135M-Instruct.Q4_K_M.gguf-Falcon-H1-0.5B-Instruct-Q4_K_M.gguf-${{ hashFiles('.github/cache-version.txt') }} @@ -617,7 +702,7 @@ jobs: - name: Save Skippy smoke model cache if: ${{ matrix.group == 'skippy-smoke' && github.ref == 'refs/heads/main' && steps.skippy_smoke_model_cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v5 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ runner.temp }}/skippy-ci-smoke-models key: ${{ steps.skippy_smoke_model_cache.outputs.cache-primary-key }} @@ -636,22 +721,21 @@ jobs: echo "::warning::sccache reported zero compile requests; check RUSTC_WRAPPER wiring if this was not a fully reused target cache." fi - linux_client_auto_boot: + linux_public_mesh_admission: needs: [changes, linux_cpu_artifact] - if: ${{ needs.linux_cpu_artifact.result == 'success' && needs.changes.outputs.linux_inference_artifact_required == 'true' && needs.changes.outputs.docs_only != 'true' }} - name: Linux client-auto boot test - runs-on: ubuntu-24.04 + if: ${{ github.event_name == 'workflow_dispatch' && needs.linux_cpu_artifact.result == 'success' }} + name: Linux public-mesh admission + runs-on: ${{ needs.changes.outputs.runner_4 }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Install client-auto smoke dependencies run: sudo apt-get update && sudo apt-get install -y curl jq python3 - - uses: actions/download-artifact@v6 + - uses: ./.github/actions/restore-smoke-inputs with: - name: ci-linux-inference-binaries - path: target/debug - - name: Make mesh-llm executable - run: chmod +x target/debug/mesh-llm - - name: Client-auto boot test + artifact_name: ci-linux-inference-binaries + artifact_path: ci-artifacts/public-mesh + staged_binary_path: target/debug/mesh-llm + - name: Public-mesh client-auto admission run: scripts/ci-client-auto-test.sh target/debug/mesh-llm hf_download_smoke: @@ -660,6 +744,7 @@ jobs: name: HuggingFace download smoke uses: ./.github/workflows/hf-download-smoke.yml with: + runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} timeout_minutes: 15 secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -672,13 +757,14 @@ jobs: artifact_name: ci-linux-inference-binaries mesh_binary_target: target/debug/mesh-llm cache_key_prefix: '' + runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} agent_live_smokes: needs: [changes, linux_cpu_artifact] if: ${{ needs.linux_cpu_artifact.result == 'success' && (vars.MESH_AGENT_BASE_URL != '' || vars.MESH_OPENCODE_BASE_URL != '') && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'openai-frontend') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-client')) && needs.changes.outputs.docs_only != 'true' }} - runs-on: ubuntu-24.04 + runs-on: ${{ needs.changes.outputs.runner_4 }} timeout-minutes: 45 env: MESH_AGENT_BASE_URL: ${{ vars.MESH_AGENT_BASE_URL || vars.MESH_OPENCODE_BASE_URL }} @@ -693,8 +779,8 @@ jobs: steps: # Keep live-agent smoke on an explicit Node version so CI does not depend # on GitHub-hosted image defaults. - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 24 - name: Install agent CLIs @@ -731,6 +817,7 @@ jobs: staged_binary_path: target/debug/mesh-llm model_cache_scope: two-node-smoke-model smoke_script: scripts/ci-two-node-client-serving-smoke.sh + runs_on: ${{ toJson(needs.changes.outputs.runner_4) }} timeout_minutes: 20 secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -745,6 +832,7 @@ jobs: staged_binary_path: target/debug/mesh-llm model_cache_scope: two-node-split-smoke-model smoke_script: scripts/ci-two-node-split-smoke.sh + runs_on: ${{ toJson(needs.changes.outputs.runner_4) }} timeout_minutes: 25 secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -759,7 +847,7 @@ jobs: artifact_path: ci-artifacts/linux staged_binary_path: target/debug/mesh-llm model_cache_scope: sdk-smoke-model - runs_on: '"ubuntu-24.04"' + runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -773,27 +861,24 @@ jobs: artifact_path: ci-artifacts/linux staged_binary_path: target/debug/mesh-llm model_cache_scope: sdk-smoke-model - runs_on: '"ubuntu-24.04"' + runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} - macos_cpu_artifact: + macos_host_input: needs: changes if: ${{ needs.changes.outputs.macos_inference_artifact_required == 'true' && needs.changes.outputs.docs_only != 'true' }} - name: macOS CPU + name: macOS immutable host runs-on: macos-15 - env: - RUN_MACOS_CPU: "true" - LLAMA_STAGE_BACKEND: metal - LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-metal - MESH_LLM_SKIP_UI: "1" steps: - - uses: actions/checkout@v5 - - uses: pnpm/action-setup@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} with: version: latest - - uses: actions/setup-node@v5 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} with: node-version: 24 @@ -804,7 +889,7 @@ jobs: - name: Restore UI dist cache id: ui-cache if: ${{ needs.changes.outputs.ui == 'true' || github.event_name == 'workflow_dispatch' }} - uses: actions/cache/restore@v4 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: crates/mesh-llm-ui/dist key: ${{ needs.changes.outputs.ui_dist_cache_key }} @@ -829,8 +914,8 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} working-directory: crates/mesh-llm-ui run: pnpm test - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: workspaces: . -> target @@ -838,45 +923,97 @@ jobs: prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} shared-key: macos save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Install build dependencies - run: brew install cmake ninja jq lld - - name: Configure macOS Rust linker - run: | - mkdir -p .cargo - lld_prefix="$(brew --prefix lld)" - cat > .cargo/config.toml < target @@ -945,12 +1065,12 @@ jobs: EOF - name: Ensure ABI cache directory run: mkdir -p "$LLAMA_STAGE_BUILD_DIR" - - name: Cache patched llama.cpp ABI build + - name: Cache static Metal ABI build id: llama_cache - uses: actions/cache@v5 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: .deps/llama.cpp/build-stage-abi-metal - key: ${{ env.CACHE_NAMESPACE }}-github-macos-15-skippy-abi-metal-${{ hashFiles('scripts/build-mac.sh', 'scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} + path: ${{ env.LLAMA_STAGE_BUILD_DIR }} + key: ${{ env.CACHE_NAMESPACE }}-github-macos-15-skippy-abi-static-metal-${{ hashFiles('scripts/build-mac.sh', 'scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - name: Prepare patched llama.cpp ABI checkout if: ${{ steps.llama_cache.outputs.cache-hit != 'true' }} run: scripts/prepare-llama.sh pinned @@ -972,17 +1092,138 @@ jobs: fi done - windows_targets: + windows_checks: needs: changes if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_rust == 'true' || needs.changes.outputs.windows_cpu == 'true' || needs.changes.outputs.windows_gpu == 'true') && needs.changes.outputs.docs_only != 'true' }} - name: Windows ${{ matrix.name }} + name: Windows lightweight checks + runs-on: windows-2022 + env: + RUN_WINDOWS_PRODUCT: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_cpu == 'true' || needs.changes.outputs.windows_gpu == 'true' }} + RUN_WINDOWS_NODE_RELEASE: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.nodejs_release == 'true' }} + RUSTC_WRAPPER: sccache + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + continue-on-error: true + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: windows-checks + save-if: "false" + - name: Check mesh-llm binary on Windows + if: ${{ env.RUN_WINDOWS_PRODUCT != 'true' }} + shell: pwsh + run: cargo check --locked -p mesh-llm --bin mesh-llm --features dynamic-native-runtime + - name: Check Node SDK addon on Windows + shell: pwsh + run: cargo check --locked -p mesh-llm-nodejs + - name: Build Node SDK addon on Windows + if: ${{ env.RUN_WINDOWS_NODE_RELEASE == 'true' }} + shell: pwsh + run: cargo build --release --locked -p mesh-llm-nodejs + + windows_host_input: + needs: changes + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_cpu == 'true' || needs.changes.outputs.windows_gpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Windows immutable debug host + runs-on: windows-2022 + env: + MESH_LLM_REQUIRE_SCCACHE: "1" + RUSTC_WRAPPER: sccache + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.x' + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + continue-on-error: true + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: windows-host-debug + save-if: "false" + - name: Initialize MSVC for neutral host + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + with: + arch: x64 + - name: Prepare immutable Windows debug host + uses: ./.github/actions/prepare-windows-host-input + with: + profile: debug + skip_ui: "true" + - name: Upload immutable Windows host input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-windows-host-input + path: host-input/* + if-no-files-found: error + retention-days: 1 + + windows_cpu_runtime_input: + needs: changes + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_cpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Windows CPU runtime input + runs-on: windows-2022 + env: + LLAMA_STAGE_BACKEND: cpu + LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-cpu + RUSTC_WRAPPER: sccache + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.x' + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + continue-on-error: true + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: windows-runtime-cpu + save-if: "false" + - name: Initialize MSVC for CPU runtime + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + with: + arch: x64 + - name: Prepare immutable Windows CPU runtime + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: cpu + target: x86_64-pc-windows-msvc + output_dir: runtime-input + - name: Upload immutable Windows CPU runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-windows-cpu-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + windows_gpu_runtime_inputs: + needs: changes + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_gpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Windows ${{ matrix.name }} runtime input runs-on: windows-2022 strategy: fail-fast: false matrix: include: - - name: CPU - backend: cpu - name: CUDA backend: cuda cuda_architectures: "75" @@ -992,58 +1233,36 @@ jobs: - name: Vulkan backend: vulkan env: - RUN_WINDOWS_CPU_FULL: ${{ matrix.backend == 'cpu' && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_cpu == 'true') }} - RUN_WINDOWS_NODE_RELEASE: ${{ matrix.backend == 'cpu' && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.nodejs_release == 'true') }} - RUN_WINDOWS_GPU: ${{ matrix.backend != 'cpu' && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_gpu == 'true') }} LLAMA_STAGE_BACKEND: ${{ matrix.backend }} LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-${{ matrix.backend }} - MESH_LLM_SKIP_UI: "1" - MESH_LLM_REQUIRE_SCCACHE: "1" RUSTC_WRAPPER: sccache WINDOWS_CUDA_VERSION: ${{ vars.CUDA_VERSION || '12.6.3' }} WINDOWS_VULKAN_SDK_VERSION: ${{ vars.VULKAN_SDK_VERSION || '1.4.328.1' }} ROCM_HIP_SDK_FILENAME: AMD-Software-PRO-Edition-25.Q3-WinSvr2022-For-HIP.exe steps: - - name: Skip Windows CPU full build - if: ${{ matrix.backend == 'cpu' && env.RUN_WINDOWS_CPU_FULL != 'true' }} - shell: pwsh - run: Write-Host "Running Windows CPU cargo check because this change does not touch Windows CPU build inputs." - - name: Skip Windows GPU build - if: ${{ matrix.backend != 'cpu' && env.RUN_WINDOWS_GPU != 'true' }} - shell: pwsh - run: Write-Host "Skipping Windows ${{ matrix.name }} because this change does not touch Windows GPU build inputs." - - uses: actions/checkout@v5 - if: ${{ matrix.backend == 'cpu' || env.RUN_WINDOWS_GPU == 'true' }} - - uses: actions/setup-python@v6 - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'vulkan' }} + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' - name: Test Windows native-runtime dependency resolver - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'vulkan' }} + if: ${{ matrix.backend == 'vulkan' }} shell: pwsh run: python -m unittest scripts.tests.test_windows_native_runtime_deps -v - - uses: dtolnay/rust-toolchain@stable - if: ${{ matrix.backend == 'cpu' || env.RUN_WINDOWS_GPU == 'true' }} - - uses: mozilla-actions/sccache-action@v0.0.9 - if: ${{ matrix.backend == 'cpu' || env.RUN_WINDOWS_GPU == 'true' }} - - uses: Swatinem/rust-cache@v2 - if: ${{ matrix.backend == 'cpu' || env.RUN_WINDOWS_GPU == 'true' }} + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: workspaces: . -> target cache-bin: "false" prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} - shared-key: windows-${{ matrix.backend }} - save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Prepare UI placeholder - if: ${{ env.RUN_WINDOWS_CPU_FULL == 'true' || env.RUN_WINDOWS_GPU == 'true' }} - shell: pwsh - run: | - New-Item -ItemType Directory -Force -Path crates/mesh-llm-ui/dist | Out-Null - '' | Set-Content -Path crates/mesh-llm-ui/dist/index.html -Encoding utf8 + shared-key: windows-runtime-${{ matrix.backend }} + save-if: "false" - name: Install CUDA toolkit - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'cuda' }} - uses: Jimver/cuda-toolkit@v0.2.35 + if: ${{ matrix.backend == 'cuda' }} + uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 with: cuda: ${{ env.WINDOWS_CUDA_VERSION }} method: network @@ -1052,28 +1271,24 @@ jobs: use-local-cache: true log-file-suffix: windows-cuda - name: Verify CUDA toolkit - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'cuda' }} + if: ${{ matrix.backend == 'cuda' }} shell: pwsh run: | if (-not $env:CUDA_PATH -or -not (Test-Path $env:CUDA_PATH)) { throw "CUDA_PATH was not configured by Jimver/cuda-toolkit." } & nvcc --version - foreach ($library in @("cuda.lib", "cudart.lib", "cublas.lib", "cublasLt.lib")) { - $path = Join-Path $env:CUDA_PATH "lib\x64\$library" - if (-not (Test-Path $path)) { - throw "Expected CUDA import library was not found: $path" - } - } + $cudaMajor = ("$env:WINDOWS_CUDA_VERSION".Trim() -split '\.', 2)[0] + "MESH_LLM_CUDA_TOOLKIT_MAJOR=$cudaMajor" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Install Vulkan SDK - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'vulkan' }} - uses: jakoch/install-vulkan-sdk-action@v1.5.2 + if: ${{ matrix.backend == 'vulkan' }} + uses: jakoch/install-vulkan-sdk-action@3c53c378c9bfbb2ea122a1cc164a837d4004c871 # v1.5.2 with: vulkan_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} cache: true stripdown: true - name: Verify Vulkan SDK - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'vulkan' }} + if: ${{ matrix.backend == 'vulkan' }} shell: pwsh run: | if (-not $env:VULKAN_SDK -or -not (Test-Path $env:VULKAN_SDK)) { @@ -1084,59 +1299,120 @@ jobs: throw "glslc.exe was not found at $glslc" } & $glslc --version - $vulkanLib = Join-Path $env:VULKAN_SDK "Lib\vulkan-1.lib" - if (-not (Test-Path $vulkanLib)) { - throw "Expected Vulkan import library was not found: $vulkanLib" - } - name: Install ROCm HIP SDK - if: ${{ env.RUN_WINDOWS_GPU == 'true' && matrix.backend == 'rocm' }} + if: ${{ matrix.backend == 'rocm' }} uses: ./.github/actions/setup-windows-rocm-sdk with: rocm-hip-sdk-filename: ${{ env.ROCM_HIP_SDK_FILENAME }} - - name: Initialize MSVC for composed product - if: ${{ env.RUN_WINDOWS_CPU_FULL == 'true' || env.RUN_WINDOWS_GPU == 'true' }} - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 + - name: Initialize MSVC for native runtime + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: x64 - - name: Build backend-neutral Windows release host - if: ${{ env.RUN_WINDOWS_CPU_FULL == 'true' || env.RUN_WINDOWS_GPU == 'true' }} - shell: pwsh - run: .\scripts\build-windows.ps1 -BuildProfile release -HostOnly - - name: Package Windows native runtime - if: ${{ env.RUN_WINDOWS_CPU_FULL == 'true' || env.RUN_WINDOWS_GPU == 'true' }} - shell: bash + - name: Prepare immutable Windows GPU runtime + uses: ./.github/actions/prepare-native-runtime-input env: LLAMA_STAGE_CUDA_ARCHITECTURES: ${{ matrix.cuda_architectures }} LLAMA_STAGE_AMDGPU_TARGETS: ${{ matrix.rocm_architectures }} - MESH_LLM_CUDA_TOOLKIT_MAJOR: "12" - run: | - scripts/package-native-runtime.sh \ - --build \ - --backend "${{ matrix.backend }}" \ - --target x86_64-pc-windows-msvc \ - --out target/release/native-runtimes - - name: Check mesh-llm binary on Windows - if: ${{ matrix.backend == 'cpu' && env.RUN_WINDOWS_CPU_FULL != 'true' }} - shell: pwsh - run: cargo check --locked -p mesh-llm --bin mesh-llm --features dynamic-native-runtime - - name: Check Node SDK addon on Windows - if: ${{ matrix.backend == 'cpu' }} - shell: pwsh - run: cargo check --locked -p mesh-llm-nodejs - - name: Build Node SDK addon on Windows - if: ${{ env.RUN_WINDOWS_NODE_RELEASE == 'true' }} - shell: pwsh - run: cargo build --release --locked -p mesh-llm-nodejs - - name: Composed Windows CLI and client readiness smoke - if: ${{ env.RUN_WINDOWS_CPU_FULL == 'true' || env.RUN_WINDOWS_GPU == 'true' }} - shell: pwsh - run: | - .\target\release\mesh-llm.exe --log-format json --version - if ($LASTEXITCODE -ne 0) { throw "mesh-llm --version failed with exit code $LASTEXITCODE" } - .\target\release\mesh-llm.exe --log-format json --help | Select-Object -First 5 - if ($LASTEXITCODE -ne 0) { throw "mesh-llm --help failed with exit code $LASTEXITCODE" } - $env:MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR = "$PWD\target\release\native-runtimes" - .\target\release\mesh-llm.exe --log-format json runtime list - if ($LASTEXITCODE -ne 0) { throw "mesh-llm runtime list failed with exit code $LASTEXITCODE" } - bash scripts/ci-client-readiness-smoke.sh target/release/mesh-llm.exe target/release/native-runtimes - if ($LASTEXITCODE -ne 0) { throw "composed Windows ${{ matrix.name }} client readiness smoke failed" } + with: + backend: ${{ matrix.backend }} + target: x86_64-pc-windows-msvc + output_dir: runtime-input + - name: Upload immutable Windows GPU runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-windows-${{ matrix.backend }}-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + windows_cpu_product: + needs: [changes, windows_host_input, windows_cpu_runtime_input] + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_cpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Windows CPU + runs-on: windows-2022 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.x' + - name: Download immutable Windows host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-windows-host-input + path: host-input + - name: Download immutable Windows CPU runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-windows-cpu-runtime-input + path: runtime-input + - name: Compose Windows CPU product + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + backend: cpu + binary_name: mesh-llm.exe + readiness_smoke: "true" + - name: Upload composed Windows CPU product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-windows-cpu-product + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 + + windows_gpu_products: + needs: [changes, windows_host_input, windows_gpu_runtime_inputs] + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_gpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Windows ${{ matrix.name }} + runs-on: windows-2022 + strategy: + fail-fast: false + matrix: + include: + - name: CUDA + backend: cuda + - name: ROCm + backend: rocm + - name: Vulkan + backend: vulkan + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.x' + - name: Download immutable Windows host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-windows-host-input + path: host-input + - name: Download immutable Windows GPU runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-windows-${{ matrix.backend }}-runtime-input + path: runtime-input + - name: Compose Windows GPU product + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + backend: ${{ matrix.backend }} + binary_name: mesh-llm.exe + readiness_smoke: "true" + - name: Upload composed Windows GPU product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-windows-${{ matrix.backend }}-product + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/pr_cleanup.yml b/.github/workflows/pr_cleanup.yml index fea359d44a..92bd24e05b 100644 --- a/.github/workflows/pr_cleanup.yml +++ b/.github/workflows/pr_cleanup.yml @@ -36,7 +36,7 @@ jobs: # or run pull request code in this workflow. - name: Plan cache deletion shards id: plan - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: PR_CACHE_REF: refs/pull/${{ github.event.inputs.pr_number || github.event.pull_request.number }}/merge PR_CACHE_CLEANUP_WORKERS: ${{ vars.PR_CACHE_CLEANUP_WORKERS }} @@ -170,7 +170,7 @@ jobs: } - name: Upload cache deletion plan - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: pr-cache-cleanup-plan-${{ github.event.inputs.pr_number || github.event.pull_request.number }} path: cache-cleanup-plan @@ -191,14 +191,14 @@ jobs: # Deletes remain serial inside each worker so every host keeps the same # rate-limited request shape as the previous single-runner cleanup. - name: Download cache deletion plan - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: pr-cache-cleanup-plan-${{ github.event.inputs.pr_number || github.event.pull_request.number }} path: cache-cleanup-plan - name: Delete cache shard id: delete-cache-shard - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: PR_CACHE_SHARD_INDEX: ${{ matrix.shard.index }} with: @@ -446,7 +446,7 @@ jobs: - name: Upload cache deletion result if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: pr-cache-cleanup-result-${{ github.event.inputs.pr_number || github.event.pull_request.number }}-${{ matrix.shard.number }} path: cache-cleanup-results @@ -467,7 +467,7 @@ jobs: steps: - name: Delete artifacts from pull request runs id: delete-pr-artifacts - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }} PR_HEAD_SHA: ${{ github.event.inputs.artifact_head_sha || github.event.pull_request.head.sha }} @@ -608,7 +608,7 @@ jobs: actions: read steps: - name: Download cache deletion results - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 continue-on-error: true with: pattern: pr-cache-cleanup-result-${{ github.event.inputs.pr_number || github.event.pull_request.number }}-* diff --git a/.github/workflows/pr_quality.yml b/.github/workflows/pr_quality.yml index 20ef664120..1bd71a4586 100644 --- a/.github/workflows/pr_quality.yml +++ b/.github/workflows/pr_quality.yml @@ -2,6 +2,12 @@ name: PR Quality Checks on: workflow_dispatch: + inputs: + use_depot: + description: Route eligible non-GPU jobs to Depot for rollout benchmarking. + required: false + default: false + type: boolean push: branches: [main] pull_request: @@ -16,6 +22,8 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: + # Native GitHub caches are merge-ref scoped on pull requests, so jobs can + # safely reuse the base branch without making PR writes visible to main. CACHE_NAMESPACE: mesh-llm CARGO_INCREMENTAL: "0" SCCACHE_GHA_ENABLED: "true" @@ -37,14 +45,24 @@ jobs: cli_surface_changed: ${{ steps.compute.outputs.cli_surface_changed }} docs_only: ${{ steps.compute.outputs.docs_only }} rust_changed: ${{ steps.compute.outputs.rust_changed }} + depot_enabled: ${{ steps.runners.outputs.depot_enabled }} + allow_depot_remote_cache: ${{ steps.runners.outputs.allow_depot_remote_cache }} + runner: ${{ steps.runners.outputs.runner }} + runner_4: ${{ steps.runners.outputs.runner_4 }} + runner_8: ${{ steps.runners.outputs.runner_8 }} + runner_16: ${{ steps.runners.outputs.runner_16 }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: fetch-depth: 0 - - name: Check CI crate-list drift - run: cargo run -p xtask -- repo-consistency ci-crate-lists - - name: Check publish crate-chain drift - run: cargo run -p xtask -- repo-consistency publish-crates + - name: Select trusted Linux runners + id: runners + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + ref: ${{ github.ref }} + depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + manual_use_depot: ${{ inputs.use_depot == true }} - uses: ./.github/actions/compute-changes id: compute with: @@ -52,29 +70,42 @@ jobs: base_sha: ${{ github.event.pull_request.base.sha || '' }} head_sha: ${{ github.event.pull_request.head.sha || '' }} + ci-consistency: + name: CI consistency + needs: changes + if: needs.changes.outputs.docs_only != 'true' + runs-on: ${{ needs.changes.outputs.runner_4 }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + continue-on-error: true + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: pr-consistency + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Check CI crate-list drift + run: cargo run -p xtask -- repo-consistency ci-crate-lists + - name: Check publish crate-chain drift + run: cargo run -p xtask -- repo-consistency publish-crates + rust-fmt: needs: changes if: needs.changes.outputs.rust_changed == 'true' - runs-on: ubuntu-24.04 - container: - image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - defaults: - run: - shell: bash + runs-on: ${{ needs.changes.outputs.runner_4 }} steps: - - uses: actions/checkout@v5 - - name: Verify prebuilt CI environment - run: verify-runner-image public + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + with: + components: rustfmt - name: Check formatting run: cargo fmt --all -- --check rust-clippy: needs: changes if: needs.changes.outputs.rust_changed == 'true' - runs-on: ubuntu-24.04 + runs-on: ${{ needs.changes.outputs.runner_8 }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d credentials: @@ -93,10 +124,13 @@ jobs: RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Verify prebuilt CI environment run: verify-runner-image public - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: workspaces: . -> target @@ -134,20 +168,27 @@ jobs: ui-quality: needs: changes if: needs.changes.outputs.ui_changed == 'true' - runs-on: ubuntu-24.04 - container: - image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + runs-on: ${{ needs.changes.outputs.runner_4 }} defaults: run: working-directory: crates/mesh-llm-ui - shell: bash steps: - - uses: actions/checkout@v5 - - name: Verify prebuilt CI environment - run: verify-runner-image public + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 + with: + version: 10 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: 24 + - name: Configure isolated pnpm store + run: pnpm config set store-dir ~/.pnpm-store + - name: Cache pnpm store + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: ~/.pnpm-store + key: ${{ env.CACHE_NAMESPACE }}-pnpm-${{ runner.os }}-${{ hashFiles('crates/mesh-llm-ui/pnpm-lock.yaml', '.github/cache-version.txt') }} + restore-keys: | + ${{ env.CACHE_NAMESPACE }}-pnpm-${{ runner.os }}- - name: Install dependencies run: pnpm i --frozen-lockfile - name: Lint @@ -160,7 +201,7 @@ jobs: cli-docs-sync: needs: changes if: needs.changes.outputs.cli_surface_changed == 'true' - runs-on: ubuntu-24.04 + runs-on: ${{ needs.changes.outputs.runner }} steps: - name: Require public website docs update env: @@ -190,13 +231,14 @@ jobs: EOF summary: - needs: [changes, rust-fmt, rust-clippy, ui-quality, cli-docs-sync] + needs: [changes, ci-consistency, rust-fmt, rust-clippy, ui-quality, cli-docs-sync] if: always() runs-on: ubuntu-24.04 steps: - name: Check quality gate env: CHANGES: ${{ needs.changes.result }} + CONSISTENCY: ${{ needs.ci-consistency.result }} FMT: ${{ needs.rust-fmt.result }} CLIPPY: ${{ needs.rust-clippy.result }} UI: ${{ needs.ui-quality.result }} @@ -208,6 +250,7 @@ jobs: echo "| Check | Result |" echo "| --- | --- |" printf '| %s | %s |\n' "changes" "$CHANGES" + printf '| %s | %s |\n' "ci-consistency" "$CONSISTENCY" printf '| %s | %s |\n' "rust-fmt" "$FMT" printf '| %s | %s |\n' "rust-clippy" "$CLIPPY" printf '| %s | %s |\n' "ui-quality" "$UI" @@ -215,7 +258,7 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" FAILED=false - for STATUS in "$CHANGES" "$FMT" "$CLIPPY" "$UI" "$CLI_DOCS"; do + for STATUS in "$CHANGES" "$CONSISTENCY" "$FMT" "$CLIPPY" "$UI" "$CLI_DOCS"; do if [[ "$STATUS" == "failure" || "$STATUS" == "cancelled" ]]; then FAILED=true fi diff --git a/.github/workflows/pr_website.yml b/.github/workflows/pr_website.yml index afb33fcec4..1f0b9b7e1e 100644 --- a/.github/workflows/pr_website.yml +++ b/.github/workflows/pr_website.yml @@ -21,7 +21,7 @@ jobs: outputs: website_changed: ${{ steps.compute.outputs.website_changed }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: fetch-depth: 0 - uses: ./.github/actions/compute-changes @@ -39,8 +39,8 @@ jobs: run: working-directory: website steps: - - uses: actions/checkout@v5 - - uses: actions/setup-node@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version: 24 cache: npm diff --git a/.github/workflows/queue-unsloth-layer-packages.yml b/.github/workflows/queue-unsloth-layer-packages.yml index 358958e43b..aadfac8817 100644 --- a/.github/workflows/queue-unsloth-layer-packages.yml +++ b/.github/workflows/queue-unsloth-layer-packages.yml @@ -63,11 +63,11 @@ jobs: SPLIT_CANDIDATE_VRAM_GIB: ${{ github.event.inputs.split_candidate_vram_gib || '8' }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 with: workspaces: . -> target prefix-key: mesh-llm-rust-${{ hashFiles('.github/cache-version.txt') }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b05c1e22cd..e097bffa4e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,14 +27,19 @@ on: required: true default: false type: boolean + use_depot: + description: Route eligible non-GPU Linux jobs to Depot for a trusted main-ref canary + required: true + default: false + type: boolean concurrency: group: release-${{ github.repository }}-${{ github.ref }} cancel-in-progress: false permissions: - contents: write - packages: write + contents: read + packages: read env: CACHE_NAMESPACE: mesh-llm @@ -55,7 +60,24 @@ jobs: skip_gpu_bundles: ${{ steps.meta.outputs.skip_gpu_bundles }} canary: ${{ steps.meta.outputs.canary }} force_hosted_runners: ${{ steps.meta.outputs.force_hosted_runners }} + depot_enabled: ${{ steps.runners.outputs.depot_enabled }} + allow_depot_remote_cache: ${{ steps.runners.outputs.allow_depot_remote_cache }} + runner: ${{ steps.runners.outputs.runner }} + runner_4: ${{ steps.runners.outputs.runner_4 }} + runner_8: ${{ steps.runners.outputs.runner_8 }} + runner_16: ${{ steps.runners.outputs.runner_16 }} steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Select trusted Linux runners + id: runners + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + ref: ${{ github.ref }} + depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + manual_use_depot: ${{ inputs.use_depot == true }} - id: meta shell: bash env: @@ -91,6 +113,8 @@ jobs: build: name: Build immutable host ${{ matrix.name }} needs: metadata + # Host attestation consumes the signing key. Keep this producer on GitHub + # until compilation and signing are split into separate jobs. runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -103,15 +127,15 @@ jobs: os: ubuntu-24.04 target: linux-x86_64 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 with: version: 10 - - uses: actions/setup-node@v5 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version: 24 cache: pnpm @@ -119,11 +143,14 @@ jobs: .github/cache-version.txt crates/mesh-llm-ui/pnpm-lock.yaml - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - - uses: taiki-e/install-action@just + - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just - - uses: mozilla-actions/sccache-action@v0.0.9 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} - name: Install Linux dependencies if: runner.os == 'Linux' @@ -139,33 +166,26 @@ jobs: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - - name: Build and attest release host + - name: Prepare release attestation keys env: MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-private-key.json MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} RELEASE_ATTESTATION_SIGNING_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE }} RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} run: | printf '%s' "$RELEASE_ATTESTATION_SIGNING_KEY" > "$MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE" printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" - just --shell bash --shell-arg -c release-host-build - mkdir -p host-input - cp target/release/mesh-llm host-input/mesh-llm - cargo run -q -p xtask -- release-attestation stamp \ - --binary host-input/mesh-llm \ - --signing-key-file "$MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE" - cargo run -q -p xtask -- release-attestation inspect \ - --binary host-input/mesh-llm \ - --public-key-file "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" \ - --json - python3 scripts/verify-host-dependencies.py \ - host-input/mesh-llm \ - --report host-input/host-imports.json - shasum -a 256 host-input/mesh-llm | awk '{print $1 " mesh-llm"}' > host-input/mesh-llm.sha256 + + - name: Build and attest release host + uses: ./.github/actions/prepare-host-input + with: + profile: release + output_dir: host-input + attestation_signing_key_file: ${{ runner.temp }}/mesh-release-attestation-private-key.json + attestation_public_key_file: ${{ runner.temp }}/mesh-release-attestation-public-key.json - name: Upload immutable host input - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: host-input-${{ matrix.target }} path: host-input/* @@ -174,7 +194,7 @@ jobs: compose_cpu_products: name: Compose CPU product ${{ matrix.name }} needs: [metadata, build, build_native_runtime] - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.host_target == 'linux-x86_64' && needs.metadata.outputs.runner_4 || matrix.os }} strategy: fail-fast: false matrix: @@ -196,87 +216,63 @@ jobs: release_arch: x86_64 artifact_name: release-linux steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@stable - - name: Install macOS attestation verifier linker - if: runner.os == 'macOS' - run: brew install lld + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - name: Download immutable host input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: host-input-${{ matrix.host_target }} path: host-input - name: Download immutable runtime input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: ${{ matrix.runtime_artifact }} path: runtime-input + - name: Prepare release verification key + env: + MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json + RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} + run: printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" + - name: Compose from verified producer inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + version: ${{ needs.metadata.outputs.tag }} + backend: ${{ matrix.flavor }} + readiness_smoke: "true" + attestation_public_key_file: ${{ runner.temp }}/mesh-release-attestation-public-key.json + + - name: Package verified product env: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} MESH_RELEASE_OS: ${{ matrix.release_os }} MESH_RELEASE_ARCH: ${{ matrix.release_arch }} MESH_RELEASE_FLAVOR: ${{ matrix.flavor }} MESH_LLM_RELEASE_BIN_DIR: ${{ github.workspace }}/host-input - MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ github.workspace }}/runtime-root + MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ steps.compose.outputs.runtime_root }} + MESH_LLM_PRECOMPOSED_PRODUCT_DIR: ${{ steps.compose.outputs.product_dir }} MESH_RELEASE_HOST_PRESTAMPED: "1" + MESH_RELEASE_ATTESTATION_PREVERIFIED: "1" MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json - RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} - run: | - set -euo pipefail - printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" - test -s host-input/mesh-llm.sha256 - (cd host-input && shasum -a 256 -c mesh-llm.sha256) - chmod +x host-input/mesh-llm - mkdir -p runtime-root - runtime_archive="" - runtime_archive_count=0 - while IFS= read -r archive; do - runtime_archive="$archive" - runtime_archive_count=$((runtime_archive_count + 1)) - done < <(find runtime-input -name '*.tar.gz' -type f -print) - test "$runtime_archive_count" -eq 1 - tar -xzf "$runtime_archive" -C runtime-root - runtime_dir="" - runtime_dir_count=0 - while IFS= read -r directory; do - runtime_dir="$directory" - runtime_dir_count=$((runtime_dir_count + 1)) - done < <(find runtime-root -mindepth 1 -maxdepth 1 -type d -print) - test "$runtime_dir_count" -eq 1 - scripts/verify-native-runtime-package.sh "$runtime_dir" - scripts/ci-client-readiness-smoke.sh host-input/mesh-llm runtime-root - if [[ "$MESH_RELEASE_OS" == "Linux" ]]; then - runtime_name="$(basename "$runtime_dir")" - rm -rf smoke-input - mkdir -p "smoke-input/native-runtimes" - cp host-input/mesh-llm smoke-input/mesh-llm - cp host-input/host-imports.json smoke-input/host-imports.json - cp -R "$runtime_dir" "smoke-input/native-runtimes/$runtime_name" - python3 scripts/compose-product-bundle.py \ - --bundle smoke-input \ - --host smoke-input/mesh-llm \ - --runtime "smoke-input/native-runtimes/$runtime_name" \ - --version "$RELEASE_TAG" \ - --backend "$MESH_RELEASE_FLAVOR" - test -s smoke-input/product-manifest.json - test -s "smoke-input/native-runtimes/$runtime_name/manifest.json" - fi - scripts/package-release.sh "$RELEASE_TAG" dist + run: scripts/package-release.sh "$RELEASE_TAG" dist - name: Upload composed release product - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: ${{ matrix.artifact_name }} path: dist/* if-no-files-found: error - name: Upload Linux composed smoke product if: matrix.artifact_name == 'release-linux' - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: - name: release-linux-inference-product - path: smoke-input/ + name: ci-release-linux-inference-product + path: ${{ steps.compose.outputs.archive_path }} if-no-files-found: error inference_smoke_tests: @@ -284,7 +280,7 @@ jobs: if: needs.compose_cpu_products.result == 'success' uses: ./.github/workflows/smoke.yml with: - artifact_name: release-linux-inference-product + artifact_name: ci-release-linux-inference-product mesh_binary_target: target/release/mesh-llm cache_key_prefix: release- release_tag: ${{ needs.metadata.outputs.tag }} @@ -294,7 +290,7 @@ jobs: build_native_sdk_runtime: name: Build native SDK runtime ${{ matrix.name }} needs: metadata - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && needs.metadata.outputs.runner_8 || matrix.os }} strategy: fail-fast: false matrix: @@ -318,13 +314,16 @@ jobs: LLAMA_STAGE_BACKEND: ${{ matrix.backend }} MESH_NATIVE_SDK_TARGET: ${{ matrix.target }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - - uses: mozilla-actions/sccache-action@v0.0.9 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} - name: Install Linux dependencies if: runner.os == 'Linux' @@ -354,20 +353,47 @@ jobs: - name: Package native SDK runtime crate run: scripts/package-native-sdk-crate.sh --out dist/native-sdk-crates dist/native-sdk/*.tar.gz + - name: Stage flat native SDK release assets + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + runtime_archives=(dist/native-sdk/*.tar.gz) + runtime_checksums=(dist/native-sdk/*.sha256) + runtime_crates=(dist/native-sdk-crates/*/target/package/*.crate) + if [[ "${#runtime_archives[@]}" -ne 1 || + "${#runtime_checksums[@]}" -ne 1 || + "${#runtime_crates[@]}" -ne 1 ]]; then + echo "expected one native SDK archive, checksum, and crate" >&2 + exit 1 + fi + + staging_dir="release-native-sdk-assets" + rm -rf "$staging_dir" + mkdir -p "$staging_dir" + for source in \ + "${runtime_archives[@]}" \ + "${runtime_checksums[@]}" \ + "${runtime_crates[@]}"; do + destination="$staging_dir/$(basename "$source")" + if [[ -e "$destination" ]]; then + echo "native SDK release asset basename collision: $destination" >&2 + exit 1 + fi + cp -p "$source" "$destination" + done + - name: Upload native SDK runtime - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-native-sdk-${{ matrix.artifact_suffix }} - path: | - dist/native-sdk/*.tar.gz - dist/native-sdk/*.sha256 - dist/native-sdk-crates/*/target/package/*.crate + path: release-native-sdk-assets/* if-no-files-found: error build_native_runtime: name: Build native runtime ${{ matrix.name }} needs: metadata - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && needs.metadata.outputs.runner_8 || matrix.os }} strategy: fail-fast: false matrix: @@ -391,13 +417,16 @@ jobs: LLAMA_STAGE_BACKEND: ${{ matrix.backend }} MESH_NATIVE_RUNTIME_TARGET: ${{ matrix.target }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - - uses: mozilla-actions/sccache-action@v0.0.9 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} - name: Install Linux dependencies if: runner.os == 'Linux' @@ -414,18 +443,14 @@ jobs: run: scripts/release-version.sh "$RELEASE_TAG" - name: Package native runtime - run: | - scripts/package-native-runtime.sh \ - --build \ - --backend "${{ matrix.backend }}" \ - --target "${{ matrix.target }}" \ - --out dist/native-runtimes - - - name: Verify native runtime artifact - run: scripts/verify-native-runtime-package.sh dist/native-runtimes/*.tar.gz + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: ${{ matrix.backend }} + target: ${{ matrix.target }} + output_dir: dist/native-runtimes - name: Upload native runtime - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-native-runtime-${{ matrix.artifact_suffix }} path: | @@ -459,7 +484,7 @@ jobs: MESH_LLM_CUDA_TOOLKIT_MAJOR: ${{ matrix.cuda_major }} SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Trust checkout directory @@ -467,22 +492,21 @@ jobs: - name: Verify prebuilt backend environment run: verify-runner-image public cuda - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' env: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - name: Package native runtime - run: | - scripts/package-native-runtime.sh \ - --build \ - --backend cuda \ - --target aarch64-unknown-linux-gnu \ - --out dist/native-runtimes - - name: Verify native runtime artifact - run: scripts/verify-native-runtime-package.sh dist/native-runtimes/*.tar.gz + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: cuda + target: aarch64-unknown-linux-gnu + output_dir: dist/native-runtimes - name: Upload native runtime - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-native-runtime-linux-aarch64-cuda-${{ matrix.cuda_major }} path: | @@ -522,7 +546,7 @@ jobs: MESH_LLM_CUDA_TOOLKIT_MAJOR: ${{ matrix.cuda_major }} SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Trust checkout directory @@ -530,8 +554,10 @@ jobs: - name: Verify prebuilt backend environment run: verify-runner-image public cuda - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} - name: Cache native runtime CUDA backend build - uses: actions/cache@v5 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ env.LLAMA_STAGE_BUILD_DIR }} key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-cuda${{ matrix.cuda_major }}-${{ matrix.cuda_version }}-dynamic-${{ matrix.cuda_architectures_cache }}-${{ hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} @@ -541,16 +567,13 @@ jobs: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - name: Package native runtime - run: | - scripts/package-native-runtime.sh \ - --build \ - --backend cuda \ - --target x86_64-unknown-linux-gnu \ - --out dist/native-runtimes - - name: Verify native runtime artifact - run: scripts/verify-native-runtime-package.sh dist/native-runtimes/*.tar.gz + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: cuda + target: x86_64-unknown-linux-gnu + output_dir: dist/native-runtimes - name: Upload native runtime - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-native-runtime-linux-x86_64-cuda-${{ matrix.cuda_major }} path: | @@ -562,7 +585,7 @@ jobs: name: Build native runtime Linux x86_64 ROCm needs: metadata if: ${{ needs.metadata.outputs.skip_gpu_bundles != 'true' }} - runs-on: ubuntu-24.04 + runs-on: ${{ needs.metadata.outputs.runner_16 }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:0e13e5d2d2c121df265ff6c69be81e468989e09f81d6b7ff049b110cc0bb0d2b credentials: @@ -575,7 +598,7 @@ jobs: MESH_NATIVE_RUNTIME_TARGET: x86_64-unknown-linux-gnu SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Trust checkout directory @@ -583,8 +606,10 @@ jobs: - name: Verify prebuilt backend environment run: verify-runner-image public rocm - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} - name: Cache native runtime ROCm backend build - uses: actions/cache@v5 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ env.LLAMA_STAGE_BUILD_DIR }} key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-rocm7.0-dynamic-gfx90a_gfx942_gfx1100_gfx1101_gfx1102_gfx1103_gfx1151_gfx1200_gfx1201-${{ hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} @@ -594,16 +619,13 @@ jobs: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - name: Package native runtime - run: | - scripts/package-native-runtime.sh \ - --build \ - --backend rocm \ - --target x86_64-unknown-linux-gnu \ - --out dist/native-runtimes - - name: Verify native runtime artifact - run: scripts/verify-native-runtime-package.sh dist/native-runtimes/*.tar.gz + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: rocm + target: x86_64-unknown-linux-gnu + output_dir: dist/native-runtimes - name: Upload native runtime - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-native-runtime-linux-x86_64-rocm path: | @@ -615,7 +637,7 @@ jobs: name: Build native runtime Linux x86_64 Vulkan needs: metadata if: ${{ needs.metadata.outputs.skip_gpu_bundles != 'true' }} - runs-on: ubuntu-24.04 + runs-on: ${{ needs.metadata.outputs.runner_16 }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:ce55fed5c680cd3184b5d4770d9a77c43a702687690906e5753efd2cea27ed80 credentials: @@ -627,7 +649,7 @@ jobs: MESH_NATIVE_RUNTIME_TARGET: x86_64-unknown-linux-gnu SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Trust checkout directory @@ -635,8 +657,10 @@ jobs: - name: Verify prebuilt backend environment run: verify-runner-image public vulkan - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} - name: Cache native runtime Vulkan backend build - uses: actions/cache@v5 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ env.LLAMA_STAGE_BUILD_DIR }} key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-vulkan-dynamic-${{ hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} @@ -646,16 +670,13 @@ jobs: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - name: Package native runtime - run: | - scripts/package-native-runtime.sh \ - --build \ - --backend vulkan \ - --target x86_64-unknown-linux-gnu \ - --out dist/native-runtimes - - name: Verify native runtime artifact - run: scripts/verify-native-runtime-package.sh dist/native-runtimes/*.tar.gz + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: vulkan + target: x86_64-unknown-linux-gnu + output_dir: dist/native-runtimes - name: Upload native runtime - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-native-runtime-linux-x86_64-vulkan path: | @@ -670,15 +691,15 @@ jobs: env: LLAMA_STAGE_BACKEND: metal steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 with: version: 10 - - uses: actions/setup-node@v5 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version: 24 cache: pnpm @@ -686,9 +707,9 @@ jobs: .github/cache-version.txt crates/mesh-llm-ui/pnpm-lock.yaml - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - - uses: mozilla-actions/sccache-action@v0.0.9 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - name: Install macOS dependencies run: brew install cmake ninja lld @@ -733,14 +754,14 @@ jobs: - name: Upload generated SwiftPM manifest if: github.event_name == 'workflow_dispatch' - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: swift-package-manifest path: Package.swift if-no-files-found: error - name: Upload SwiftPM binary artifact - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-swift-sdk path: dist/MeshLLMFFI.xcframework.zip @@ -754,21 +775,21 @@ jobs: LLAMA_STAGE_BACKEND: cpu MESH_RELEASE_ARCH: aarch64 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 with: version: 10 - - uses: actions/setup-node@v5 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version: 24 cache: pnpm cache-dependency-path: | .github/cache-version.txt crates/mesh-llm-ui/pnpm-lock.yaml - - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@just + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y build-essential cmake ninja-build pkg-config libssl-dev libdbus-1-dev curl lld - name: Prepare dispatched release version @@ -776,27 +797,25 @@ jobs: env: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - - name: Build and attest ARM64 release host + - name: Prepare release attestation keys env: MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-private-key.json MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} RELEASE_ATTESTATION_SIGNING_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE }} RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} run: | printf '%s' "$RELEASE_ATTESTATION_SIGNING_KEY" > "$MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE" printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" - just --shell bash --shell-arg -c release-host-build - mkdir -p host-input - cp target/release/mesh-llm host-input/mesh-llm - cargo run -q -p xtask -- release-attestation stamp \ - --binary host-input/mesh-llm \ - --signing-key-file "$MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE" - python3 scripts/verify-host-dependencies.py \ - host-input/mesh-llm \ - --report host-input/host-imports.json - shasum -a 256 host-input/mesh-llm | awk '{print $1 " mesh-llm"}' > host-input/mesh-llm.sha256 - - uses: actions/upload-artifact@v6 + + - name: Build and attest ARM64 release host + uses: ./.github/actions/prepare-host-input + with: + profile: release + output_dir: host-input + attestation_signing_key_file: ${{ runner.temp }}/mesh-release-attestation-private-key.json + attestation_public_key_file: ${{ runner.temp }}/mesh-release-attestation-public-key.json + + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: host-input-linux-aarch64 path: host-input/* @@ -807,44 +826,51 @@ jobs: needs: [metadata, build_linux_arm64, build_native_runtime] runs-on: ubuntu-24.04-arm steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@stable - - uses: actions/download-artifact@v7 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: host-input-linux-aarch64 path: host-input - - uses: actions/download-artifact@v7 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-native-runtime-linux-aarch64-cpu path: runtime-input + - name: Prepare release verification key + env: + MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json + RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} + run: printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" + - name: Compose from verified producer inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + version: ${{ needs.metadata.outputs.tag }} + backend: cpu + readiness_smoke: "true" + attestation_public_key_file: ${{ runner.temp }}/mesh-release-attestation-public-key.json + + - name: Package verified ARM64 CPU product env: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} MESH_RELEASE_OS: Linux MESH_RELEASE_ARCH: aarch64 MESH_RELEASE_FLAVOR: cpu MESH_LLM_RELEASE_BIN_DIR: ${{ github.workspace }}/host-input - MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ github.workspace }}/runtime-root + MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ steps.compose.outputs.runtime_root }} + MESH_LLM_PRECOMPOSED_PRODUCT_DIR: ${{ steps.compose.outputs.product_dir }} MESH_RELEASE_HOST_PRESTAMPED: "1" + MESH_RELEASE_ATTESTATION_PREVERIFIED: "1" MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json - RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} - run: | - set -euo pipefail - printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" - (cd host-input && sha256sum -c mesh-llm.sha256) - chmod +x host-input/mesh-llm - mkdir -p runtime-root - mapfile -t runtime_archives < <(find runtime-input -name '*.tar.gz' -type f -print) - test "${#runtime_archives[@]}" -eq 1 - tar -xzf "${runtime_archives[0]}" -C runtime-root - mapfile -t runtime_dirs < <(find runtime-root -mindepth 1 -maxdepth 1 -type d -print) - test "${#runtime_dirs[@]}" -eq 1 - scripts/verify-native-runtime-package.sh "${runtime_dirs[0]}" - scripts/ci-client-readiness-smoke.sh host-input/mesh-llm runtime-root - scripts/package-release.sh "$RELEASE_TAG" dist - - uses: actions/upload-artifact@v6 + run: scripts/package-release.sh "$RELEASE_TAG" dist + + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-linux-arm64 path: dist/* @@ -856,7 +882,7 @@ jobs: if: ${{ needs.compose_linux_arm64_cpu.result == 'success' }} runs-on: ${{ fromJson(vars.USE_SELF_HOSTED == 'true' && needs.metadata.outputs.force_hosted_runners != 'true' && '["self-hosted","Linux","ARM64"]' || '["ubuntu-24.04-arm"]') }} steps: - - uses: actions/download-artifact@v7 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-linux-arm64 path: release-linux-arm64 @@ -888,8 +914,8 @@ jobs: "$binary" --help | head -5 "$binary" runtime --help | head -20 - build_linux_aarch64_cuda: - name: Build Linux aarch64 CUDA (${{ matrix.cuda_version }}) + compose_linux_aarch64_cuda: + name: Compose Linux aarch64 CUDA (${{ matrix.cuda_version }}) needs: [metadata, build_linux_arm64, build_native_runtime_linux_aarch64_cuda] if: ${{ needs.metadata.outputs.skip_gpu_bundles != 'true' }} runs-on: ubuntu-24.04-arm @@ -907,26 +933,22 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} env: - LLAMA_STAGE_BACKEND: cuda - MESH_RELEASE_ARCH: aarch64 MESH_CUDA_VERSION: ${{ matrix.cuda_version }} - SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Trust checkout directory run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Verify prebuilt backend environment run: verify-runner-image public cuda - - uses: ./.github/actions/configure-sccache-gha - name: Download immutable host input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: host-input-linux-aarch64 path: host-input - name: Download immutable runtime input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-native-runtime-linux-aarch64-cuda-${{ matrix.cuda_version == '12.9.2' && '12' || '13' }} path: runtime-input @@ -935,50 +957,59 @@ jobs: env: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - - name: Compose aarch64 CUDA release bundle from producer inputs - shell: bash + + - name: Prepare release verification key env: MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} + run: printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" + + - name: Compose aarch64 CUDA product from producer inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + version: ${{ needs.metadata.outputs.tag }} + backend: cuda + readiness_smoke: "true" + attestation_public_key_file: ${{ runner.temp }}/mesh-release-attestation-public-key.json + + - name: Package verified aarch64 CUDA product + env: + RELEASE_TAG: ${{ needs.metadata.outputs.tag }} + MESH_RELEASE_OS: Linux + MESH_RELEASE_ARCH: aarch64 + MESH_RELEASE_FLAVOR: cuda MESH_LLM_RELEASE_BIN_DIR: ${{ github.workspace }}/host-input - MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ github.workspace }}/runtime-root + MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ steps.compose.outputs.runtime_root }} + MESH_LLM_PRECOMPOSED_PRODUCT_DIR: ${{ steps.compose.outputs.product_dir }} MESH_RELEASE_HOST_PRESTAMPED: "1" - run: | - printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" - (cd host-input && sha256sum -c mesh-llm.sha256) - chmod +x host-input/mesh-llm - mkdir -p runtime-root - mapfile -t runtime_archives < <(find runtime-input -name '*.tar.gz' -type f -print) - test "${#runtime_archives[@]}" -eq 1 - tar -xzf "${runtime_archives[0]}" -C runtime-root - mapfile -t runtime_dirs < <(find runtime-root -mindepth 1 -maxdepth 1 -type d -print) - test "${#runtime_dirs[@]}" -eq 1 - scripts/verify-native-runtime-package.sh "${runtime_dirs[0]}" - scripts/ci-client-readiness-smoke.sh host-input/mesh-llm runtime-root - MESH_RELEASE_FLAVOR=cuda scripts/package-release.sh "$RELEASE_TAG" dist - - uses: actions/upload-artifact@v6 + MESH_RELEASE_ATTESTATION_PREVERIFIED: "1" + MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json + run: scripts/package-release.sh "$RELEASE_TAG" dist + + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-linux-aarch64-cuda-${{ matrix.cuda_version }} path: dist/* if-no-files-found: error - build_linux_cuda: - name: Build Linux CUDA (${{ matrix.cuda_version }}) + compose_linux_cuda: + name: Compose Linux CUDA (${{ matrix.cuda_version }}) needs: [metadata, build, build_native_runtime_linux_x86_64_cuda] if: ${{ needs.metadata.outputs.skip_gpu_bundles != 'true' }} - runs-on: ${{ fromJson(vars.USE_SELF_HOSTED == 'true' && needs.metadata.outputs.force_hosted_runners != 'true' && '["self-hosted","Linux","X64","amd64","gpu-nvidia"]' || '["ubuntu-24.04"]') }} + runs-on: ${{ needs.metadata.outputs.runner_4 }} strategy: fail-fast: false matrix: include: - cuda_version: '12.9.2' cuda_major: '12' - cuda_architectures_cache: '75_80_86_87_89_90' runner_image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd - cuda_version: '13.1.2' cuda_major: '13' - cuda_architectures_cache: '75_80_86_87_89_90_100_103_120_121' runner_image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:6b87598605f5d8deeafecfb1a55027e0ca9e47f4fc6f230d030487c450c31aa6 container: image: ${{ matrix.runner_image }} @@ -986,62 +1017,63 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} env: - LLAMA_STAGE_BACKEND: cuda - LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-cuda-sm${{ matrix.cuda_architectures_cache }} MESH_CUDA_VERSION: ${{ matrix.cuda_version }} - SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Trust checkout directory run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Verify prebuilt backend environment run: verify-runner-image public cuda - - uses: ./.github/actions/configure-sccache-gha - name: Download immutable host input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: host-input-linux-x86_64 path: host-input - name: Download immutable runtime input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-native-runtime-linux-x86_64-cuda-${{ matrix.cuda_major }} path: runtime-input - - name: Cache release CUDA backend build - uses: actions/cache@v5 - with: - path: ${{ env.LLAMA_STAGE_BUILD_DIR }} - key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-cuda${{ matrix.cuda_major }}-${{ matrix.cuda_version }}-dynamic-${{ matrix.cuda_architectures_cache }}-${{ hashFiles('scripts/package-native-runtime.sh', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'Justfile', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' env: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - - name: Compose CUDA release bundle from producer inputs - shell: bash + - name: Prepare release verification key env: MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} + run: printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" + + - name: Compose CUDA product from producer inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + version: ${{ needs.metadata.outputs.tag }} + backend: cuda + readiness_smoke: "true" + attestation_public_key_file: ${{ runner.temp }}/mesh-release-attestation-public-key.json + + - name: Package verified CUDA product + env: + RELEASE_TAG: ${{ needs.metadata.outputs.tag }} + MESH_RELEASE_OS: Linux + MESH_RELEASE_ARCH: x86_64 + MESH_RELEASE_FLAVOR: cuda MESH_LLM_RELEASE_BIN_DIR: ${{ github.workspace }}/host-input - MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ github.workspace }}/runtime-root + MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ steps.compose.outputs.runtime_root }} + MESH_LLM_PRECOMPOSED_PRODUCT_DIR: ${{ steps.compose.outputs.product_dir }} MESH_RELEASE_HOST_PRESTAMPED: "1" - run: | - printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" - (cd host-input && sha256sum -c mesh-llm.sha256) - chmod +x host-input/mesh-llm - mkdir -p runtime-root - mapfile -t runtime_archives < <(find runtime-input -name '*.tar.gz' -type f -print) - test "${#runtime_archives[@]}" -eq 1 - tar -xzf "${runtime_archives[0]}" -C runtime-root - mapfile -t runtime_dirs < <(find runtime-root -mindepth 1 -maxdepth 1 -type d -print) - test "${#runtime_dirs[@]}" -eq 1 - scripts/verify-native-runtime-package.sh "${runtime_dirs[0]}" - scripts/ci-client-readiness-smoke.sh host-input/mesh-llm runtime-root - MESH_RELEASE_FLAVOR=cuda scripts/package-release.sh "$RELEASE_TAG" dist - - uses: actions/upload-artifact@v6 + MESH_RELEASE_ATTESTATION_PREVERIFIED: "1" + MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json + run: scripts/package-release.sh "$RELEASE_TAG" dist + + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-linux-cuda-${{ matrix.cuda_version }} path: dist/* @@ -1049,223 +1081,207 @@ jobs: - build_linux_rocm: - name: Build Linux ROCm + compose_linux_rocm: + name: Compose Linux ROCm needs: [metadata, build, build_native_runtime_linux_x86_64_rocm] if: ${{ needs.metadata.outputs.skip_gpu_bundles != 'true' }} - runs-on: ubuntu-24.04 + runs-on: ${{ needs.metadata.outputs.runner_4 }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:0e13e5d2d2c121df265ff6c69be81e468989e09f81d6b7ff049b110cc0bb0d2b credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - env: - LLAMA_STAGE_BACKEND: rocm - LLAMA_STAGE_AMDGPU_TARGETS: gfx90a;gfx942;gfx1100;gfx1101;gfx1102;gfx1103;gfx1151;gfx1200;gfx1201 - LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-rocm-gfx90a_gfx942_gfx1100_gfx1101_gfx1102_gfx1103_gfx1151_gfx1200_gfx1201 - SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Trust checkout directory run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Verify prebuilt backend environment run: verify-runner-image public rocm - - uses: ./.github/actions/configure-sccache-gha - name: Download immutable host input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: host-input-linux-x86_64 path: host-input - name: Download immutable runtime input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-native-runtime-linux-x86_64-rocm path: runtime-input - - name: Cache release ROCm backend build - uses: actions/cache@v5 - with: - path: ${{ env.LLAMA_STAGE_BUILD_DIR }} - key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-rocm7.0-dynamic-gfx90a_gfx942_gfx1100_gfx1101_gfx1102_gfx1103_gfx1151_gfx1200_gfx1201-${{ hashFiles('scripts/package-native-runtime.sh', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'Justfile', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' env: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - - name: Compose ROCm release bundle from producer inputs - shell: bash + - name: Prepare release verification key env: MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} + run: printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" + + - name: Compose ROCm product from producer inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + version: ${{ needs.metadata.outputs.tag }} + backend: rocm + readiness_smoke: "true" + attestation_public_key_file: ${{ runner.temp }}/mesh-release-attestation-public-key.json + + - name: Package verified ROCm product + env: + RELEASE_TAG: ${{ needs.metadata.outputs.tag }} + MESH_RELEASE_OS: Linux + MESH_RELEASE_ARCH: x86_64 + MESH_RELEASE_FLAVOR: rocm MESH_LLM_RELEASE_BIN_DIR: ${{ github.workspace }}/host-input - MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ github.workspace }}/runtime-root + MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ steps.compose.outputs.runtime_root }} + MESH_LLM_PRECOMPOSED_PRODUCT_DIR: ${{ steps.compose.outputs.product_dir }} MESH_RELEASE_HOST_PRESTAMPED: "1" - run: | - printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" - (cd host-input && sha256sum -c mesh-llm.sha256) - chmod +x host-input/mesh-llm - mkdir -p runtime-root - mapfile -t runtime_archives < <(find runtime-input -name '*.tar.gz' -type f -print) - test "${#runtime_archives[@]}" -eq 1 - tar -xzf "${runtime_archives[0]}" -C runtime-root - mapfile -t runtime_dirs < <(find runtime-root -mindepth 1 -maxdepth 1 -type d -print) - test "${#runtime_dirs[@]}" -eq 1 - scripts/verify-native-runtime-package.sh "${runtime_dirs[0]}" - scripts/ci-client-readiness-smoke.sh host-input/mesh-llm runtime-root - MESH_RELEASE_FLAVOR=rocm scripts/package-release.sh "$RELEASE_TAG" dist - - uses: actions/upload-artifact@v6 + MESH_RELEASE_ATTESTATION_PREVERIFIED: "1" + MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json + run: scripts/package-release.sh "$RELEASE_TAG" dist + + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-linux-rocm path: dist/* if-no-files-found: error - build_linux_vulkan: - name: Build Linux Vulkan + compose_linux_vulkan: + name: Compose Linux Vulkan needs: [metadata, build, build_native_runtime_linux_x86_64_vulkan] if: ${{ needs.metadata.outputs.skip_gpu_bundles != 'true' }} - runs-on: ubuntu-24.04 + runs-on: ${{ needs.metadata.outputs.runner_4 }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:ce55fed5c680cd3184b5d4770d9a77c43a702687690906e5753efd2cea27ed80 credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - env: - LLAMA_STAGE_BACKEND: vulkan - LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-vulkan - SCCACHE_GHA_ENABLED: "false" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - name: Trust checkout directory run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Verify prebuilt backend environment run: verify-runner-image public vulkan - - uses: ./.github/actions/configure-sccache-gha - name: Download immutable host input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: host-input-linux-x86_64 path: host-input - name: Download immutable runtime input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-native-runtime-linux-x86_64-vulkan path: runtime-input - - name: Cache release Vulkan backend build - uses: actions/cache@v5 - with: - path: ${{ env.LLAMA_STAGE_BUILD_DIR }} - key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-vulkan-dynamic-${{ hashFiles('scripts/package-native-runtime.sh', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'Justfile', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' env: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - - name: Compose Vulkan release bundle from producer inputs - shell: bash + - name: Prepare release verification key env: MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} + run: printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" + + - name: Compose Vulkan product from producer inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + version: ${{ needs.metadata.outputs.tag }} + backend: vulkan + readiness_smoke: "true" + attestation_public_key_file: ${{ runner.temp }}/mesh-release-attestation-public-key.json + + - name: Package verified Vulkan product + env: + RELEASE_TAG: ${{ needs.metadata.outputs.tag }} + MESH_RELEASE_OS: Linux + MESH_RELEASE_ARCH: x86_64 + MESH_RELEASE_FLAVOR: vulkan MESH_LLM_RELEASE_BIN_DIR: ${{ github.workspace }}/host-input - MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ github.workspace }}/runtime-root + MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ steps.compose.outputs.runtime_root }} + MESH_LLM_PRECOMPOSED_PRODUCT_DIR: ${{ steps.compose.outputs.product_dir }} MESH_RELEASE_HOST_PRESTAMPED: "1" - run: | - printf '%s' "$RELEASE_ATTESTATION_PUBLIC_KEY" > "$MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" - (cd host-input && sha256sum -c mesh-llm.sha256) - chmod +x host-input/mesh-llm - mkdir -p runtime-root - mapfile -t runtime_archives < <(find runtime-input -name '*.tar.gz' -type f -print) - test "${#runtime_archives[@]}" -eq 1 - tar -xzf "${runtime_archives[0]}" -C runtime-root - mapfile -t runtime_dirs < <(find runtime-root -mindepth 1 -maxdepth 1 -type d -print) - test "${#runtime_dirs[@]}" -eq 1 - scripts/verify-native-runtime-package.sh "${runtime_dirs[0]}" - scripts/ci-client-readiness-smoke.sh host-input/mesh-llm runtime-root - MESH_RELEASE_FLAVOR=vulkan scripts/package-release.sh "$RELEASE_TAG" dist - - uses: actions/upload-artifact@v6 + MESH_RELEASE_ATTESTATION_PREVERIFIED: "1" + MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json + run: scripts/package-release.sh "$RELEASE_TAG" dist + + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-linux-vulkan path: dist/* if-no-files-found: error - build_windows_cpu: + windows_host_input: name: Build immutable host Windows x86_64 needs: metadata runs-on: windows-2022 env: - LLAMA_STAGE_BACKEND: cpu + MESH_LLM_REQUIRE_SCCACHE: "1" + RUSTC_WRAPPER: sccache steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 with: version: 10 - - uses: actions/setup-node@v5 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version: 24 cache: pnpm cache-dependency-path: | .github/cache-version.txt crates/mesh-llm-ui/pnpm-lock.yaml - - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v6 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' - - uses: taiki-e/install-action@just - - uses: mozilla-actions/sccache-action@v0.0.9 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' shell: bash env: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - - name: Build and attest Windows release host + - name: Prepare release attestation keys shell: pwsh env: MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-private-key.json MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} - MESH_LLM_REQUIRE_SCCACHE: "1" RELEASE_ATTESTATION_SIGNING_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE }} RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} run: | Set-Content -Path $env:MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE -Value $env:RELEASE_ATTESTATION_SIGNING_KEY -NoNewline Set-Content -Path $env:MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE -Value $env:RELEASE_ATTESTATION_PUBLIC_KEY -NoNewline - just release-host-build-windows - New-Item -ItemType Directory -Path host-input -Force | Out-Null - Copy-Item target\release\mesh-llm.exe host-input\mesh-llm.exe -Force - cargo run -q -p xtask -- release-attestation stamp --binary host-input\mesh-llm.exe --signing-key-file $env:MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE - cargo run -q -p xtask -- release-attestation inspect --binary host-input\mesh-llm.exe --public-key-file $env:MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE --json - Copy-Item target\debug\xtask.exe host-input\release-attestation-verifier.exe -Force - $verifierHash = (Get-FileHash -Algorithm SHA256 host-input\release-attestation-verifier.exe).Hash.ToLowerInvariant() - Set-Content -Path host-input\release-attestation-verifier.exe.sha256 -Value "$verifierHash release-attestation-verifier.exe" -NoNewline - python scripts\verify-host-dependencies.py host-input\mesh-llm.exe --report host-input\host-imports.json - $hash = (Get-FileHash -Algorithm SHA256 host-input\mesh-llm.exe).Hash.ToLowerInvariant() - Set-Content -Path host-input\mesh-llm.exe.sha256 -Value "$hash mesh-llm.exe" -NoNewline - - uses: actions/upload-artifact@v6 + - name: Build and attest Windows release host + uses: ./.github/actions/prepare-windows-host-input + with: + profile: release + attestation_signing_key_file: ${{ runner.temp }}/mesh-release-attestation-private-key.json + attestation_public_key_file: ${{ runner.temp }}/mesh-release-attestation-public-key.json + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: host-input-windows-x86_64 path: host-input/* if-no-files-found: error - build_windows_gpu: + compose_windows_gpu: name: Compose Windows ${{ matrix.name }} product - needs: [metadata, build_windows_cpu, build_native_runtime_windows_gpu] + needs: [metadata, windows_host_input, build_native_runtime_windows_gpu] if: ${{ needs.metadata.outputs.skip_gpu_bundles != 'true' }} runs-on: windows-2022 - env: - ROCM_HIP_SDK_FILENAME: AMD-Software-PRO-Edition-25.Q3-WinSvr2022-For-HIP.exe - # Pin Windows CUDA to a version sccache 0.15.0 supports. - # `choco install cuda` (used previously via install-windows-sdk.ps1) - # pulls latest = CUDA 13.2, which deterministically crashes sccache - # on nvcc output (see mozilla/sccache#2470). CI pins to the same - # version via Jimver/cuda-toolkit; mirror that here. - WINDOWS_CUDA_VERSION: ${{ vars.CUDA_VERSION || '12.9.2' }} strategy: fail-fast: false matrix: @@ -1283,56 +1299,51 @@ jobs: runtime_artifact: release-native-runtime-windows-x86_64-vulkan artifact_name: release-windows-vulkan steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' - - uses: dtolnay/rust-toolchain@stable - - name: Prepare dispatched release version - if: github.event_name == 'workflow_dispatch' - shell: bash - env: - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} - run: scripts/release-version.sh "$RELEASE_TAG" - name: Download immutable host input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: host-input-windows-x86_64 path: host-input - name: Download immutable runtime input - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: ${{ matrix.runtime_artifact }} path: runtime-input - - name: Compose Windows GPU release bundle from producer inputs + - name: Prepare release verification key shell: pwsh env: MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} - MESH_LLM_RELEASE_BIN_DIR: ${{ github.workspace }}\host-input - MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ github.workspace }}\runtime-root + run: Set-Content -Path $env:MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE -Value $env:RELEASE_ATTESTATION_PUBLIC_KEY -NoNewline + - name: Compose Windows GPU product from producer inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + version: ${{ needs.metadata.outputs.tag }} + backend: ${{ matrix.backend }} + binary_name: mesh-llm.exe + readiness_smoke: "true" + attestation_public_key_file: ${{ runner.temp }}/mesh-release-attestation-public-key.json + attestation_verifier: host-input/release-attestation-verifier.exe + - name: Package verified Windows GPU product + shell: pwsh + env: + RELEASE_TAG: ${{ needs.metadata.outputs.tag }} + MESH_LLM_PRECOMPOSED_PRODUCT_DIR: ${{ steps.compose.outputs.product_dir }} MESH_RELEASE_HOST_PRESTAMPED: "1" - MESH_RELEASE_ATTESTATION_VERIFIER: ${{ github.workspace }}\host-input\release-attestation-verifier.exe - run: | - Set-Content -Path $env:MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE -Value $env:RELEASE_ATTESTATION_PUBLIC_KEY -NoNewline - $expectedHash = (Get-Content host-input\mesh-llm.exe.sha256).Split()[0] - $actualHash = (Get-FileHash -Algorithm SHA256 host-input\mesh-llm.exe).Hash.ToLowerInvariant() - if ($actualHash -ne $expectedHash) { throw "immutable host checksum mismatch" } - $archives = @(Get-ChildItem runtime-input -Recurse -Filter '*.tar.gz' -File) - if ($archives.Count -ne 1) { throw "expected exactly one runtime archive, found $($archives.Count)" } - New-Item -ItemType Directory -Path runtime-root -Force | Out-Null - tar -xzf $archives[0].FullName -C runtime-root - $runtimeDirs = @(Get-ChildItem runtime-root -Directory) - if ($runtimeDirs.Count -ne 1) { throw "expected exactly one extracted runtime directory, found $($runtimeDirs.Count)" } - bash scripts/verify-native-runtime-package.sh $runtimeDirs[0].FullName - if ($LASTEXITCODE -ne 0) { throw "native runtime verification failed" } - bash scripts/ci-client-readiness-smoke.sh host-input/mesh-llm.exe runtime-root - if ($LASTEXITCODE -ne 0) { throw "composed client readiness smoke failed" } - .\scripts\package-release.ps1 -Version $env:RELEASE_TAG -OutputDir dist -Flavor "${{ matrix.backend }}" - - uses: actions/upload-artifact@v6 + MESH_RELEASE_ATTESTATION_PREVERIFIED: "1" + MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json + run: .\scripts\package-release.ps1 -Version $env:RELEASE_TAG -OutputDir dist -Flavor "${{ matrix.backend }}" + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: ${{ matrix.artifact_name }} path: dist/* @@ -1340,51 +1351,52 @@ jobs: compose_windows_cpu: name: Compose Windows CPU product - needs: [metadata, build_windows_cpu, build_native_runtime_windows_cpu] + needs: [metadata, windows_host_input, build_native_runtime_windows_cpu] runs-on: windows-2022 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' - - uses: dtolnay/rust-toolchain@stable - - uses: actions/download-artifact@v7 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: host-input-windows-x86_64 path: host-input - - uses: actions/download-artifact@v7 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-native-runtime-windows-x86_64-cpu path: runtime-input - - name: Compose Windows CPU release bundle from producer inputs + - name: Prepare release verification key shell: pwsh env: MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} RELEASE_ATTESTATION_PUBLIC_KEY: ${{ secrets.MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE }} - MESH_LLM_RELEASE_BIN_DIR: ${{ github.workspace }}\host-input - MESH_LLM_NATIVE_RUNTIME_ROOT: ${{ github.workspace }}\runtime-root + run: Set-Content -Path $env:MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE -Value $env:RELEASE_ATTESTATION_PUBLIC_KEY -NoNewline + - name: Compose Windows CPU product from producer inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + version: ${{ needs.metadata.outputs.tag }} + backend: cpu + binary_name: mesh-llm.exe + readiness_smoke: "true" + attestation_public_key_file: ${{ runner.temp }}/mesh-release-attestation-public-key.json + attestation_verifier: host-input/release-attestation-verifier.exe + - name: Package verified Windows CPU product + shell: pwsh + env: + RELEASE_TAG: ${{ needs.metadata.outputs.tag }} + MESH_LLM_PRECOMPOSED_PRODUCT_DIR: ${{ steps.compose.outputs.product_dir }} MESH_RELEASE_HOST_PRESTAMPED: "1" - MESH_RELEASE_ATTESTATION_VERIFIER: ${{ github.workspace }}\host-input\release-attestation-verifier.exe - run: | - Set-Content -Path $env:MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE -Value $env:RELEASE_ATTESTATION_PUBLIC_KEY -NoNewline - $expectedHash = (Get-Content host-input\mesh-llm.exe.sha256).Split()[0] - $actualHash = (Get-FileHash -Algorithm SHA256 host-input\mesh-llm.exe).Hash.ToLowerInvariant() - if ($actualHash -ne $expectedHash) { throw "immutable host checksum mismatch" } - $archives = @(Get-ChildItem runtime-input -Recurse -Filter '*.tar.gz' -File) - if ($archives.Count -ne 1) { throw "expected exactly one runtime archive, found $($archives.Count)" } - New-Item -ItemType Directory -Path runtime-root -Force | Out-Null - tar -xzf $archives[0].FullName -C runtime-root - $runtimeDirs = @(Get-ChildItem runtime-root -Directory) - if ($runtimeDirs.Count -ne 1) { throw "expected exactly one extracted runtime directory, found $($runtimeDirs.Count)" } - bash scripts/verify-native-runtime-package.sh $runtimeDirs[0].FullName - if ($LASTEXITCODE -ne 0) { throw "native runtime verification failed" } - bash scripts/ci-client-readiness-smoke.sh host-input/mesh-llm.exe runtime-root - if ($LASTEXITCODE -ne 0) { throw "composed client readiness smoke failed" } - .\scripts\package-release.ps1 -Version $env:RELEASE_TAG -OutputDir dist -Flavor cpu - - uses: actions/upload-artifact@v6 + MESH_RELEASE_ATTESTATION_PREVERIFIED: "1" + MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE: ${{ runner.temp }}/mesh-release-attestation-public-key.json + run: .\scripts\package-release.ps1 -Version $env:RELEASE_TAG -OutputDir dist -Flavor cpu + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-windows path: dist/* @@ -1398,16 +1410,16 @@ jobs: LLAMA_STAGE_BACKEND: cpu MESH_NATIVE_RUNTIME_TARGET: x86_64-pc-windows-msvc steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' - - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@v0.0.9 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - name: Initialize MSVC for native runtime - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: x64 - name: Prepare dispatched release version @@ -1417,18 +1429,13 @@ jobs: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" - name: Package native runtime - shell: bash - run: | - scripts/package-native-runtime.sh \ - --build \ - --backend cpu \ - --target x86_64-pc-windows-msvc \ - --out dist/native-runtimes - - name: Verify native runtime artifact - shell: bash - run: scripts/verify-native-runtime-package.sh dist/native-runtimes/*.tar.gz + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: cpu + target: x86_64-pc-windows-msvc + output_dir: dist/native-runtimes - name: Upload native runtime - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-native-runtime-windows-x86_64-cpu path: | @@ -1467,23 +1474,32 @@ jobs: backend: vulkan artifact_name: release-native-runtime-windows-x86_64-vulkan steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' - - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@v0.0.9 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' shell: bash env: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: scripts/release-version.sh "$RELEASE_TAG" + - name: Validate CUDA 12 artifact contract + if: ${{ matrix.backend == 'cuda' }} + shell: pwsh + run: | + $cudaVersion = "$env:WINDOWS_CUDA_VERSION".Trim() + $cudaMajor = ($cudaVersion -split '\.', 2)[0] + if ($cudaMajor -ne '12') { + throw "CUDA_VERSION must select CUDA 12.x because this release lane publishes cuda12-labelled artifacts; got '$cudaVersion'." + } - name: Install CUDA toolkit if: ${{ matrix.backend == 'cuda' }} - uses: Jimver/cuda-toolkit@v0.2.35 + uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 with: cuda: ${{ env.WINDOWS_CUDA_VERSION }} method: network @@ -1500,7 +1516,7 @@ jobs: } & nvcc --version - name: Initialize MSVC for native runtime - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: x64 - name: Install backend SDK @@ -1512,24 +1528,19 @@ jobs: -RocmHipSdkFilename "$env:ROCM_HIP_SDK_FILENAME" ` -InstallerCacheDir "$HOME\sdk-installer-cache" - name: Package native runtime - shell: bash env: LLAMA_STAGE_BACKEND: ${{ matrix.backend }} LLAMA_STAGE_CUDA_ARCHITECTURES: ${{ matrix.cuda_architectures }} LLAMA_STAGE_AMDGPU_TARGETS: ${{ matrix.rocm_architectures }} LLAMA_STAGE_USE_SCCACHE: ${{ matrix.use_sccache || '1' }} MESH_LLM_CUDA_TOOLKIT_MAJOR: '12' - run: | - scripts/package-native-runtime.sh \ - --build \ - --backend "${{ matrix.backend }}" \ - --target x86_64-pc-windows-msvc \ - --out dist/native-runtimes - - name: Verify native runtime artifact - shell: bash - run: scripts/verify-native-runtime-package.sh dist/native-runtimes/*.tar.gz + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: ${{ matrix.backend }} + target: x86_64-pc-windows-msvc + output_dir: dist/native-runtimes - name: Upload native runtime - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: ${{ matrix.artifact_name }} path: | @@ -1554,29 +1565,31 @@ jobs: - build_linux_arm64 - compose_linux_arm64_cpu - smoke_linux_arm64_artifact - - build_linux_aarch64_cuda - - build_linux_cuda - - build_linux_rocm - - build_linux_vulkan - - build_windows_cpu + - compose_linux_aarch64_cuda + - compose_linux_cuda + - compose_linux_rocm + - compose_linux_vulkan + - windows_host_input - compose_windows_cpu - - build_windows_gpu + - compose_windows_gpu - build_native_runtime_windows_cpu - build_native_runtime_windows_gpu - if: ${{ always() && needs.metadata.result == 'success' && needs.metadata.outputs.canary != 'true' && needs.compose_cpu_products.result == 'success' && needs.inference_smoke_tests.result == 'success' && needs.build_native_sdk_runtime.result == 'success' && needs.build_native_runtime.result == 'success' && (needs.build_native_runtime_linux_aarch64_cuda.result == 'success' || needs.build_native_runtime_linux_aarch64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_cuda.result == 'success' || needs.build_native_runtime_linux_x86_64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_rocm.result == 'success' || needs.build_native_runtime_linux_x86_64_rocm.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_vulkan.result == 'success' || needs.build_native_runtime_linux_x86_64_vulkan.result == 'skipped') && needs.build_swift_sdk_artifact.result == 'success' && needs.compose_linux_arm64_cpu.result == 'success' && needs.smoke_linux_arm64_artifact.result == 'success' && (needs.build_linux_aarch64_cuda.result == 'success' || needs.build_linux_aarch64_cuda.result == 'skipped') && (needs.build_linux_cuda.result == 'success' || needs.build_linux_cuda.result == 'skipped') && (needs.build_linux_rocm.result == 'success' || needs.build_linux_rocm.result == 'skipped') && (needs.build_linux_vulkan.result == 'success' || needs.build_linux_vulkan.result == 'skipped') && needs.compose_windows_cpu.result == 'success' && (needs.build_windows_gpu.result == 'success' || needs.build_windows_gpu.result == 'skipped') && needs.build_native_runtime_windows_cpu.result == 'success' && (needs.build_native_runtime_windows_gpu.result == 'success' || needs.build_native_runtime_windows_gpu.result == 'skipped') }} + if: ${{ always() && needs.metadata.result == 'success' && needs.metadata.outputs.canary != 'true' && needs.compose_cpu_products.result == 'success' && needs.inference_smoke_tests.result == 'success' && needs.build_native_sdk_runtime.result == 'success' && needs.build_native_runtime.result == 'success' && (needs.build_native_runtime_linux_aarch64_cuda.result == 'success' || needs.build_native_runtime_linux_aarch64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_cuda.result == 'success' || needs.build_native_runtime_linux_x86_64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_rocm.result == 'success' || needs.build_native_runtime_linux_x86_64_rocm.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_vulkan.result == 'success' || needs.build_native_runtime_linux_x86_64_vulkan.result == 'skipped') && needs.build_swift_sdk_artifact.result == 'success' && needs.compose_linux_arm64_cpu.result == 'success' && needs.smoke_linux_arm64_artifact.result == 'success' && (needs.compose_linux_aarch64_cuda.result == 'success' || needs.compose_linux_aarch64_cuda.result == 'skipped') && (needs.compose_linux_cuda.result == 'success' || needs.compose_linux_cuda.result == 'skipped') && (needs.compose_linux_rocm.result == 'success' || needs.compose_linux_rocm.result == 'skipped') && (needs.compose_linux_vulkan.result == 'success' || needs.compose_linux_vulkan.result == 'skipped') && needs.compose_windows_cpu.result == 'success' && (needs.compose_windows_gpu.result == 'success' || needs.compose_windows_gpu.result == 'skipped') && needs.build_native_runtime_windows_cpu.result == 'success' && (needs.build_native_runtime_windows_gpu.result == 'success' || needs.build_native_runtime_windows_gpu.result == 'skipped') }} runs-on: ubuntu-24.04 + permissions: + contents: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: true fetch-depth: 0 - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 if: github.event_name == 'workflow_dispatch' with: version: 10 - - uses: actions/setup-node@v5 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 if: github.event_name == 'workflow_dispatch' with: node-version: 24 @@ -1585,11 +1598,11 @@ jobs: .github/cache-version.txt crates/mesh-llm-ui/pnpm-lock.yaml - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 if: github.event_name == 'workflow_dispatch' - name: Download release artifacts - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: release-artifacts pattern: release-* @@ -1638,7 +1651,7 @@ jobs: - name: Download generated SwiftPM manifest if: github.event_name == 'workflow_dispatch' - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: swift-package-manifest path: generated-swift-manifest @@ -1678,7 +1691,7 @@ jobs: git push origin "refs/tags/$RELEASE_TAG" - name: Publish GitHub release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 with: tag_name: ${{ needs.metadata.outputs.tag }} prerelease: ${{ needs.metadata.outputs.prerelease }} @@ -1731,11 +1744,11 @@ jobs: if: ${{ needs.metadata.outputs.prerelease != 'true' && needs.metadata.outputs.canary != 'true' }} runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ needs.metadata.outputs.tag }} persist-credentials: false - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' env: @@ -1752,11 +1765,11 @@ jobs: if: ${{ needs.metadata.outputs.prerelease != 'true' && needs.metadata.outputs.canary != 'true' }} runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ needs.metadata.outputs.tag }} persist-credentials: false - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' env: diff --git a/.github/workflows/reset-caches.yml b/.github/workflows/reset-caches.yml index 2429ccc488..4a67c93f76 100644 --- a/.github/workflows/reset-caches.yml +++ b/.github/workflows/reset-caches.yml @@ -27,7 +27,7 @@ jobs: - name: Delete all repository caches id: delete-caches - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: result-encoding: string script: | diff --git a/.github/workflows/scripted-binary-smoke.yml b/.github/workflows/scripted-binary-smoke.yml index 29e800690c..316125129b 100644 --- a/.github/workflows/scripted-binary-smoke.yml +++ b/.github/workflows/scripted-binary-smoke.yml @@ -61,7 +61,7 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Install smoke dependencies run: sudo apt-get update && sudo apt-get install -y curl jq lsof diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 0be36337d5..ebb1b3d6ef 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -58,30 +58,30 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 if: ${{ inputs.sdk_kind == 'rust' }} with: python-version: "3.12" - - uses: actions/setup-java@v5 + - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 if: ${{ inputs.sdk_kind == 'kotlin' }} with: distribution: temurin java-version: '21' - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 with: version: 10 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version: "24" cache: pnpm cache-dependency-path: crates/mesh-llm-ui/pnpm-lock.yaml - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - name: Install Linux SDK dependencies if: ${{ runner.os == 'Linux' }} @@ -95,10 +95,9 @@ jobs: # `-fuse-ld=/opt/homebrew/bin/ld64.lld`. Apple clang accepts # that flag only when the binary actually exists on disk; # without `lld` installed the link step fails with - # `clang: error: invalid linker name in argument`. The - # `macos_targets` job already does `brew install ... lld` for - # the same reason — install it here too so the swift smoke - # lane matches. + # `clang: error: invalid linker name in argument`. The macOS + # host and unit-test jobs install `lld` for the same + # reason. Install it here too so the Swift smoke lane matches. run: brew install jq lld - name: Configure Linux Rust linker diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 6f5dc95e4c..9bb027e1e7 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -37,13 +37,13 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Prepare dispatched release version if: inputs.release_tag != '' run: scripts/release-version.sh "${{ inputs.release_tag }}" - - uses: actions/setup-python@v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" cache: pip @@ -51,7 +51,7 @@ jobs: .github/cache-version.txt ci/requirements-ci-python.txt - - uses: actions/setup-node@v5 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version: 24 package-manager-cache: false diff --git a/.github/workflows/stale-prs.yml b/.github/workflows/stale-prs.yml index 2544fbb1a1..ba548cf4e5 100644 --- a/.github/workflows/stale-prs.yml +++ b/.github/workflows/stale-prs.yml @@ -45,7 +45,7 @@ jobs: echo "days_before_close=$PR_WARNING_DAYS" >> "$GITHUB_OUTPUT" - name: Close stale pull requests - uses: actions/stale@v10 + uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 with: repo-token: ${{ github.token }} days-before-issue-stale: -1 diff --git a/.github/workflows/website-pages.yml b/.github/workflows/website-pages.yml index 6b2db8bcd0..90da0731d6 100644 --- a/.github/workflows/website-pages.yml +++ b/.github/workflows/website-pages.yml @@ -27,10 +27,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Set up Node.js - uses: actions/setup-node@v5 + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version: 24 cache: npm @@ -70,7 +70,7 @@ jobs: cp -R docs/pagefind "$artifact_dir/" - name: Upload Pages artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 with: path: public-website-artifact @@ -91,4 +91,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 diff --git a/.github/workflows/windows-warm-caches.yml b/.github/workflows/windows-warm-caches.yml index 51de74b429..d9186d79be 100644 --- a/.github/workflows/windows-warm-caches.yml +++ b/.github/workflows/windows-warm-caches.yml @@ -48,11 +48,11 @@ jobs: MESH_LLM_REQUIRE_SCCACHE: "1" RUSTC_WRAPPER: sccache steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@just - - uses: mozilla-actions/sccache-action@v0.0.9 - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 with: workspaces: . -> target prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} @@ -68,7 +68,7 @@ jobs: run: New-Item -ItemType Directory -Force -Path $env:LLAMA_STAGE_BUILD_DIR | Out-Null - name: Restore Windows CPU ABI cache id: llama_cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: .deps/llama.cpp/build-stage-abi-cpu key: ${{ env.CACHE_NAMESPACE }}-windows-2022-skippy-abi-cpu--cpu-${{ hashFiles('scripts/build-windows.ps1', 'scripts/install-windows-sdk.ps1', '.github/actions/setup-windows-rocm-sdk/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} @@ -86,7 +86,7 @@ jobs: $libs | ForEach-Object { Write-Host $_.FullName } - name: Save Windows CPU ABI cache if: steps.llama_cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v5 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: .deps/llama.cpp/build-stage-abi-cpu key: ${{ steps.llama_cache.outputs.cache-primary-key }} @@ -116,11 +116,11 @@ jobs: build_recipe: release-build-vulkan-windows build_args: "" steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@just - - uses: mozilla-actions/sccache-action@v0.0.9 - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 with: workspaces: . -> target prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} @@ -136,13 +136,13 @@ jobs: run: New-Item -ItemType Directory -Force -Path $env:LLAMA_STAGE_BUILD_DIR | Out-Null - name: Restore Windows GPU ABI cache id: llama_cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: .deps/llama.cpp/build-stage-abi-${{ matrix.backend }} key: ${{ env.CACHE_NAMESPACE }}-windows-2022-skippy-abi-${{ matrix.backend }}-${{ matrix.build_args }}-${{ matrix.backend == 'cuda' && format('cuda-{0}-Jimver-v0.2.35', env.WINDOWS_CUDA_VERSION) || matrix.backend == 'vulkan' && format('vulkan-{0}-jakoch-v1.5.2', env.WINDOWS_VULKAN_SDK_VERSION) || format('rocm-{0}', env.ROCM_HIP_SDK_FILENAME) }}-${{ hashFiles('scripts/build-windows.ps1', 'scripts/install-windows-sdk.ps1', '.github/actions/setup-windows-rocm-sdk/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - name: Install CUDA toolkit if: ${{ matrix.backend == 'cuda' && steps.llama_cache.outputs.cache-hit != 'true' }} - uses: Jimver/cuda-toolkit@v0.2.35 + uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 with: cuda: ${{ env.WINDOWS_CUDA_VERSION }} method: network @@ -166,7 +166,7 @@ jobs: } - name: Install Vulkan SDK if: ${{ matrix.backend == 'vulkan' && steps.llama_cache.outputs.cache-hit != 'true' }} - uses: jakoch/install-vulkan-sdk-action@v1.5.2 + uses: jakoch/install-vulkan-sdk-action@3c53c378c9bfbb2ea122a1cc164a837d4004c871 # v1.5.2 with: vulkan_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} cache: true @@ -213,7 +213,7 @@ jobs: $libs | ForEach-Object { Write-Host $_.FullName } - name: Save Windows GPU ABI cache if: steps.llama_cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v5 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: .deps/llama.cpp/build-stage-abi-${{ matrix.backend }} key: ${{ steps.llama_cache.outputs.cache-primary-key }} @@ -227,7 +227,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Prune old Windows ABI caches - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: CACHE_NAMESPACE: ${{ env.CACHE_NAMESPACE }} RETENTION: ${{ env.LLAMA_WINDOWS_CACHE_RETENTION }} diff --git a/README.md b/README.md index dba6790fc7..f6b7c2f901 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,20 @@ binary to `invalid`, but default startup still allows it. | [docs/specs/layer-package-repos.md](docs/specs/layer-package-repos.md) | Manifest and artifact format spec | | [docs/specs/mesh-setup-installer.md](docs/specs/mesh-setup-installer.md) | Installer/bootstrap and setup command behavior spec | +## CI infrastructure + + + + + + Depot + + + +Mesh LLM is adopting [Depot's managed GitHub Actions +runners](https://depot.dev/docs/github-actions/overview) for non-GPU CI builds. +Hardware-qualified GPU tests remain on dedicated runners. + ## Community Mesh LLM is experimental distributed-systems software. When you report bugs, diff --git a/ci/DEPOT_MIGRATION.md b/ci/DEPOT_MIGRATION.md new file mode 100644 index 0000000000..edf2965e9e --- /dev/null +++ b/ci/DEPOT_MIGRATION.md @@ -0,0 +1,365 @@ +# Depot CI migration and build-graph plan + +This document is the migration contract for moving MeshLLM's non-hardware CI +jobs to Depot while restructuring builds around immutable, reusable artifacts. +The implementation must optimize elapsed feedback time without allowing PR, +main, and release products to drift. + +## Baseline and targets + +Use [`scripts/collect-ci-metrics.py`](../scripts/collect-ci-metrics.py) and the +methodology in [`METRICS.md`](METRICS.md) for every before/after comparison. +The initial mixed-change-class baseline is: + +| Workflow/cohort | Sample | p50 wall | p95 wall | Maximum | +| --- | ---: | ---: | ---: | ---: | +| PR Builds, successful | 20 | 33m 12s | 55m 33s | 69m 01s | +| PR Builds, successful | 50 | 35m 00s | 82m 42s | 87m 00s | +| Main CI, successful | 50 | 34m 12s | 112m 42s | 125m 54s | +| PR Quality, successful | 30 | 12m 36s | 27m 30s | — | + +The 20-run PR cohort's largest job-family p95 values were Windows CUDA +42m 09s, Windows ROCm 39m 23s, Windows CPU 32m 06s, and Swift SDK smoke +27m 12s. Swift finished last in 8 of those 20 runs. A representative full PR +run took 42m 12s; its Windows CPU row took 41m 30s. A representative main run +spent about 77 runner-minutes rebuilding the same Windows neutral host across +backend rows. + +These numbers mix change classes. The rollout must compare like-for-like +cohorts and report queue time separately from execution time. + +Target service levels: + +| Signal | Target | +| --- | --- | +| PR routing/format signal | p95 under 2 minutes | +| Typical Rust PR required signal | p50 under 10 minutes, p95 under 20 minutes | +| Backend-affecting PR | p95 under 30 minutes | +| Main full composed-product graph | p95 under 45 minutes | +| Warm-cache no-op compile request | at least 80% sccache hit rate | +| Artifact consumer rebuilds | zero | + +## Product graph + +Every supported platform follows one graph: + +```mermaid +flowchart LR + Source["Source revision"] + UI["Prepared UI assets"] + Host["Neutral host producer"] + CPU["CPU/Metal runtime producer"] + Backends["CUDA / ROCm / Vulkan runtime producers"] + Product["Product-v2 composer"] + Smoke["Smoke / SDK / packaging consumers"] + Publish["Release publication"] + + Source --> UI --> Host + Source --> CPU + Source --> Backends + Host --> Product + CPU --> Product + Backends --> Product + Product --> Smoke + Product --> Publish +``` + +The shared implementation primitives are: + +- `.github/actions/prepare-host-input`: build one backend-neutral host, then + attest when requested, import-check it, and write a checksum. +- `.github/actions/prepare-windows-host-input`: perform the same immutable host + preparation for Windows debug/release profiles and include a prebuilt + attestation verifier for release consumers. +- `.github/actions/prepare-native-runtime-input`: build/package exactly one + runtime archive and run the release-grade runtime verifier. +- `.github/actions/compose-product-input`: checksum and verify producer inputs, + compose product-v2 without compiling, and run client readiness. + +`scripts/build-host.sh` is the only Unix host builder. +`scripts/build-release.sh` is a compatibility wrapper. Backend recipes and +workflows must never build a host as a side effect of producing a runtime. + +Artifact contracts: + +| Layer | Required contents | Mutation rule | +| --- | --- | --- | +| Host input | executable, `.sha256`, `host-imports.json`; release adds attestation | immutable after checksum | +| Runtime input | runtime directory, archive, archive checksum, `manifest.json` | immutable after verification | +| Product input | host, host imports, `product-manifest.json`, one `native-runtimes/` | composer never compiles | +| Static ABI test input | one tarred CPU llama ABI build keyed by patch queue and build recipe | one producer per workflow; test rows only restore | + +PR artifacts are unstamped, retained for one day, and cannot be promoted into a +release. Main and release exercise the same actions; release adds version +preparation, signing, public packaging, and publication around them. + +## PR, main, and release policy + +PRs optimize for the earliest reliable signal: + +- route from changed files before compiling consistency checks; +- run formatting directly on a standard runner instead of pulling the large + backend image; +- build one Linux host and one CPU runtime independently only when an inference + artifact is needed, then compose them without compiling; +- build backend products only for ABI/backend inputs; +- use the debug profile for the ordinary PR CPU signal and the release profile + for manual, benchmark, and backend-affecting runs; +- fan that one exact host artifact into the CPU and every selected backend + runtime row; +- build or restore the static CPU llama ABI once, archive it, and fan those + exact bytes into every crate-test and grouped-test row instead of compiling + the same C++ graph concurrently; +- run runner-image contract checks only when their workflow, cache version, or + cache integration changes; +- make SDK smokes consume the staged product runtime and reject hidden rebuilds; +- keep public-mesh admission out of required PR checks. It remains an explicit + manual integration probe, while product readiness uses hermetic local mDNS. + +Main is the exhaustive trust boundary: + +- run all workspace crate-test batches; +- use the same single static-ABI producer/fan-out contract as PR builds; +- build one Linux release host on every non-doc main change; +- build CPU, CUDA, ROCm, and Vulkan runtime products from that host; +- run the longer integration and SDK consumers from uploaded product bytes; +- retain hardware qualification as a separate lane. + +Release uses the same host/runtime/product actions. Signing and publishing are +release-only wrappers and never change the underlying compilation process. +The first Depot release phase routes only a `workflow_dispatch` from `main`: +Linux x86 CPU native SDK/runtime producers, compile-only ROCm/Vulkan runtime +producers, and Linux product composers. Tag-triggered releases, metadata, +publishing, attested host producers, inference jobs carrying `HF_TOKEN`, +macOS, Windows, ARM, and hardware-qualified GPU work stay on their existing +runners. The attested host cannot move until unsigned compilation and +GitHub-hosted signing are separate jobs. + +## Depot runner rollout + +Depot runners are selected with a single label such as +`depot-ubuntu-24.04-8`. MeshLLM uses: + +| Workload | Initial label | Reason | +| --- | --- | --- | +| routing, summaries, CLI docs | `depot-ubuntu-24.04` or `-4` | short, low-memory | +| format and UI quality | `depot-ubuntu-24.04-4` | avoid backend image pull | +| Rust check/test/clippy and unsigned host builds | `depot-ubuntu-24.04-8` | CPU-bound compile | +| runtime builds without hardware execution | `depot-ubuntu-24.04-8` | CPU/I/O-bound C++ build | +| measured high-parallelism runtime builds | `depot-ubuntu-24.04-16` | compare wall time, peak disk, and cost before adopting | +| hardware-qualified CUDA tests | dedicated GPU runner | requires a real device | + +The current runner selector has one effective repository gate: + +- `DEPOT_RUNNERS_ENABLED=true` enables eligible trusted `main` push and + `main`-ref dispatch jobs. Tag pushes and every other ref remain hosted. +- Every `pull_request` event selects GitHub-hosted runners, even for a + same-repository branch and even if `DEPOT_PR_RUNNERS_ENABLED` is set. + `DEPOT_PR_RUNNERS_ENABLED` is ignored while automatic Depot Cache is enabled. + +Trusted `workflow_dispatch` runs accept `use_depot=true` for a bounded canary, +but the selector requires `github.ref == 'refs/heads/main'`. The manual input is +never authority to run feature-branch code on Depot. The selector emits one +typed cache permission from the same decision, so a caller cannot select a +hosted runner while independently enabling Depot WebDAV. + +This selector is defense in depth, not the primary security boundary. The +current pull-request workflows and repository-local actions are evaluated from +PR-controlled code, so a pull request can modify or bypass the selector itself. +Consequently, repository variables and same-repository comparisons cannot make +the current PR workflow safe for Depot. + +Activation prerequisites: + +1. The Depot GitHub Apps remain connected to `Mesh-LLM`. +2. While public-repository access is still disabled, change GitHub's + organization `Default` runner group to selected repository + `Mesh-LLM/mesh-llm` and selected workflow + `Mesh-LLM/mesh-llm/.github/workflows/depot-canary.yml@refs/heads/main`. +3. Only after both restrictions are saved, enable public repositories for the + `Default` group. Depot-managed ephemeral runners register in that group. +4. Dispatch `depot-canary.yml` from `refs/heads/main` twice. Verify all four + runner sizes and a cold-to-warm cache hit without printing credentials. +5. Dispatch the canary from a feature ref, prove that it cannot acquire a + Depot runner, and cancel that exact queued run. +6. Add exact default-branch workflow refs only as their phase starts. Reusable + workflows whose jobs run on Depot must be listed separately. +7. Set `DEPOT_RUNNERS_ENABLED=true` only after comparable trusted canaries meet + the rollout targets. + +The initial main allowlist is: + +```text +Mesh-LLM/mesh-llm/.github/workflows/ci.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/pr_quality.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/hf-download-smoke.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/smoke.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/scripted-binary-smoke.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/sdk-smoke.yml@refs/heads/main +``` + +Add `pr_builds.yml@refs/heads/main` only for its trusted manual benchmark and +`release.yml@refs/heads/main` only for the non-publishing release phase. Never +select a feature ref, `refs/pull/*`, or “all workflows.” + +Depot PR execution is intentionally out of the current rollout. Automatic +Depot Cache injects repository-scoped cache authority into the whole job, with +no branch isolation. A default-branch-pinned reusable workflow and separate +cache-key conventions cannot stop malicious checked-out PR code from using +that authority directly. PR code may run on Depot only after automatic cache +injection is disabled and complete token/API isolation is proven, or after +Depot provides a comparably strong per-PR cache boundary. Until then, required +and optional PR-event jobs remain GitHub-hosted; `pr_builds.yml` can be +benchmarked only by a trusted manual dispatch from `main`. + +Do not use `pull_request_target` to build or execute PR content. A +default-branch-pinned reusable workflow preserves the normal `pull_request` +event while keeping the runner-owning workflow definition trusted. + +As of 2026-07-29, the Depot dashboard reports the `Mesh-LLM` GitHub connection +active with automatic Depot Cache and registry authentication enabled. GitHub's +organization installation API lists `depot-managed-runners` and +`depot-code-access` for all repositories. A read-only organization-settings +inspection found: + +- `Default`: all repositories, public repositories disabled, all workflows, + and no persistent runner; +- `mesh-llm`: two dedicated GPU scale sets, selected repositories including + public repositories, and all workflows. + +The current `Default` state safely prevents Depot from serving this public +repository, so a canary will queue until the ordered restriction changes above +are made. The GPU group is separate from Depot and its all-workflows policy +must also be reviewed before treating those devices as a trusted-only pool. + +Depot redirects every GitHub Actions cache API consumer on its runners, +including `actions/cache`, `actions/setup-node`, and third-party cache actions. +Its namespace is repository-scoped and is not isolated by branch. Therefore: + +- current pull-request jobs never run on Depot and may use the normal + `mesh-llm` key namespace in GitHub's native cache because GitHub scopes PR + writes to the merge ref and trusted main jobs do not restore from that ref; +- a local sccache disk-only setting protects only that sccache child process; + it does not remove the Depot token or prevent another cache API consumer + from reading or poisoning the repository cache; +- no untrusted PR code may run on Depot while automatic cache injection is + enabled; +- trusted main/release jobs may explicitly enable the `disk,webdav` chain and + fall back to job-local disk. + +GitHub-hosted jobs retain the existing disk/GitHub Actions cache path. Never +print a cache token. + +Relevant Depot documentation: + +- [GitHub Actions runner overview](https://depot.dev/docs/github-actions/overview) +- [Runner quickstart](https://depot.dev/docs/github-actions/quickstart) +- [Runner types and sizes](https://depot.dev/docs/github-actions/runner-types) +- [GitHub Actions cache behavior](https://depot.dev/docs/cache/integrations/github-actions) +- [sccache integration](https://depot.dev/docs/cache/integrations/sccache) +- [Actions analytics](https://depot.dev/docs/github-actions/observability/github-actions-metrics) +- [GitHub runner-group selected-workflow API](https://docs.github.com/en/rest/actions/self-hosted-runner-groups?apiVersion=2022-11-28) +- [GitHub guidance for self-hosted runners in public repositories](https://docs.github.com/en/actions/how-tos/manage-runners/self-hosted-runners/manage-access) + +## Cross-repository responsibilities + +### `Mesh-LLM/mesh-llm-runner-images` + +Runner images own stable tools and backend SDKs, not commit-specific products. +The next image revision should: + +1. publish a `public-rust-host` image with Rust/Cargo/sccache, host libraries, + CMake/Ninja, lld, and only Cargo dependency warming; +2. publish `public-native-{cpu,cuda,rocm,vulkan}` images with the C/C++ SDK and + packaging tools but no Rust registry, Node, pnpm, UI, website, or Python + application dependencies; +3. publish a `public-compose` image with only Bash, Python standard library, + tar/coreutils, runtime libraries, and artifact verifiers. Composition jobs + must never pull a multi-gigabyte backend SDK; +4. remove the duplicate Actions runner installation from self-hosted images; +5. make one content-addressed architecture base feed every backend overlay, + and move source-revision provenance after dependency-warm layers; +6. build each architecture once, verify that exact staged digest, and assemble + manifests/tags without rebuilding; +7. gate updates on compressed size and cold-pull measurements. + +The current pinned CPU image is about 1.53 GB compressed, while the latest +candidate is about 1.92 GB (+25.5%). Do not bump that candidate until it beats +the current image's size/cold-start contract. Existing cold initialization +medians were approximately 80s CPU, 45–49s CUDA, 124–155s ROCm, and 78–81s +Vulkan. Initial acceptance targets are at most 1.0 GB for `public-rust-host`, +500 MB for a CPU native builder, 250 MB and 20s cold-pull p50 for +`public-compose`, at least 1 GB removed from each backend image, and image +publication reduced from 39 minutes to 25 minutes. + +GHCR remains canonical. After the split, a trusted Depot canary may compare a +Depot Registry pull-through reference and the containerd layer store against +the exact same GHCR index/child digests. Adopt either only with at least 20% +and 10 seconds of median pull improvement; never expose the registry or cache +token to PR code. + +### `Mesh-LLM/mesh-packaging` + +Packaging already consumes product-v2 and must not rebuild the CLI/runtime. +After the MeshLLM graph is stable: + +1. replace the global native-package matrix barrier with a reusable per-row + package → QA → image → QA pipeline; +2. publish a canonical release artifact index from MeshLLM containing source + SHA, product/host/runtime/addon assets, schemas, and digests; +3. publish Node addon producer artifacts from MeshLLM so packaging assembles + npm instead of rebuilding five targets; +4. build each runtime image once at an immutable staging digest, test it, then + promote that digest; +5. route CPU-only packaging rows to Depot after the main repository canary. + +The first product-v2 release candidate is the compatibility baseline for that +work; older v0.74 assets predate the contract. + +## Measurement and rollout gates + +For each phase, save raw observations and label them with provider, runner size, +image digest, and change class: + +```bash +python3 scripts/collect-ci-metrics.py \ + --repo Mesh-LLM/mesh-llm \ + --workflow pr_builds.yml \ + --event pull_request \ + --limit 30 \ + --label provider=depot \ + --label runner=depot-ubuntu-24.04-8 \ + --raw-out /tmp/pr-depot-runs.json \ + --json-out /tmp/pr-depot-metrics.json \ + --markdown-out /tmp/pr-depot-metrics.md +``` + +Rollout sequence: + +1. restrict the `Default` runner group to the repository and only + `depot-canary.yml@refs/heads/main`, then enable public-repository access; +2. run the allowed-main and denied-feature-ref canaries; +3. compare `-4`, `-8`, and `-16` using Depot CPU/memory/disk utilization data; +4. allowlist main CI plus every directly invoked reusable workflow, then + canary routing, quality, and the Linux product graph from `main`; +5. collect five comparable green main canaries; +6. set `DEPOT_RUNNERS_ENABLED=true` for trusted main jobs after those canaries + meet the targets; +7. allowlist `release.yml@refs/heads/main` and exercise the non-publishing, + non-secret runtime/composition producers. Tag-push publishing remains hosted; +8. keep all PR-event code hosted while automatic Depot Cache is enabled; +9. migrate packaging only after product-v2 and addon contracts are stable. + +Rollback for the currently implemented trusted lanes is one +repository-variable change: + +```bash +gh variable set DEPOT_RUNNERS_ENABLED --repo Mesh-LLM/mesh-llm --body false +``` + +`DEPOT_PR_RUNNERS_ENABLED` does not activate Depot in the current PR workflows. +Leave it unset or `false`; any later PR phase requires automatic-cache +isolation plus its own explicit rollback control before activation. + +Disabling Depot changes runner placement only. It must not change the build +graph, action inputs, cache keys, or artifact contracts. diff --git a/ci/METRICS.md b/ci/METRICS.md new file mode 100644 index 0000000000..0c771ad418 --- /dev/null +++ b/ci/METRICS.md @@ -0,0 +1,78 @@ +# CI timing metrics + +Use `scripts/collect-ci-metrics.py` to establish repeatable timing baselines for +PR Builds, main CI, or exact workflow runs. The script is dependency-free beyond +Python and the GitHub CLI, and every GitHub operation it performs is read-only. + +Collect successful PR Builds and save the raw observations as well as JSON and +Markdown summaries: + +```bash +python3 scripts/collect-ci-metrics.py \ + --repo Mesh-LLM/mesh-llm \ + --workflow pr_builds.yml \ + --event pull_request \ + --limit 30 \ + --label provider=github \ + --raw-out /tmp/pr-builds-runs.json \ + --json-out /tmp/pr-builds-metrics.json \ + --markdown-out /tmp/pr-builds-metrics.md +``` + +Collect main CI over a bounded date range: + +```bash +python3 scripts/collect-ci-metrics.py \ + --repo Mesh-LLM/mesh-llm \ + --workflow ci.yml \ + --branch main \ + --created '>=2026-07-01' \ + --limit 50 \ + --json-out /tmp/main-ci-metrics.json \ + --markdown-out /tmp/main-ci-metrics.md +``` + +Analyze exact runs or reprocess saved observations without another API request: + +```bash +python3 scripts/collect-ci-metrics.py \ + --run-id 30435682397 \ + --run-id 30460057494 + +python3 scripts/collect-ci-metrics.py \ + --input /tmp/pr-builds-runs.json \ + --json-out /tmp/pr-builds-recomputed.json \ + --markdown-out /tmp/pr-builds-recomputed.md +``` + +When no output path is supplied, the Markdown report is written to stdout. Use +`--json-out -` for machine-readable stdout. + +## Timing definitions + +- Workflow wall time is GitHub's run `created_at` to `updated_at`. +- Workflow queue time is run `created_at` to `started_at`. +- Workflow wall time, workflow queue time, and job start delay exclude rerun + attempts. GitHub retains the original run-level timestamps while its jobs API + returns the latest attempt, so combining them would create false queue and + wall measurements. Job duration and job queue remain valid for the latest + attempt. +- Job duration is job `started_at` to `completed_at`. +- Job queue time is job `created_at` to `started_at`. Live collection uses the + read-only jobs API because `gh run view --json jobs` omits job creation times. +- Job start delay is workflow `created_at` to job `started_at`; it includes + dependency wait and must not be presented as runner queue time. It is only + reported for first attempts. +- A terminal job is the last non-skipped job to finish. It is a critical-path + candidate, not a reconstructed Actions dependency graph. + +The JSON output includes p50, p90, and p95 summaries; exact slow observations; +job-family summaries; terminal-job counts; runner labels; and individual run +metadata. Raw output intentionally excludes logs and step output. + +For before/after comparisons, use the same workflow, event, change class, run +conclusion, and sample size. Keep documentation-only and full native-build PRs +in separate cohorts, and record the runner provider/image revision with +`--label`. Compare both wall time and queue time: a faster compiler does not +explain provider-capacity delays, and a shorter routed workflow is not evidence +that an unchanged build became faster. diff --git a/ci/ci.md b/ci/ci.md index cbd612197d..66568d71c6 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -67,17 +67,26 @@ flowchart TD WebsiteDocs --> CLIDocsSync CLIDocs --> CLIDocsSync -subgraph PRCI["pr_builds.yml · PR Builds"] + subgraph PRCI["pr_builds.yml · PR Builds"] direction TB - subgraph Producers["top-level target jobs"] - LinuxCPU["linux_cpu_artifact\ndebug mesh-llm · CLI smoke\n→ ci-linux-inference-binaries"] + subgraph Producers["producers, checks, and composers"] + LinuxHost["linux_host_input\none immutable neutral host"] + LinuxCPU["linux_cpu_runtime_input\none CPU runtime"] + LinuxProduct["linux_cpu_artifact\ncompose host + CPU runtime\n→ ci-linux-inference-binaries"] + StaticABI["linux_static_abi_input\none immutable CPU llama ABI"] RustCrateTests["rust_crate_tests matrix\nmetadata-derived crate suites"] LinuxTests["linux_test_groups matrix\nprotocol · Skippy smoke"] - LinuxTargets["linux_targets matrix\nCUDA / ROCm / Vulkan rows build when backend_changed"] - WindowsTargets["windows_targets matrix\nCPU / CUDA / ROCm / Vulkan\nfull builds only for Windows inputs"] - MacCPU["macos_cpu_artifact\nmacOS Metal build · CLI smoke\n→ ci-macos-inference-binaries"] + LinuxTargets["linux_targets matrix\nruntime-only CUDA / ROCm / Vulkan\ncompose with shared host"] + WindowsChecks["windows_checks\nlightweight broad-Rust signal"] + WindowsHost["windows_host_input\none immutable debug host"] + WindowsCPURuntime["windows_cpu_runtime_input\none CPU runtime"] + WindowsGPURuntimes["windows_gpu_runtime_inputs matrix\nCUDA / ROCm / Vulkan runtimes"] + WindowsCPUProduct["windows_cpu_product\ncompose host + CPU runtime"] + WindowsGPUProducts["windows_gpu_products matrix\ncompose shared host + GPU runtime"] + MacHost["macos_host_input\none immutable neutral host"] + MacRuntime["macos_metal_runtime_input\none Metal runtime"] + MacCPU["macos_cpu_artifact\ncompose host + Metal runtime\n→ ci-macos-inference-binaries"] MacTests["macos_unit_tests"] - MacTargets["macos_targets matrix\nCUDA / ROCm / Vulkan explicit skips"] end subgraph Smokes["artifact-consuming smokes"] @@ -89,16 +98,31 @@ subgraph PRCI["pr_builds.yml · PR Builds"] end Docs -. "true: gate heavy jobs" .-> PRCI + InferenceArtifact --> LinuxHost InferenceArtifact --> LinuxCPU + LinuxHost --> LinuxProduct + LinuxCPU --> LinuxProduct Affected --> LinuxTests - TestBins --> RustCrateTests - InferenceArtifact --> MacCPU + TestBins --> StaticABI + StaticABI --> RustCrateTests + StaticABI --> LinuxTests + InferenceArtifact --> MacHost + InferenceArtifact --> MacRuntime + MacHost --> MacCPU + MacRuntime --> MacCPU Affected --> MacTests + LinuxHost --> LinuxTargets Backend --> LinuxTargets - WindowsCPU --> WindowsTargets - WindowsGPU --> WindowsTargets - Backend --> MacTargets - LinuxCPU -- "artifact: ci-linux-inference-binaries" --> Restore + Affected --> WindowsChecks + WindowsCPU --> WindowsHost + WindowsCPU --> WindowsCPURuntime + WindowsGPU --> WindowsHost + WindowsGPU --> WindowsGPURuntimes + WindowsHost --> WindowsCPUProduct + WindowsCPURuntime --> WindowsCPUProduct + WindowsHost --> WindowsGPUProducts + WindowsGPURuntimes --> WindowsGPUProducts + LinuxProduct -- "artifact: ci-linux-inference-binaries" --> Restore MacCPU -- "artifact: ci-macos-inference-binaries" --> Restore Restore --> Inference Restore --> Scripted @@ -140,14 +164,23 @@ subgraph PRCI["pr_builds.yml · PR Builds"] - `ci.yml` validates the same composed product shape on trusted main pushes and manual dispatches: a backend-neutral host plus one separately packaged native - runtime. Linux and macOS debug artifact producers upload both layers, and the - Linux consumer reruns the JSON client-readiness smoke from the downloaded - host/runtime bytes without rebuilding either one. -- CUDA, ROCm, Vulkan, and Windows rows build the host independently from the - selected runtime, require `--version`, `runtime list`, and client readiness, - and do so without a CUDA/ROCm/Vulkan driver, loader-path injection, or a - driver stub. GPU availability remains additional hardware qualification, not - a reason to skip the composed client-start check. + runtime. The Linux host and CPU runtime build independently, then a + composition-only job uploads product-v2 for every downstream smoke. SDK + smokes consume the staged runtime instead of compiling a private replacement. +- Main builds immutable Linux, macOS, and Windows release hosts independently + from their CPU, Metal, CUDA, ROCm, and Vulkan runtimes. Composition-only jobs + verify and combine those exact producer inputs. Linux and Windows backend + rows build only their runtime and reuse the platform host. Each product + requires `--version`, `runtime list`, and client readiness without a driver + stub. GPU availability remains separate hardware qualification. +- `.github/actions/prepare-host-input`, + `.github/actions/prepare-windows-host-input`, + `.github/actions/prepare-native-runtime-input`, and + `.github/actions/compose-product-input` are the shared PR/main/release + primitives. The composer never compiles either producer input. +- Linux crate-test and grouped-test matrices restore one tarred static CPU + llama ABI from `linux_static_abi_input`; individual rows never rebuild the + same patch queue concurrently. ### Current PR Builds contract @@ -155,8 +188,10 @@ subgraph PRCI["pr_builds.yml · PR Builds"] React console, and CLI-documentation feedback: formatting, React console UI quality when relevant, the CLI-docs sync guard when Rust CLI definitions change, and deterministic clippy bins from - `scripts/plan-clippy-batches.sh`. Its summary job writes a Markdown table to - `$GITHUB_STEP_SUMMARY` instead of printing a terminal-only table. + `scripts/plan-clippy-batches.sh`. Routing no longer waits for the compiled + consistency checks; `ci-consistency` runs beside it and remains part of the + summary gate. Formatting and UI quality run directly on the selected runner + instead of paying the public backend-image pull cost. - `pr_website.yml` is named **PR Website Checks** and owns the public website PR canary. It uses `.github/actions/compute-changes` and runs `website-build` only when `website_changed` is true, or when manually @@ -178,7 +213,15 @@ subgraph PRCI["pr_builds.yml · PR Builds"] Every affected Rust workspace crate is assigned to a generated `rust_crate_tests` matrix and runs its complete `cargo test -p ` suite; protocol compatibility and Skippy smoke remain separate integration rows. - Linux/macOS backend matrices remain separate from the CPU artifact producers. + Linux host/CPU-runtime and macOS host/Metal-runtime producers run + independently, and their product composers never compile. Linux backend rows + consume the same immutable host artifact and build only their selected + runtime. Windows follows the same graph: one debug neutral host, independent + CPU/CUDA/ROCm/Vulkan runtime inputs, and composition-only products. + Unsupported macOS CUDA, ROCm, and Vulkan rows are omitted. +- Product readiness starts a local mDNS client and never depends on the mutable + public mesh. The public `client --auto` admission probe is manual-only, so an + external peer outage cannot block a pull request or release. - Pull requests test affected crates plus their reverse dependents. Main pushes and manual dispatches assign every Cargo workspace member to the matrix, so a targeted-routing mistake cannot permanently hide a crate suite. @@ -199,12 +242,13 @@ subgraph PRCI["pr_builds.yml · PR Builds"] that can affect native ABI/backend products, such as `third_party/llama.cpp/**`, `crates/skippy-ffi/**`, backend build scripts, backend-relevant Justfile hunks, and `.github/cache-version.txt`. -- Windows target jobs use compute-changes' `windows_cpu_build_required` and - `windows_gpu_build_required` outputs for full platform builds. The CPU row can - still run lightweight Windows cargo checks for broad Rust changes, but - CUDA/ROCm/Vulkan rows stay skipped unless Windows GPU inputs changed, - backend-relevant Justfile hunks changed, or the workflow is manually - dispatched. +- Windows broad-Rust changes run lightweight Cargo checks. The immutable debug + host, CPU runtime, and CPU product run only when + `windows_cpu_build_required` is true or the workflow is manually dispatched. + CUDA/ROCm/Vulkan runtime producers and composition-only product jobs run only + when `windows_gpu_build_required` is true, backend-relevant Justfile hunks + changed, or the workflow is manually dispatched. All products consume the + same host artifact and use the release composer contract. - `pr_cleanup.yml` deletes PR merge-ref caches and artifacts from positively matched PR workflow runs when a pull request closes. Cache cleanup first plans deterministic shards, then fans deletion out across @@ -303,13 +347,14 @@ while runtime GPU assertions require a matching restricted self-hosted pool. Linux workflow-local toolchain and package setup blocks are migration debt and must be removed when their lane adopts an image, not copied elsewhere. -PR Builds runs `public_runner_image_contract` inside the public image and a -two-row `arc_runner_image_contract` matrix directly on `mesh-llm-amd64` and -`mesh-llm-arm64`. The public job validates the baked dependency/tool contract. -The ARC job checks the native machine architecture, validates the self-hosted -image, and performs a small Rust check. It has no hosted fallback by design: it -is the pull-request gate that detects ARC, K3s scheduling, multi-architecture -image, and runner startup regressions. +PR Builds runs `public_runner_image_contract` inside the public image when the +runner workflow, cache integration, or cache version changes (and on manual +dispatch). Ordinary source/docs PRs do not pay this infrastructure canary. +Trusted main CI owns the two-row `arc_runner_image_contract` matrix directly on +`mesh-llm-amd64` and `mesh-llm-arm64`; untrusted PR-event jobs never request +those labels. The public job validates the baked dependency/tool contract. The +ARC job checks the native machine architecture, validates the self-hosted +image, and performs a small Rust check. It has no hosted fallback by design. Repository visibility and GHCR package visibility are separate controls. If an anonymous pull still returns `401` or `403`, public-container jobs must grant @@ -317,18 +362,45 @@ anonymous pull still returns `401` or `403`, public-container jobs must grant and `secrets.GITHUB_TOKEN`. Making `mesh-llm-runner-images` public does not by itself prove that an existing package is anonymously readable. -The public image already contains `sccache`. Public-image Rust jobs start with -its GHA remote backend disabled, then use the repository-local -`configure-sccache-gha` action to export the ephemeral Actions cache URL/token -and start a `disk,gha` multi-level cache. The disk cache serves as L0, GHA is a -best-effort L1, and the disk tier uses a job-local directory beside the checkout -so it does not make release sources appear dirty. Cache read failures degrade to -misses, and cache write failures only emit warnings. Compiler invocations also -fall back locally if the sccache server becomes unavailable. A failed initial -remote probe stops the remote-configured server and restarts `sccache` with -disk-only storage. Persistent Cargo target and ABI caches continue through the -existing cache actions. Do not download a second sccache binary just to -configure the GHA backend. +The public image already contains `sccache`. In trusted jobs, the +repository-local `configure-sccache-gha` action may use Depot's injected WebDAV +endpoint/token and start a `disk,webdav` cache. Current pull-request jobs remain +GitHub-hosted. When the typed Depot permission is false, the cache action starts +the sccache child with a credential-free, job-local disk backend. That isolates +only sccache: Depot's automatically injected job token and transparent +GitHub-cache API redirection remain available to other code on a Depot runner. +Consequently, no untrusted PR code may run on Depot while automatic cache +injection is enabled. GitHub-hosted jobs retain the existing best-effort +`disk,gha` path or explicit disk-only mode. Cache read failures degrade to +misses, cache write failures only warn, and a failed remote probe restarts +`sccache` with disk-only storage. + +## Depot rollout + +Every current `pull_request` job selects GitHub-hosted runners, regardless of +repository ownership or `DEPOT_PR_RUNNERS_ENABLED`; that variable is ignored. +The selector is only defense in depth +because PR workflow and local-action files are themselves PR-controlled. + +Trusted main/release jobs use `DEPOT_RUNNERS_ENABLED`, and a trusted main-ref +manual dispatch can use `use_depot=true` for a bounded canary. The selector +requires `refs/heads/main`; tag pushes and feature refs fall back to hosted +runners. Depot-managed runners register in the organization `Default` runner +group. Before enabling public access, restrict that group to +`Mesh-LLM/mesh-llm` and exact default-branch workflow refs, beginning with +`depot-canary.yml@refs/heads/main`. The existing `mesh-llm` runner group owns +the dedicated GPU scale sets and is not the Depot group. + +Current GitHub-hosted PR jobs may share the `mesh-llm` key namespace because +GitHub scopes PR cache writes to the merge ref; trusted main does not restore +from that ref. Depot's cache is repository-scoped instead, so cache-key +conventions or a trusted reusable caller are not sufficient protection from +malicious checked-out PR code. PR events stay hosted while automatic Depot +Cache is enabled. Runner placement does not alter build action inputs or +artifact contracts. Hardware-qualified GPU execution stays on dedicated +runners. See +[`DEPOT_MIGRATION.md`](DEPOT_MIGRATION.md) for activation prerequisites, +baseline metrics, target service levels, and the cross-repository plan. ## Public website deployment @@ -396,7 +468,7 @@ Use these checks when reviewing PR CI wall-clock regressions: treating a slow Windows miss as expected. - **Runner routing**: platform-specific work should run on its native runner class (Blacksmith Windows 2025 for Windows ABI products, Blacksmith macOS for Swift/Metal, Linux - for Linux backends) and skip unsupported combinations explicitly. + for Linux backends) and omit unsupported combinations. For canonical agent-facing CI rules, start with `.agents/skills/manage-ci/SKILL.md`. The scoped `.github/AGENTS.md` file routes diff --git a/scripts/build-host.sh b/scripts/build-host.sh index d9728bb019..4539b37668 100755 --- a/scripts/build-host.sh +++ b/scripts/build-host.sh @@ -84,6 +84,36 @@ configure_rust_cache() { fi } +stamp_build_version() { + local release_version="" + local pkgid="" + + if [[ -n "${MESH_LLM_BUILD_VERSION:-}" ]]; then + echo "Using preset MESH_LLM_BUILD_VERSION: $MESH_LLM_BUILD_VERSION" + return 0 + fi + + if ! pkgid="$(cd "$REPO_ROOT" && cargo pkgid -p mesh-llm 2>/dev/null)"; then + echo "Warning: unable to derive build version; cargo pkgid unavailable." >&2 + unset MESH_LLM_BUILD_VERSION || true + return 0 + fi + release_version="${pkgid##*#}" + if [[ -z "$release_version" || "$release_version" == "$pkgid" ]]; then + echo "Warning: unable to derive build version; cargo pkgid output was unexpected." >&2 + unset MESH_LLM_BUILD_VERSION || true + return 0 + fi + + export MESH_LLM_BUILD_VERSION="$release_version" + echo "Using release MESH_LLM_BUILD_VERSION: $MESH_LLM_BUILD_VERSION" +} + +if [[ "${MESH_LLM_DYNAMIC_NATIVE_RUNTIME:-1}" != "1" ]]; then + echo "Host builds must use dynamic native runtimes; MESH_LLM_DYNAMIC_NATIVE_RUNTIME=0 is unsupported." >&2 + exit 1 +fi + configure_lld_linker configure_rust_cache @@ -97,6 +127,7 @@ fi cargo_args=(build --locked -p mesh-llm --bin mesh-llm --no-default-features \ --features "web-ui,dynamic-native-runtime") if [[ "$BUILD_PROFILE" == "release" ]]; then + stamp_build_version cargo_args=(build --release --locked -p mesh-llm --bin mesh-llm --no-default-features \ --features "web-ui,dynamic-native-runtime") fi diff --git a/scripts/build-release.sh b/scripts/build-release.sh index 1d08804cd7..6166e238f7 100755 --- a/scripts/build-release.sh +++ b/scripts/build-release.sh @@ -1,146 +1,8 @@ #!/usr/bin/env bash +# Compatibility entry point. All Unix host profiles share build-host.sh so +# development, PR, main, and release builds cannot drift. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" - -UI_DIR="$REPO_ROOT/crates/mesh-llm-ui" - -append_rustflag() { - local flag="$1" - case " ${RUSTFLAGS:-} " in - *" $flag "*) ;; - *) export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }$flag" ;; - esac -} - -stamp_build_version() { - local release_version="" - local pkgid="" - - if [[ -n "${MESH_LLM_BUILD_VERSION:-}" ]]; then - echo "Using preset MESH_LLM_BUILD_VERSION: $MESH_LLM_BUILD_VERSION" - return 0 - fi - - if ! pkgid="$(cd "$REPO_ROOT" && cargo pkgid -p mesh-llm 2>/dev/null)"; then - echo "Warning: unable to derive build version; cargo pkgid unavailable." >&2 - unset MESH_LLM_BUILD_VERSION || true - return 0 - fi - release_version="${pkgid##*#}" - if [[ -z "$release_version" || "$release_version" == "$pkgid" ]]; then - echo "Warning: unable to derive build version; cargo pkgid output was unexpected." >&2 - unset MESH_LLM_BUILD_VERSION || true - return 0 - fi - - export MESH_LLM_BUILD_VERSION="$release_version" - echo "Using release MESH_LLM_BUILD_VERSION: $MESH_LLM_BUILD_VERSION" - return 0 -} - -configure_lld_linker() { - case "$(uname -s)" in - Linux) - if ! command -v ld.lld >/dev/null 2>&1; then - cat >&2 <<'EOF' -Error: LLVM ld.lld was not found. - -lld is required for faster Rust builds (measured up to 26% faster locally). - -Install lld, then rerun the just command. Common Linux packages: - Ubuntu/Debian: sudo apt-get update && sudo apt-get install -y lld - Fedora: sudo dnf install lld - Arch Linux: sudo pacman -S lld - openSUSE: sudo zypper install lld - -The build requires ld.lld to be available on PATH. -EOF - exit 1 - fi - append_rustflag "-C link-arg=-fuse-ld=lld" - echo "Using Rust linker: $(command -v ld.lld)" - ;; - Darwin) - local lld="" - local lld_prefix="" - if command -v ld64.lld >/dev/null 2>&1; then - lld="$(command -v ld64.lld)" - elif command -v brew >/dev/null 2>&1; then - lld_prefix="$(brew --prefix lld 2>/dev/null || true)" - if [[ -n "$lld_prefix" && -x "$lld_prefix/bin/ld64.lld" ]]; then - lld="$lld_prefix/bin/ld64.lld" - fi - fi - if [[ -z "$lld" ]]; then - for candidate in /opt/homebrew/opt/lld/bin/ld64.lld /usr/local/opt/lld/bin/ld64.lld; do - if [[ -x "$candidate" ]]; then - lld="$candidate" - break - fi - done - fi - if [[ -z "$lld" ]]; then - cat >&2 <<'EOF' -Error: LLVM ld64.lld was not found. - -lld is required for faster Rust builds (measured up to 26% faster locally). - -Install lld, then rerun the just command: - brew install lld - -If Homebrew installed lld but it is not on PATH, Mesh-LLM also checks: - $(brew --prefix lld)/bin/ld64.lld - /opt/homebrew/opt/lld/bin/ld64.lld - /usr/local/opt/lld/bin/ld64.lld -EOF - exit 1 - fi - append_rustflag "-C link-arg=-fuse-ld=$lld" - echo "Using Rust linker: $lld" - ;; - *) - echo "Unsupported OS for release build: $(uname -s)" >&2 - exit 1 - ;; - esac -} - -configure_rust_cache() { - if [[ -n "${RUSTC_WRAPPER:-}" ]]; then - echo "Using Rust compiler wrapper: $RUSTC_WRAPPER" - elif command -v sccache >/dev/null 2>&1; then - export RUSTC_WRAPPER="$(command -v sccache)" - echo "Using Rust compiler wrapper: $RUSTC_WRAPPER" - fi -} - -case "$(uname -s)" in - Darwin|Linux) - ;; - *) - echo "Unsupported OS for release build: $(uname -s)" >&2 - exit 1 - ;; -esac - -if [[ "${MESH_LLM_DYNAMIC_NATIVE_RUNTIME:-1}" != "1" ]]; then - echo "Release hosts must use dynamic native runtimes; MESH_LLM_DYNAMIC_NATIVE_RUNTIME=0 is unsupported." >&2 - exit 1 -fi - -configure_lld_linker -configure_rust_cache - -echo "Building backend-neutral host with dynamic native-runtime support." - -echo "Building UI..." -MESH_LLM_BUILD_PROFILE=release "$SCRIPT_DIR/build-ui.sh" "$UI_DIR" - -echo "Building mesh-llm..." -stamp_build_version -(cd "$REPO_ROOT" && cargo build --release --locked -p mesh-llm \ - --no-default-features \ - --features web-ui,dynamic-native-runtime) +exec "$SCRIPT_DIR/build-host.sh" --profile release diff --git a/scripts/build-windows.ps1 b/scripts/build-windows.ps1 index da4bcee292..02c3fb447a 100755 --- a/scripts/build-windows.ps1 +++ b/scripts/build-windows.ps1 @@ -1023,11 +1023,17 @@ if ($HostOnly) { } } Set-BuildVersionStamp - $hostArgs = @("build", "--release", "--locked", "-p", "mesh-llm", "--bin", "mesh-llm", "--no-default-features", "--features", "web-ui,dynamic-native-runtime") + $hostArgs = @("build") + $hostOutputProfile = "debug" + if ($buildProfile -eq "release") { + $hostArgs += "--release" + $hostOutputProfile = "release" + } + $hostArgs += @("--locked", "-p", "mesh-llm", "--bin", "mesh-llm", "--no-default-features", "--features", "web-ui,dynamic-native-runtime") Invoke-NativeCommand "cargo" $hostArgs - Write-Host "Mesh backend-neutral host: target\\release\\mesh-llm.exe" + Write-Host "Mesh backend-neutral host: target\\$hostOutputProfile\\mesh-llm.exe" } - exit 0 + return } switch ($backendName) { diff --git a/scripts/ci-client-readiness-process.py b/scripts/ci-client-readiness-process.py index cdd0eaa9e0..8a0c006bca 100644 --- a/scripts/ci-client-readiness-process.py +++ b/scripts/ci-client-readiness-process.py @@ -52,13 +52,19 @@ def request_ctrl_break(pid: int) -> None: os.kill(pid, ctrl_break) -def is_running(pid: int) -> bool: +def is_running(pid: int, *, is_windows: bool) -> bool: try: os.kill(pid, 0) except ProcessLookupError: return False except PermissionError: return True + except OSError as error: + # Windows reports ERROR_INVALID_PARAMETER after a process has exited + # instead of raising ProcessLookupError for os.kill(pid, 0). + if is_windows and getattr(error, "winerror", None) == 87: + return False + raise return True @@ -97,7 +103,7 @@ def main() -> int: request_ctrl_break(args.pid) return 0 if args.command == "is-running": - return 0 if is_running(args.pid) else 1 + return 0 if is_running(args.pid, is_windows=os.name == "nt") else 1 if args.command == "force-stop": force_stop(args.pid) return 0 diff --git a/scripts/ci-compose-product-input.sh b/scripts/ci-compose-product-input.sh new file mode 100755 index 0000000000..ba92c2d44b --- /dev/null +++ b/scripts/ci-compose-product-input.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE is required}" +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" +: "${INPUT_HOST_INPUT_DIR:?INPUT_HOST_INPUT_DIR is required}" +: "${INPUT_RUNTIME_INPUT_DIR:?INPUT_RUNTIME_INPUT_DIR is required}" +: "${INPUT_OUTPUT_DIR:?INPUT_OUTPUT_DIR is required}" +: "${INPUT_BACKEND:?INPUT_BACKEND is required}" +: "${INPUT_BINARY_NAME:?INPUT_BINARY_NAME is required}" +: "${INPUT_READINESS_SMOKE:?INPUT_READINESS_SMOKE is required}" +INPUT_ATTESTATION_PUBLIC_KEY_FILE="${INPUT_ATTESTATION_PUBLIC_KEY_FILE:-}" +INPUT_ATTESTATION_VERIFIER="${INPUT_ATTESTATION_VERIFIER:-}" + +if command -v python3 >/dev/null 2>&1; then + python_bin="python3" +elif command -v python >/dev/null 2>&1; then + python_bin="python" +else + echo "python3 or python is required to compose a product input" >&2 + exit 1 +fi + +canonical_paths=() +while IFS= read -r path; do + canonical_paths+=("$path") +done < <( + "$python_bin" - \ + "$GITHUB_WORKSPACE" \ + "$INPUT_HOST_INPUT_DIR" \ + "$INPUT_RUNTIME_INPUT_DIR" \ + "$INPUT_OUTPUT_DIR" <<'PY' +import sys +from pathlib import Path + + +def resolve_in_workspace(workspace: Path, raw: str, *, require_dir: bool) -> Path: + candidate = Path(raw) + if not candidate.is_absolute(): + candidate = workspace / candidate + candidate = candidate.resolve(strict=False) + try: + candidate.relative_to(workspace) + except ValueError as error: + raise SystemExit( + f"CI artifact path escapes GITHUB_WORKSPACE: {raw} -> {candidate}" + ) from error + if require_dir and not candidate.is_dir(): + raise SystemExit(f"CI producer input is not a directory: {candidate}") + return candidate + + +def overlaps(left: Path, right: Path) -> bool: + return ( + left == right + or left in right.parents + or right in left.parents + ) + + +workspace = Path(sys.argv[1]).resolve(strict=True) +host_input = resolve_in_workspace(workspace, sys.argv[2], require_dir=True) +runtime_input = resolve_in_workspace(workspace, sys.argv[3], require_dir=True) +output = resolve_in_workspace(workspace, sys.argv[4], require_dir=False) + +if output == workspace: + raise SystemExit(f"product output cannot be GITHUB_WORKSPACE: {output}") +for label, producer_input in ( + ("host", host_input), + ("runtime", runtime_input), +): + if overlaps(output, producer_input): + raise SystemExit( + f"product output overlaps {label} producer input: " + f"{output} and {producer_input}" + ) + +print(host_input) +print(runtime_input) +print(output) +PY +) + +if [[ "${#canonical_paths[@]}" -ne 3 ]]; then + echo "failed to canonicalize CI artifact paths" >&2 + exit 1 +fi + +host_input_dir="${canonical_paths[0]}" +runtime_input_dir="${canonical_paths[1]}" +output_dir="${canonical_paths[2]}" +host="$host_input_dir/$INPUT_BINARY_NAME" +host_imports="$host_input_dir/host-imports.json" +host_checksum="$host_input_dir/$INPUT_BINARY_NAME.sha256" + +test -f "$host" +chmod +x "$host" +test -s "$host_imports" +test -s "$host_checksum" +expected_host_checksum="$(awk 'NR == 1 {print $1}' "$host_checksum")" +if command -v sha256sum >/dev/null 2>&1; then + actual_host_checksum="$(sha256sum "$host" | awk '{print $1}')" +else + actual_host_checksum="$(shasum -a 256 "$host" | awk '{print $1}')" +fi +test "$actual_host_checksum" = "$expected_host_checksum" + +if [[ -n "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" ]]; then + attestation_verifier="${INPUT_ATTESTATION_VERIFIER:-$host_input_dir/release-attestation-verifier}" + verifier_checksum="$attestation_verifier.sha256" + test -s "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" + test -f "$attestation_verifier" + test -s "$verifier_checksum" + expected_verifier_checksum="$(awk 'NR == 1 {print $1}' "$verifier_checksum")" + if command -v sha256sum >/dev/null 2>&1; then + actual_verifier_checksum="$(sha256sum "$attestation_verifier" | awk '{print $1}')" + else + actual_verifier_checksum="$(shasum -a 256 "$attestation_verifier" | awk '{print $1}')" + fi + test "$actual_verifier_checksum" = "$expected_verifier_checksum" + chmod +x "$attestation_verifier" + "$attestation_verifier" release-attestation inspect \ + --binary "$host" \ + --public-key-file "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" \ + --json +elif [[ -n "$INPUT_ATTESTATION_VERIFIER" ]]; then + echo "INPUT_ATTESTATION_VERIFIER requires INPUT_ATTESTATION_PUBLIC_KEY_FILE" >&2 + exit 1 +fi + +rm -rf -- "$output_dir" +mkdir -p "$output_dir/native-runtimes" +cp "$host" "$output_dir/$INPUT_BINARY_NAME" +chmod +x "$output_dir/$INPUT_BINARY_NAME" +cp "$host_imports" "$output_dir/host-imports.json" + +runtime_archives=() +while IFS= read -r archive; do + runtime_archives+=("$archive") +done < <(find "$runtime_input_dir" -type f -name '*.tar.gz' -print) +if [[ "${#runtime_archives[@]}" -gt 1 ]]; then + echo "expected at most one runtime archive; found ${#runtime_archives[@]}" >&2 + exit 1 +elif [[ "${#runtime_archives[@]}" -eq 1 ]]; then + scripts/verify-native-runtime-package.sh "${runtime_archives[0]}" + tar -xzf "${runtime_archives[0]}" -C "$output_dir/native-runtimes" +else + runtime_dirs=() + while IFS= read -r manifest; do + runtime_dirs+=("$(dirname "$manifest")") + done < <( + find "$runtime_input_dir" \ + -mindepth 2 \ + -maxdepth 2 \ + -type f \ + -name manifest.json \ + -print + ) + if [[ "${#runtime_dirs[@]}" -ne 1 ]]; then + echo "expected exactly one extracted runtime; found ${#runtime_dirs[@]}" >&2 + exit 1 + fi + cp -a "${runtime_dirs[0]}" "$output_dir/native-runtimes/" +fi + +composed_runtime_dirs=() +while IFS= read -r manifest; do + composed_runtime_dirs+=("$(dirname "$manifest")") +done < <( + find "$output_dir/native-runtimes" \ + -mindepth 2 \ + -maxdepth 2 \ + -type f \ + -name manifest.json \ + -print +) +if [[ "${#composed_runtime_dirs[@]}" -ne 1 ]]; then + echo "expected exactly one composed runtime; found ${#composed_runtime_dirs[@]}" >&2 + exit 1 +fi +runtime_dir="${composed_runtime_dirs[0]}" +scripts/verify-native-runtime-package.sh "$runtime_dir" + +version="${INPUT_VERSION:-}" +if [[ -z "$version" ]]; then + version="$( + "$python_bin" - "$runtime_dir/manifest.json" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as handle: + print(json.load(handle)["runtime"]["mesh_version"]) +PY + )" +fi +version="${version#v}" +host_version_output="$("$output_dir/$INPUT_BINARY_NAME" --version)" +host_version="$(awk '{print $NF}' <<<"$host_version_output")" +if [[ "$host_version" != "$version" ]]; then + echo "composed host version mismatch: expected $version, got ${host_version:-}" >&2 + echo "Output: $host_version_output" >&2 + exit 1 +fi +"$python_bin" scripts/compose-product-bundle.py \ + --bundle "$output_dir" \ + --host "$output_dir/$INPUT_BINARY_NAME" \ + --runtime "$runtime_dir" \ + --version "$version" \ + --backend "$INPUT_BACKEND" +test -s "$output_dir/product-manifest.json" + +if [[ "$INPUT_READINESS_SMOKE" == "true" ]]; then + MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR="$output_dir/native-runtimes" \ + "$output_dir/$INPUT_BINARY_NAME" --log-format json --version + MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR="$output_dir/native-runtimes" \ + "$output_dir/$INPUT_BINARY_NAME" --log-format json runtime list + scripts/ci-client-readiness-smoke.sh \ + "$output_dir/$INPUT_BINARY_NAME" \ + "$output_dir/native-runtimes" +elif [[ "$INPUT_READINESS_SMOKE" != "false" ]]; then + echo "INPUT_READINESS_SMOKE must be true or false" >&2 + exit 1 +fi + +product_dir="$(cd "$output_dir" && pwd -P)" +runtime_name="$(basename "$runtime_dir")" +archive_path="$product_dir.tar.gz" +rm -f -- "$archive_path" +tar -C "$product_dir" -czf "$archive_path" . +test -s "$archive_path" +{ + echo "product_dir=$product_dir" + echo "binary_path=$product_dir/$INPUT_BINARY_NAME" + echo "runtime_root=$product_dir/native-runtimes" + echo "runtime_dir=$product_dir/native-runtimes/$runtime_name" + echo "archive_path=$archive_path" +} >> "$GITHUB_OUTPUT" diff --git a/scripts/ci-kotlin-sdk-smoke.sh b/scripts/ci-kotlin-sdk-smoke.sh index 0ffbb7a499..55dfff34a0 100755 --- a/scripts/ci-kotlin-sdk-smoke.sh +++ b/scripts/ci-kotlin-sdk-smoke.sh @@ -68,9 +68,15 @@ print(os.path.dirname(manifest.get("uniffi_library") or manifest["library"])) PY )" export MESHLLM_KOTLIN_JNA_LIBRARY_PATH="$native_sdk_artifact_dir/$native_sdk_uniffi_library" -native_runtime_dir="$(scripts/ci-prepare-native-runtime.sh "$REPO_ROOT/target/kotlin-native-runtime" cpu)" +native_runtime_dir="$( + scripts/ci-prepare-native-runtime.sh \ + "$REPO_ROOT/target/kotlin-native-runtime" \ + cpu \ + --reuse-from-binary "$1" +)" export MESHLLM_NATIVE_RUNTIME_ARTIFACT_DIR="$native_runtime_dir" +# shellcheck disable=SC2016 # The nested shell expands exported fixture variables. scripts/ci-sdk-fixture.sh "$1" "$2" "$3" -- \ bash -lc ' set -euo pipefail diff --git a/scripts/ci-prepare-native-runtime.sh b/scripts/ci-prepare-native-runtime.sh index 8354f6c7c1..5b74f1dd7c 100755 --- a/scripts/ci-prepare-native-runtime.sh +++ b/scripts/ci-prepare-native-runtime.sh @@ -1,18 +1,237 @@ #!/usr/bin/env bash set -euo pipefail -if [[ "$#" -lt 1 || "$#" -gt 2 ]]; then - echo "Usage: $0 [backend]" >&2 +usage() { + cat >&2 <<'EOF' +Usage: scripts/ci-prepare-native-runtime.sh [backend] [options] + +Options: + --reuse-from-binary PATH + Prefer a compatible packaged runtime under PATH's adjacent + native-runtimes directory. A present but incompatible bundle is an error. + +Environment: + MESH_SDK_NATIVE_RUNTIME_BUILD_FALLBACK + Set to 1 to build when --reuse-from-binary has no adjacent bundle, or 0 + to reject that fallback. The default is 0 in CI and 1 for local runs. +EOF +} + +if [[ "$#" -lt 1 ]]; then + usage exit 1 fi REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" OUT_DIR="$1" -BACKEND="${2:-cpu}" +shift + +BACKEND="cpu" +if [[ "$#" -gt 0 && "$1" != --* ]]; then + BACKEND="$1" + shift +fi + +REUSE_BINARY="" +while [[ "$#" -gt 0 ]]; do + case "$1" in + --reuse-from-binary) + REUSE_BINARY="${2:?missing binary path}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage + exit 1 + ;; + esac +done + BUILD_DIR="$REPO_ROOT/.deps/llama-build/build-stage-abi-ci-runtime-${BACKEND}" +TEMP_ROOT="" + +cleanup() { + if [[ -n "$TEMP_ROOT" ]]; then + rm -rf -- "$TEMP_ROOT" + fi +} +trap cleanup EXIT + +build_fallback_enabled() { + local configured="${MESH_SDK_NATIVE_RUNTIME_BUILD_FALLBACK:-}" + if [[ -z "$configured" ]]; then + [[ "${CI:-}" != "true" ]] + return + fi + case "$configured" in + 1|true) return 0 ;; + 0|false) return 1 ;; + *) + echo "MESH_SDK_NATIVE_RUNTIME_BUILD_FALLBACK must be 0, 1, false, or true" >&2 + exit 1 + ;; + esac +} + +select_compatible_runtime() { + local binary="$1" + local runtime_root="$2" + local compatibility_json + local expected_skippy_abi + local runtime_dir + + if [[ ! -x "$binary" ]]; then + echo "Missing executable mesh-llm binary for runtime reuse: $binary" >&2 + return 1 + fi + + TEMP_ROOT="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mesh-sdk-runtime-compat.XXXXXX")" + compatibility_json="$TEMP_ROOT/available.json" + expected_skippy_abi="$( + python3 - "$REPO_ROOT/crates/skippy-ffi/src/lib.rs" <<'PY' +import re +import sys + +values = {} +for line in open(sys.argv[1], encoding="utf-8"): + match = re.match( + r"pub const ABI_VERSION_(MAJOR|MINOR|PATCH): u32 = ([0-9]+);", + line.strip(), + ) + if match: + values[match.group(1)] = match.group(2) +try: + print("{}.{}.{}".format(values["MAJOR"], values["MINOR"], values["PATCH"])) +except KeyError as error: + raise SystemExit(f"missing Skippy ABI constant in {sys.argv[1]}: {error}") from error +PY + )" + env \ + -u MESH_LLM_CONFIG \ + -u MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR \ + -u MESH_LLM_NATIVE_RUNTIME_CACHE_DIR \ + HOME="$TEMP_ROOT/home" \ + XDG_CACHE_HOME="$TEMP_ROOT/xdg-cache" \ + XDG_CONFIG_HOME="$TEMP_ROOT/xdg-config" \ + "$binary" \ + --log-format json \ + runtime list \ + --available \ + --bundle-dir "$runtime_root" \ + --cache-dir "$TEMP_ROOT/cache" \ + --json >"$compatibility_json" + + runtime_dir="$( + python3 - \ + "$runtime_root" \ + "$BACKEND" \ + "$compatibility_json" \ + "$expected_skippy_abi" <<'PY' +import json +import sys +from pathlib import Path + +runtime_root = Path(sys.argv[1]).resolve() +requested_backend = {"cuda-blackwell": "cuda", "hip": "rocm"}.get( + sys.argv[2], sys.argv[2] +) +expected_skippy_abi = sys.argv[4] +with open(sys.argv[3], encoding="utf-8") as fh: + rows = json.load(fh) + +if not isinstance(rows, list): + raise SystemExit("native runtime compatibility output must be a JSON list") + +supported = [row for row in rows if row.get("supported") is True] +preferred = [row for row in supported if row.get("backend") == requested_backend] +if len(preferred) == 1: + selected = preferred[0] +elif len(supported) == 1: + selected = supported[0] +else: + rendered = ", ".join( + f"{row.get('id', '')}:{row.get('backend', '')}" + for row in supported + ) or "none" + raise SystemExit( + "expected exactly one compatible adjacent native runtime " + f"(preferred backend {requested_backend}); found {rendered}" + ) + +runtime_id = selected.get("id") +if not isinstance(runtime_id, str) or not runtime_id.strip(): + raise SystemExit("compatible native runtime is missing its id") + +matches = [] +manifest_paths = [] +if (runtime_root / "manifest.json").is_file(): + manifest_paths.append(runtime_root / "manifest.json") +manifest_paths.extend(sorted(runtime_root.glob("*/manifest.json"))) +for manifest_path in manifest_paths: + with manifest_path.open(encoding="utf-8") as fh: + manifest = json.load(fh) + runtime = manifest.get("runtime") or {} + if runtime.get("id") != runtime_id: + continue + if runtime.get("skippy_abi") != expected_skippy_abi: + raise SystemExit( + f"adjacent native runtime {runtime_id} has Skippy ABI " + f"{runtime.get('skippy_abi')}, expected {expected_skippy_abi}" + ) + matches.append(manifest_path.parent.resolve()) + +if len(matches) != 1: + rendered = ", ".join(str(path) for path in matches) or "none" + raise SystemExit( + f"expected one adjacent artifact directory for runtime {runtime_id}; " + f"found {rendered}" + ) +try: + matches[0].relative_to(runtime_root) +except ValueError as error: + raise SystemExit( + f"selected native runtime escapes adjacent bundle root: {matches[0]}" + ) from error +print(matches[0]) +PY + )" + + scripts/verify-native-runtime-package.sh "$runtime_dir" >&2 + echo "Reusing compatible native runtime beside the smoke binary:" >&2 + echo " runtime: $runtime_dir" >&2 + printf '%s\n' "$runtime_dir" +} + +resolve_reuse_binary() { + local binary_dir + binary_dir="$(cd "$(dirname "$REUSE_BINARY")" && pwd)" + printf '%s/%s\n' "$binary_dir" "$(basename "$REUSE_BINARY")" +} + +if [[ -n "$REUSE_BINARY" ]]; then + REUSE_BINARY="$(resolve_reuse_binary)" +fi cd "$REPO_ROOT" +if [[ -n "$REUSE_BINARY" ]]; then + adjacent_runtime_root="$(dirname "$REUSE_BINARY")/native-runtimes" + if [[ -d "$adjacent_runtime_root" ]]; then + select_compatible_runtime "$REUSE_BINARY" "$adjacent_runtime_root" + exit 0 + fi + if ! build_fallback_enabled; then + echo "Adjacent native runtime bundle is required in CI: $adjacent_runtime_root" >&2 + echo "Set MESH_SDK_NATIVE_RUNTIME_BUILD_FALLBACK=1 only for an explicit standalone fallback." >&2 + exit 1 + fi + echo "No adjacent native runtime bundle found; building the standalone fallback." >&2 +fi + rm -rf "$OUT_DIR" LLAMA_STAGE_LINK_MODE=dynamic \ LLAMA_STAGE_BACKEND="$BACKEND" \ diff --git a/scripts/ci-rust-sdk-smoke.sh b/scripts/ci-rust-sdk-smoke.sh index 30acd20b63..a7ed1f75ee 100755 --- a/scripts/ci-rust-sdk-smoke.sh +++ b/scripts/ci-rust-sdk-smoke.sh @@ -13,7 +13,12 @@ scripts/check-sdk-contract.sh scripts/package-sdk-console-assets.sh --sdk node scripts/verify-sdk-console-assets.sh --sdk node -native_runtime_dir="$(scripts/ci-prepare-native-runtime.sh "$REPO_ROOT/target/rust-native-runtime" cpu)" +native_runtime_dir="$( + scripts/ci-prepare-native-runtime.sh \ + "$REPO_ROOT/target/rust-native-runtime" \ + cpu \ + --reuse-from-binary "$1" +)" export MESHLLM_NATIVE_RUNTIME_ARTIFACT_DIR="$native_runtime_dir" scripts/ci-sdk-fixture.sh "$1" "$2" "$3" -- \ diff --git a/scripts/ci-swift-sdk-smoke.sh b/scripts/ci-swift-sdk-smoke.sh index 4397253b2e..0fb607b1d9 100755 --- a/scripts/ci-swift-sdk-smoke.sh +++ b/scripts/ci-swift-sdk-smoke.sh @@ -31,9 +31,15 @@ ditto -c -k --sequesterRsrc --keepParent \ scripts/verify-swift-release-artifact.sh \ "$SWIFT_ARTIFACT_SMOKE_DIR/MeshLLMFFI.xcframework.zip" -native_runtime_dir="$(scripts/ci-prepare-native-runtime.sh "$REPO_ROOT/target/swift-native-runtime" cpu)" +native_runtime_dir="$( + scripts/ci-prepare-native-runtime.sh \ + "$REPO_ROOT/target/swift-native-runtime" \ + cpu \ + --reuse-from-binary "$1" +)" export MESHLLM_NATIVE_RUNTIME_ARTIFACT_DIR="$native_runtime_dir" +# shellcheck disable=SC2016 # The nested shell expands exported fixture variables. scripts/ci-sdk-fixture.sh "$1" "$2" "$3" -- \ bash -lc ' set -euo pipefail diff --git a/scripts/collect-ci-metrics.py b/scripts/collect-ci-metrics.py new file mode 100755 index 0000000000..def026b4c2 --- /dev/null +++ b/scripts/collect-ci-metrics.py @@ -0,0 +1,661 @@ +#!/usr/bin/env python3 +"""Collect dependency-free GitHub Actions timing summaries.""" + +from __future__ import annotations + +import argparse +import collections +import datetime as dt +import json +import math +import pathlib +import subprocess +import sys +from typing import Any + + +RUN_FIELDS = ",".join( + ( + "databaseId", + "attempt", + "workflowName", + "displayTitle", + "event", + "status", + "conclusion", + "createdAt", + "startedAt", + "updatedAt", + "url", + "headSha", + "headBranch", + ) +) +SKIPPED = "skipped" + + +def pick(data: dict[str, Any], *names: str, default: Any = None) -> Any: + for name in names: + if name in data: + return data[name] + return default + + +def timestamp(value: Any) -> dt.datetime | None: + if not isinstance(value, str) or not value: + return None + value = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + parsed = dt.datetime.fromisoformat(value) + except ValueError as error: + raise ValueError(f"invalid timestamp {value!r}") from error + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=dt.timezone.utc) + parsed = parsed.astimezone(dt.timezone.utc) + return parsed if parsed.year > 1970 else None + + +def elapsed(start: dt.datetime | None, end: dt.datetime | None) -> float | None: + if start is None or end is None: + return None + seconds = (end - start).total_seconds() + return seconds if seconds >= 0 else None + + +def percentile(values: list[float], quantile: float) -> float: + position = (len(values) - 1) * quantile + low, high = math.floor(position), math.ceil(position) + if low == high: + return values[low] + return values[low] * (high - position) + values[high] * (position - low) + + +def summarize(values: list[float | None]) -> dict[str, float | int | None]: + samples = sorted(value for value in values if value is not None) + if not samples: + return { + "count": 0, + "min": None, + "mean": None, + "p50": None, + "p90": None, + "p95": None, + "max": None, + } + + def rounded(value: float) -> float: + return round(value, 3) + + return { + "count": len(samples), + "min": rounded(samples[0]), + "mean": rounded(sum(samples) / len(samples)), + "p50": rounded(percentile(samples, 0.50)), + "p90": rounded(percentile(samples, 0.90)), + "p95": rounded(percentile(samples, 0.95)), + "max": rounded(samples[-1]), + } + + +def normalize_job(raw: dict[str, Any]) -> dict[str, Any]: + labels = pick(raw, "labels", "runner_labels", default=[]) + return { + "id": pick(raw, "id", "databaseId", "database_id"), + "name": str(pick(raw, "name", default="unknown job")), + "conclusion": str(pick(raw, "conclusion", default="")), + "created": timestamp(pick(raw, "created_at", "createdAt")), + "started": timestamp(pick(raw, "started_at", "startedAt")), + "completed": timestamp(pick(raw, "completed_at", "completedAt")), + "url": str(pick(raw, "html_url", "url", default="")), + "labels": [str(label) for label in labels] + if isinstance(labels, list) + else [], + } + + +def normalize_run(raw: dict[str, Any]) -> dict[str, Any]: + if not isinstance(raw.get("jobs"), list): + run_id = pick(raw, "id", "databaseId", "database_id", default="unknown") + raise ValueError( + f"run {run_id} has no jobs array; use --raw-out or save " + "'gh run view --json ...,jobs' output" + ) + raw_attempt = pick(raw, "attempt", "run_attempt", default=1) + try: + attempt = int(raw_attempt) + except (TypeError, ValueError): + attempt = 1 + return { + "id": pick(raw, "id", "databaseId", "database_id"), + "attempt": max(attempt, 1), + "workflow": str(pick(raw, "workflow_name", "workflowName", default="")), + "title": str(pick(raw, "title", "displayTitle", default="")), + "event": str(pick(raw, "event", default="")), + "status": str(pick(raw, "status", default="")), + "conclusion": str(pick(raw, "conclusion", default="")), + "created": timestamp(pick(raw, "created_at", "createdAt")), + "started": timestamp(pick(raw, "started_at", "startedAt")), + "updated": timestamp(pick(raw, "updated_at", "updatedAt")), + "url": str(pick(raw, "html_url", "url", default="")), + "sha": str(pick(raw, "head_sha", "headSha", default="")), + "branch": str(pick(raw, "head_branch", "headBranch", default="")), + "jobs": [normalize_job(job) for job in raw["jobs"]], + } + + +def load_runs(path: str) -> list[dict[str, Any]]: + if path == "-": + data = json.load(sys.stdin) + else: + with open(path, encoding="utf-8") as handle: + data = json.load(handle) + if isinstance(data, dict) and isinstance(data.get("runs"), list): + data = data["runs"] + elif isinstance(data, dict): + data = [data] + if not isinstance(data, list) or not all(isinstance(run, dict) for run in data): + raise ValueError("input must be a run object, run array, or object.runs") + return data + + +def gh_json(arguments: list[str]) -> Any: + command = ["gh", *arguments] + try: + result = subprocess.run( + command, + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as error: + raise RuntimeError("gh is required for live collection") from error + except subprocess.CalledProcessError as error: + detail = error.stderr.strip() or error.stdout.strip() or "unknown error" + raise RuntimeError(f"{' '.join(command)} failed: {detail}") from error + try: + return json.loads(result.stdout) + except json.JSONDecodeError as error: + raise RuntimeError(f"{' '.join(command)} returned invalid JSON") from error + + +def fetch_jobs(repository: str, run_id: int) -> list[dict[str, Any]]: + jobs: list[dict[str, Any]] = [] + page = 1 + while True: + response = gh_json( + [ + "api", + "--method", + "GET", + f"repos/{repository}/actions/runs/{run_id}/jobs", + "-f", + "filter=latest", + "-f", + "per_page=100", + "-f", + f"page={page}", + ] + ) + if not isinstance(response, dict) or not isinstance(response.get("jobs"), list): + raise RuntimeError(f"invalid jobs response for run {run_id}") + page_jobs = response["jobs"] + jobs.extend(page_jobs) + total = response.get("total_count") + if len(page_jobs) < 100 or (isinstance(total, int) and len(jobs) >= total): + return jobs + page += 1 + + +def fetch_exact_run(repository: str, run_id: int) -> dict[str, Any]: + run = gh_json( + [ + "run", + "view", + str(run_id), + "--repo", + repository, + "--json", + RUN_FIELDS, + ] + ) + if not isinstance(run, dict): + raise RuntimeError(f"invalid run response for {run_id}") + run["jobs"] = fetch_jobs(repository, run_id) + return run + + +def fetch_runs(args: argparse.Namespace) -> list[dict[str, Any]]: + if args.run_id: + return [fetch_exact_run(args.repo, run_id) for run_id in args.run_id] + command = [ + "run", + "list", + "--repo", + args.repo, + "--workflow", + args.workflow, + "--limit", + str(args.limit), + "--json", + RUN_FIELDS, + ] + for flag, value in ( + ("--status", None if args.status == "all" else args.status), + ("--branch", args.branch), + ("--event", args.event), + ("--created", args.created), + ): + if value: + command.extend([flag, value]) + runs = gh_json(command) + if not isinstance(runs, list): + raise RuntimeError("invalid run list response") + for run in runs: + run_id = run.get("databaseId") if isinstance(run, dict) else None + if not isinstance(run_id, int): + raise RuntimeError("run list contains an invalid run") + run["jobs"] = fetch_jobs(args.repo, run_id) + return runs + + +def observation(run: dict[str, Any], job: dict[str, Any]) -> dict[str, Any]: + return { + "run_id": run["id"], + "run_attempt": run["attempt"], + "run_url": run["url"], + "job_id": job["id"], + "job_url": job["url"], + "name": job["name"], + "conclusion": job["conclusion"], + "duration_seconds": elapsed(job["started"], job["completed"]), + "queue_seconds": elapsed(job["created"], job["started"]), + "start_delay_seconds": ( + elapsed(run["created"], job["started"]) + if run["attempt"] == 1 + else None + ), + "runner_labels": job["labels"], + } + + +def included(run: dict[str, Any], requested: str) -> tuple[bool, str]: + if run["status"] != "completed": + return False, "not_completed" + if requested in ("all", "completed") or run["conclusion"] == requested: + return True, "" + return False, f"conclusion_{run['conclusion'] or 'missing'}" + + +def analyze( + runs: list[dict[str, Any]], + requested_status: str, + top: int, + source: dict[str, Any], + labels: dict[str, str], +) -> dict[str, Any]: + selected = [] + skipped = collections.Counter() + for run in runs: + matches, reason = included(run, requested_status) + if matches: + selected.append(run) + else: + skipped[reason] += 1 + if not selected: + raise ValueError("no completed workflow runs matched the requested status") + + observations: list[dict[str, Any]] = [] + run_reports = [] + terminal_counts = collections.Counter() + wall_times = [] + queue_times = [] + workflow_timing_excluded_reruns = 0 + for run in selected: + jobs = [job for job in run["jobs"] if job["conclusion"] != SKIPPED] + samples = [observation(run, job) for job in jobs] + observations.extend(samples) + completed = [job for job in jobs if job["completed"] is not None] + terminal = ( + max(completed, key=lambda job: job["completed"]) if completed else None + ) + if terminal: + terminal_counts[terminal["name"]] += 1 + longest = max( + (sample for sample in samples if sample["duration_seconds"] is not None), + key=lambda sample: sample["duration_seconds"], + default=None, + ) + workflow_timing_eligible = run["attempt"] == 1 + wall = ( + elapsed(run["created"], run["updated"]) + if workflow_timing_eligible + else None + ) + queue = ( + elapsed(run["created"], run["started"]) + if workflow_timing_eligible + else None + ) + if workflow_timing_eligible: + wall_times.append(wall) + queue_times.append(queue) + else: + workflow_timing_excluded_reruns += 1 + run_reports.append( + { + "id": run["id"], + "attempt": run["attempt"], + "url": run["url"], + "workflow": run["workflow"], + "title": run["title"], + "event": run["event"], + "conclusion": run["conclusion"], + "head_sha": run["sha"], + "head_branch": run["branch"], + "wall_seconds": wall, + "queue_seconds": queue, + "workflow_timing_excluded": not workflow_timing_eligible, + "executed_job_count": len(jobs), + "skipped_job_count": len(run["jobs"]) - len(jobs), + "terminal_job": terminal["name"] if terminal else None, + "longest_job": longest["name"] if longest else None, + "longest_job_seconds": longest["duration_seconds"] if longest else None, + } + ) + + groups: dict[str, list[dict[str, Any]]] = collections.defaultdict(list) + for sample in observations: + groups[sample["name"]].append(sample) + by_name = [ + { + "name": name, + "sample_count": len(samples), + "duration_seconds": summarize( + [sample["duration_seconds"] for sample in samples] + ), + "queue_seconds": summarize( + [sample["queue_seconds"] for sample in samples] + ), + "start_delay_seconds": summarize( + [sample["start_delay_seconds"] for sample in samples] + ), + "terminal_count": terminal_counts[name], + "conclusions": dict( + sorted(collections.Counter(s["conclusion"] for s in samples).items()) + ), + } + for name, samples in groups.items() + ] + by_name.sort( + key=lambda item: ( + item["duration_seconds"]["p95"] or -1, + item["duration_seconds"]["mean"] or -1, + ), + reverse=True, + ) + slowest = sorted( + ( + sample + for sample in observations + if sample["duration_seconds"] is not None + ), + key=lambda sample: sample["duration_seconds"], + reverse=True, + )[:top] + critical = [ + { + "name": name, + "terminal_count": count, + "share": round(count / len(selected), 4), + } + for name, count in terminal_counts.most_common(top) + ] + return { + "schema_version": 1, + "generated_at": dt.datetime.now(dt.timezone.utc) + .isoformat() + .replace("+00:00", "Z"), + "source": source, + "benchmark_labels": labels, + "definitions": { + "workflow_wall_seconds": ( + "first-attempt run created_at to updated_at; reruns excluded" + ), + "workflow_queue_seconds": ( + "first-attempt run created_at to started_at; reruns excluded" + ), + "job_duration_seconds": "job started_at to completed_at", + "job_queue_seconds": ( + "job created_at to started_at; unavailable without job created_at" + ), + "job_start_delay_seconds": ( + "first-attempt workflow created_at to job started_at; includes " + "dependency wait; reruns excluded" + ), + "terminal_job": ( + "last non-skipped job to finish; a critical-path candidate" + ), + }, + "selection": { + "requested_status": requested_status, + "seen_run_count": len(runs), + "included_run_count": len(selected), + "workflow_timing_excluded_reruns": workflow_timing_excluded_reruns, + "skipped_runs": dict(sorted(skipped.items())), + }, + "workflow": { + "wall_seconds": summarize(wall_times), + "queue_seconds": summarize(queue_times), + }, + "jobs": { + "sample_count": len(observations), + "duration_seconds": summarize( + [sample["duration_seconds"] for sample in observations] + ), + "queue_seconds": summarize( + [sample["queue_seconds"] for sample in observations] + ), + "start_delay_seconds": summarize( + [sample["start_delay_seconds"] for sample in observations] + ), + "by_name": by_name, + "critical_finish_candidates": critical, + "slowest_observations": slowest, + }, + "runs": run_reports, + } + + +def human(seconds: float | int | None) -> str: + if seconds is None: + return "n/a" + total = int(round(seconds)) + hours, remainder = divmod(total, 3600) + minutes, seconds = divmod(remainder, 60) + if hours: + return f"{hours}h {minutes}m {seconds}s" + return f"{minutes}m {seconds}s" if minutes else f"{seconds}s" + + +def markdown_escape(value: Any) -> str: + return str(value).replace("|", "\\|").replace("\n", " ") + + +def render_markdown(report: dict[str, Any], top: int) -> str: + workflow = report["workflow"] + jobs = report["jobs"] + lines = [ + "# CI timing summary", + "", + ( + f"Analyzed **{report['selection']['included_run_count']}** completed " + f"run(s) from `{markdown_escape(report['source']['description'])}`." + ), + "", + ] + excluded_reruns = report["selection"]["workflow_timing_excluded_reruns"] + if excluded_reruns: + lines += [ + ( + f"> Excluded workflow wall, workflow queue, and job start-delay " + f"timing from **{excluded_reruns}** rerun attempt(s). GitHub " + "retains the original run timestamps when jobs are rerun." + ), + "", + ] + lines += [ + "## Workflow timing", + "", + "| Timing | Samples | p50 | p90 | p95 | Max |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + for label, key in (("Wall time", "wall_seconds"), ("Queue", "queue_seconds")): + stats = workflow[key] + lines.append( + f"| {label} | {stats['count']} | {human(stats['p50'])} | " + f"{human(stats['p90'])} | {human(stats['p95'])} | " + f"{human(stats['max'])} |" + ) + lines += [ + "", + "## Slow job families", + "", + "| Job | Samples | Duration p50 | Duration p95 | Queue p50 | Queue p95 |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + for item in jobs["by_name"][:top]: + duration, queue = item["duration_seconds"], item["queue_seconds"] + lines.append( + f"| {markdown_escape(item['name'])} | {item['sample_count']} | " + f"{human(duration['p50'])} | {human(duration['p95'])} | " + f"{human(queue['p50'])} | {human(queue['p95'])} |" + ) + lines += [ + "", + "## Terminal jobs (critical-path candidates)", + "", + "| Job | Runs finishing last | Share |", + "| --- | ---: | ---: |", + ] + for item in jobs["critical_finish_candidates"][:top]: + lines.append( + f"| {markdown_escape(item['name'])} | {item['terminal_count']} | " + f"{item['share']:.1%} |" + ) + lines += [ + "", + "## Slowest observations", + "", + "| Run | Job | Duration | Queue |", + "| --- | --- | ---: | ---: |", + ] + for item in jobs["slowest_observations"][:top]: + run_id = item["run_id"] + run = f"[{run_id}]({item['run_url']})" if item["run_url"] else str(run_id) + lines.append( + f"| {run} | {markdown_escape(item['name'])} | " + f"{human(item['duration_seconds'])} | {human(item['queue_seconds'])} |" + ) + lines += [ + "", + ( + "_Job queue uses GitHub's job creation-to-start interval. Offline " + "`gh run view` data may omit creation times; those samples are n/a. " + "Workflow and start-delay timing excludes rerun attempts because " + "their run timestamps belong to the original attempt. Terminal jobs " + "are candidates only; the DAG is not reconstructed._" + ), + "", + ] + return "\n".join(lines) + + +def write(path: str, content: str) -> None: + if path == "-": + sys.stdout.write(content) + return + output = pathlib.Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(content, encoding="utf-8") + + +def labels(values: list[str]) -> dict[str, str]: + result = {} + for value in values: + key, separator, label = value.partition("=") + if not key or not separator: + raise ValueError(f"benchmark label must be KEY=VALUE, got {value!r}") + result[key] = label + return dict(sorted(result.items())) + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Collect read-only GitHub Actions wall, queue, and job metrics." + ) + parser.add_argument("--repo", default="Mesh-LLM/mesh-llm") + parser.add_argument("--workflow") + parser.add_argument("--run-id", type=int, action="append", default=[]) + parser.add_argument("--input", help="Detailed run JSON, --raw-out JSON, or -") + parser.add_argument("--limit", type=int, default=20) + parser.add_argument("--status", default="success") + parser.add_argument("--branch") + parser.add_argument("--event") + parser.add_argument("--created", help="GitHub date filter, e.g. >=2026-07-01") + parser.add_argument("--top", type=int, default=10) + parser.add_argument("--label", action="append", default=[], metavar="KEY=VALUE") + parser.add_argument("--json-out") + parser.add_argument("--markdown-out") + parser.add_argument("--raw-out", help="Save detailed inputs for offline analysis") + args = parser.parse_args(argv) + if args.input and (args.workflow or args.run_id): + parser.error("--input cannot be combined with --workflow or --run-id") + if args.workflow and args.run_id: + parser.error("--workflow cannot be combined with --run-id") + if not args.input and not args.workflow and not args.run_id: + parser.error("one of --input, --workflow, or --run-id is required") + if args.limit < 1 or args.top < 1: + parser.error("--limit and --top must be at least 1") + outputs = (args.json_out, args.markdown_out, args.raw_out) + if sum(path == "-" for path in outputs) > 1: + parser.error("only one output may use stdout (-)") + return args + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + try: + if args.input: + raw_runs = load_runs(args.input) + source = {"kind": "file", "description": args.input} + else: + raw_runs = fetch_runs(args) + target = ",".join(map(str, args.run_id)) if args.run_id else args.workflow + source = { + "kind": "github", + "description": f"{args.repo}:{target}", + "repository": args.repo, + "workflow": args.workflow, + } + runs = [normalize_run(run) for run in raw_runs] + report = analyze(runs, args.status, args.top, source, labels(args.label)) + if args.raw_out: + write( + args.raw_out, + json.dumps({"schema_version": 1, "runs": raw_runs}, indent=2) + "\n", + ) + if args.json_out: + write(args.json_out, json.dumps(report, indent=2, sort_keys=True) + "\n") + summary = render_markdown(report, args.top) + if args.markdown_out: + write(args.markdown_out, summary) + if not args.json_out and not args.markdown_out: + sys.stdout.write(summary) + except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error: + print(f"ci metrics error: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/compose-product-bundle.py b/scripts/compose-product-bundle.py index 5ac95a5821..f8000d6834 100644 --- a/scripts/compose-product-bundle.py +++ b/scripts/compose-product-bundle.py @@ -121,6 +121,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--runtime", type=Path, required=True) parser.add_argument("--version", required=True) parser.add_argument("--backend", required=True) + parser.add_argument( + "--check", + action="store_true", + help="Validate the existing product manifest without rewriting it.", + ) return parser.parse_args() @@ -129,10 +134,18 @@ def main() -> int: manifest = compose_manifest( args.bundle, args.host, args.runtime, args.version, args.backend ) - (args.bundle / "product-manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) + manifest_path = args.bundle / "product-manifest.json" + if args.check: + existing = json.loads(manifest_path.read_text(encoding="utf-8")) + if existing != manifest: + raise ValueError( + f"product manifest does not match composed bytes: {manifest_path}" + ) + else: + manifest_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) return 0 diff --git a/scripts/package-release.ps1 b/scripts/package-release.ps1 index a3315188aa..8c706f0977 100755 --- a/scripts/package-release.ps1 +++ b/scripts/package-release.ps1 @@ -22,6 +22,12 @@ $nativeRuntimeRoot = if ($env:MESH_LLM_NATIVE_RUNTIME_ROOT) { $attestationSigningKeyFile = $env:MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE $attestationPublicKeyFile = $env:MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE $attestationVerifier = $env:MESH_RELEASE_ATTESTATION_VERIFIER +$precomposedProductDir = $env:MESH_LLM_PRECOMPOSED_PRODUCT_DIR +$attestationPreverified = if ([string]::IsNullOrWhiteSpace($env:MESH_RELEASE_ATTESTATION_PREVERIFIED)) { + "0" +} else { + $env:MESH_RELEASE_ATTESTATION_PREVERIFIED.Trim() +} Add-Type -AssemblyName System.IO.Compression.FileSystem @@ -250,7 +256,32 @@ function Test-HasValue { return -not [string]::IsNullOrWhiteSpace($Value) } +function Resolve-RepositoryPath { + param([string]$Path) + + # Bash composite actions expose Git-for-Windows paths such as + # /d/a/mesh-llm/mesh-llm/product-input through GITHUB_OUTPUT. Convert that + # form before handing it to the Windows .NET path APIs. + if ($Path -match '^/(?[A-Za-z])(?:/(?.*))?$') { + $drive = $Matches.drive.ToUpperInvariant() + $tail = "$($Matches.tail)".Replace("/", "\") + return [System.IO.Path]::GetFullPath("${drive}:\${tail}") + } + if ([System.IO.Path]::IsPathRooted($Path)) { + return [System.IO.Path]::GetFullPath($Path) + } + return [System.IO.Path]::GetFullPath((Join-Path $repoRoot $Path)) +} + function Assert-AttestationConfig { + if ($attestationPreverified -eq "1") { + if ($env:MESH_RELEASE_HOST_PRESTAMPED -ne "1" -or -not (Test-HasValue $precomposedProductDir)) { + throw "MESH_RELEASE_ATTESTATION_PREVERIFIED=1 requires a pre-stamped precomposed product" + } + } elseif ($attestationPreverified -ne "0") { + throw "MESH_RELEASE_ATTESTATION_PREVERIFIED must be 0 or 1" + } + if ($env:MESH_RELEASE_HOST_PRESTAMPED -eq "1") { if (-not (Test-HasValue $attestationPublicKeyFile)) { throw "MESH_RELEASE_HOST_PRESTAMPED=1 requires MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" @@ -275,6 +306,10 @@ function Invoke-ReleaseAttestationStamp { if (-not (Test-Path $attestationPublicKeyFile) -or (Get-Item $attestationPublicKeyFile).Length -eq 0) { throw "MESH_RELEASE_HOST_PRESTAMPED=1 requires a non-empty MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" } + if ($attestationPreverified -eq "1") { + Write-Host "Release attestation: verified by immutable product composer" + return + } if (Test-HasValue $attestationVerifier) { Assert-FileChecksum -Path $attestationVerifier -ChecksumPath "${attestationVerifier}.sha256" $inspectJson = & $attestationVerifier release-attestation inspect ` @@ -347,6 +382,67 @@ function Invoke-ReleaseAttestationStamp { } } +function Copy-AndVerifyPrecomposedProduct { + param( + [string]$SourceDir, + [string]$BundleDir, + [string]$ExpectedVersion, + [string]$ExpectedBackend, + [string]$VerificationReport + ) + + $resolvedSourceDir = Resolve-RepositoryPath $SourceDir + if (-not (Test-Path -LiteralPath $resolvedSourceDir -PathType Container)) { + throw "Precomposed product directory does not exist: $resolvedSourceDir" + } + + Require-File (Join-Path $resolvedSourceDir "product-manifest.json") + Require-File (Join-Path $resolvedSourceDir "host-imports.json") + foreach ($entry in [System.IO.Directory]::GetFileSystemEntries($resolvedSourceDir)) { + Copy-Item -LiteralPath $entry -Destination $BundleDir -Recurse -Force + } + + $bundleBinary = Join-Path $BundleDir "mesh-llm.exe" + Require-File $bundleBinary + Assert-MeshBinaryVersion -Path $bundleBinary -ExpectedVersion $ExpectedVersion + Invoke-ReleaseAttestationStamp -BinaryPath $bundleBinary + + $python = Get-PythonCommand + & $python (Join-Path $scriptDir "verify-host-dependencies.py") ` + $bundleBinary ` + --report $VerificationReport + if ($LASTEXITCODE -ne 0) { + throw "backend-neutral host dependency verification failed" + } + + $runtimeRoot = Join-Path $BundleDir "native-runtimes" + if (-not (Test-Path -LiteralPath $runtimeRoot -PathType Container)) { + throw "Precomposed product is missing native-runtimes: $runtimeRoot" + } + $runtimeDirs = @(Get-ChildItem -LiteralPath $runtimeRoot -Directory) + if ($runtimeDirs.Count -ne 1) { + throw "Precomposed product must contain exactly one native runtime; found $($runtimeDirs.Count)" + } + $runtimeDir = $runtimeDirs[0].FullName + Require-File (Join-Path $runtimeDir "manifest.json") + + & bash (Join-Path $scriptDir "verify-native-runtime-package.sh") $runtimeDir + if ($LASTEXITCODE -ne 0) { + throw "native runtime verification failed" + } + + & $python (Join-Path $scriptDir "compose-product-bundle.py") ` + --bundle $BundleDir ` + --host $bundleBinary ` + --runtime $runtimeDir ` + --version $ExpectedVersion ` + --backend $ExpectedBackend ` + --check + if ($LASTEXITCODE -ne 0) { + throw "precomposed product manifest does not match the packaged bytes" + } +} + $Version = Normalize-RecipeArgument $Version @("version") $OutputDir = Normalize-RecipeArgument $OutputDir @("output", "output_dir", "outputdir") $Flavor = Normalize-RecipeArgument $Flavor @("flavor", "backend") @@ -364,7 +460,9 @@ $versionedAsset = New-ReleaseAssetName -Prefix "mesh-llm-$Version" -TargetTriple $meshBinary = Join-Path $releaseBinDir "mesh-llm.exe" -Require-File $meshBinary +if (-not (Test-HasValue $precomposedProductDir)) { + Require-File $meshBinary +} $resolvedOutputDir = if ([System.IO.Path]::IsPathRooted($OutputDir)) { [System.IO.Path]::GetFullPath($OutputDir) @@ -378,62 +476,72 @@ $bundleDir = Join-Path $stagingRoot "mesh-bundle" New-Item -ItemType Directory -Path $bundleDir -Force | Out-Null try { - $bundleBinary = Join-Path $bundleDir (Get-BundleBinaryName "mesh-llm" $binaryFlavor) - Copy-Item $meshBinary -Destination $bundleBinary -Force - Assert-MeshBinaryVersion -Path $bundleBinary -ExpectedVersion $Version - - Invoke-ReleaseAttestationStamp -BinaryPath $bundleBinary - $python = Get-PythonCommand - $hostReport = Join-Path $bundleDir "host-imports.json" - & $python (Join-Path $scriptDir "verify-host-dependencies.py") $bundleBinary --report $hostReport - if ($LASTEXITCODE -ne 0) { - throw "backend-neutral host dependency verification failed" - } - - $cudaMajor = if ($env:MESH_LLM_CUDA_TOOLKIT_MAJOR) { - $env:MESH_LLM_CUDA_TOOLKIT_MAJOR - } elseif ($env:MESH_CUDA_VERSION) { - ($env:MESH_CUDA_VERSION -split '\.')[0] + if (Test-HasValue $precomposedProductDir) { + Copy-AndVerifyPrecomposedProduct ` + -SourceDir $precomposedProductDir ` + -BundleDir $bundleDir ` + -ExpectedVersion $Version ` + -ExpectedBackend $releaseFlavor ` + -VerificationReport (Join-Path $stagingRoot "host-imports.verify.json") } else { - "" - } - $selectorArgs = @( - (Join-Path $scriptDir "select-native-runtime.py") - "--root" - $nativeRuntimeRoot - "--os" - "windows" - "--arch" - "x86_64" - "--backend" - $releaseFlavor - ) - if (Test-HasValue $cudaMajor) { - $selectorArgs += @("--cuda-major", $cudaMajor) - } - $selectorOutput = & $python @selectorArgs - $selectorExitCode = $LASTEXITCODE - if ($selectorExitCode -ne 0) { - throw "failed to select the packaged Windows native runtime" - } - $runtimeDir = $selectorOutput | ForEach-Object { $_.Trim() } | Where-Object { $_ } | Select-Object -Last 1 - if (-not $runtimeDir) { - throw "failed to select the packaged Windows native runtime" - } - $runtimeDestinationRoot = Join-Path $bundleDir "native-runtimes" - $runtimeDestination = Join-Path $runtimeDestinationRoot (Split-Path -Leaf $runtimeDir) - New-Item -ItemType Directory -Path $runtimeDestinationRoot -Force | Out-Null - Copy-Item $runtimeDir -Destination $runtimeDestination -Recurse -Force + $bundleBinary = Join-Path $bundleDir (Get-BundleBinaryName "mesh-llm" $binaryFlavor) + Copy-Item $meshBinary -Destination $bundleBinary -Force + Assert-MeshBinaryVersion -Path $bundleBinary -ExpectedVersion $Version + + Invoke-ReleaseAttestationStamp -BinaryPath $bundleBinary + $python = Get-PythonCommand + $hostReport = Join-Path $bundleDir "host-imports.json" + & $python (Join-Path $scriptDir "verify-host-dependencies.py") $bundleBinary --report $hostReport + if ($LASTEXITCODE -ne 0) { + throw "backend-neutral host dependency verification failed" + } - & $python (Join-Path $scriptDir "compose-product-bundle.py") ` - --bundle $bundleDir ` - --host $bundleBinary ` - --runtime $runtimeDestination ` - --version $Version ` - --backend $releaseFlavor - if ($LASTEXITCODE -ne 0) { - throw "failed to write the product-v2 bundle manifest" + $cudaMajor = if ($env:MESH_LLM_CUDA_TOOLKIT_MAJOR) { + $env:MESH_LLM_CUDA_TOOLKIT_MAJOR + } elseif ($env:MESH_CUDA_VERSION) { + ($env:MESH_CUDA_VERSION -split '\.')[0] + } else { + "" + } + $selectorArgs = @( + (Join-Path $scriptDir "select-native-runtime.py") + "--root" + $nativeRuntimeRoot + "--os" + "windows" + "--arch" + "x86_64" + "--backend" + $releaseFlavor + ) + if (Test-HasValue $cudaMajor) { + $selectorArgs += @("--cuda-major", $cudaMajor) + } + $selectorOutput = & $python @selectorArgs + $selectorExitCode = $LASTEXITCODE + if ($selectorExitCode -ne 0) { + throw "failed to select the packaged Windows native runtime" + } + $runtimeDir = $selectorOutput | ForEach-Object { $_.Trim() } | Where-Object { $_ } | Select-Object -Last 1 + if (-not $runtimeDir) { + throw "failed to select the packaged Windows native runtime" + } + $runtimeDestinationRoot = Join-Path $bundleDir "native-runtimes" + $runtimeDestination = Join-Path $runtimeDestinationRoot (Split-Path -Leaf $runtimeDir) + New-Item -ItemType Directory -Path $runtimeDestinationRoot -Force | Out-Null + Copy-Item $runtimeDir -Destination $runtimeDestination -Recurse -Force + + & $python (Join-Path $scriptDir "compose-product-bundle.py") ` + --bundle $bundleDir ` + --host $bundleBinary ` + --runtime $runtimeDestination ` + --version $Version ` + --backend $releaseFlavor + if ($LASTEXITCODE -ne 0) { + throw "failed to write the product-v2 bundle manifest" + } } + $versionedPath = Join-Path $resolvedOutputDir $versionedAsset $stablePath = Join-Path $resolvedOutputDir $stableAsset diff --git a/scripts/package-release.sh b/scripts/package-release.sh index 33a462fb52..9cf28138c6 100755 --- a/scripts/package-release.sh +++ b/scripts/package-release.sh @@ -12,6 +12,8 @@ RELEASE_BIN_DIR="${MESH_LLM_RELEASE_BIN_DIR:-$REPO_ROOT/target/release}" NATIVE_RUNTIME_ROOT="${MESH_LLM_NATIVE_RUNTIME_ROOT:-$REPO_ROOT/dist/native-runtimes}" ATTESTATION_SIGNING_KEY_FILE="${MESH_RELEASE_ATTESTATION_SIGNING_KEY_FILE:-}" ATTESTATION_PUBLIC_KEY_FILE="${MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE:-}" +PRECOMPOSED_PRODUCT_DIR="${MESH_LLM_PRECOMPOSED_PRODUCT_DIR:-}" +ATTESTATION_PREVERIFIED="${MESH_RELEASE_ATTESTATION_PREVERIFIED:-0}" python_bin() { if command -v python3 >/dev/null 2>&1; then @@ -194,6 +196,15 @@ write_product_manifest() { } validate_attestation_env() { + if [[ "$ATTESTATION_PREVERIFIED" == "1" ]]; then + if [[ "${MESH_RELEASE_HOST_PRESTAMPED:-0}" != "1" || -z "$PRECOMPOSED_PRODUCT_DIR" ]]; then + echo "MESH_RELEASE_ATTESTATION_PREVERIFIED=1 requires a pre-stamped precomposed product" >&2 + exit 1 + fi + elif [[ "$ATTESTATION_PREVERIFIED" != "0" ]]; then + echo "MESH_RELEASE_ATTESTATION_PREVERIFIED must be 0 or 1" >&2 + exit 1 + fi if [[ "${MESH_RELEASE_HOST_PRESTAMPED:-0}" == "1" ]]; then if [[ -z "$ATTESTATION_PUBLIC_KEY_FILE" ]]; then echo "MESH_RELEASE_HOST_PRESTAMPED=1 requires MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" >&2 @@ -222,6 +233,10 @@ stamp_bundle_binary() { echo "MESH_RELEASE_HOST_PRESTAMPED=1 requires a non-empty MESH_RELEASE_ATTESTATION_PUBLIC_KEY_FILE" >&2 exit 1 fi + if [[ "$ATTESTATION_PREVERIFIED" == "1" ]]; then + echo "Release attestation: verified by immutable product composer" + return 0 + fi inspect_json="$( cd "$REPO_ROOT" cargo run -q -p xtask -- release-attestation inspect \ @@ -472,6 +487,56 @@ verify_mesh_binary_version() { fi } +copy_and_verify_precomposed_product() { + local source_dir="$1" + local bundle_dir="$2" + local version="$3" + local flavor="$4" + local runtime_dir="" + local runtime_count=0 + local verification_report="$_STAGING_DIR/host-imports.verify.json" + local bundle_binary="" + bundle_binary="$bundle_dir/$(bundle_bin_name mesh-llm)" + + if [[ ! -d "$source_dir" ]]; then + echo "Precomposed product directory does not exist: $source_dir" >&2 + exit 1 + fi + test -s "$source_dir/product-manifest.json" + mkdir -p "$bundle_dir" + cp -a "$source_dir/." "$bundle_dir/" + + stamp_bundle_binary "$bundle_binary" + verify_mesh_binary_version "$bundle_binary" "$version" + "$(python_bin)" "$SCRIPT_DIR/verify-host-dependencies.py" \ + "$bundle_binary" \ + --report "$verification_report" + + while IFS= read -r manifest; do + runtime_dir="$(dirname "$manifest")" + runtime_count=$((runtime_count + 1)) + done < <( + find "$bundle_dir/native-runtimes" \ + -mindepth 2 \ + -maxdepth 2 \ + -type f \ + -name manifest.json \ + -print + ) + if [[ "$runtime_count" -ne 1 ]]; then + echo "Precomposed product must contain exactly one native runtime; found $runtime_count" >&2 + exit 1 + fi + "$SCRIPT_DIR/verify-native-runtime-package.sh" "$runtime_dir" + "$(python_bin)" "$SCRIPT_DIR/compose-product-bundle.py" \ + --bundle "$bundle_dir" \ + --host "$bundle_binary" \ + --runtime "$runtime_dir" \ + --version "$version" \ + --backend "$flavor" \ + --check +} + main() { if [[ $# -lt 1 || -z "${1:-}" ]]; then usage @@ -501,31 +566,39 @@ main() { _STAGING_DIR="$(mktemp -d)" bundle_dir="$_STAGING_DIR/mesh-bundle" - mkdir -p "$bundle_dir" + if [[ -n "$PRECOMPOSED_PRODUCT_DIR" ]]; then + copy_and_verify_precomposed_product \ + "$PRECOMPOSED_PRODUCT_DIR" \ + "$bundle_dir" \ + "$version" \ + "$(effective_release_flavor)" + else + mkdir -p "$bundle_dir" - bundle_binary="$bundle_dir/$(bundle_bin_name mesh-llm)" - cp "$RELEASE_BIN_DIR/mesh-llm${BIN_EXT}" "$bundle_binary" + bundle_binary="$bundle_dir/$(bundle_bin_name mesh-llm)" + cp "$RELEASE_BIN_DIR/mesh-llm${BIN_EXT}" "$bundle_binary" - if [[ "$os_name" == "Darwin" && -f "$bundle_binary" ]]; then - install_name_tool -add_rpath @executable_path/ "$bundle_binary" 2>/dev/null || true - fi + if [[ "$os_name" == "Darwin" && -f "$bundle_binary" ]]; then + install_name_tool -add_rpath @executable_path/ "$bundle_binary" 2>/dev/null || true + fi - stamp_bundle_binary "$bundle_binary" - verify_mesh_binary_version "$bundle_binary" "$version" - "$(python_bin)" "$SCRIPT_DIR/verify-host-dependencies.py" \ - "$bundle_binary" \ - --report "$bundle_dir/host-imports.json" - - runtime_dir="$(select_native_runtime_dir)" - bundled_runtime="$bundle_dir/native-runtimes/$(basename "$runtime_dir")" - mkdir -p "$(dirname "$bundled_runtime")" - cp -R "$runtime_dir" "$bundled_runtime" - write_product_manifest \ - "$bundle_dir" \ - "$bundle_binary" \ - "$bundled_runtime" \ - "$version" \ - "$(effective_release_flavor)" + stamp_bundle_binary "$bundle_binary" + verify_mesh_binary_version "$bundle_binary" "$version" + "$(python_bin)" "$SCRIPT_DIR/verify-host-dependencies.py" \ + "$bundle_binary" \ + --report "$bundle_dir/host-imports.json" + + runtime_dir="$(select_native_runtime_dir)" + bundled_runtime="$bundle_dir/native-runtimes/$(basename "$runtime_dir")" + mkdir -p "$(dirname "$bundled_runtime")" + cp -R "$runtime_dir" "$bundled_runtime" + write_product_manifest \ + "$bundle_dir" \ + "$bundle_binary" \ + "$bundled_runtime" \ + "$version" \ + "$(effective_release_flavor)" + fi create_archive "$bundle_dir" "$output_dir/$versioned_asset" "$ARCHIVE_EXT" write_checksum_sidecar "$output_dir/$versioned_asset" diff --git a/scripts/tests/test_build_release.py b/scripts/tests/test_build_release.py index 792cfa1167..8b5fb3607a 100644 --- a/scripts/tests/test_build_release.py +++ b/scripts/tests/test_build_release.py @@ -12,23 +12,29 @@ ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "build-release.sh" +HOST_SCRIPT = ROOT / "scripts" / "build-host.sh" class BuildReleaseScriptTests(unittest.TestCase): def test_release_host_never_enables_backend_gpu_features(self) -> None: - script = SCRIPT.read_text(encoding="utf-8") + script = HOST_SCRIPT.read_text(encoding="utf-8") self.assertNotIn("gpu-bench-cuda", script) self.assertNotIn("gpu-bench-hip", script) self.assertNotIn("build-llama.sh", script) def test_dynamic_native_runtime_feature_is_required(self) -> None: - script = SCRIPT.read_text(encoding="utf-8") + script = HOST_SCRIPT.read_text(encoding="utf-8") - self.assertIn("--features web-ui,dynamic-native-runtime", script) + self.assertIn('"web-ui,dynamic-native-runtime"', script) self.assertIn("--no-default-features", script) self.assertIn("MESH_LLM_DYNAMIC_NATIVE_RUNTIME=0 is unsupported", script) + def test_release_entry_point_delegates_to_canonical_host_builder(self) -> None: + script = SCRIPT.read_text(encoding="utf-8") + + self.assertIn('exec "$SCRIPT_DIR/build-host.sh" --profile release', script) + def test_cuda_selection_does_not_change_host_build(self) -> None: cargo_log = self.run_build_release_with_backend("cuda") @@ -62,6 +68,11 @@ def run_build_release_with_backend( copied_script = scripts_dir / "build-release.sh" shutil.copy(SCRIPT, copied_script) copied_script.chmod(copied_script.stat().st_mode | stat.S_IXUSR) + copied_host_script = scripts_dir / "build-host.sh" + shutil.copy(HOST_SCRIPT, copied_host_script) + copied_host_script.chmod( + copied_host_script.stat().st_mode | stat.S_IXUSR + ) self.write_executable( scripts_dir / "build-ui.sh", diff --git a/scripts/tests/test_build_windows.py b/scripts/tests/test_build_windows.py index 9af4e728ae..d491435bb2 100644 --- a/scripts/tests/test_build_windows.py +++ b/scripts/tests/test_build_windows.py @@ -48,17 +48,42 @@ def test_all_normal_profiles_build_a_dynamic_host_and_adjacent_runtime(self) -> script, ) - def test_windows_packaged_cli_smoke_checks_each_native_command(self) -> None: + def test_host_only_build_honors_debug_and_release_profiles(self) -> None: + script = SCRIPT.read_text(encoding="utf-8") + start = script.index("if ($HostOnly) {") + end = script.index("\nswitch ($backendName)", start) + host_only = script[start:end] + + self.assertIn('$hostArgs = @("build")', host_only) + self.assertIn('if ($buildProfile -eq "release")', host_only) + self.assertIn('$hostArgs += "--release"', host_only) + self.assertIn('$hostOutputProfile = "debug"', host_only) + self.assertIn('$hostOutputProfile = "release"', host_only) + self.assertNotIn( + '@("build", "--release", "--locked"', + host_only, + ) + self.assertIn("\n return\n", host_only) + self.assertNotIn("exit 0", host_only) + + def test_windows_products_use_the_shared_composition_and_smoke_contract( + self, + ) -> None: workflow = PR_BUILDS.read_text(encoding="utf-8") - start = workflow.index(" - name: Composed Windows CLI and client readiness smoke") - smoke = workflow[start:] + cpu_start = workflow.index(" windows_cpu_product:") + gpu_start = workflow.index(" windows_gpu_products:", cpu_start) + products = (workflow[cpu_start:gpu_start], workflow[gpu_start:]) - self.assertIn(".\\target\\release\\mesh-llm.exe --log-format json --version", smoke) - self.assertIn(".\\target\\release\\mesh-llm.exe --log-format json runtime list", smoke) - self.assertIn("mesh-llm --version failed with exit code $LASTEXITCODE", smoke) - self.assertIn("mesh-llm --help failed with exit code $LASTEXITCODE", smoke) - self.assertIn("mesh-llm runtime list failed with exit code $LASTEXITCODE", smoke) - self.assertIn("failed with exit code $LASTEXITCODE", smoke) + for product in products: + with self.subTest(job=product.splitlines()[0].strip()): + self.assertIn( + "uses: ./.github/actions/compose-product-input", + product, + ) + self.assertIn("binary_name: mesh-llm.exe", product) + self.assertIn('readiness_smoke: "true"', product) + self.assertNotIn("cargo ", product) + self.assertNotIn("build-windows.ps1", product) if __name__ == "__main__": diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py new file mode 100644 index 0000000000..99b9a46909 --- /dev/null +++ b/scripts/tests/test_ci_artifact_actions.py @@ -0,0 +1,514 @@ +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import tarfile +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +ACTIONS = ROOT / ".github" / "actions" +COMPOSE_SCRIPT = ROOT / "scripts" / "ci-compose-product-input.sh" + + +class CiArtifactActionTests(unittest.TestCase): + def read_action(self, name: str) -> str: + return (ACTIONS / name / "action.yml").read_text(encoding="utf-8") + + def test_external_actions_have_sha_and_release_provenance(self) -> None: + action_files = sorted(ACTIONS.glob("*/action.yml")) + workflow_files = sorted( + (ROOT / ".github" / "workflows").glob("*.yml"), + ) + exact_pin = re.compile( + r"^[^@\s]+@[0-9a-f]{40}\s+#\s+\S", + ) + + for path in (*action_files, *workflow_files): + for line_number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), + start=1, + ): + if "uses:" not in line: + continue + value = line.split("uses:", maxsplit=1)[1].strip() + if value.startswith("./"): + continue + with self.subTest( + path=path.relative_to(ROOT), + line=line_number, + ): + self.assertRegex(value, exact_pin) + + def write_fake_product_inputs( + self, + workspace: Path, + *, + host_version: str = "1.2.3", + ) -> tuple[Path, Path]: + host_input = workspace / "host-input" + runtime_input = workspace / "runtime-input" + host_input.mkdir() + runtime_input.mkdir() + + host = host_input / "mesh-llm" + host.write_text( + "#!/usr/bin/env bash\n" + f"printf 'mesh-llm {host_version}\\n'\n", + encoding="utf-8", + ) + host.chmod(0o755) + host_digest = hashlib.sha256(host.read_bytes()).hexdigest() + (host_input / "mesh-llm.sha256").write_text( + f"{host_digest} mesh-llm\n", + encoding="utf-8", + ) + (host_input / "host-imports.json").write_text( + "{}\n", + encoding="utf-8", + ) + + runtime_id = "meshllm-native-runtime-test-x86_64-cpu" + runtime = runtime_input / runtime_id + (runtime / "lib").mkdir(parents=True) + (runtime / "tools").mkdir() + library = runtime / "lib" / "libmesh_fake.a" + library.write_bytes(b"fake static library") + tool = runtime / "tools" / "mesh-runtime-bench" + tool.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + tool.chmod(0o755) + library_digest = hashlib.sha256(library.read_bytes()).hexdigest() + tool_digest = hashlib.sha256(tool.read_bytes()).hexdigest() + manifest = { + "runtime": { + "id": runtime_id, + "mesh_version": "1.2.3", + "skippy_abi": {"major": 1, "minor": 0, "patch": 0}, + "platform": {"os": "test", "arch": "x86_64"}, + "backend": {"kind": "cpu"}, + "libraries": ["lib/libmesh_fake.a"], + "tools": {"tools/mesh-runtime-bench": tool_digest}, + }, + "build": { + "backend": "cpu", + "primary_library": "lib/libmesh_fake.a", + "library_sha256": library_digest, + }, + } + (runtime / "manifest.json").write_text( + json.dumps(manifest) + "\n", + encoding="utf-8", + ) + return host_input, runtime_input + + def run_product_composer( + self, + workspace: Path, + *, + host_version: str = "1.2.3", + ) -> subprocess.CompletedProcess[str]: + host_input, runtime_input = self.write_fake_product_inputs( + workspace, + host_version=host_version, + ) + return subprocess.run( + [str(COMPOSE_SCRIPT)], + cwd=ROOT, + env={ + **os.environ, + "GITHUB_WORKSPACE": str(workspace), + "GITHUB_OUTPUT": str(workspace / "github-output"), + "INPUT_HOST_INPUT_DIR": str(host_input), + "INPUT_RUNTIME_INPUT_DIR": str(runtime_input), + "INPUT_OUTPUT_DIR": str(workspace / "product-input"), + "INPUT_BACKEND": "cpu", + "INPUT_VERSION": "1.2.3", + "INPUT_BINARY_NAME": "mesh-llm", + "INPUT_READINESS_SMOKE": "false", + }, + check=False, + capture_output=True, + text=True, + ) + + def run_runner_selector( + self, + *, + event_name: str, + ref: str, + main_enabled: str, + manual_enabled: str, + ) -> dict[str, str]: + action = self.read_action("select-ci-runners") + run_block = action.split(" run: |\n", maxsplit=1)[1] + script = "\n".join( + line[8:] if line.startswith(" ") else line + for line in run_block.splitlines() + ) + with tempfile.TemporaryDirectory() as temp_dir: + output = Path(temp_dir) / "github-output" + result = subprocess.run( + ["bash", "-c", script], + cwd=ROOT, + env={ + **os.environ, + "GITHUB_OUTPUT": str(output), + "INPUT_EVENT_NAME": event_name, + "INPUT_REF": ref, + "INPUT_DEPOT_MAIN_ENABLED": main_enabled, + "INPUT_MANUAL_USE_DEPOT": manual_enabled, + }, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return dict( + line.split("=", maxsplit=1) + for line in output.read_text(encoding="utf-8").splitlines() + ) + + def test_host_action_uses_canonical_dynamic_host_builder(self) -> None: + action = self.read_action("prepare-host-input") + + self.assertIn('scripts/build-host.sh --profile "$INPUT_PROFILE"', action) + self.assertIn("scripts/verify-host-dependencies.py", action) + self.assertNotIn("package-native-runtime.sh", action) + + def test_windows_host_action_owns_the_neutral_host_integrity_contract( + self, + ) -> None: + action = self.read_action("prepare-windows-host-input") + + self.assertIn( + "& .\\scripts\\build-windows.ps1 -BuildProfile $profile -HostOnly", + action, + ) + self.assertIn("scripts\\verify-host-dependencies.py", action) + self.assertIn("mesh-llm.exe.sha256", action) + self.assertIn("cargo build -q -p xtask --bin xtask", action) + self.assertIn("release-attestation stamp", action) + self.assertIn("release-attestation inspect", action) + self.assertIn('"$attestationVerifierPath.sha256"', action) + self.assertIn( + '"$verifierHash release-attestation-verifier.exe"', + action, + ) + self.assertNotIn("package-native-runtime.sh", action) + self.assertNotIn("compose-product", action) + + def test_windows_debug_host_uses_the_package_version_for_composition( + self, + ) -> None: + action = self.read_action("prepare-windows-host-input") + + debug = action[ + action.index('if ($profile -eq "debug")') + : action.index('if ($env:INPUT_SKIP_UI -eq "true")') + ] + self.assertIn("cargo pkgid -p mesh-llm", debug) + self.assertIn("$env:MESH_LLM_BUILD_VERSION", debug) + self.assertNotIn("git ", debug) + + def test_windows_routes_cover_every_shared_product_primitive(self) -> None: + action = self.read_action("compute-changes") + routing = action[ + action.index("WINDOWS_CPU_INPUTS=") + : action.index("# SDK smokes are consumer tests") + ] + + for primitive in ( + "prepare-windows-host-input", + "prepare-native-runtime-input", + "compose-product-input", + "package-native-runtime", + "verify-native-runtime-package", + "compose-product-bundle", + "ci-compose-product-input", + "ci-client-readiness-smoke", + ): + with self.subTest(primitive=primitive): + self.assertIn(primitive, routing) + + def test_runtime_action_never_builds_the_host(self) -> None: + action = self.read_action("prepare-native-runtime-input") + + self.assertIn('scripts/package-native-runtime.sh "${args[@]}"', action) + self.assertIn("scripts/verify-native-runtime-package.sh", action) + self.assertNotIn("build-host.sh", action) + self.assertNotIn("build-release.sh", action) + + def test_product_action_only_composes_verified_inputs(self) -> None: + action = self.read_action("compose-product-input") + + self.assertIn("scripts/ci-compose-product-input.sh", action) + self.assertNotIn("cargo build", action) + self.assertNotIn("package-native-runtime.sh", action) + script = COMPOSE_SCRIPT.read_text(encoding="utf-8") + self.assertIn("scripts/compose-product-bundle.py", script) + self.assertIn("scripts/verify-native-runtime-package.sh", script) + self.assertIn("scripts/ci-client-readiness-smoke.sh", script) + self.assertIn('archive_path="$product_dir.tar.gz"', script) + self.assertIn('tar -C "$product_dir" -czf "$archive_path" .', script) + + def test_product_archive_preserves_verified_executable_modes(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + workspace = Path(temp_dir) + result = self.run_product_composer(workspace) + + self.assertEqual(result.returncode, 0, result.stderr) + archive = workspace / "product-input.tar.gz" + self.assertTrue(archive.is_file()) + with tarfile.open(archive, "r:gz") as bundle: + host = next( + member + for member in bundle.getmembers() + if member.name.endswith("/mesh-llm") + ) + tool = next( + member + for member in bundle.getmembers() + if member.name.endswith( + "/tools/mesh-runtime-bench" + ) + ) + self.assertNotEqual(host.mode & 0o111, 0) + self.assertNotEqual(tool.mode & 0o111, 0) + output = (workspace / "github-output").read_text(encoding="utf-8") + self.assertIn(f"archive_path={archive.resolve()}", output) + + def test_product_composer_rejects_host_version_drift(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + result = self.run_product_composer( + Path(temp_dir), + host_version="9.9.9", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("composed host version mismatch", result.stderr) + + def test_release_attestation_is_verified_without_compiling_in_composer( + self, + ) -> None: + host_action = self.read_action("prepare-host-input") + product_action = self.read_action("compose-product-input") + product_script = COMPOSE_SCRIPT.read_text(encoding="utf-8") + + self.assertIn("cargo build -q -p xtask --bin xtask", host_action) + self.assertIn("release-attestation-verifier.sha256", host_action) + self.assertNotIn("cargo ", product_action) + self.assertIn( + '"$attestation_verifier" release-attestation inspect', + product_script, + ) + self.assertIn( + 'test "$actual_verifier_checksum" = ' + '"$expected_verifier_checksum"', + product_script, + ) + + def test_smoke_restore_rechecks_the_archived_product(self) -> None: + action = self.read_action("restore-smoke-inputs") + + self.assertIn("expected exactly one composed product archive", action) + self.assertIn("tar -xzf", action) + self.assertIn("scripts/verify-native-runtime-package.sh", action) + self.assertIn("--check", action) + + def test_smoke_restore_model_is_optional(self) -> None: + action = self.read_action("restore-smoke-inputs") + model_inputs_present = ( + "inputs.model_url != '' && inputs.model_file != ''" + ) + + self.assertEqual(action.count(model_inputs_present), 4) + self.assertIn( + f"if: ${{{{ {model_inputs_present} }}}}\n" + " id: cache-model", + action, + ) + self.assertIn( + f"if: ${{{{ {model_inputs_present} }}}}\n" + " id: model-file", + action, + ) + + def test_product_action_rejects_destructive_output_paths(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + workspace = Path(temp_dir) + host_input = workspace / "inputs" / "host" + runtime_input = workspace / "inputs" / "runtime" + host_input.mkdir(parents=True) + runtime_input.mkdir(parents=True) + sentinel = workspace / "sentinel" + sentinel.write_text("keep", encoding="utf-8") + outside = workspace.parent / f"{workspace.name}-outside" + dangerous_outputs = ( + ".", + "./", + "product/..", + str(workspace), + str(outside), + str(host_input), + str(host_input / "product"), + str(workspace / "inputs"), + ) + + for output in dangerous_outputs: + with self.subTest(output=output): + result = subprocess.run( + [str(COMPOSE_SCRIPT)], + cwd=workspace, + env={ + **os.environ, + "GITHUB_WORKSPACE": str(workspace), + "GITHUB_OUTPUT": str(workspace / "github-output"), + "INPUT_HOST_INPUT_DIR": str(host_input), + "INPUT_RUNTIME_INPUT_DIR": str(runtime_input), + "INPUT_OUTPUT_DIR": output, + "INPUT_BACKEND": "cpu", + "INPUT_VERSION": "", + "INPUT_BINARY_NAME": "mesh-llm", + "INPUT_READINESS_SMOKE": "false", + }, + check=False, + capture_output=True, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(sentinel.read_text(encoding="utf-8"), "keep") + + def test_sccache_prefers_depot_webdav_with_disk_fallback(self) -> None: + action = self.read_action("configure-sccache-gha") + + self.assertIn("allow_depot_remote_cache", action) + self.assertIn('default: "false"', action) + self.assertIn("SCCACHE_WEBDAV_ENDPOINT", action) + self.assertIn("DEPOT_CACHE_TOKEN", action) + self.assertIn("'disk,webdav'", action) + self.assertIn("'disk'", action) + self.assertIn( + "Depot cache is present but disabled for this trust context", + action, + ) + self.assertIn( + "'Unable to start baked sccache with its trust-isolated disk cache.'", + action, + ) + self.assertIn("env: diskOnlyEnvironment()", action) + self.assertNotIn( + "core.exportVariable('ACTIONS_RUNTIME_TOKEN', '')", + action, + ) + + def test_runner_selection_never_routes_pull_requests_to_depot(self) -> None: + action = self.read_action("select-ci-runners") + + self.assertIn("depot_main_enabled", action) + self.assertNotIn("depot_pr_enabled", action) + self.assertNotIn("head_repository", action) + self.assertNotIn("\n repository:", action) + self.assertIn("\n ref:", action) + + runtime = action.split("runs:", maxsplit=1)[1] + pull_request_case = runtime.split( + "pull_request|pull_request_target)", + maxsplit=1, + )[1].split(";;", maxsplit=1)[0] + self.assertIn("depot_enabled=false", pull_request_case) + self.assertNotIn("depot_enabled=true", pull_request_case) + self.assertNotIn("INPUT_DEPOT_PR_ENABLED", runtime) + self.assertNotIn("INPUT_HEAD_REPOSITORY", runtime) + self.assertNotIn("INPUT_REPOSITORY", runtime) + + dispatch_case = runtime.split( + "workflow_dispatch)", + maxsplit=1, + )[1].split(";;", maxsplit=1)[0] + self.assertIn("INPUT_DEPOT_MAIN_ENABLED", dispatch_case) + self.assertIn("INPUT_MANUAL_USE_DEPOT", dispatch_case) + self.assertIn('INPUT_REF" == "refs/heads/main"', dispatch_case) + self.assertIn("depot_enabled=true", dispatch_case) + + push_case = runtime.split( + "push)", + maxsplit=1, + )[1].split(";;", maxsplit=1)[0] + self.assertIn("INPUT_DEPOT_MAIN_ENABLED", push_case) + self.assertIn('INPUT_REF" == "refs/heads/main"', push_case) + self.assertIn("depot_enabled=true", push_case) + + default_case = runtime.split( + "*)", + maxsplit=1, + )[1].split(";;", maxsplit=1)[0] + self.assertIn("depot_enabled=false", default_case) + self.assertNotIn("depot_enabled=true", default_case) + self.assertIn("depot-ubuntu-24.04-16", action) + + cases = ( + ("pull_request", "refs/pull/12/merge", "true", "true", "false", "ubuntu-24.04"), + ("pull_request_target", "refs/heads/main", "true", "true", "false", "ubuntu-24.04"), + ("workflow_dispatch", "refs/heads/main", "false", "true", "true", "depot-ubuntu-24.04"), + ("workflow_dispatch", "refs/heads/feature", "true", "true", "false", "ubuntu-24.04"), + ("push", "refs/heads/main", "true", "false", "true", "depot-ubuntu-24.04"), + ("push", "refs/heads/feature", "true", "false", "false", "ubuntu-24.04"), + ("push", "refs/tags/v1.2.3", "true", "false", "false", "ubuntu-24.04"), + ("push", "refs/heads/main", "false", "false", "false", "ubuntu-24.04"), + ("schedule", "refs/heads/main", "true", "true", "false", "ubuntu-24.04"), + ) + for event_name, ref, main, manual, enabled, runner in cases: + with self.subTest(event_name=event_name, ref=ref): + outputs = self.run_runner_selector( + event_name=event_name, + ref=ref, + main_enabled=main, + manual_enabled=manual, + ) + self.assertEqual(outputs["depot_enabled"], enabled) + self.assertEqual(outputs["allow_depot_remote_cache"], enabled) + self.assertEqual(outputs["runner"], runner) + + def test_pr_caches_rely_on_github_ref_scoping_while_depot_is_blocked( + self, + ) -> None: + for workflow_name in ("pr_builds.yml", "pr_quality.yml"): + workflow = ( + ROOT / ".github" / "workflows" / workflow_name + ).read_text(encoding="utf-8") + with self.subTest(workflow=workflow_name): + self.assertIn("CACHE_NAMESPACE: mesh-llm", workflow) + self.assertNotIn("CACHE_NAMESPACE: mesh-llm-pr", workflow) + self.assertNotIn("'mesh-llm-pr'", workflow) + self.assertIn( + "save-if: ${{ github.ref == 'refs/heads/main' }}", + workflow, + ) + + quality = ( + ROOT / ".github" / "workflows" / "pr_quality.yml" + ).read_text(encoding="utf-8") + builds = ( + ROOT / ".github" / "workflows" / "pr_builds.yml" + ).read_text(encoding="utf-8") + self.assertIn( + "allow_depot_remote_cache: " + "${{ needs.changes.outputs.allow_depot_remote_cache }}", + builds, + ) + self.assertIn( + "allow_depot_remote_cache: " + "${{ needs.changes.outputs.allow_depot_remote_cache }}", + quality, + ) + self.assertIn("${{ env.CACHE_NAMESPACE }}-pnpm-", quality) + self.assertNotIn("cache: pnpm", quality) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_ci_client_readiness_process.py b/scripts/tests/test_ci_client_readiness_process.py index 6193714bf6..5a42d0e26a 100644 --- a/scripts/tests/test_ci_client_readiness_process.py +++ b/scripts/tests/test_ci_client_readiness_process.py @@ -71,6 +71,21 @@ def test_ctrl_break_targets_the_native_process_group(self): kill.assert_called_once_with(4242, 1) + def test_windows_invalid_parameter_means_process_has_exited(self): + error = OSError("invalid parameter") + error.winerror = 87 + with mock.patch.object(PROCESS.os, "kill", side_effect=error): + running = PROCESS.is_running(4242, is_windows=True) + + self.assertFalse(running) + + def test_unexpected_liveness_error_is_not_hidden(self): + error = OSError("unexpected") + error.winerror = 5 + with mock.patch.object(PROCESS.os, "kill", side_effect=error): + with self.assertRaises(OSError): + PROCESS.is_running(4242, is_windows=True) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_ci_prepare_native_runtime.py b/scripts/tests/test_ci_prepare_native_runtime.py new file mode 100644 index 0000000000..baf049a0b8 --- /dev/null +++ b/scripts/tests/test_ci_prepare_native_runtime.py @@ -0,0 +1,306 @@ +import hashlib +import json +import os +import platform +import shutil +import stat +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts/ci-prepare-native-runtime.sh" +SDK_SMOKE_SCRIPTS = ( + ROOT / "scripts/ci-rust-sdk-smoke.sh", + ROOT / "scripts/ci-kotlin-sdk-smoke.sh", + ROOT / "scripts/ci-swift-sdk-smoke.sh", +) + + +def host_os() -> str: + if sys.platform == "darwin": + return "macos" + if sys.platform.startswith("linux"): + return "linux" + if sys.platform in {"win32", "cygwin"}: + return "windows" + raise RuntimeError(f"unsupported test platform: {sys.platform}") + + +def host_arch() -> str: + machine = platform.machine().lower() + if machine in {"arm64", "aarch64"}: + return "aarch64" + if machine in {"amd64", "x86_64"}: + return "x86_64" + return machine + + +def make_executable(path: Path) -> None: + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + +def current_skippy_abi() -> str: + values = {} + constants = ROOT / "crates/skippy-ffi/src/lib.rs" + for line in constants.read_text(encoding="utf-8").splitlines(): + for part in ("MAJOR", "MINOR", "PATCH"): + prefix = f"pub const ABI_VERSION_{part}: u32 = " + if line.startswith(prefix): + values[part] = line.removeprefix(prefix).removesuffix(";") + return f"{values['MAJOR']}.{values['MINOR']}.{values['PATCH']}" + + +class CiPrepareNativeRuntimeTests(unittest.TestCase): + def write_runtime( + self, + root: Path, + runtime_id: str = "meshllm-native-runtime-test", + backend: str = "cpu", + skippy_abi: str | None = None, + ) -> Path: + artifact = root / runtime_id + library = artifact / "lib/runtime.bin" + library.parent.mkdir(parents=True) + library.write_bytes(b"verified runtime fixture") + digest = hashlib.sha256(library.read_bytes()).hexdigest() + manifest = { + "runtime": { + "id": runtime_id, + "mesh_version": "0.72.1", + "skippy_abi": skippy_abi or current_skippy_abi(), + "platform": {"os": host_os(), "arch": host_arch()}, + "backend": {"kind": backend}, + "libraries": ["lib/runtime.bin"], + "files": {"lib/runtime.bin": digest}, + } + } + (artifact / "manifest.json").write_text( + json.dumps(manifest), encoding="utf-8" + ) + return artifact + + def write_fake_binary( + self, + product: Path, + runtime_id: str, + backend: str, + supported: bool, + ) -> Path: + binary = product / "mesh-llm" + binary.parent.mkdir(parents=True, exist_ok=True) + rows = [ + { + "id": runtime_id, + "mesh_version": "0.72.1", + "skippy_abi": current_skippy_abi(), + "backend": backend, + "os": host_os(), + "arch": host_arch(), + "supported": supported, + "rejection_reasons": [] if supported else ["mesh version mismatch"], + "url": None, + } + ] + rows_path = product / "runtime-rows.json" + rows_path.write_text(json.dumps(rows), encoding="utf-8") + binary.write_text( + """#!/usr/bin/env bash +set -euo pipefail +if [[ "$*" != *"runtime list --available"* ]]; then + echo "unexpected fake mesh-llm invocation: $*" >&2 + exit 2 +fi +cat "$(dirname "$0")/runtime-rows.json" +""", + encoding="utf-8", + ) + make_executable(binary) + return binary + + def run_script( + self, + out: Path, + binary: Path, + *, + extra_env: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env["CI"] = "true" + if extra_env: + env.update(extra_env) + return subprocess.run( + [ + str(SCRIPT), + str(out), + "cpu", + "--reuse-from-binary", + str(binary), + ], + cwd=ROOT, + env=env, + check=False, + capture_output=True, + text=True, + ) + + def test_reuses_compatible_runtime_beside_binary(self) -> None: + with tempfile.TemporaryDirectory() as directory: + product = Path(directory) / "product" + runtime = self.write_runtime(product / "native-runtimes") + binary = self.write_fake_binary(product, runtime.name, "cpu", True) + out = Path(directory) / "fallback" + + result = self.run_script(out, binary) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(Path(result.stdout.strip()), runtime.resolve()) + self.assertIn("verified native runtime artifact", result.stderr) + self.assertIn("Reusing compatible native runtime", result.stderr) + self.assertFalse(out.exists()) + + def test_reuses_sole_compatible_product_backend_before_cpu_fallback(self) -> None: + with tempfile.TemporaryDirectory() as directory: + product = Path(directory) / "product" + runtime = self.write_runtime( + product / "native-runtimes", + backend="metal", + ) + binary = self.write_fake_binary(product, runtime.name, "metal", True) + out = Path(directory) / "fallback" + + result = self.run_script(out, binary) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(Path(result.stdout.strip()), runtime.resolve()) + self.assertFalse(out.exists()) + + def test_rejects_incompatible_staged_runtime_without_building(self) -> None: + with tempfile.TemporaryDirectory() as directory: + product = Path(directory) / "product" + runtime = self.write_runtime(product / "native-runtimes") + binary = self.write_fake_binary(product, runtime.name, "cpu", False) + out = Path(directory) / "fallback" + + result = self.run_script( + out, + binary, + extra_env={"MESH_SDK_NATIVE_RUNTIME_BUILD_FALLBACK": "1"}, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("expected exactly one compatible", result.stderr) + self.assertFalse(out.exists()) + + def test_rejects_staged_runtime_with_wrong_sdk_abi(self) -> None: + with tempfile.TemporaryDirectory() as directory: + product = Path(directory) / "product" + runtime = self.write_runtime( + product / "native-runtimes", + skippy_abi="99.99.99", + ) + binary = self.write_fake_binary(product, runtime.name, "cpu", True) + out = Path(directory) / "fallback" + + result = self.run_script( + out, + binary, + extra_env={"MESH_SDK_NATIVE_RUNTIME_BUILD_FALLBACK": "1"}, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("has Skippy ABI 99.99.99", result.stderr) + self.assertFalse(out.exists()) + + def test_ci_requires_adjacent_runtime_without_explicit_fallback(self) -> None: + with tempfile.TemporaryDirectory() as directory: + product = Path(directory) / "product" + binary = self.write_fake_binary(product, "unused", "cpu", True) + out = Path(directory) / "fallback" + + result = self.run_script(out, binary) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("Adjacent native runtime bundle is required in CI", result.stderr) + self.assertFalse(out.exists()) + + def test_explicit_ci_fallback_preserves_standalone_build_path(self) -> None: + with tempfile.TemporaryDirectory() as directory: + fake_root = Path(directory) / "repo" + scripts = fake_root / "scripts" + scripts.mkdir(parents=True) + helper = scripts / SCRIPT.name + shutil.copy2(SCRIPT, helper) + make_executable(helper) + + package_script = scripts / "package-native-runtime.sh" + package_script.write_text( + """#!/usr/bin/env bash +set -euo pipefail +out="" +while [[ "$#" -gt 0 ]]; do + if [[ "$1" == "--out" ]]; then + out="$2" + shift 2 + else + shift + fi +done +runtime="$out/meshllm-native-runtime-fallback" +mkdir -p "$runtime" +printf '{}\n' > "$runtime/manifest.json" +printf 'archive\n' > "$out/meshllm-native-runtime-fallback.tar.gz" +""", + encoding="utf-8", + ) + make_executable(package_script) + verify_script = scripts / "verify-native-runtime-package.sh" + verify_script.write_text("#!/usr/bin/env bash\nset -euo pipefail\n", encoding="utf-8") + make_executable(verify_script) + + product = Path(directory) / "product" + binary = self.write_fake_binary(product, "unused", "cpu", True) + out = Path(directory) / "fallback" + env = os.environ.copy() + env.update( + { + "CI": "true", + "MESH_SDK_NATIVE_RUNTIME_BUILD_FALLBACK": "1", + } + ) + + result = subprocess.run( + [ + str(helper), + str(out), + "cpu", + "--reuse-from-binary", + str(binary), + ], + cwd=fake_root, + env=env, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("building the standalone fallback", result.stderr) + self.assertEqual( + Path(result.stdout.strip()), + out / "meshllm-native-runtime-fallback", + ) + + def test_all_sdk_smokes_prefer_the_binary_runtime(self) -> None: + for script in SDK_SMOKE_SCRIPTS: + with self.subTest(script=script.name): + contents = script.read_text(encoding="utf-8") + self.assertIn("--reuse-from-binary", contents) + self.assertIn('"$1"', contents) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_ci_workflow_artifacts.py b/scripts/tests/test_ci_workflow_artifacts.py new file mode 100644 index 0000000000..3b44db207d --- /dev/null +++ b/scripts/tests/test_ci_workflow_artifacts.py @@ -0,0 +1,363 @@ +from __future__ import annotations + +import re +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" + + +def job_section(workflow: str, job_name: str) -> str: + marker = f" {job_name}:\n" + start = workflow.index(marker) + next_job = re.search(r"(?m)^ [a-zA-Z0-9_]+:\n", workflow[start + len(marker) :]) + if next_job is None: + return workflow[start:] + return workflow[start : start + len(marker) + next_job.start()] + + +class CiWorkflowArtifactTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.workflow = CI_WORKFLOW.read_text(encoding="utf-8") + + def test_release_host_has_one_neutral_producer(self) -> None: + host = job_section(self.workflow, "linux_host_input") + + self.assertIn("name: Linux immutable release host", host) + self.assertIn("uses: ./.github/actions/prepare-host-input", host) + self.assertIn("profile: release", host) + self.assertIn("name: ci-linux-host-input", host) + self.assertNotIn("prepare-native-runtime-input", host) + self.assertNotIn("compose-product-input", host) + self.assertNotIn("linux_release_host_input:", self.workflow) + self.assertNotIn("ci-linux-release-host-input", self.workflow) + + def test_arc_runner_contract_is_trusted_main_only(self) -> None: + arc = job_section(self.workflow, "arc_runner_image_contract") + pr_workflow = ( + ROOT / ".github" / "workflows" / "pr_builds.yml" + ).read_text(encoding="utf-8") + + self.assertIn("github.ref == 'refs/heads/main'", arc) + self.assertIn("runner: mesh-llm-amd64", arc) + self.assertIn("runner: mesh-llm-arm64", arc) + self.assertNotIn("arc_runner_image_contract:", pr_workflow) + self.assertNotIn("runner: mesh-llm-amd64", pr_workflow) + self.assertNotIn("runner: mesh-llm-arm64", pr_workflow) + + def test_cpu_runtime_is_an_independent_producer(self) -> None: + runtime = job_section(self.workflow, "linux_cpu_runtime_input") + + self.assertIn("needs: changes", runtime) + self.assertIn( + "if: ${{ needs.changes.outputs.docs_only != 'true' }}", + runtime, + ) + self.assertIn("uses: ./.github/actions/prepare-native-runtime-input", runtime) + self.assertIn("backend: cpu", runtime) + self.assertIn("name: ci-linux-cpu-runtime-input", runtime) + self.assertNotIn("linux_host_input", runtime) + self.assertNotIn("prepare-host-input", runtime) + self.assertNotIn("compose-product-input", runtime) + + def test_cpu_product_only_composes_immutable_inputs(self) -> None: + product = job_section(self.workflow, "linux_cpu_artifact") + + self.assertIn( + "needs: [changes, linux_host_input, linux_cpu_runtime_input]", + product, + ) + self.assertIn("name: ci-linux-host-input", product) + self.assertIn("name: ci-linux-cpu-runtime-input", product) + self.assertIn("uses: ./.github/actions/compose-product-input", product) + self.assertIn("name: ci-linux-inference-binaries", product) + self.assertNotIn("prepare-host-input", product) + self.assertNotIn("prepare-native-runtime-input", product) + self.assertNotIn("scripts/build-host.sh", product) + self.assertNotIn("scripts/package-native-runtime.sh", product) + self.assertNotIn("configure-sccache-gha", product) + + def test_gpu_products_reuse_the_neutral_host(self) -> None: + artifacts = { + "linux_cuda": "ci-linux-cuda-product", + "linux_rocm": "ci-linux-rocm-product", + "linux_vulkan": "ci-linux-vulkan-product", + } + + for job_name, product_artifact in artifacts.items(): + with self.subTest(job=job_name): + job = job_section(self.workflow, job_name) + self.assertIn("needs: [changes, linux_host_input]", job) + self.assertIn("name: ci-linux-host-input", job) + self.assertIn( + "uses: ./.github/actions/prepare-native-runtime-input", + job, + ) + self.assertIn("uses: ./.github/actions/compose-product-input", job) + self.assertIn(f"name: {product_artifact}", job) + self.assertNotIn("prepare-host-input", job) + self.assertNotIn("scripts/build-host.sh", job) + + def test_linux_tests_share_one_static_abi_producer(self) -> None: + producer = job_section(self.workflow, "linux_static_abi_input") + crate_tests = job_section(self.workflow, "rust_crate_tests") + grouped_tests = job_section(self.workflow, "linux_test_groups") + + self.assertIn("run: scripts/build-llama.sh", producer) + self.assertIn("name: ci-linux-static-abi-input", producer) + self.assertIn("mesh-llm-static-abi.tar.gz", producer) + for consumer in (crate_tests, grouped_tests): + with self.subTest(consumer=consumer.splitlines()[0].strip()): + self.assertIn("linux_static_abi_input", consumer) + self.assertIn("name: ci-linux-static-abi-input", consumer) + self.assertIn("Restore immutable static ABI input", consumer) + self.assertNotIn("run: scripts/build-llama.sh", consumer) + self.assertNotIn("Cache patched llama.cpp ABI build", consumer) + + def test_macos_host_and_runtime_are_independent_producers(self) -> None: + route = ( + "if: ${{ (github.event_name == 'workflow_dispatch' || " + "needs.changes.outputs.rust == 'true' || " + "needs.changes.outputs.ui == 'true' || " + "needs.changes.outputs.benchmarks == 'true') && " + "needs.changes.outputs.docs_only != 'true' }}" + ) + host = job_section(self.workflow, "macos_host_input") + runtime = job_section(self.workflow, "macos_metal_runtime_input") + + self.assertIn(route, host) + self.assertIn("name: macOS immutable release host", host) + self.assertIn("uses: ./.github/actions/prepare-host-input", host) + self.assertIn("profile: release", host) + self.assertIn("name: ci-macos-host-input", host) + self.assertNotIn("prepare-native-runtime-input", host) + self.assertNotIn("compose-product-input", host) + + self.assertIn(route, runtime) + self.assertIn( + "uses: ./.github/actions/prepare-native-runtime-input", + runtime, + ) + self.assertIn("backend: metal", runtime) + self.assertIn("name: ci-macos-metal-runtime-input", runtime) + self.assertNotIn("prepare-host-input", runtime) + self.assertNotIn("compose-product-input", runtime) + self.assertNotIn("\n macos:\n", self.workflow) + + def test_macos_product_only_composes_immutable_inputs(self) -> None: + product = job_section(self.workflow, "macos_cpu_artifact") + + self.assertIn("name: macOS Metal release product", product) + self.assertIn( + "needs: [changes, macos_host_input, macos_metal_runtime_input]", + product, + ) + self.assertIn("name: ci-macos-host-input", product) + self.assertIn("name: ci-macos-metal-runtime-input", product) + self.assertIn("uses: ./.github/actions/compose-product-input", product) + self.assertIn("name: ci-macos-inference-binaries", product) + self.assertNotIn("prepare-host-input", product) + self.assertNotIn("prepare-native-runtime-input", product) + self.assertNotIn("rust-toolchain", product) + self.assertNotIn("rust-cache", product) + self.assertNotIn("brew install", product) + self.assertNotIn("cargo ", product) + + def test_new_macos_jobs_pin_their_external_actions(self) -> None: + host = job_section(self.workflow, "macos_host_input") + runtime = job_section(self.workflow, "macos_metal_runtime_input") + product = job_section(self.workflow, "macos_cpu_artifact") + unit_tests = job_section(self.workflow, "macos_unit_tests") + checkout = ( + "actions/checkout@" + "fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09" + ) + + for job in (host, runtime, product, unit_tests): + with self.subTest(job=job.splitlines()[0].strip()): + self.assertIn(checkout, job) + + self.assertIn( + "pnpm/action-setup@" + "b906affcce14559ad1aafd4ab0e942779e9f58b1", + host, + ) + self.assertIn( + "actions/setup-node@" + "a0853c24544627f65ddf259abe73b1d18a591444", + host, + ) + rust_toolchain = ( + "dtolnay/rust-toolchain@" + "4cda84d5c5c54efe2404f9d843567869ab1699d4" + ) + rust_cache = ( + "Swatinem/rust-cache@" + "e18b497796c12c097a38f9edb9d0641fb99eee32" + ) + actions_cache = "caa296126883cff596d87d8935842f9db880ef25" + for job in (host, unit_tests): + self.assertIn(rust_toolchain, job) + self.assertIn(rust_cache, job) + self.assertIn(actions_cache, job) + + def test_macos_unit_tests_keep_static_abi_separate(self) -> None: + unit_tests = job_section(self.workflow, "macos_unit_tests") + + self.assertIn( + "LLAMA_STAGE_BUILD_DIR: " + ".deps/llama-build/build-stage-abi-static-metal", + unit_tests, + ) + self.assertIn("name: Cache static Metal ABI build", unit_tests) + self.assertIn("run: scripts/build-llama.sh", unit_tests) + self.assertIn("cargo test -p \"$c\" --lib", unit_tests) + self.assertNotIn("prepare-host-input", unit_tests) + self.assertNotIn("prepare-native-runtime-input", unit_tests) + self.assertNotIn("compose-product-input", unit_tests) + + def test_windows_main_reuses_one_release_host_for_all_products(self) -> None: + host = job_section(self.workflow, "windows_host_input") + + self.assertIn("needs.changes.outputs.rust == 'true'", host) + self.assertIn("needs.changes.outputs.windows_cpu == 'true'", host) + self.assertIn("needs.changes.outputs.windows_gpu == 'true'", host) + self.assertIn( + "uses: ./.github/actions/prepare-windows-host-input", + host, + ) + self.assertIn("profile: release", host) + self.assertIn("name: ci-windows-host-input", host) + self.assertNotIn("prepare-native-runtime-input", host) + self.assertNotIn("compose-product-input", host) + self.assertNotIn("\n windows_cpu:\n", self.workflow) + self.assertNotIn("\n windows_gpu:\n", self.workflow) + + def test_windows_main_runtime_inputs_are_independent_producers(self) -> None: + cpu = job_section(self.workflow, "windows_cpu_runtime_input") + gpu = job_section(self.workflow, "windows_gpu_runtime_inputs") + + self.assertIn("needs.changes.outputs.rust == 'true'", cpu) + self.assertIn("needs.changes.outputs.windows_cpu == 'true'", cpu) + self.assertNotIn("needs.changes.outputs.windows_gpu == 'true'", cpu) + self.assertIn( + "uses: ./.github/actions/prepare-native-runtime-input", + cpu, + ) + self.assertIn("backend: cpu", cpu) + self.assertIn("name: ci-windows-cpu-runtime-input", cpu) + + self.assertIn("needs.changes.outputs.windows_gpu == 'true'", gpu) + self.assertNotIn("needs.changes.outputs.rust == 'true'", gpu) + self.assertNotIn("needs.changes.outputs.windows_cpu == 'true'", gpu) + for backend in ("cuda", "rocm", "vulkan"): + self.assertIn(f"backend: {backend}", gpu) + self.assertIn( + "uses: ./.github/actions/prepare-native-runtime-input", + gpu, + ) + self.assertIn( + "name: ci-windows-${{ matrix.backend }}-runtime-input", + gpu, + ) + + for producer in (cpu, gpu): + with self.subTest(producer=producer.splitlines()[0].strip()): + self.assertNotIn("prepare-windows-host-input", producer) + self.assertNotIn("compose-product-input", producer) + + def test_windows_main_products_are_composition_only(self) -> None: + cpu = job_section(self.workflow, "windows_cpu_product") + gpu = job_section(self.workflow, "windows_gpu_products") + products = ( + ( + cpu, + "ci-windows-cpu-runtime-input", + "backend: cpu", + ), + ( + gpu, + "ci-windows-${{ matrix.backend }}-runtime-input", + "backend: ${{ matrix.backend }}", + ), + ) + + self.assertIn("needs.changes.outputs.rust == 'true'", cpu) + self.assertIn("needs.changes.outputs.windows_cpu == 'true'", cpu) + self.assertIn("needs.changes.outputs.windows_gpu == 'true'", gpu) + self.assertNotIn("needs.changes.outputs.rust == 'true'", gpu) + + for product, runtime_artifact, backend in products: + with self.subTest(product=product.splitlines()[0].strip()): + self.assertIn("name: ci-windows-host-input", product) + self.assertIn(f"name: {runtime_artifact}", product) + self.assertIn( + "uses: ./.github/actions/compose-product-input", + product, + ) + self.assertIn(backend, product) + self.assertIn("binary_name: mesh-llm.exe", product) + self.assertIn('readiness_smoke: "true"', product) + self.assertNotIn("prepare-windows-host-input", product) + self.assertNotIn("prepare-native-runtime-input", product) + self.assertNotIn("rust-toolchain", product) + self.assertNotIn("rust-cache", product) + self.assertNotIn("sccache-action", product) + self.assertNotIn("cargo ", product) + self.assertNotIn("build-windows.ps1", product) + + def test_windows_node_checks_remain_separate_from_product_builds(self) -> None: + checks = job_section(self.workflow, "windows_node_checks") + + self.assertIn("name: Windows Node SDK checks", checks) + self.assertIn("cargo check --locked -p mesh-llm-nodejs", checks) + self.assertNotIn("prepare-windows-host-input", checks) + self.assertNotIn("prepare-native-runtime-input", checks) + self.assertNotIn("compose-product-input", checks) + + def test_swift_smoke_uses_composed_macos_product(self) -> None: + swift = job_section(self.workflow, "swift_sdk_smoke") + + self.assertIn( + "needs: [changes, macos_cpu_artifact, macos_unit_tests]", + swift, + ) + self.assertIn("needs.macos_cpu_artifact.result == 'success'", swift) + self.assertIn("needs.macos_unit_tests.result == 'success'", swift) + self.assertIn("needs.macos_unit_tests.result == 'skipped'", swift) + self.assertIn("artifact_name: ci-macos-inference-binaries", swift) + self.assertIn("staged_binary_path: target/release/mesh-llm", swift) + + def test_main_runner_policy_is_selected_once(self) -> None: + changes = job_section(self.workflow, "changes") + + self.assertIn("runs-on: ubuntu-24.04", changes) + self.assertIn("uses: ./.github/actions/select-ci-runners", changes) + self.assertIn( + "depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }}", + changes, + ) + self.assertIn("ref: ${{ github.ref }}", changes) + self.assertIn( + "allow_depot_remote_cache: " + "${{ steps.runners.outputs.allow_depot_remote_cache }}", + changes, + ) + self.assertNotIn( + "(vars.DEPOT_RUNNERS_ENABLED == 'true' || " + "inputs.use_depot == true)", + self.workflow, + ) + + def test_linux_product_consumers_stage_the_release_profile(self) -> None: + linux_consumers = self.workflow[: self.workflow.index(" swift_sdk_smoke:")] + + self.assertNotIn("target/debug/mesh-llm", linux_consumers) + self.assertIn("target/release/mesh-llm", linux_consumers) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_collect_ci_metrics.py b/scripts/tests/test_collect_ci_metrics.py new file mode 100644 index 0000000000..0fe1167915 --- /dev/null +++ b/scripts/tests/test_collect_ci_metrics.py @@ -0,0 +1,312 @@ +import importlib.util +import json +import pathlib +import subprocess +import sys +import tempfile +import unittest + + +SCRIPT = pathlib.Path(__file__).resolve().parents[1] / "collect-ci-metrics.py" + + +def load_collector(): + spec = importlib.util.spec_from_file_location("collect_ci_metrics", SCRIPT) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def job( + name, + created_at, + started_at, + completed_at, + *, + conclusion="success", + labels=None, +): + return { + "id": hash((name, started_at)) & 0xFFFF, + "name": name, + "status": "completed", + "conclusion": conclusion, + "created_at": created_at, + "started_at": started_at, + "completed_at": completed_at, + "html_url": f"https://example.test/jobs/{name}", + "labels": labels or ["ubuntu-24.04"], + } + + +def run( + run_id, + created_at, + started_at, + updated_at, + jobs, + *, + conclusion="success", + attempt=1, +): + return { + "databaseId": run_id, + "attempt": attempt, + "workflowName": "PR Builds", + "displayTitle": f"run {run_id}", + "event": "pull_request", + "status": "completed", + "conclusion": conclusion, + "createdAt": created_at, + "startedAt": started_at, + "updatedAt": updated_at, + "url": f"https://example.test/runs/{run_id}", + "headSha": f"sha-{run_id}", + "headBranch": "feature", + "jobs": jobs, + } + + +class CollectCiMetricsTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.collector = load_collector() + + def sample_runs(self): + return [ + run( + 101, + "2026-07-01T00:00:00Z", + "2026-07-01T00:00:05Z", + "2026-07-01T00:10:00Z", + [ + job( + "build", + "2026-07-01T00:00:10Z", + "2026-07-01T00:00:20Z", + "2026-07-01T00:08:20Z", + ), + job( + "smoke", + "2026-07-01T00:08:20Z", + "2026-07-01T00:08:25Z", + "2026-07-01T00:09:50Z", + ), + job( + "unused", + "2026-07-01T00:00:00Z", + "2026-07-01T00:10:00Z", + "2026-07-01T00:10:00Z", + conclusion="skipped", + ), + ], + ), + run( + 102, + "2026-07-02T00:00:00Z", + "2026-07-02T00:00:15Z", + "2026-07-02T00:20:00Z", + [ + job( + "build", + "2026-07-02T00:00:20Z", + "2026-07-02T00:00:50Z", + "2026-07-02T00:16:50Z", + ), + job( + "summary", + "2026-07-02T00:16:50Z", + "2026-07-02T00:16:52Z", + "2026-07-02T00:19:52Z", + ), + ], + ), + ] + + def test_percentiles_use_linear_interpolation(self): + summary = self.collector.summarize([10.0, 20.0, 30.0, 40.0]) + + self.assertEqual(summary["p50"], 25.0) + self.assertEqual(summary["p90"], 37.0) + self.assertEqual(summary["p95"], 38.5) + self.assertEqual(summary["mean"], 25.0) + + def test_analysis_reports_wall_queue_slow_and_terminal_jobs(self): + runs = [self.collector.normalize_run(item) for item in self.sample_runs()] + + report = self.collector.analyze( + runs, + requested_status="success", + top=5, + source={"description": "fixture"}, + labels={"provider": "github"}, + ) + + self.assertEqual(report["selection"]["included_run_count"], 2) + self.assertEqual(report["workflow"]["wall_seconds"]["p50"], 900.0) + self.assertEqual(report["workflow"]["queue_seconds"]["p50"], 10.0) + by_name = {item["name"]: item for item in report["jobs"]["by_name"]} + self.assertEqual(by_name["build"]["duration_seconds"]["p50"], 720.0) + self.assertEqual(by_name["build"]["queue_seconds"]["p50"], 20.0) + self.assertNotIn("unused", by_name) + self.assertEqual( + report["jobs"]["critical_finish_candidates"], + [ + {"name": "smoke", "terminal_count": 1, "share": 0.5}, + {"name": "summary", "terminal_count": 1, "share": 0.5}, + ], + ) + self.assertEqual( + report["jobs"]["slowest_observations"][0]["name"], + "build", + ) + + def test_gh_run_view_json_has_unavailable_job_queue_not_fake_queue(self): + raw = self.sample_runs()[0] + raw["jobs"][0].pop("created_at") + raw["jobs"][0]["startedAt"] = raw["jobs"][0].pop("started_at") + raw["jobs"][0]["completedAt"] = raw["jobs"][0].pop("completed_at") + normalized = self.collector.normalize_run(raw) + + report = self.collector.analyze( + [normalized], + requested_status="success", + top=5, + source={"description": "gh run view"}, + labels={}, + ) + build = next( + item for item in report["jobs"]["by_name"] if item["name"] == "build" + ) + + self.assertEqual(build["queue_seconds"]["count"], 0) + self.assertEqual(build["start_delay_seconds"]["p50"], 20.0) + + def test_reruns_exclude_workflow_and_start_delay_timing(self): + first_attempt = self.sample_runs()[0] + rerun = run( + 103, + "2026-07-01T00:00:00Z", + "2026-07-02T00:00:05Z", + "2026-07-02T00:10:00Z", + [ + job( + "build", + "2026-07-02T00:00:10Z", + "2026-07-02T00:00:20Z", + "2026-07-02T00:08:20Z", + ) + ], + attempt=2, + ) + runs = [ + self.collector.normalize_run(item) for item in (first_attempt, rerun) + ] + + report = self.collector.analyze( + runs, + requested_status="success", + top=5, + source={"description": "fixture"}, + labels={}, + ) + + self.assertEqual( + report["selection"]["workflow_timing_excluded_reruns"], + 1, + ) + self.assertEqual(report["workflow"]["wall_seconds"]["count"], 1) + self.assertEqual(report["workflow"]["queue_seconds"]["count"], 1) + rerun_report = next(item for item in report["runs"] if item["id"] == 103) + self.assertEqual(rerun_report["attempt"], 2) + self.assertTrue(rerun_report["workflow_timing_excluded"]) + self.assertIsNone(rerun_report["wall_seconds"]) + build = next( + item for item in report["jobs"]["by_name"] if item["name"] == "build" + ) + self.assertEqual(build["duration_seconds"]["count"], 2) + self.assertEqual(build["start_delay_seconds"]["count"], 1) + self.assertIn("Excluded workflow wall", self.collector.render_markdown(report, 5)) + + def test_non_matching_and_in_progress_runs_are_reported_as_skipped(self): + successful, failed = self.sample_runs() + failed["conclusion"] = "failure" + pending = self.sample_runs()[0] + pending["databaseId"] = 103 + pending["status"] = "in_progress" + runs = [ + self.collector.normalize_run(item) + for item in (successful, failed, pending) + ] + + report = self.collector.analyze( + runs, + requested_status="success", + top=5, + source={"description": "fixture"}, + labels={}, + ) + + self.assertEqual(report["selection"]["included_run_count"], 1) + self.assertEqual( + report["selection"]["skipped_runs"], + {"conclusion_failure": 1, "not_completed": 1}, + ) + + def test_cli_reads_saved_json_and_writes_both_report_formats(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + input_path = root / "runs.json" + json_path = root / "metrics.json" + markdown_path = root / "metrics.md" + input_path.write_text( + json.dumps({"runs": self.sample_runs()}), + encoding="utf-8", + ) + + subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--input", + str(input_path), + "--json-out", + str(json_path), + "--markdown-out", + str(markdown_path), + "--label", + "provider=fixture", + ], + check=True, + capture_output=True, + text=True, + ) + + report = json.loads(json_path.read_text(encoding="utf-8")) + markdown = markdown_path.read_text(encoding="utf-8") + self.assertEqual(report["schema_version"], 1) + self.assertEqual(report["benchmark_labels"], {"provider": "fixture"}) + self.assertIn("# CI timing summary", markdown) + self.assertIn("Slow job families", markdown) + self.assertIn("20m 0s", markdown) + + def test_cli_rejects_run_list_json_without_detailed_jobs(self): + with tempfile.TemporaryDirectory() as directory: + input_path = pathlib.Path(directory) / "runs.json" + raw = self.sample_runs()[0] + raw.pop("jobs") + input_path.write_text(json.dumps([raw]), encoding="utf-8") + + result = subprocess.run( + [sys.executable, str(SCRIPT), "--input", str(input_path)], + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 2) + self.assertIn("has no jobs array", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_depot_canary_workflow.py b/scripts/tests/test_depot_canary_workflow.py new file mode 100644 index 0000000000..5c9bf9f8fb --- /dev/null +++ b/scripts/tests/test_depot_canary_workflow.py @@ -0,0 +1,44 @@ +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github" / "workflows" / "depot-canary.yml" + + +class DepotCanaryWorkflowTests(unittest.TestCase): + def setUp(self) -> None: + self.workflow = WORKFLOW.read_text(encoding="utf-8") + + def test_canary_has_no_code_or_credential_access(self) -> None: + self.assertIn("permissions: {}", self.workflow) + self.assertNotIn("actions/checkout", self.workflow) + self.assertNotIn("secrets.", self.workflow) + self.assertNotIn("pull_request", self.workflow) + self.assertNotIn("push:", self.workflow) + + def test_canary_covers_measured_depot_sizes(self) -> None: + for runner in ( + "depot-ubuntu-24.04", + "depot-ubuntu-24.04-4", + "depot-ubuntu-24.04-8", + "depot-ubuntu-24.04-16", + ): + with self.subTest(runner=runner): + self.assertIn(f"- {runner}", self.workflow) + + def test_canary_uses_a_pinned_cache_action_without_printing_tokens( + self, + ) -> None: + self.assertIn( + "actions/cache@caa296126883cff596d87d8935842f9db880ef25 " + "# v5.1.0", + self.workflow, + ) + self.assertIn("${DEPOT_CACHE_TOKEN:-}", self.workflow) + self.assertNotIn("echo \"$DEPOT_CACHE_TOKEN\"", self.workflow) + self.assertNotIn("printenv", self.workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_package_release_ps1.py b/scripts/tests/test_package_release_ps1.py index acfc86f7e1..eb116a1733 100644 --- a/scripts/tests/test_package_release_ps1.py +++ b/scripts/tests/test_package_release_ps1.py @@ -10,6 +10,104 @@ class PackageReleasePowerShellTests(unittest.TestCase): + def test_precomposed_product_reuses_exact_verified_tree(self) -> None: + contents = SCRIPT.read_text(encoding="utf-8") + precomposed = self.function_block( + contents, + "Copy-AndVerifyPrecomposedProduct", + "$Version = Normalize-RecipeArgument", + ) + + self.assertIn( + "$precomposedProductDir = $env:MESH_LLM_PRECOMPOSED_PRODUCT_DIR", + contents, + ) + self.assertIn( + "$resolvedSourceDir = Resolve-RepositoryPath $SourceDir", + precomposed, + ) + self.assertIn( + "$attestationPreverified", + contents, + ) + self.assertIn( + "[System.IO.Directory]::GetFileSystemEntries($resolvedSourceDir)", + precomposed, + ) + self.assertIn( + 'verify-host-dependencies.py"', + precomposed, + ) + self.assertIn( + 'verify-native-runtime-package.sh"', + precomposed, + ) + self.assertIn( + 'compose-product-bundle.py"', + precomposed, + ) + self.assertIn("--check", precomposed) + + def test_composer_output_accepts_git_for_windows_path(self) -> None: + contents = SCRIPT.read_text(encoding="utf-8") + resolver = self.function_block( + contents, + "Resolve-RepositoryPath", + "function Assert-AttestationConfig", + ) + + self.assertIn( + r"^/(?[A-Za-z])(?:/(?.*))?$", + resolver, + ) + self.assertIn( + 'GetFullPath("${drive}:\\${tail}")', + resolver, + ) + + def test_preverified_attestation_requires_immutable_composer_contract(self) -> None: + contents = SCRIPT.read_text(encoding="utf-8") + config = self.function_block( + contents, + "Assert-AttestationConfig", + "function Invoke-ReleaseAttestationStamp", + ) + attestation = self.function_block( + contents, + "Invoke-ReleaseAttestationStamp", + "function Copy-AndVerifyPrecomposedProduct", + ) + + self.assertIn( + "MESH_RELEASE_ATTESTATION_PREVERIFIED=1 requires a pre-stamped precomposed product", + config, + ) + self.assertIn( + '$env:MESH_RELEASE_HOST_PRESTAMPED -ne "1"', + config, + ) + self.assertIn( + 'if ($attestationPreverified -eq "1")', + attestation, + ) + self.assertIn( + "Release attestation: verified by immutable product composer", + attestation, + ) + self.assertLess( + attestation.index('if ($attestationPreverified -eq "1")'), + attestation.index("cargo run -q -p xtask"), + ) + + def test_legacy_runtime_selection_path_is_preserved(self) -> None: + contents = SCRIPT.read_text(encoding="utf-8") + main = contents[contents.index("try {") :] + + self.assertIn("if (Test-HasValue $precomposedProductDir)", main) + self.assertIn("} else {", main) + self.assertIn("$selectorArgs = @(", main) + self.assertIn("failed to select the packaged Windows native runtime", main) + def test_selector_arguments_are_built_as_tokens_with_optional_cuda_major(self) -> None: selector = self.selector_block() @@ -69,6 +167,12 @@ def selector_block(self) -> str: end = contents.index("$runtimeDestinationRoot =", start) return contents[start:end] + @staticmethod + def function_block(contents: str, function_name: str, end_marker: str) -> str: + start = contents.index(f"function {function_name}") + end = contents.index(end_marker, start) + return contents[start:end] + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_pr_workflow_artifacts.py b/scripts/tests/test_pr_workflow_artifacts.py new file mode 100644 index 0000000000..39a718fd15 --- /dev/null +++ b/scripts/tests/test_pr_workflow_artifacts.py @@ -0,0 +1,467 @@ +from __future__ import annotations + +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +PR_WORKFLOW = ROOT / ".github" / "workflows" / "pr_builds.yml" + + +def job_section( + workflow: str, + job_name: str, + next_job_name: str | None = None, +) -> str: + start = workflow.index(f" {job_name}:") + if next_job_name is None: + return workflow[start:] + end = workflow.index(f" {next_job_name}:", start) + return workflow[start:end] + + +class PrWorkflowArtifactTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.workflow = PR_WORKFLOW.read_text(encoding="utf-8") + cls.host = job_section( + cls.workflow, + "linux_host_input", + "linux_cpu_runtime_input", + ) + cls.cpu_runtime = job_section( + cls.workflow, + "linux_cpu_runtime_input", + "linux_cpu_artifact", + ) + cls.cpu_product = job_section( + cls.workflow, + "linux_cpu_artifact", + "linux_targets", + ) + cls.backend_products = job_section( + cls.workflow, + "linux_targets", + "rust_crate_tests", + ) + cls.macos_host = job_section( + cls.workflow, + "macos_host_input", + "macos_metal_runtime_input", + ) + cls.macos_runtime = job_section( + cls.workflow, + "macos_metal_runtime_input", + "macos_cpu_artifact", + ) + cls.macos_product = job_section( + cls.workflow, + "macos_cpu_artifact", + "swift_sdk_smoke", + ) + cls.windows_checks = job_section( + cls.workflow, + "windows_checks", + "windows_host_input", + ) + cls.windows_host = job_section( + cls.workflow, + "windows_host_input", + "windows_cpu_runtime_input", + ) + cls.windows_cpu_runtime = job_section( + cls.workflow, + "windows_cpu_runtime_input", + "windows_gpu_runtime_inputs", + ) + cls.windows_gpu_runtimes = job_section( + cls.workflow, + "windows_gpu_runtime_inputs", + "windows_cpu_product", + ) + cls.windows_cpu_product = job_section( + cls.workflow, + "windows_cpu_product", + "windows_gpu_products", + ) + cls.windows_gpu_products = job_section( + cls.workflow, + "windows_gpu_products", + ) + + def test_host_profile_covers_every_backend_product_route(self) -> None: + self.assertIn( + "needs.changes.outputs.linux_inference_artifact_required == 'true' " + "|| needs.changes.outputs.benchmarks == 'true'", + self.host, + ) + self.assertIn( + "needs.changes.outputs.backend_changed == 'true' " + "|| needs.changes.outputs.benchmarks == 'true'", + self.host, + ) + self.assertIn("&& 'release' || 'debug'", self.host) + + self.assertIn( + "github.event_name == 'workflow_dispatch' " + "|| needs.changes.outputs.backend_changed == 'true' " + "|| needs.changes.outputs.benchmarks == 'true'", + self.backend_products, + ) + + def test_cpu_runtime_only_runs_for_cpu_product_consumers(self) -> None: + condition = ( + "if: ${{ needs.changes.outputs.linux_inference_artifact_required " + "== 'true' && needs.changes.outputs.docs_only != 'true' }}" + ) + self.assertIn(condition, self.cpu_runtime) + self.assertNotIn("benchmarks", self.cpu_runtime) + + def test_cpu_product_uses_matching_immutable_inputs(self) -> None: + self.assertIn("name: pr-linux-host-input", self.host) + self.assertIn("name: pr-linux-cpu-runtime-input", self.cpu_runtime) + self.assertIn( + "needs: [changes, linux_host_input, linux_cpu_runtime_input]", + self.cpu_product, + ) + self.assertIn("name: pr-linux-host-input", self.cpu_product) + self.assertIn("path: host-input", self.cpu_product) + self.assertIn("name: pr-linux-cpu-runtime-input", self.cpu_product) + self.assertIn("path: runtime-input", self.cpu_product) + self.assertIn("output_dir: ci-product", self.cpu_product) + self.assertIn( + "path: ${{ steps.compose.outputs.archive_path }}", + self.cpu_product, + ) + + def test_backend_products_reuse_the_same_host_artifact(self) -> None: + self.assertIn("needs: [changes, linux_host_input]", self.backend_products) + self.assertIn("name: pr-linux-host-input", self.backend_products) + self.assertIn("path: host-input", self.backend_products) + self.assertNotIn("pr-linux-release-host-input", self.workflow) + self.assertIn("output_dir: product-input", self.backend_products) + self.assertIn( + "path: ${{ steps.compose.outputs.archive_path }}", + self.backend_products, + ) + + def test_cuda_runtime_uses_the_production_multiarch_image(self) -> None: + self.assertIn( + "sha256:c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd", + self.workflow, + ) + self.assertNotIn( + "sha256:295341c6c9f17c9eb69281fd454bda953799406d6915f472c914fb5f024a88ed", + self.workflow, + ) + + def test_public_mesh_admission_is_manual_not_a_pr_gate(self) -> None: + admission = job_section( + self.workflow, + "linux_public_mesh_admission", + "hf_download_smoke", + ) + + self.assertIn("github.event_name == 'workflow_dispatch'", admission) + self.assertNotIn("linux_client_auto_boot:", self.workflow) + self.assertIn("scripts/ci-client-auto-test.sh", admission) + self.assertIn("uses: ./.github/actions/restore-smoke-inputs", admission) + self.assertIn( + "artifact_name: ci-linux-inference-binaries", + admission, + ) + self.assertIn( + "staged_binary_path: target/debug/mesh-llm", + admission, + ) + self.assertNotIn("uses: actions/download-artifact@", admission) + self.assertNotIn("chmod +x target/debug/mesh-llm", admission) + + def test_linux_test_groups_use_the_same_dynamic_plan_as_main(self) -> None: + groups = job_section( + self.workflow, + "linux_test_groups", + "linux_public_mesh_admission", + ) + + self.assertIn( + "linux_test_groups_json: " + "${{ steps.compute.outputs.linux_test_groups_json }}", + self.workflow, + ) + self.assertIn( + "needs: [changes, linux_static_abi_input]", + groups, + ) + self.assertNotIn("linux_cpu_artifact", groups) + self.assertIn( + "include: " + "${{ fromJson(needs.changes.outputs.linux_test_groups_json) }}", + groups, + ) + self.assertNotIn("- group: protocol", groups) + self.assertNotIn("- group: skippy-smoke", groups) + + def test_linux_tests_share_one_static_abi_producer(self) -> None: + producer = job_section( + self.workflow, + "linux_static_abi_input", + "rust_crate_tests", + ) + crate_tests = job_section( + self.workflow, + "rust_crate_tests", + "linux_test_groups", + ) + grouped_tests = job_section( + self.workflow, + "linux_test_groups", + "linux_public_mesh_admission", + ) + + self.assertIn("run: scripts/build-llama.sh", producer) + self.assertIn("name: pr-linux-static-abi-input", producer) + self.assertIn("mesh-llm-static-abi.tar.gz", producer) + for consumer in (crate_tests, grouped_tests): + with self.subTest(consumer=consumer.splitlines()[0].strip()): + self.assertIn("linux_static_abi_input", consumer) + self.assertIn("name: pr-linux-static-abi-input", consumer) + self.assertIn("Restore immutable static ABI input", consumer) + self.assertNotIn("run: scripts/build-llama.sh", consumer) + self.assertNotIn("Cache patched llama.cpp ABI build", consumer) + + def test_macos_producers_keep_the_existing_product_route(self) -> None: + route = ( + "if: ${{ needs.changes.outputs.macos_inference_artifact_required " + "== 'true' && needs.changes.outputs.docs_only != 'true' }}" + ) + + self.assertIn("needs: changes", self.macos_host) + self.assertIn(route, self.macos_host) + self.assertIn("needs: changes", self.macos_runtime) + self.assertIn(route, self.macos_runtime) + + def test_macos_host_and_runtime_are_independent_producers(self) -> None: + self.assertIn( + "uses: ./.github/actions/prepare-host-input", + self.macos_host, + ) + self.assertIn("profile: debug", self.macos_host) + self.assertIn("name: pr-macos-host-input", self.macos_host) + self.assertNotIn("prepare-native-runtime-input", self.macos_host) + self.assertNotIn("compose-product-input", self.macos_host) + + self.assertIn( + "uses: ./.github/actions/prepare-native-runtime-input", + self.macos_runtime, + ) + self.assertIn( + "LLAMA_STAGE_BUILD_DIR: " + ".deps/llama-build/build-stage-abi-dynamic-metal", + self.macos_runtime, + ) + self.assertIn("backend: metal", self.macos_runtime) + self.assertIn("target: aarch64-apple-darwin", self.macos_runtime) + self.assertIn( + "name: pr-macos-metal-runtime-input", + self.macos_runtime, + ) + self.assertNotIn("macos_host_input", self.macos_runtime) + self.assertNotIn("prepare-host-input", self.macos_runtime) + self.assertNotIn("compose-product-input", self.macos_runtime) + + def test_macos_product_only_composes_immutable_inputs(self) -> None: + self.assertIn( + "needs: [changes, macos_host_input, macos_metal_runtime_input]", + self.macos_product, + ) + self.assertIn( + "needs.macos_host_input.result == 'success' " + "&& needs.macos_metal_runtime_input.result == 'success'", + self.macos_product, + ) + self.assertIn("name: pr-macos-host-input", self.macos_product) + self.assertIn( + "name: pr-macos-metal-runtime-input", + self.macos_product, + ) + self.assertIn( + "uses: ./.github/actions/compose-product-input", + self.macos_product, + ) + self.assertIn("backend: metal", self.macos_product) + self.assertIn("output_dir: ci-product", self.macos_product) + self.assertIn( + "name: ci-macos-inference-binaries", + self.macos_product, + ) + self.assertIn( + "path: ${{ steps.compose.outputs.archive_path }}", + self.macos_product, + ) + self.assertNotIn("prepare-host-input", self.macos_product) + self.assertNotIn("prepare-native-runtime-input", self.macos_product) + self.assertNotIn("scripts/build-host.sh", self.macos_product) + self.assertNotIn( + "scripts/package-native-runtime.sh", + self.macos_product, + ) + self.assertNotIn("brew install", self.macos_product) + self.assertNotIn("Swatinem/rust-cache", self.macos_product) + + def test_macos_swift_gate_and_supported_targets_are_preserved(self) -> None: + swift = job_section( + self.workflow, + "swift_sdk_smoke", + "macos_unit_tests", + ) + unit_tests = job_section( + self.workflow, + "macos_unit_tests", + "windows_checks", + ) + + self.assertIn( + "needs: [changes, macos_cpu_artifact, macos_unit_tests]", + swift, + ) + self.assertIn("always()", swift) + self.assertIn("needs.macos_cpu_artifact.result == 'success'", swift) + self.assertIn("needs.macos_unit_tests.result == 'success'", swift) + self.assertIn("needs.macos_unit_tests.result == 'skipped'", swift) + self.assertIn("artifact_name: ci-macos-inference-binaries", swift) + self.assertIn("needs: changes", unit_tests) + self.assertNotIn("macos_cpu_artifact", unit_tests) + self.assertIn( + "LLAMA_STAGE_BUILD_DIR: " + ".deps/llama-build/build-stage-abi-static-metal", + unit_tests, + ) + self.assertIn( + "actions/checkout@" + "fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09", + unit_tests, + ) + self.assertIn( + "dtolnay/rust-toolchain@" + "4cda84d5c5c54efe2404f9d843567869ab1699d4", + unit_tests, + ) + self.assertIn( + "Swatinem/rust-cache@" + "e18b497796c12c097a38f9edb9d0641fb99eee32", + unit_tests, + ) + self.assertIn( + "actions/cache@" + "caa296126883cff596d87d8935842f9db880ef25", + unit_tests, + ) + self.assertNotIn(" macos_targets:", self.workflow) + self.assertNotIn("Skip unsupported macOS GPU backend", self.workflow) + + def test_windows_pr_keeps_broad_rust_signals_lightweight(self) -> None: + self.assertIn("needs.changes.outputs.all_rust == 'true'", self.windows_checks) + self.assertIn("name: Windows lightweight checks", self.windows_checks) + self.assertIn("cargo check --locked -p mesh-llm --bin mesh-llm", self.windows_checks) + self.assertNotIn("prepare-windows-host-input", self.windows_checks) + self.assertNotIn("prepare-native-runtime-input", self.windows_checks) + self.assertNotIn("compose-product-input", self.windows_checks) + + for producer in ( + self.windows_host, + self.windows_cpu_runtime, + self.windows_gpu_runtimes, + ): + with self.subTest(producer=producer.splitlines()[0].strip()): + self.assertNotIn("needs.changes.outputs.all_rust", producer) + + def test_windows_pr_builds_one_debug_host_and_independent_runtimes(self) -> None: + self.assertIn( + "uses: ./.github/actions/prepare-windows-host-input", + self.windows_host, + ) + self.assertIn("profile: debug", self.windows_host) + self.assertIn("name: pr-windows-host-input", self.windows_host) + self.assertNotIn("prepare-native-runtime-input", self.windows_host) + self.assertNotIn("compose-product-input", self.windows_host) + + self.assertIn( + "needs.changes.outputs.windows_cpu == 'true'", + self.windows_cpu_runtime, + ) + self.assertNotIn( + "needs.changes.outputs.windows_gpu == 'true'", + self.windows_cpu_runtime, + ) + self.assertIn( + "uses: ./.github/actions/prepare-native-runtime-input", + self.windows_cpu_runtime, + ) + self.assertIn("backend: cpu", self.windows_cpu_runtime) + self.assertIn( + "name: pr-windows-cpu-runtime-input", + self.windows_cpu_runtime, + ) + self.assertNotIn("prepare-windows-host-input", self.windows_cpu_runtime) + self.assertNotIn("compose-product-input", self.windows_cpu_runtime) + + self.assertIn( + "needs.changes.outputs.windows_gpu == 'true'", + self.windows_gpu_runtimes, + ) + self.assertNotIn( + "needs.changes.outputs.windows_cpu == 'true'", + self.windows_gpu_runtimes, + ) + for backend in ("cuda", "rocm", "vulkan"): + self.assertIn(f"backend: {backend}", self.windows_gpu_runtimes) + self.assertIn( + "uses: ./.github/actions/prepare-native-runtime-input", + self.windows_gpu_runtimes, + ) + self.assertIn( + "name: pr-windows-${{ matrix.backend }}-runtime-input", + self.windows_gpu_runtimes, + ) + self.assertNotIn("prepare-windows-host-input", self.windows_gpu_runtimes) + self.assertNotIn("compose-product-input", self.windows_gpu_runtimes) + + def test_windows_pr_products_only_compose_matching_inputs(self) -> None: + products = ( + ( + self.windows_cpu_product, + "pr-windows-cpu-runtime-input", + "backend: cpu", + ), + ( + self.windows_gpu_products, + "pr-windows-${{ matrix.backend }}-runtime-input", + "backend: ${{ matrix.backend }}", + ), + ) + + for product, runtime_artifact, backend in products: + with self.subTest(product=product.splitlines()[0].strip()): + self.assertIn("name: pr-windows-host-input", product) + self.assertIn(f"name: {runtime_artifact}", product) + self.assertIn( + "uses: ./.github/actions/compose-product-input", + product, + ) + self.assertIn(backend, product) + self.assertIn("binary_name: mesh-llm.exe", product) + self.assertIn('readiness_smoke: "true"', product) + self.assertNotIn("prepare-windows-host-input", product) + self.assertNotIn("prepare-native-runtime-input", product) + self.assertNotIn("rust-toolchain", product) + self.assertNotIn("rust-cache", product) + self.assertNotIn("sccache-action", product) + self.assertNotIn("cargo ", product) + self.assertNotIn("build-windows.ps1", product) + + self.assertNotIn(" windows_targets:", self.workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_release_workflow_artifacts.py b/scripts/tests/test_release_workflow_artifacts.py index d3bb64ee6d..16da4b75ba 100644 --- a/scripts/tests/test_release_workflow_artifacts.py +++ b/scripts/tests/test_release_workflow_artifacts.py @@ -8,84 +8,318 @@ RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "release.yml" +def job_block(workflow: str, job_name: str, next_job_name: str) -> str: + start = workflow.index(f" {job_name}:") + end = workflow.index(f" {next_job_name}:", start) + return workflow[start:end] + + class ReleaseWorkflowArtifactTests(unittest.TestCase): - def test_macos_cpu_composer_installs_attestation_linker(self) -> None: + def test_release_depot_policy_is_main_ref_only_and_selected_once( + self, + ) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") - composer = self.job_block(workflow, "compose_cpu_products", "inference_smoke_tests") + metadata = job_block(workflow, "metadata", "build") - self.assertIn("Install macOS attestation verifier linker", composer) - self.assertIn("if: runner.os == 'macOS'", composer) - self.assertIn("run: brew install lld", composer) + self.assertIn("use_depot:", workflow[: workflow.index("\njobs:\n")]) + self.assertIn( + "uses: ./.github/actions/select-ci-runners", + metadata, + ) + self.assertIn("ref: ${{ github.ref }}", metadata) + self.assertIn( + "depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }}", + metadata, + ) + self.assertIn( + "manual_use_depot: ${{ inputs.use_depot == true }}", + metadata, + ) + self.assertEqual( + workflow.count("uses: ./.github/actions/select-ci-runners"), + 1, + ) - def test_container_product_composers_run_bash(self) -> None: + def test_release_routes_only_initial_non_secret_linux_lanes( + self, + ) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + host = job_block(workflow, "build", "compose_cpu_products") + sdk_runtime = job_block( + workflow, + "build_native_sdk_runtime", + "build_native_runtime", + ) + native_runtime = job_block( + workflow, + "build_native_runtime", + "build_native_runtime_linux_aarch64_cuda", + ) + rocm = job_block( + workflow, + "build_native_runtime_linux_x86_64_rocm", + "build_native_runtime_linux_x86_64_vulkan", + ) + vulkan = job_block( + workflow, + "build_native_runtime_linux_x86_64_vulkan", + "build_swift_sdk_artifact", + ) + publish = job_block( + workflow, + "publish", + "dispatch_packaging_release", + ) + + self.assertIn("runs-on: ${{ matrix.os }}", host) + self.assertIn("RELEASE_ATTESTATION_SIGNING_KEY", host) + for producer in (sdk_runtime, native_runtime): + self.assertIn( + "matrix.target == 'x86_64-unknown-linux-gnu'", + producer, + ) + self.assertIn( + "needs.metadata.outputs.runner_8", + producer, + ) + for producer in (rocm, vulkan): + self.assertIn( + "runs-on: ${{ needs.metadata.outputs.runner_16 }}", + producer, + ) + self.assertIn( + "allow_depot_remote_cache: " + "${{ needs.metadata.outputs.allow_depot_remote_cache }}", + producer, + ) + self.assertIn("runs-on: ubuntu-24.04", publish) + self.assertNotIn("needs.metadata.outputs.runner", publish) - for step_name in ( - "Compose aarch64 CUDA release bundle from producer inputs", - "Compose CUDA release bundle from producer inputs", - "Compose ROCm release bundle from producer inputs", - "Compose Vulkan release bundle from producer inputs", - ): - step_start = workflow.index(f"- name: {step_name}") - env_start = workflow.index(" env:", step_start) - self.assertIn(" shell: bash", workflow[step_start:env_start]) - - def test_windows_composers_reuse_checksum_verified_host_verifier(self) -> None: + def test_inference_smoke_consumes_composed_product(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + self.assertEqual( + workflow.count("ci-release-linux-inference-product"), + 2, + ) + self.assertNotIn("release-linux-inference-binary", workflow) self.assertIn( - "Copy-Item target\\debug\\xtask.exe " - "host-input\\release-attestation-verifier.exe -Force", + "uses: ./.github/actions/compose-product-input", workflow, ) + self.assertIn("output_dir: product-input", workflow) self.assertIn( - "host-input\\release-attestation-verifier.exe.sha256", + "path: ${{ steps.compose.outputs.archive_path }}", workflow, ) - self.assertEqual( - workflow.count("MESH_RELEASE_ATTESTATION_VERIFIER:"), - 2, + + def test_release_permissions_are_least_privilege(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + header = workflow[: workflow.index("\njobs:\n")] + publish = job_block( + workflow, + "publish", + "dispatch_packaging_release", ) - def test_unix_composition_restores_downloaded_host_executable_bit(self) -> None: + self.assertIn( + "permissions:\n contents: read\n packages: read", + header, + ) + self.assertNotIn("contents: write", header) + self.assertNotIn("packages: write", header) + self.assertIn( + " permissions:\n contents: write", + publish, + ) + self.assertNotIn("packages: write", publish) + + def test_native_sdk_assets_are_staged_flat_for_publishing(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") - readiness_command = ( - "scripts/ci-client-readiness-smoke.sh " - "host-input/mesh-llm runtime-root" + producer = job_block( + workflow, + "build_native_sdk_runtime", + "build_native_runtime", + ) + upload = producer[producer.index("- name: Upload native SDK runtime") :] + publish = job_block( + workflow, + "publish", + "dispatch_packaging_release", ) - self.assertEqual(workflow.count(readiness_command), 6) - self.assertEqual( - workflow.count("chmod +x host-input/mesh-llm"), - workflow.count(readiness_command), + self.assertIn("- name: Stage flat native SDK release assets", producer) + self.assertIn( + "native SDK release asset basename collision", + producer, ) + self.assertIn("path: release-native-sdk-assets/*", upload) + self.assertNotIn("dist/native-sdk/", upload) + self.assertNotIn("dist/native-sdk-crates/", upload) + self.assertIn("files: release-artifacts/*", publish) - def test_inference_smoke_consumes_composed_product(self) -> None: + def test_windows_host_publishes_prebuilt_attestation_verifier(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + producer = job_block( + workflow, + "windows_host_input", + "compose_windows_gpu", + ) - self.assertEqual( - workflow.count("release-linux-inference-product"), - 2, + self.assertIn( + "uses: ./.github/actions/prepare-windows-host-input", + producer, ) - self.assertNotIn("release-linux-inference-binary", workflow) - for required_path in ( - "smoke-input/mesh-llm", - "smoke-input/host-imports.json", - 'smoke-input/native-runtimes/$runtime_name', - "smoke-input/product-manifest.json", - ): - self.assertIn(required_path, workflow) + self.assertIn("profile: release", producer) + self.assertIn( + "attestation_signing_key_file:", + producer, + ) + self.assertIn( + "attestation_public_key_file:", + producer, + ) + self.assertIn("path: host-input/*", producer) + self.assertNotIn("prepare-native-runtime-input", producer) + self.assertNotIn("compose-product-input", producer) + def test_windows_composers_use_shared_verified_product_action(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + jobs = ( + ("compose_windows_gpu", "compose_windows_cpu"), + ("compose_windows_cpu", "build_native_runtime_windows_cpu"), + ) + + for job_name, next_job_name in jobs: + with self.subTest(job=job_name): + job = job_block(workflow, job_name, next_job_name) + composition = job.index( + "uses: ./.github/actions/compose-product-input", + ) + packaging = job.index( + "- name: Package verified Windows", + ) + self.assertLess(composition, packaging) + self.assertIn( + "binary_name: mesh-llm.exe", + job, + ) + self.assertIn( + "attestation_verifier: host-input/release-attestation-verifier.exe", + job, + ) + self.assertIn( + "version: ${{ needs.metadata.outputs.tag }}", + job, + ) + expected_backend = ( + "backend: ${{ matrix.backend }}" + if job_name == "compose_windows_gpu" + else "backend: cpu" + ) + self.assertIn(expected_backend, job) + self.assertIn('readiness_smoke: "true"', job) + self.assertIn( + "MESH_LLM_PRECOMPOSED_PRODUCT_DIR: ${{ steps.compose.outputs.product_dir }}", + job, + ) + self.assertIn( + 'MESH_RELEASE_ATTESTATION_PREVERIFIED: "1"', + job, + ) + self.assertNotIn("Verify immutable runtime archive", job) + self.assertNotIn("tar -xzf", job) + self.assertNotIn("ci-client-readiness-smoke.sh", job) + self.assertNotIn("cargo run", job) + self.assertNotIn("dtolnay/rust-toolchain", job) + self.assertNotIn("sccache-action", job) + + def test_windows_cuda12_label_rejects_other_toolkit_majors(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + producer = job_block( + workflow, + "build_native_runtime_windows_gpu", + "publish", + ) + + validation = producer.index("- name: Validate CUDA 12 artifact contract") + installation = producer.index("- name: Install CUDA toolkit") + self.assertLess(validation, installation) + self.assertIn("$cudaMajor -ne '12'", producer) self.assertIn( - "python3 scripts/compose-product-bundle.py", + "release-native-runtime-windows-x86_64-cuda12", + producer, + ) + + def test_linux_cuda_composition_uses_hosted_runner(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + composer = job_block( + workflow, + "compose_linux_cuda", + "compose_linux_rocm", + ) + job_header = composer[: composer.index(" steps:")] + + self.assertIn( + " runs-on: ${{ needs.metadata.outputs.runner_4 }}", + job_header, + ) + self.assertNotIn("self-hosted", job_header) + self.assertNotIn("USE_SELF_HOSTED", job_header) + + def test_release_uses_shared_host_and_runtime_producers(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + + self.assertEqual( + workflow.count("uses: ./.github/actions/prepare-host-input"), + 2, + ) + self.assertEqual( + workflow.count( + "uses: ./.github/actions/prepare-windows-host-input", + ), + 1, + ) + self.assertGreaterEqual( + workflow.count("uses: ./.github/actions/prepare-native-runtime-input"), + 5, + ) + self.assertEqual( + workflow.count("uses: ./.github/actions/compose-product-input"), + 8, + ) + self.assertNotIn( + "scripts/ci-client-readiness-smoke.sh host-input/mesh-llm runtime-root", workflow, ) - @staticmethod - def job_block(workflow: str, start_job: str, next_job: str) -> str: - start = workflow.index(f" {start_job}:") - end = workflow.index(f" {next_job}:", start) - return workflow[start:end] + def test_release_product_jobs_do_not_restore_compiler_caches(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + product_jobs = ( + "compose_linux_aarch64_cuda", + "compose_linux_cuda", + "compose_linux_rocm", + "compose_linux_vulkan", + ) + + for index, job_name in enumerate(product_jobs): + start = workflow.index(f" {job_name}:") + next_starts = [ + workflow.find(f" {other_job}:", start + 1) + for other_job in product_jobs[index + 1 :] + ] + next_starts = [position for position in next_starts if position >= 0] + end = min(next_starts) if next_starts else len(workflow) + job = workflow[start:end] + self.assertIn( + "uses: ./.github/actions/compose-product-input", + job, + ) + self.assertNotIn( + "uses: ./.github/actions/configure-sccache-gha", + job, + ) + self.assertNotIn("uses: actions/cache@", job) if __name__ == "__main__": diff --git a/scripts/tests/test_verify_host_dependencies.py b/scripts/tests/test_verify_host_dependencies.py index 73a690231d..41f38a8452 100644 --- a/scripts/tests/test_verify_host_dependencies.py +++ b/scripts/tests/test_verify_host_dependencies.py @@ -15,17 +15,32 @@ class VerifyHostDependenciesTests(unittest.TestCase): - def test_release_workflow_uses_python_for_non_executable_verifier(self) -> None: + def test_shared_host_actions_invoke_non_executable_verifier_with_python( + self, + ) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + unix_action = ( + ROOT / ".github" / "actions" / "prepare-host-input" / "action.yml" + ).read_text(encoding="utf-8") + windows_action = ( + ROOT + / ".github" + / "actions" + / "prepare-windows-host-input" + / "action.yml" + ).read_text(encoding="utf-8") self.assertEqual( - workflow.count("python3 scripts/verify-host-dependencies.py"), - 2, + unix_action.count("python3 scripts/verify-host-dependencies.py"), + 1, ) - self.assertNotIn( - "\n scripts/verify-host-dependencies.py", - workflow, + self.assertEqual( + windows_action.count( + r"& python scripts\verify-host-dependencies.py", + ), + 1, ) + self.assertNotIn("verify-host-dependencies.py", workflow) def test_parses_elf_needed_entries(self) -> None: imports = MODULE.parse_elf_imports( diff --git a/tools/xtask/src/publish_consistency.rs b/tools/xtask/src/publish_consistency.rs index f6304ca778..31b3723941 100644 --- a/tools/xtask/src/publish_consistency.rs +++ b/tools/xtask/src/publish_consistency.rs @@ -403,11 +403,11 @@ fn check_publish_workflow_invariants(repo_root: &Path) -> DynResult<()> { if: ${{ needs.metadata.outputs.prerelease != 'true' && needs.metadata.outputs.canary != 'true' }} runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ needs.metadata.outputs.tag }} persist-credentials: false - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' env: diff --git a/tools/xtask/src/workflow_checks.rs b/tools/xtask/src/workflow_checks.rs index 0ea088e53e..c4995a8ae1 100644 --- a/tools/xtask/src/workflow_checks.rs +++ b/tools/xtask/src/workflow_checks.rs @@ -25,6 +25,14 @@ pub(crate) fn check_docs_and_workflow_invariants(repo_root: &Path) -> DynResult< fs::read_to_string(repo_root.join(".github/actions/compute-changes/action.yml"))?; let configure_sccache_action = fs::read_to_string(repo_root.join(".github/actions/configure-sccache-gha/action.yml"))?; + let prepare_windows_host_action = fs::read_to_string( + repo_root.join(".github/actions/prepare-windows-host-input/action.yml"), + )?; + let prepare_native_runtime_action = fs::read_to_string( + repo_root.join(".github/actions/prepare-native-runtime-input/action.yml"), + )?; + let compose_product_action = + fs::read_to_string(repo_root.join(".github/actions/compose-product-input/action.yml"))?; let affected_crates_script = fs::read_to_string(repo_root.join("scripts/affected-crates.sh"))?; let ci_docs = fs::read_to_string(repo_root.join("ci/ci.md"))?; let pr_cleanup_workflow = @@ -77,28 +85,28 @@ pub(crate) fn check_docs_and_workflow_invariants(repo_root: &Path) -> DynResult< )?; ensure_contains( &release_workflow, - "- build_linux_aarch64_cuda", + "- compose_linux_aarch64_cuda", "release workflow aarch64 CUDA publish need", )?; ensure_contains( &release_workflow, - "build_windows_cpu:", - "release workflow Windows CPU build", + "windows_host_input:", + "release workflow immutable Windows host build", )?; ensure_contains( &release_workflow, - "build_windows_gpu:", - "release workflow Windows GPU build", + "compose_windows_gpu:", + "release workflow Windows GPU composition", )?; ensure_contains( &release_workflow, - "- build_windows_cpu", - "release workflow Windows CPU publish need", + "- windows_host_input", + "release workflow immutable Windows host publish need", )?; ensure_contains( &release_workflow, - "- build_windows_gpu", - "release workflow Windows GPU publish need", + "- compose_windows_gpu", + "release workflow Windows GPU composition publish need", )?; ensure_contains( &justfile, @@ -190,6 +198,11 @@ pub(crate) fn check_docs_and_workflow_invariants(repo_root: &Path) -> DynResult< "windows_gpu_build_required", "compute-changes Windows GPU build output", )?; + ensure_contains( + &compute_changes_action, + "runner_contract_required", + "compute-changes runner contract output", + )?; ensure_contains( &compute_changes_action, "build-linux-rocm", @@ -265,6 +278,11 @@ pub(crate) fn check_docs_and_workflow_invariants(repo_root: &Path) -> DynResult< "steps.compute.outputs.windows_gpu_build_required", "PR Builds Windows GPU compute route", )?; + ensure_contains( + &pr_builds_workflow, + "steps.compute.outputs.runner_contract_required", + "PR Builds runner contract route", + )?; ensure_contains( &ci_docs, "website_changed?", @@ -333,12 +351,12 @@ pub(crate) fn check_docs_and_workflow_invariants(repo_root: &Path) -> DynResult< )?; ensure_contains( &website_pages_workflow, - "actions/upload-pages-artifact@v3", + "actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa", "public website Pages artifact upload", )?; ensure_contains( &website_pages_workflow, - "actions/deploy-pages@v4", + "actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e", "public website Pages deploy action", )?; ensure_contains( @@ -371,7 +389,13 @@ pub(crate) fn check_docs_and_workflow_invariants(repo_root: &Path) -> DynResult< "push:\n branches: [main]", "main CI push trigger", )?; - check_windows_dynamic_runtime_contract(&ci_workflow, &pr_builds_workflow)?; + check_windows_dynamic_runtime_contract( + &ci_workflow, + &pr_builds_workflow, + &prepare_windows_host_action, + &prepare_native_runtime_action, + &compose_product_action, + )?; check_release_dispatch_version_preparation(&release_workflow)?; check_release_container_contracts(&release_workflow, &configure_sccache_action)?; check_ci_crate_test_coverage(&ci_workflow, &pr_builds_workflow, &compute_changes_action)?; @@ -385,12 +409,11 @@ fn check_release_dispatch_version_preparation(release_workflow: &str) -> DynResu "build_native_sdk_runtime", "build_swift_sdk_artifact", "build_linux_arm64", - "build_linux_aarch64_cuda", - "build_linux_cuda", - "build_linux_rocm", - "build_linux_vulkan", - "build_windows_cpu", - "build_windows_gpu", + "compose_linux_aarch64_cuda", + "compose_linux_cuda", + "compose_linux_rocm", + "compose_linux_vulkan", + "windows_host_input", ]; const REQUIRED_STEP: &str = "Prepare dispatched release version"; const REQUIRED_COMMAND: &str = "scripts/release-version.sh \"$RELEASE_TAG\""; @@ -427,6 +450,8 @@ fn check_release_container_contracts( const REQUIRED_COMMAND: &str = "git config --global --add safe.directory \"$GITHUB_WORKSPACE\""; const LOCAL_SCCACHE_ENV: &str = " SCCACHE_GHA_ENABLED: \"false\""; const CONFIGURE_SCCACHE_ACTION: &str = " - uses: ./.github/actions/configure-sccache-gha"; + const COMPOSE_PRODUCT_ACTION: &str = "uses: ./.github/actions/compose-product-input"; + const PREPARE_RUNTIME_ACTION: &str = "uses: ./.github/actions/prepare-native-runtime-input"; const PINNED_GITHUB_SCRIPT: &str = "uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd"; @@ -486,6 +511,15 @@ fn check_release_container_contracts( "core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk,gha')", "sccache GHA action cache chain", ), + ( + "process.env.SCCACHE_WEBDAV_ENDPOINT", + "sccache Depot WebDAV endpoint", + ), + ("process.env.DEPOT_CACHE_TOKEN", "sccache Depot job token"), + ( + "core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk,webdav')", + "sccache Depot cache chain", + ), ( "core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk')", "sccache GHA action disk-only fallback", @@ -519,6 +553,25 @@ fn check_release_container_contracts( REQUIRED_COMMAND, &format!("release workflow `{job_name}` safe-directory command"), )?; + let composition_only = + job.contains(COMPOSE_PRODUCT_ACTION) && !job.contains(PREPARE_RUNTIME_ACTION); + if composition_only { + ensure_not_contains( + job, + CONFIGURE_SCCACHE_ACTION.trim(), + &format!( + "release workflow `{job_name}` composition must not configure a compiler cache" + ), + )?; + ensure_not_contains( + job, + "uses: actions/cache@", + &format!( + "release workflow `{job_name}` composition must not restore a compiler cache" + ), + )?; + continue; + } if !job.lines().any(|line| line == LOCAL_SCCACHE_ENV) { return Err(format!( "release workflow `{job_name}`: missing job-level `{}`", @@ -557,67 +610,119 @@ fn release_container_job_names(release_workflow: &str) -> Vec<&str> { fn check_windows_dynamic_runtime_contract( ci_workflow: &str, pr_builds_workflow: &str, + prepare_windows_host_action: &str, + prepare_native_runtime_action: &str, + compose_product_action: &str, ) -> DynResult<()> { - let ci_windows_cpu = workflow_job_section(ci_workflow, "windows_cpu") - .ok_or("main CI workflow: missing `windows_cpu` job")?; - let ci_windows_gpu = workflow_job_section(ci_workflow, "windows_gpu") - .ok_or("main CI workflow: missing `windows_gpu` job")?; - let pr_windows_targets = workflow_job_section(pr_builds_workflow, "windows_targets") - .ok_or("PR Builds workflow: missing `windows_targets` job")?; - - for (workflow, context) in [ - (ci_windows_cpu, "main CI Windows CPU"), - (ci_windows_gpu, "main CI Windows GPU"), - (pr_windows_targets, "PR Builds Windows targets"), - ] { - ensure_contains( - workflow, - "uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756", - &format!("{context} persistent MSVC environment"), - )?; - } + ensure_contains( + prepare_windows_host_action, + r"& .\scripts\build-windows.ps1 -BuildProfile $profile -HostOnly", + "shared Windows host action canonical host-only build", + )?; + ensure_contains( + prepare_windows_host_action, + r"scripts\verify-host-dependencies.py", + "shared Windows host action import-policy verification", + )?; + ensure_not_contains( + prepare_windows_host_action, + "package-native-runtime.sh", + "shared Windows host action must not build a native runtime", + )?; + ensure_contains( + prepare_native_runtime_action, + r#"scripts/package-native-runtime.sh "${args[@]}""#, + "shared native-runtime action canonical runtime builder", + )?; + ensure_not_contains( + prepare_native_runtime_action, + "build-windows.ps1", + "shared native-runtime action must not build the Windows host", + )?; + ensure_contains( + compose_product_action, + "scripts/ci-compose-product-input.sh", + "shared product action canonical composition script", + )?; + + for (workflow, workflow_name) in [(ci_workflow, "main CI"), (pr_builds_workflow, "PR Builds")] { + let host = workflow_job_section(workflow, "windows_host_input") + .ok_or_else(|| format!("{workflow_name}: missing `windows_host_input` job"))?; + let cpu_runtime = workflow_job_section(workflow, "windows_cpu_runtime_input") + .ok_or_else(|| format!("{workflow_name}: missing `windows_cpu_runtime_input` job"))?; + let gpu_runtimes = workflow_job_section(workflow, "windows_gpu_runtime_inputs") + .ok_or_else(|| format!("{workflow_name}: missing `windows_gpu_runtime_inputs` job"))?; + let cpu_product = workflow_job_section(workflow, "windows_cpu_product") + .ok_or_else(|| format!("{workflow_name}: missing `windows_cpu_product` job"))?; + let gpu_products = workflow_job_section(workflow, "windows_gpu_products") + .ok_or_else(|| format!("{workflow_name}: missing `windows_gpu_products` job"))?; - for (workflow, context) in [ - (ci_workflow, "main CI"), - (pr_windows_targets, "PR Builds Windows targets"), - ] { - ensure_contains( - workflow, - "build-windows.ps1 -BuildProfile release -HostOnly", - &format!("{context} backend-neutral Windows host build"), - )?; - ensure_contains( - workflow, - "scripts/package-native-runtime.sh \\", - &format!("{context} packaged Windows native runtime"), - )?; - ensure_contains( - workflow, - "--target x86_64-pc-windows-msvc", - &format!("{context} Windows runtime target"), - )?; ensure_contains( - workflow, - "MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR", - &format!("{context} composed Windows runtime discovery"), + host, + "uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756", + &format!("{workflow_name} persistent MSVC host environment"), )?; ensure_contains( - workflow, - "scripts/ci-client-readiness-smoke.sh target/release/mesh-llm.exe target/release/native-runtimes", - &format!("{context} Windows client readiness smoke"), + host, + "uses: ./.github/actions/prepare-windows-host-input", + &format!("{workflow_name} shared immutable Windows host producer"), )?; + + for (runtime, runtime_name) in [ + (cpu_runtime, "CPU runtime"), + (gpu_runtimes, "GPU runtime matrix"), + ] { + ensure_contains( + runtime, + "uses: ./.github/actions/prepare-native-runtime-input", + &format!("{workflow_name} shared Windows {runtime_name} producer"), + )?; + ensure_contains( + runtime, + "target: x86_64-pc-windows-msvc", + &format!("{workflow_name} Windows {runtime_name} target"), + )?; + } + + for (product, product_name) in [ + (cpu_product, "CPU product"), + (gpu_products, "GPU product matrix"), + ] { + ensure_contains( + product, + "uses: ./.github/actions/compose-product-input", + &format!("{workflow_name} shared Windows {product_name} composer"), + )?; + ensure_contains( + product, + "binary_name: mesh-llm.exe", + &format!("{workflow_name} Windows {product_name} executable"), + )?; + ensure_contains( + product, + "readiness_smoke: \"true\"", + &format!("{workflow_name} Windows {product_name} readiness"), + )?; + + for forbidden in [ + "cargo ", + "dtolnay/rust-toolchain", + "Swatinem/rust-cache", + "mozilla-actions/sccache-action", + "scripts/build-windows.ps1", + "scripts/package-native-runtime.sh", + "prepare-windows-host-input", + "prepare-native-runtime-input", + ] { + ensure_not_contains( + product, + forbidden, + &format!("{workflow_name} Windows {product_name} composition-only contract"), + )?; + } + } } - ensure_not_contains( - pr_windows_targets, - "steps.llama_cache.outputs.cache-hit", - "PR Builds Windows target must not skip composition or startup on ABI-cache hits", - )?; - ensure_not_contains( - pr_windows_targets, - "Skipping Windows ${{ matrix.name }} launch smoke", - "PR Builds Windows target must not skip GPU client readiness", - )?; Ok(()) } @@ -742,6 +847,9 @@ core.exportVariable('SCCACHE_GHA_ENABLED', 'true') core.exportVariable('SCCACHE_GHA_ENABLED', 'false') core.exportVariable('SCCACHE_IGNORE_SERVER_IO_ERROR', '1') core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk,gha') +process.env.SCCACHE_WEBDAV_ENDPOINT +process.env.DEPOT_CACHE_TOKEN +core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk,webdav') core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk') core.exportVariable('SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'ignore') ['--start-server'] @@ -768,11 +876,38 @@ jobs: runs-on: ubuntu-24.04 "#; + const VALID_COMPOSITION_CONTAINER_WORKFLOW: &str = r#"env: + SCCACHE_DIR: ${{ github.workspace }}/../.sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + SCCACHE_MULTILEVEL_CHAIN: disk,gha + SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY: ignore +jobs: + compose_linux_cuda: + container: + image: example.invalid/runner@sha256:digest + steps: + - uses: actions/checkout@v5 + - name: Trust checkout directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - uses: ./.github/actions/compose-product-input + publish: + runs-on: ubuntu-24.04 +"#; + #[test] fn release_container_contract_accepts_remote_sccache_with_local_fallback() { check_release_container_contracts(VALID_CONTAINER_WORKFLOW, VALID_SCCACHE_ACTION).unwrap(); } + #[test] + fn release_container_contract_accepts_cache_free_product_composition() { + check_release_container_contracts( + VALID_COMPOSITION_CONTAINER_WORKFLOW, + VALID_SCCACHE_ACTION, + ) + .unwrap(); + } + #[test] fn release_container_contract_requires_safe_checkout() { let workflow = VALID_CONTAINER_WORKFLOW.replace( From 532f4f2dbf48229ebbccc5f8d7c9dda44934dcba Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 15:09:51 -0400 Subject: [PATCH 02/23] ci: configure job-local sccache storage --- .github/actions/configure-sccache-gha/action.yml | 14 ++++++++++++++ scripts/tests/test_ci_artifact_actions.py | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/.github/actions/configure-sccache-gha/action.yml b/.github/actions/configure-sccache-gha/action.yml index 4977c8db6b..9849219318 100644 --- a/.github/actions/configure-sccache-gha/action.yml +++ b/.github/actions/configure-sccache-gha/action.yml @@ -33,6 +33,19 @@ runs: process.env.SCCACHE_WEBDAV_TOKEN || process.env.DEPOT_CACHE_TOKEN || ''; + const diskCacheDirectory = + process.env.SCCACHE_DIR || + (process.env.RUNNER_TEMP + ? `${process.env.RUNNER_TEMP}/mesh-llm-sccache` + : ''); + if (!diskCacheDirectory) { + core.setFailed( + 'SCCACHE_DIR or RUNNER_TEMP is required for the job-local disk cache.', + ); + return; + } + await io.mkdirP(diskCacheDirectory); + core.exportVariable('SCCACHE_DIR', diskCacheDirectory); const diskOnlyEnvironment = () => ({ ...process.env, ACTIONS_CACHE_URL: '', @@ -42,6 +55,7 @@ runs: SCCACHE_WEBDAV_TOKEN: '', SCCACHE_WEBDAV_USERNAME: '', SCCACHE_WEBDAV_PASSWORD: '', + SCCACHE_DIR: diskCacheDirectory, }); if ( diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index 99b9a46909..66d1d604d1 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -390,6 +390,14 @@ def test_sccache_prefers_depot_webdav_with_disk_fallback(self) -> None: self.assertIn('default: "false"', action) self.assertIn("SCCACHE_WEBDAV_ENDPOINT", action) self.assertIn("DEPOT_CACHE_TOKEN", action) + self.assertIn("process.env.SCCACHE_DIR", action) + self.assertIn("process.env.RUNNER_TEMP", action) + self.assertIn("await io.mkdirP(diskCacheDirectory)", action) + self.assertIn( + "core.exportVariable('SCCACHE_DIR', diskCacheDirectory)", + action, + ) + self.assertIn("SCCACHE_DIR: diskCacheDirectory", action) self.assertIn("'disk,webdav'", action) self.assertIn("'disk'", action) self.assertIn( From 55a4d9c72de02e2e0ba9fade340a064a64121524 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 15:36:03 -0400 Subject: [PATCH 03/23] fix: harden Windows artifact composition --- scripts/ci-compose-product-input.sh | 96 +++++++++++++++++++---- scripts/tests/test_ci_artifact_actions.py | 42 +++++++++- 2 files changed, 120 insertions(+), 18 deletions(-) diff --git a/scripts/ci-compose-product-input.sh b/scripts/ci-compose-product-input.sh index ba92c2d44b..2fcc2797ea 100755 --- a/scripts/ci-compose-product-input.sh +++ b/scripts/ci-compose-product-input.sh @@ -21,9 +21,55 @@ else exit 1 fi +to_shell_path() { + local path="${1%$'\r'}" + if command -v cygpath >/dev/null 2>&1; then + cygpath -u "$path" + else + printf '%s\n' "$path" + fi +} + +to_workflow_path() { + local path="$1" + if command -v cygpath >/dev/null 2>&1; then + cygpath -m "$path" + else + printf '%s\n' "$path" + fi +} + +require_file() { + local label="$1" + local path="$2" + if [[ ! -f "$path" ]]; then + echo "$label is missing: $path" >&2 + exit 1 + fi +} + +require_nonempty_file() { + local label="$1" + local path="$2" + if [[ ! -s "$path" ]]; then + echo "$label is missing or empty: $path" >&2 + exit 1 + fi +} + +require_checksum_match() { + local label="$1" + local expected="$2" + local actual="$3" + if [[ "$actual" != "$expected" ]]; then + echo "$label checksum mismatch: expected $expected, got $actual" >&2 + exit 1 + fi +} + canonical_paths=() while IFS= read -r path; do - canonical_paths+=("$path") + canonical_paths+=("$(to_shell_path "$path")") done < <( "$python_bin" - \ "$GITHUB_WORKSPACE" \ @@ -93,31 +139,46 @@ host="$host_input_dir/$INPUT_BINARY_NAME" host_imports="$host_input_dir/host-imports.json" host_checksum="$host_input_dir/$INPUT_BINARY_NAME.sha256" -test -f "$host" +GITHUB_OUTPUT="$(to_shell_path "$GITHUB_OUTPUT")" +if [[ -n "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" ]]; then + INPUT_ATTESTATION_PUBLIC_KEY_FILE="$( + to_shell_path "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" + )" +fi +if [[ -n "$INPUT_ATTESTATION_VERIFIER" ]]; then + INPUT_ATTESTATION_VERIFIER="$(to_shell_path "$INPUT_ATTESTATION_VERIFIER")" +fi + +require_file "immutable host" "$host" chmod +x "$host" -test -s "$host_imports" -test -s "$host_checksum" +require_nonempty_file "host import report" "$host_imports" +require_nonempty_file "host checksum" "$host_checksum" expected_host_checksum="$(awk 'NR == 1 {print $1}' "$host_checksum")" if command -v sha256sum >/dev/null 2>&1; then actual_host_checksum="$(sha256sum "$host" | awk '{print $1}')" else actual_host_checksum="$(shasum -a 256 "$host" | awk '{print $1}')" fi -test "$actual_host_checksum" = "$expected_host_checksum" +require_checksum_match "immutable host" "$expected_host_checksum" "$actual_host_checksum" if [[ -n "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" ]]; then attestation_verifier="${INPUT_ATTESTATION_VERIFIER:-$host_input_dir/release-attestation-verifier}" verifier_checksum="$attestation_verifier.sha256" - test -s "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" - test -f "$attestation_verifier" - test -s "$verifier_checksum" + require_nonempty_file \ + "release attestation public key" \ + "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" + require_file "release attestation verifier" "$attestation_verifier" + require_nonempty_file "release attestation verifier checksum" "$verifier_checksum" expected_verifier_checksum="$(awk 'NR == 1 {print $1}' "$verifier_checksum")" if command -v sha256sum >/dev/null 2>&1; then actual_verifier_checksum="$(sha256sum "$attestation_verifier" | awk '{print $1}')" else actual_verifier_checksum="$(shasum -a 256 "$attestation_verifier" | awk '{print $1}')" fi - test "$actual_verifier_checksum" = "$expected_verifier_checksum" + require_checksum_match \ + "release attestation verifier" \ + "$expected_verifier_checksum" \ + "$actual_verifier_checksum" chmod +x "$attestation_verifier" "$attestation_verifier" release-attestation inspect \ --binary "$host" \ @@ -207,7 +268,7 @@ fi --runtime "$runtime_dir" \ --version "$version" \ --backend "$INPUT_BACKEND" -test -s "$output_dir/product-manifest.json" +require_nonempty_file "composed product manifest" "$output_dir/product-manifest.json" if [[ "$INPUT_READINESS_SMOKE" == "true" ]]; then MESH_LLM_NATIVE_RUNTIME_BUNDLE_DIR="$output_dir/native-runtimes" \ @@ -227,11 +288,14 @@ runtime_name="$(basename "$runtime_dir")" archive_path="$product_dir.tar.gz" rm -f -- "$archive_path" tar -C "$product_dir" -czf "$archive_path" . -test -s "$archive_path" +require_nonempty_file "composed product archive" "$archive_path" { - echo "product_dir=$product_dir" - echo "binary_path=$product_dir/$INPUT_BINARY_NAME" - echo "runtime_root=$product_dir/native-runtimes" - echo "runtime_dir=$product_dir/native-runtimes/$runtime_name" - echo "archive_path=$archive_path" + printf 'product_dir=%s\n' "$(to_workflow_path "$product_dir")" + printf 'binary_path=%s\n' \ + "$(to_workflow_path "$product_dir/$INPUT_BINARY_NAME")" + printf 'runtime_root=%s\n' \ + "$(to_workflow_path "$product_dir/native-runtimes")" + printf 'runtime_dir=%s\n' \ + "$(to_workflow_path "$product_dir/native-runtimes/$runtime_name")" + printf 'archive_path=%s\n' "$(to_workflow_path "$archive_path")" } >> "$GITHUB_OUTPUT" diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index 66d1d604d1..ea1c11c2d2 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -8,12 +8,15 @@ import subprocess import tarfile import tempfile +import tomllib import unittest ROOT = Path(__file__).resolve().parents[2] ACTIONS = ROOT / ".github" / "actions" COMPOSE_SCRIPT = ROOT / "scripts" / "ci-compose-product-input.sh" +RELEASE_FOOTER_MANIFEST = ROOT / "crates" / "mesh-llm-release-footer" / "Cargo.toml" +XTASK_MANIFEST = ROOT / "tools" / "xtask" / "Cargo.toml" class CiArtifactActionTests(unittest.TestCase): @@ -202,6 +205,19 @@ def test_windows_host_action_owns_the_neutral_host_integrity_contract( self.assertNotIn("package-native-runtime.sh", action) self.assertNotIn("compose-product", action) + def test_windows_attestation_verifier_stays_native_abi_free(self) -> None: + xtask = tomllib.loads(XTASK_MANIFEST.read_text(encoding="utf-8")) + xtask_dependencies = xtask["dependencies"] + self.assertEqual( + xtask_dependencies["mesh-llm-release-footer"], + {"workspace": True}, + ) + self.assertNotIn("mesh-llm-system", xtask_dependencies) + self.assertNotIn("skippy-ffi", xtask_dependencies) + + footer = tomllib.loads(RELEASE_FOOTER_MANIFEST.read_text(encoding="utf-8")) + self.assertEqual(set(footer["dependencies"]), {"hex", "sha2"}) + def test_windows_debug_host_uses_the_package_version_for_composition( self, ) -> None: @@ -221,6 +237,11 @@ def test_windows_routes_cover_every_shared_product_primitive(self) -> None: action.index("WINDOWS_CPU_INPUTS=") : action.index("# SDK smokes are consumer tests") ] + cpu_routing = routing[: routing.index("WINDOWS_GPU_INPUTS=")] + gpu_routing = routing[routing.index("WINDOWS_GPU_INPUTS=") :] + + self.assertIn("^crates/mesh-llm-release-footer/", cpu_routing) + self.assertNotIn("^crates/mesh-llm-release-footer/", gpu_routing) for primitive in ( "prepare-windows-host-input", @@ -256,6 +277,23 @@ def test_product_action_only_composes_verified_inputs(self) -> None: self.assertIn('archive_path="$product_dir.tar.gz"', script) self.assertIn('tar -C "$product_dir" -czf "$archive_path" .', script) + def test_product_composer_normalizes_windows_shell_boundaries(self) -> None: + script = COMPOSE_SCRIPT.read_text(encoding="utf-8") + + self.assertIn("local path=\"${1%$'\\r'}\"", script) + self.assertIn('cygpath -u "$path"', script) + self.assertIn('cygpath -m "$path"', script) + self.assertIn( + 'canonical_paths+=("$(to_shell_path "$path")")', + script, + ) + self.assertIn( + 'GITHUB_OUTPUT="$(to_shell_path "$GITHUB_OUTPUT")"', + script, + ) + self.assertIn('require_file "immutable host" "$host"', script) + self.assertNotIn('test -f "$host"', script) + def test_product_archive_preserves_verified_executable_modes(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: workspace = Path(temp_dir) @@ -307,8 +345,8 @@ def test_release_attestation_is_verified_without_compiling_in_composer( product_script, ) self.assertIn( - 'test "$actual_verifier_checksum" = ' - '"$expected_verifier_checksum"', + '"$expected_verifier_checksum" \\\n' + ' "$actual_verifier_checksum"', product_script, ) From aa51db670f6a0dd68b3e3a8eb417616006bceaf6 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 15:36:07 -0400 Subject: [PATCH 04/23] test: assert pinned nightly artifact action --- crates/mesh-llm/tests/qa_nightly_stability.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/mesh-llm/tests/qa_nightly_stability.rs b/crates/mesh-llm/tests/qa_nightly_stability.rs index 3f65496f91..c6ea18ecd9 100644 --- a/crates/mesh-llm/tests/qa_nightly_stability.rs +++ b/crates/mesh-llm/tests/qa_nightly_stability.rs @@ -86,7 +86,7 @@ fn nightly_stability_reusable_workflow_owns_execution() { "scripts/qa-nightly-stability.py", "Publish run summary", "$GITHUB_STEP_SUMMARY", - "actions/upload-artifact@v6", + "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0", ] { assert!( reusable.contains(expected), From 73f2ccd45e87cd7551494ee011395480bcd077d1 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 15:48:32 -0400 Subject: [PATCH 05/23] ci: allow superseded SDK smokes to cancel --- .github/workflows/ci.yml | 2 +- .github/workflows/pr_builds.yml | 2 +- scripts/tests/test_ci_workflow_artifacts.py | 2 ++ scripts/tests/test_pr_workflow_artifacts.py | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b13855a80b..42d9c5999e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -819,7 +819,7 @@ jobs: swift_sdk_smoke: needs: [changes, macos_cpu_artifact, macos_unit_tests] - if: ${{ always() && needs.macos_cpu_artifact.result == 'success' && (needs.macos_unit_tests.result == 'success' || needs.macos_unit_tests.result == 'skipped') && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.sdk == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ !cancelled() && needs.macos_cpu_artifact.result == 'success' && (needs.macos_unit_tests.result == 'success' || needs.macos_unit_tests.result == 'skipped') && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.sdk == 'true') && needs.changes.outputs.docs_only != 'true' }} permissions: contents: read uses: ./.github/workflows/sdk-smoke.yml diff --git a/.github/workflows/pr_builds.yml b/.github/workflows/pr_builds.yml index 19f5944a1a..890e4366a0 100644 --- a/.github/workflows/pr_builds.yml +++ b/.github/workflows/pr_builds.yml @@ -1013,7 +1013,7 @@ jobs: swift_sdk_smoke: needs: [changes, macos_cpu_artifact, macos_unit_tests] - if: ${{ always() && needs.macos_cpu_artifact.result == 'success' && (needs.macos_unit_tests.result == 'success' || needs.macos_unit_tests.result == 'skipped') && needs.changes.outputs.macos_inference_artifact_required == 'true' && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + if: ${{ !cancelled() && needs.macos_cpu_artifact.result == 'success' && (needs.macos_unit_tests.result == 'success' || needs.macos_unit_tests.result == 'skipped') && needs.changes.outputs.macos_inference_artifact_required == 'true' && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} uses: ./.github/workflows/sdk-smoke.yml with: sdk_kind: swift diff --git a/scripts/tests/test_ci_workflow_artifacts.py b/scripts/tests/test_ci_workflow_artifacts.py index 3b44db207d..00ab002807 100644 --- a/scripts/tests/test_ci_workflow_artifacts.py +++ b/scripts/tests/test_ci_workflow_artifacts.py @@ -325,6 +325,8 @@ def test_swift_smoke_uses_composed_macos_product(self) -> None: "needs: [changes, macos_cpu_artifact, macos_unit_tests]", swift, ) + self.assertIn("!cancelled()", swift) + self.assertNotIn("always()", swift) self.assertIn("needs.macos_cpu_artifact.result == 'success'", swift) self.assertIn("needs.macos_unit_tests.result == 'success'", swift) self.assertIn("needs.macos_unit_tests.result == 'skipped'", swift) diff --git a/scripts/tests/test_pr_workflow_artifacts.py b/scripts/tests/test_pr_workflow_artifacts.py index 39a718fd15..531c5efead 100644 --- a/scripts/tests/test_pr_workflow_artifacts.py +++ b/scripts/tests/test_pr_workflow_artifacts.py @@ -325,7 +325,8 @@ def test_macos_swift_gate_and_supported_targets_are_preserved(self) -> None: "needs: [changes, macos_cpu_artifact, macos_unit_tests]", swift, ) - self.assertIn("always()", swift) + self.assertIn("!cancelled()", swift) + self.assertNotIn("always()", swift) self.assertIn("needs.macos_cpu_artifact.result == 'success'", swift) self.assertIn("needs.macos_unit_tests.result == 'success'", swift) self.assertIn("needs.macos_unit_tests.result == 'skipped'", swift) From eeae8172c29e90b1b4dc186bb6d60dfa4cb2f023 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 16:01:24 -0400 Subject: [PATCH 06/23] docs: document cancellable CI fan-in gates --- ci/DEPOT_MIGRATION.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/DEPOT_MIGRATION.md b/ci/DEPOT_MIGRATION.md index edf2965e9e..78e33a314c 100644 --- a/ci/DEPOT_MIGRATION.md +++ b/ci/DEPOT_MIGRATION.md @@ -115,6 +115,9 @@ PRs optimize for the earliest reliable signal: - make SDK smokes consume the staged product runtime and reject hidden rebuilds; - keep public-mesh admission out of required PR checks. It remains an explicit manual integration probe, while product readiness uses hermetic local mDNS. +- gate fan-in jobs that must tolerate skipped dependencies with + [`!cancelled()`](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-cancellation), + never `always()`, so cancelling a superseded run releases its runner capacity. Main is the exhaustive trust boundary: From ac5e517d4064ce45b3387d5973ffc4064b83f796 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 18:14:54 -0400 Subject: [PATCH 07/23] ci: harden composable build graph and metrics --- .agents/skills/manage-ci/SKILL.md | 8 +- .../manage-ci/references/current-inventory.md | 30 +- .github/actionlint.yaml | 4 - .../actions/capture-sccache-stats/action.yml | 57 + .../actions/capture-sccache-stats/capture.py | 189 ++ .github/actions/compute-changes/action.yml | 44 +- .../actions/configure-sccache-gha/action.yml | 37 + .../actions/restore-smoke-inputs/action.yml | 158 +- .../restore-windows-abi-cache/action.yml | 190 ++ .github/workflows/ci.yml | 309 +-- .github/workflows/llama-upstream-canary.yml | 2 +- .github/workflows/nightly-stability-run.yml | 4 +- .github/workflows/pr_builds.yml | 381 +++- .github/workflows/pr_cleanup.yml | 6 +- .github/workflows/pr_quality.yml | 41 +- .github/workflows/pr_website.yml | 2 +- .github/workflows/release.yml | 135 +- .github/workflows/windows-warm-caches.yml | 148 +- ci/DEPOT_MIGRATION.md | 23 +- ci/METRICS.md | 90 + ci/ci.md | 56 +- ci/metrics/2026-07-29-pr-builds-baseline.json | 1747 +++++++++++++++++ scripts/build-windows.ps1 | 5 +- scripts/ci-compose-product-input.sh | 47 +- ...enerate-native-runtime-release-manifest.sh | 78 +- scripts/package-native-sdk.sh | 1 - scripts/safe-extract-tar.py | 186 ++ scripts/summarize-sccache-stats.py | 168 ++ scripts/tests/test_build_windows.py | 105 +- scripts/tests/test_ci_artifact_actions.py | 408 +++- .../tests/test_ci_prepare_native_runtime.py | 36 +- scripts/tests/test_ci_workflow_artifacts.py | 123 +- ...enerate_native_runtime_release_manifest.py | 200 +- .../tests/test_native_artifact_verifiers.py | 531 +++++ scripts/tests/test_pr_workflow_artifacts.py | 155 +- .../tests/test_release_workflow_artifacts.py | 111 ++ scripts/tests/test_safe_extract_tar.py | 238 +++ scripts/tests/test_sccache_evidence.py | 297 +++ .../tests/test_windows_native_runtime_deps.py | 58 +- scripts/verify-checksum-sidecar.py | 68 + scripts/verify-native-runtime-package.sh | 267 ++- scripts/verify-native-sdk-package.sh | 218 +- 42 files changed, 6237 insertions(+), 724 deletions(-) create mode 100644 .github/actions/capture-sccache-stats/action.yml create mode 100644 .github/actions/capture-sccache-stats/capture.py create mode 100644 .github/actions/restore-windows-abi-cache/action.yml create mode 100644 ci/metrics/2026-07-29-pr-builds-baseline.json create mode 100755 scripts/safe-extract-tar.py create mode 100644 scripts/summarize-sccache-stats.py create mode 100644 scripts/tests/test_native_artifact_verifiers.py create mode 100644 scripts/tests/test_safe_extract_tar.py create mode 100644 scripts/tests/test_sccache_evidence.py create mode 100755 scripts/verify-checksum-sidecar.py diff --git a/.agents/skills/manage-ci/SKILL.md b/.agents/skills/manage-ci/SKILL.md index e0ce80e130..4bbbed0faf 100644 --- a/.agents/skills/manage-ci/SKILL.md +++ b/.agents/skills/manage-ci/SKILL.md @@ -283,6 +283,10 @@ update the skill resources in the same change. reuse unsafe: OS, architecture, backend/toolchain, relevant lockfiles, `.github/cache-version.txt`, and build inputs. Do not broaden restore keys across incompatible or untrusted contexts. +- Windows native-runtime producers and the trusted warmer must use + `.github/actions/restore-windows-abi-cache`. Keep CPU, CUDA, ROCm, and Vulkan + architecture/toolchain identities exact; do not duplicate its key expression + in individual workflows or add broad restore prefixes. - GitHub-hosted PR jobs may share the normal key namespace with main because GitHub scopes PR writes to the merge ref and trusted main does not restore them. Do not assume that isolation applies to another cache provider. @@ -298,7 +302,9 @@ update the skill resources in the same change. logs. - Use `retention-days: 1` for PR and smoke-only artifacts unless a documented debugging or release requirement needs longer retention. Release evidence - follows the release policy, not the PR default. + follows the release policy, not the PR default. Sccache migration evidence is + retained for 14 days so cold/warm samples cover the configured Depot + cache-retention window. - Restore producer artifacts through `.github/actions/restore-smoke-inputs`. Reuse `smoke.yml`, `scripted-binary-smoke.yml`, `sdk-smoke.yml`, and `hf-download-smoke.yml`; do not rebuild MeshLLM, native runtimes, or duplicate diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 69f80789ba..2441822539 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -57,10 +57,11 @@ prebuilt verifier for the immutable host instead of compiling workspace code. `ci.yml` applies the same executable-product rule to trusted main validation. Linux and macOS build immutable release-profile hosts and separately packaged CPU or Metal runtimes, then upload complete product-v2 trees from -composition-only jobs. Linux CUDA, ROCm, and Vulkan rows download the same -Linux host, build only one selected runtime, and compose a product through the -shared action. SDK consumers reuse the producer's adjacent runtime and fail if -CI would silently rebuild it. Windows likewise builds one immutable +composition-only jobs. Linux CUDA, ROCm, and Vulkan each use an independent +runtime producer plus a thin composer that downloads the same immutable Linux +host; no backend waits on a matrix-wide fan-in. SDK consumers reuse the +producer's adjacent runtime and fail if CI would silently rebuild it. Windows +likewise builds one immutable release-profile host, independent CPU/CUDA/ROCm/Vulkan runtime inputs, and composition-only products. Broad main Rust changes exercise the Windows CPU product; Windows GPU products remain limited to GPU/backend inputs or manual @@ -68,9 +69,9 @@ dispatch. Every composed backend product requires `runtime list` plus no-driver client readiness; hosted GPU rows neither inject a driver stub nor skip startup because no device is present. -`pr_builds.yml` uses the same split producer/composer shape for Linux CPU and -macOS Metal products while retaining debug-profile hosts for lightweight PR -iteration. Windows broad-Rust validation stays at lightweight Cargo checks; +`pr_builds.yml` uses the same split producer/composer shape for Linux CPU/GPU +and macOS Metal products while retaining debug-profile hosts for lightweight +PR iteration. Windows broad-Rust validation stays at lightweight Cargo checks; the debug host plus CPU or GPU runtime/product graph runs only for its platform/backend input or manual dispatch. Unsupported macOS CUDA, ROCm, and Vulkan combinations are omitted rather than emitted as no-op jobs. @@ -87,8 +88,12 @@ Local actions: permission is derived from the same typed trust decision. - `.github/actions/configure-sccache-gha` exports ephemeral Actions cache credentials to the baked `sccache`, permits Depot WebDAV only for an explicit - trusted call, and uses disk-only storage if a future pull-request trust - context is ever evaluated on Depot. + trusted call, uses disk-only storage if a future pull-request trust context is + ever evaluated on Depot, and resets counters after configuring the server. +- `.github/actions/capture-sccache-stats` validates and uploads one + machine-readable sccache evidence artifact per instrumented job or matrix + row. Evidence is retained for 14 days so cold/warm samples span the configured + Depot cache-retention window. - `.github/actions/prepare-host-input` owns Unix neutral-host build, optional release attestation, import-policy verification, and checksumming. - `.github/actions/prepare-windows-host-input` owns the equivalent Windows @@ -100,6 +105,10 @@ Local actions: product-v2 tree without compiling, and runs CLI/client readiness. - `.github/actions/restore-smoke-inputs` owns producer artifact staging and model restoration for smoke consumers. +- `.github/actions/restore-windows-abi-cache` owns the exact Windows CPU, + CUDA, ROCm, and Vulkan ABI cache identity shared by the trusted warmer and + PR/main/release runtime producers. Architecture sets and toolchain versions + are compatibility boundaries; the action never uses restore prefixes. - `.github/actions/setup-windows-rocm-sdk` owns reusable Windows ROCm setup. Routing and test-planning scripts: @@ -112,6 +121,9 @@ Routing and test-planning scripts: workflow-owned test allowlist. - `scripts/test-portable.sh` owns the portable non-Cargo test aggregate used by the local `test-all` path. +- `scripts/summarize-sccache-stats.py` aggregates downloaded sccache JSON + evidence offline and can enforce the migration hit-rate threshold without + GitHub or network access. ## Runner and image contract diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 31dd3e4cc5..04f45ef4d7 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -8,10 +8,6 @@ self-hosted-runner: - gpu-nvidia - mesh-llm-amd64 - mesh-llm-arm64 - - blacksmith-4vcpu-ubuntu-2404 - - blacksmith-6vcpu-macos-15 - - blacksmith-4vcpu-ubuntu-2404-arm - - blacksmith-8vcpu-windows-2025 - depot-ubuntu-24.04 - depot-ubuntu-24.04-4 - depot-ubuntu-24.04-8 diff --git a/.github/actions/capture-sccache-stats/action.yml b/.github/actions/capture-sccache-stats/action.yml new file mode 100644 index 0000000000..5119d2702c --- /dev/null +++ b/.github/actions/capture-sccache-stats/action.yml @@ -0,0 +1,57 @@ +name: Capture sccache statistics +description: Validate sccache counters and upload machine-readable build-cache evidence. + +inputs: + artifact_name: + description: Unique artifact name for this workflow job and matrix row. + required: true + +outputs: + stats_file: + description: Absolute path to the captured sccache JSON document. + value: ${{ steps.capture.outputs.stats_file }} + compile_requests: + description: Number of compiler requests observed since sccache was configured. + value: ${{ steps.capture.outputs.compile_requests }} + requests_executed: + description: Number of compiler requests executed since sccache was configured. + value: ${{ steps.capture.outputs.requests_executed }} + cache_hits: + description: Number of cache hits observed since sccache was configured. + value: ${{ steps.capture.outputs.cache_hits }} + cache_misses: + description: Number of cache misses observed since sccache was configured. + value: ${{ steps.capture.outputs.cache_misses }} + cache_writes: + description: Number of cache writes observed since sccache was configured. + value: ${{ steps.capture.outputs.cache_writes }} + cache_read_errors: + description: Number of cache read errors observed since sccache was configured. + value: ${{ steps.capture.outputs.cache_read_errors }} + cache_write_errors: + description: Number of cache write errors observed since sccache was configured. + value: ${{ steps.capture.outputs.cache_write_errors }} + +runs: + using: composite + steps: + - name: Capture and validate sccache counters + id: capture + shell: bash + env: + SCCACHE_STATS_ARTIFACT_NAME: ${{ inputs.artifact_name }} + SCCACHE_STATS_OUTPUT_DIR: ${{ runner.temp }}/mesh-llm-sccache-evidence + run: | + set -euo pipefail + python3 "$GITHUB_ACTION_PATH/capture.py" \ + --artifact-name "$SCCACHE_STATS_ARTIFACT_NAME" \ + --output "$SCCACHE_STATS_OUTPUT_DIR/$SCCACHE_STATS_ARTIFACT_NAME/sccache-stats.json" \ + --github-output "$GITHUB_OUTPUT" + + - name: Upload machine-readable sccache evidence + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ inputs.artifact_name }} + path: ${{ steps.capture.outputs.stats_file }} + if-no-files-found: error + retention-days: 14 diff --git a/.github/actions/capture-sccache-stats/capture.py b/.github/actions/capture-sccache-stats/capture.py new file mode 100644 index 0000000000..d09b2a61ea --- /dev/null +++ b/.github/actions/capture-sccache-stats/capture.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Capture and validate sccache statistics for CI evidence.""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + + +ARTIFACT_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +REQUIRED_COUNTERS = ( + "compile_requests", + "requests_executed", + "compilations", + "cache_writes", + "cache_read_errors", + "cache_write_errors", +) +REQUIRED_COUNT_MAPS = ("cache_hits", "cache_misses", "cache_errors") + + +class EvidenceError(RuntimeError): + """Raised when sccache cannot provide trustworthy evidence.""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--artifact-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--github-output", type=Path) + return parser.parse_args() + + +def require_counter(stats: dict[str, Any], name: str) -> int: + value = stats.get(name) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise EvidenceError( + f"sccache JSON field stats.{name} must be a non-negative integer", + ) + return value + + +def validate_count_tree(value: Any, field: str) -> int: + if isinstance(value, bool): + raise EvidenceError(f"sccache JSON field {field} contains a boolean") + if isinstance(value, int): + if value < 0: + raise EvidenceError( + f"sccache JSON field {field} contains a negative counter", + ) + return value + if isinstance(value, dict): + return sum( + validate_count_tree(child, f"{field}.{name}") + for name, child in value.items() + ) + raise EvidenceError( + f"sccache JSON field {field} must contain only counter maps and integers", + ) + + +def require_count_map(stats: dict[str, Any], name: str) -> int: + value = stats.get(name) + if not isinstance(value, dict): + raise EvidenceError(f"sccache JSON field stats.{name} must be an object") + counts = value.get("counts") + if not isinstance(counts, dict): + raise EvidenceError( + f"sccache JSON field stats.{name}.counts must be an object", + ) + validate_count_tree(value, f"stats.{name}") + return validate_count_tree(counts, f"stats.{name}.counts") + + +def validate_stats(payload: Any) -> dict[str, int]: + if not isinstance(payload, dict): + raise EvidenceError("sccache JSON root must be an object") + stats = payload.get("stats") + if not isinstance(stats, dict): + raise EvidenceError("sccache JSON field stats must be an object") + + counters = {name: require_counter(stats, name) for name in REQUIRED_COUNTERS} + counters.update( + {name: require_count_map(stats, name) for name in REQUIRED_COUNT_MAPS}, + ) + return counters + + +def run_sccache(arguments: list[str], *, capture: bool = False) -> str: + result = subprocess.run( + ["sccache", *arguments], + check=False, + capture_output=capture, + text=True, + ) + if result.returncode != 0: + detail = result.stderr.strip() if capture else "" + suffix = f": {detail}" if detail else "" + raise EvidenceError( + f"sccache {' '.join(arguments)} failed with " + f"exit code {result.returncode}{suffix}", + ) + return result.stdout if capture else "" + + +def write_github_outputs( + destination: Path | None, + stats_file: Path, + counters: dict[str, int], +) -> None: + if destination is None: + return + with destination.open("a", encoding="utf-8") as output: + output.write(f"stats_file={stats_file}\n") + for name in ( + "compile_requests", + "requests_executed", + "cache_hits", + "cache_misses", + "cache_writes", + "cache_read_errors", + "cache_write_errors", + ): + output.write(f"{name}={counters[name]}\n") + + +def main() -> int: + arguments = parse_args() + try: + if not ARTIFACT_NAME_PATTERN.fullmatch(arguments.artifact_name): + raise EvidenceError( + "artifact name must contain only letters, numbers, dots, " + "underscores, and hyphens", + ) + if shutil.which("sccache") is None: + raise EvidenceError("sccache is required to capture build-cache evidence") + + print("::group::Human-readable sccache statistics", flush=True) + try: + run_sccache(["--show-stats"]) + finally: + print("::endgroup::", flush=True) + + raw_json = run_sccache( + ["--show-stats", "--stats-format", "json"], + capture=True, + ) + try: + payload = json.loads(raw_json) + except json.JSONDecodeError as error: + raise EvidenceError(f"sccache returned invalid JSON: {error}") from error + counters = validate_stats(payload) + + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + raw_json.rstrip("\n") + "\n", + encoding="utf-8", + ) + stats_file = arguments.output.resolve() + write_github_outputs(arguments.github_output, stats_file, counters) + + print( + "sccache evidence: " + f"requests={counters['compile_requests']} " + f"executed={counters['requests_executed']} " + f"hits={counters['cache_hits']} " + f"misses={counters['cache_misses']} " + f"writes={counters['cache_writes']}", + ) + if counters["compile_requests"] == 0: + print( + "::warning title=sccache reported zero compile requests::" + "Check RUSTC_WRAPPER wiring unless this job fully reused its " + "restored target cache.", + ) + except EvidenceError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/actions/compute-changes/action.yml b/.github/actions/compute-changes/action.yml index e22d7d3412..fa9de33eb4 100644 --- a/.github/actions/compute-changes/action.yml +++ b/.github/actions/compute-changes/action.yml @@ -6,11 +6,11 @@ inputs: description: 'GitHub event name (pull_request, push, workflow_dispatch)' required: true base_sha: - description: 'PR base SHA (empty for non-PR events)' + description: 'Comparison base SHA for pull_request or push events' required: false default: '' head_sha: - description: 'PR head SHA (empty for non-PR events)' + description: 'Comparison head SHA for pull_request or push events' required: false default: '' @@ -90,8 +90,18 @@ runs: # For pull_request: use base_sha...head_sha git diff --name-only "${{ inputs.base_sha }}...${{ inputs.head_sha }}" > /tmp/changed_files.txt elif [[ "${{ inputs.event_name }}" == "push" ]]; then - # For push: use HEAD^ HEAD - git diff --name-only HEAD^ HEAD > /tmp/changed_files.txt + # A push may contain more than one commit. Diff the complete event + # range; never classify only the final commit. New branches, missing + # history, and malformed payloads fail open to the full CI graph. + base_sha="${{ inputs.base_sha }}" + head_sha="${{ inputs.head_sha }}" + if [[ -z "$base_sha" || -z "$head_sha" || "$base_sha" =~ ^0+$ ]] || + ! git cat-file -e "$base_sha^{commit}" 2>/dev/null || + ! git cat-file -e "$head_sha^{commit}" 2>/dev/null; then + echo "__force_all__" > /tmp/changed_files.txt + elif ! git diff --name-only "$base_sha" "$head_sha" > /tmp/changed_files.txt; then + echo "__force_all__" > /tmp/changed_files.txt + fi elif [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]; then # For workflow_dispatch: force all by using a known Rust file echo "__force_all__" > /tmp/changed_files.txt @@ -142,6 +152,11 @@ runs: TEST_BATCHES=$(bash scripts/plan-test-batches.sh --crates-json "$AFFECTED_CRATES" --bins 4) fi + FORCE_ALL="false" + if grep -qx "__force_all__" /tmp/changed_files.txt; then + FORCE_ALL="true" + fi + # Read changed files for docs_only and rust_changed logic CHANGED_FILES=$(cat /tmp/changed_files.txt | grep -v "^__force_all__$" || true) @@ -149,7 +164,7 @@ runs: if [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]; then RUNNER_CONTRACT_REQUIRED="true" elif [[ -n "$CHANGED_FILES" ]]; then - RUNNER_CONTRACT_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^\.github/cache-version\.txt$|^\.github/actionlint\.yaml$|^\.github/actions/(configure-sccache-gha|select-ci-runners)/|^\.github/workflows/(ci|depot-canary|pr_builds|pr_quality|release)\.yml$)' || true) + RUNNER_CONTRACT_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^\.github/cache-version\.txt$|^\.github/actionlint\.yaml$|^\.github/actions/(capture-sccache-stats|configure-sccache-gha|select-ci-runners)/|^\.github/workflows/(ci|depot-canary|pr_builds|pr_quality|release)\.yml$)' || true) if [[ -n "$RUNNER_CONTRACT_INPUTS" ]]; then RUNNER_CONTRACT_REQUIRED="true" fi @@ -207,7 +222,7 @@ runs: } function is_backend_recipe(name) { - return name ~ /^(with-lld|build|build-dev|build-mac|build-linux|build-runtime|release-build|release-build-[[:alnum:]-]+|llama-prepare|llama-prepare-latest|llama-build|bundle|release-bundle|release-bundle-[[:alnum:]-]+)$/ + return name ~ /^(with-lld|build|build-dev|build-mac|build-linux|build-runtime|release-host-build|release-host-build-windows|release-runtime-build|release-build|release-build-[[:alnum:]-]+|llama-prepare|llama-prepare-latest|llama-build|bundle|release-bundle|release-bundle-[[:alnum:]-]+)$/ } { @@ -267,8 +282,8 @@ runs: JUSTFILE_BASE_AVAILABLE="true" fi elif [[ "${{ inputs.event_name }}" == "push" ]]; then - JUSTFILE_DIFF=$(git diff -U0 HEAD^ HEAD -- Justfile || true) - if git show HEAD^:Justfile > "$JUSTFILE_BASE" 2>/dev/null; then + JUSTFILE_DIFF=$(git diff -U0 "${{ inputs.base_sha }}" "${{ inputs.head_sha }}" -- Justfile || true) + if git show "${{ inputs.base_sha }}:Justfile" > "$JUSTFILE_BASE" 2>/dev/null; then JUSTFILE_BASE_AVAILABLE="true" fi fi @@ -300,7 +315,7 @@ runs: if [[ "$ALL_RUST" == "true" ]]; then BACKEND_CHANGED="true" elif [[ -n "$CHANGED_FILES" ]]; then - BACKEND_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^third_party/llama\.cpp/|^crates/skippy-ffi/|^scripts/(build-llama|prepare-llama|build-linux|build-linux-rocm|build-mac|build-windows|install-windows-sdk|build-host|build-release|package-native-runtime|verify-native-runtime-package|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^\.github/actions/(prepare-host-input|prepare-windows-host-input|prepare-native-runtime-input|compose-product-input|restore-smoke-inputs|setup-windows-rocm-sdk)/|^\.github/workflows/(ci|pr_builds|release|sdk-smoke|smoke)\.yml$|^\.github/cache-version\.txt$)' || true) + BACKEND_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^third_party/llama\.cpp/|^crates/skippy-ffi/|^scripts/(build-llama|prepare-llama|build-linux|build-linux-rocm|build-mac|build-windows|install-windows-sdk|build-host|build-release|package-release|package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^\.github/actions/(prepare-host-input|prepare-windows-host-input|prepare-native-runtime-input|compose-product-input|restore-smoke-inputs|restore-windows-abi-cache|setup-windows-rocm-sdk)/|^\.github/workflows/(ci|pr_builds|release|sdk-smoke|smoke)\.yml$|^\.github/cache-version\.txt$)' || true) if [[ -n "$BACKEND_INPUTS" ]] || [[ "$BACKEND_RECIPE_CHANGED" == "true" ]]; then BACKEND_CHANGED="true" fi @@ -308,9 +323,12 @@ runs: WINDOWS_CPU_BUILD_REQUIRED="false" WINDOWS_GPU_BUILD_REQUIRED="false" - if [[ -n "$CHANGED_FILES" ]]; then - WINDOWS_CPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/mesh-llm-release-footer/|^crates/mesh-llm-nodejs/|^crates/skippy-ffi/|^scripts/build-windows\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^third_party/llama\.cpp/|^Cargo\.toml$|^Cargo\.lock$|^\.github/cache-version\.txt$|^\.github/actions/(compute-changes/action\.yml$|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/))' || true) - WINDOWS_GPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/skippy-ffi/|^scripts/(build-windows|install-windows-sdk)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^scripts/windows-native-runtime-deps\.py$|^scripts/tests/test_windows_native_runtime_deps\.py$|^third_party/llama\.cpp/|^\.github/cache-version\.txt$|^\.github/actions/(compute-changes/action\.yml$|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|setup-windows-rocm-sdk/))' || true) + if [[ "$FORCE_ALL" == "true" ]]; then + WINDOWS_CPU_BUILD_REQUIRED="true" + WINDOWS_GPU_BUILD_REQUIRED="true" + elif [[ -n "$CHANGED_FILES" ]]; then + WINDOWS_CPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/mesh-llm-release-footer/|^crates/mesh-llm-nodejs/|^crates/skippy-ffi/|^scripts/(build-windows|package-release)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^third_party/llama\.cpp/|^Cargo\.toml$|^Cargo\.lock$|^\.github/cache-version\.txt$|^\.github/workflows/(ci|pr_builds|release|windows-warm-caches)\.yml$|^\.github/actions/(compute-changes/action\.yml$|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|restore-windows-abi-cache/))' || true) + WINDOWS_GPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/skippy-ffi/|^scripts/(build-windows|install-windows-sdk|package-release)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^scripts/windows-native-runtime-deps\.py$|^scripts/tests/test_windows_native_runtime_deps\.py$|^third_party/llama\.cpp/|^\.github/cache-version\.txt$|^\.github/workflows/(ci|pr_builds|release|windows-warm-caches)\.yml$|^\.github/actions/(compute-changes/action\.yml$|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|restore-windows-abi-cache/|setup-windows-rocm-sdk/))' || true) if [[ -n "$WINDOWS_CPU_INPUTS" ]] || [[ "$BACKEND_RECIPE_CHANGED" == "true" ]]; then WINDOWS_CPU_BUILD_REQUIRED="true" fi @@ -325,7 +343,7 @@ runs: if [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]; then SDK_SMOKE_REQUIRED="true" elif [[ -n "$CHANGED_FILES" ]]; then - DIRECT_SDK_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^sdk/|^Package\.swift$|^scripts/ci-(rust|kotlin|swift)-sdk-smoke\.sh$|^scripts/ci-prepare-native-runtime\.sh$|^scripts/ci-sdk-fixture\.sh$|^\.github/workflows/sdk-smoke\.yml$)' || true) + DIRECT_SDK_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^sdk/|^Package\.swift$|^scripts/ci-(rust|kotlin|swift)-sdk-smoke\.sh$|^scripts/ci-prepare-native-runtime\.sh$|^scripts/ci-sdk-fixture\.sh$|^scripts/(check-sdk-contract|package-sdk-console-assets|verify-sdk-console-assets|verify-swift-privacy-manifest|verify-swift-release-artifact|prepare-llama|build-llama)\.sh$|^scripts/(package-native-sdk|package-native-sdk-crate|verify-native-sdk-package|verify-checksum-sidecar|safe-extract-tar)\.(sh|py)$|^\.github/actions/restore-smoke-inputs/|^\.github/workflows/sdk-smoke\.yml$)' || true) if [[ -n "$DIRECT_SDK_INPUTS" ]]; then SDK_SMOKE_REQUIRED="true" elif echo "$AFFECTED_CRATES" | jq -e 'index("mesh-llm-client") or index("mesh-llm-api-client") or index("mesh-llm-api-server") or index("mesh-llm-config") or index("mesh-llm-console-server") or index("mesh-llm-ffi") or index("mesh-llm-native-runtime") or index("mesh-llm-protocol") or index("mesh-llm-routing") or index("mesh-llm-types")' >/dev/null; then diff --git a/.github/actions/configure-sccache-gha/action.yml b/.github/actions/configure-sccache-gha/action.yml index 9849219318..2e278886c8 100644 --- a/.github/actions/configure-sccache-gha/action.yml +++ b/.github/actions/configure-sccache-gha/action.yml @@ -57,6 +57,23 @@ runs: SCCACHE_WEBDAV_PASSWORD: '', SCCACHE_DIR: diskCacheDirectory, }); + const resetStatistics = async (environment = process.env) => { + const resetExitCode = await exec.exec( + 'sccache', + ['--zero-stats'], + { + env: environment, + ignoreReturnCode: true, + }, + ); + if (resetExitCode !== 0) { + core.setFailed( + 'Unable to reset sccache statistics after configuring the server.', + ); + return false; + } + return true; + }; if ( allowDepotRemoteCache === 'true' && @@ -77,6 +94,9 @@ runs: ignoreReturnCode: true, }); if (depotExitCode === 0) { + if (!(await resetStatistics())) { + return; + } core.info( 'Baked sccache is using job-local disk with Depot WebDAV cache.', ); @@ -99,6 +119,10 @@ runs: ); if (depotFallbackExitCode !== 0) { core.setFailed('Unable to start baked sccache with Depot or job-local storage.'); + return; + } + if (!(await resetStatistics(diskOnlyEnvironment()))) { + return; } return; } @@ -132,6 +156,10 @@ runs: core.setFailed( 'Unable to start baked sccache with its trust-isolated disk cache.', ); + return; + } + if (!(await resetStatistics(diskOnlyEnvironment()))) { + return; } return; } @@ -166,6 +194,10 @@ runs: ); if (diskOnlyExitCode !== 0) { core.setFailed('Unable to start baked sccache with its disk-only cache.'); + return; + } + if (!(await resetStatistics(diskOnlyEnvironment()))) { + return; } return; } @@ -194,6 +226,9 @@ runs: ignoreReturnCode: true, }); if (remoteExitCode === 0) { + if (!(await resetStatistics())) { + return; + } core.info( 'Baked sccache is using job-local disk with a best-effort GitHub Actions cache.', ); @@ -219,4 +254,6 @@ runs: ); if (localExitCode !== 0) { core.setFailed('Unable to start baked sccache with either remote or job-local storage.'); + return; } + await resetStatistics(diskOnlyEnvironment()); diff --git a/.github/actions/restore-smoke-inputs/action.yml b/.github/actions/restore-smoke-inputs/action.yml index 8d815b0ef5..ab84da72a5 100644 --- a/.github/actions/restore-smoke-inputs/action.yml +++ b/.github/actions/restore-smoke-inputs/action.yml @@ -60,36 +60,168 @@ runs: echo "expected exactly one composed product archive in $artifact_path; found $archive_count" >&2 exit 1 fi - tar -xzf "$archive" -C "$artifact_path" - rm -f -- "$archive" - product_manifest="$artifact_path/product-manifest.json" + extract_root="$(mktemp -d)" + trap 'rm -rf -- "$extract_root"' EXIT + python3 scripts/safe-extract-tar.py "$archive" "$extract_root" + + product_manifest="$extract_root/product-manifest.json" test -s "$product_manifest" IFS=$'\t' read -r version backend host_path runtime_path < <( - python3 - "$product_manifest" <<'PY' + python3 - \ + "$product_manifest" \ + "$extract_root" \ + "${{ inputs.binary_name }}" <<'PY' import json + from pathlib import Path, PurePosixPath + import re import sys - with open(sys.argv[1], encoding="utf-8") as handle: + manifest_path, product_root_raw, binary_name = sys.argv[1:4] + product_root = Path(product_root_raw).resolve(strict=True) + windows_drive = re.compile(r"^[A-Za-z]:") + + + def product_path(label, raw_path, *, require_directory): + if ( + not isinstance(raw_path, str) + or not raw_path + or any(character in raw_path for character in ("\0", "\r", "\n", "\t")) + or "\\" in raw_path + ): + raise SystemExit(f"{label} must be a portable relative path") + relative = PurePosixPath(raw_path) + if ( + relative.is_absolute() + or windows_drive.match(raw_path) + or ".." in relative.parts + ): + raise SystemExit( + f"{label} escapes the composed product: {raw_path}" + ) + candidate = product_root.joinpath(*relative.parts) + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(product_root) + except (OSError, ValueError): + raise SystemExit( + f"{label} is missing or escapes the composed product: " + f"{raw_path}" + ) from None + if candidate.is_symlink(): + raise SystemExit(f"{label} cannot be a symlink: {raw_path}") + if require_directory != resolved.is_dir(): + expected = "directory" if require_directory else "file" + raise SystemExit(f"{label} must resolve to a {expected}: {raw_path}") + return relative.as_posix() + + + with open(manifest_path, encoding="utf-8") as handle: manifest = json.load(handle) + if not isinstance(manifest, dict): + raise SystemExit("product manifest must be an object") + if manifest.get("schema_version") != 2: + raise SystemExit("product manifest schema_version must be 2") + if manifest.get("contract") != "mesh-llm-product-v2": + raise SystemExit("product manifest contract is invalid") + + version = manifest.get("mesh_version") + backend = manifest.get("backend") + if not isinstance(version, str) or not version: + raise SystemExit("product mesh_version must be a non-empty string") + if backend not in { + "cpu", + "metal", + "cuda", + "cuda-blackwell", + "rocm", + "hip", + "vulkan", + }: + raise SystemExit(f"unsupported product backend: {backend!r}") + + host = manifest.get("host") + runtime = manifest.get("runtime") + if not isinstance(host, dict) or not isinstance(runtime, dict): + raise SystemExit("product host and runtime entries must be objects") + host_path = product_path( + "product host", + host.get("path"), + require_directory=False, + ) + if host_path != binary_name: + raise SystemExit( + f"product host path must be {binary_name!r}, got {host_path!r}" + ) + product_path( + "host import report", + "host-imports.json", + require_directory=False, + ) + runtime_path = product_path( + "product runtime", + runtime.get("path"), + require_directory=True, + ) + runtime_parts = PurePosixPath(runtime_path).parts + if len(runtime_parts) != 2 or runtime_parts[0] != "native-runtimes": + raise SystemExit( + "product runtime must be one direct child of native-runtimes" + ) + allowed_top_level = { + binary_name, + "host-imports.json", + "native-runtimes", + "product-manifest.json", + } + actual_top_level = {entry.name for entry in product_root.iterdir()} + if actual_top_level != allowed_top_level: + unexpected = sorted(actual_top_level - allowed_top_level) + missing = sorted(allowed_top_level - actual_top_level) + raise SystemExit( + "product top-level contents are not canonical: " + f"unexpected={unexpected}, missing={missing}" + ) + top_level_links = sorted( + entry.name for entry in product_root.iterdir() if entry.is_symlink() + ) + if top_level_links: + raise SystemExit( + f"product top-level entries cannot be symlinks: {top_level_links}" + ) + runtime_entries = list((product_root / "native-runtimes").iterdir()) + selected_runtime = product_root.joinpath( + *PurePosixPath(runtime_path).parts + ) + if ( + len(runtime_entries) != 1 + or runtime_entries[0].resolve(strict=True) + != selected_runtime.resolve(strict=True) + ): + raise SystemExit( + "product must contain exactly its manifest-selected runtime" + ) print( - manifest["mesh_version"], - manifest["backend"], - manifest["host"]["path"], - manifest["runtime"]["path"], + version, + backend, + host_path, + runtime_path, sep="\t", ) PY ) - scripts/verify-native-runtime-package.sh "$artifact_path/$runtime_path" + scripts/verify-native-runtime-package.sh "$extract_root/$runtime_path" python3 scripts/compose-product-bundle.py \ - --bundle "$artifact_path" \ - --host "$artifact_path/$host_path" \ - --runtime "$artifact_path/$runtime_path" \ + --bundle "$extract_root" \ + --host "$extract_root/$host_path" \ + --runtime "$extract_root/$runtime_path" \ --version "$version" \ --backend "$backend" \ --check + rm -f -- "$archive" + cp -a "$extract_root/." "$artifact_path/" + - name: Stage mesh-llm binary shell: bash run: | diff --git a/.github/actions/restore-windows-abi-cache/action.yml b/.github/actions/restore-windows-abi-cache/action.yml new file mode 100644 index 0000000000..82e81e88f1 --- /dev/null +++ b/.github/actions/restore-windows-abi-cache/action.yml @@ -0,0 +1,190 @@ +name: Restore Windows native ABI cache +description: Validate and restore one architecture- and toolchain-specific Windows native ABI build tree outside the replaceable llama.cpp worktree. + +inputs: + backend: + description: Native backend to restore (cpu, cuda, rocm, or vulkan). + required: true + build_dir: + description: Repository-relative Windows native ABI build directory. + required: true + architecture_set: + description: Canonical backend architecture set; CUDA and ROCm require a non-empty value. + required: false + default: "" + cuda_toolchain_version: + description: CUDA toolkit version; required for the CUDA backend. + required: false + default: "" + vulkan_toolchain_version: + description: Vulkan SDK version; required for the Vulkan backend. + required: false + default: "" + rocm_toolchain_version: + description: Windows ROCm HIP SDK filename/version; required for the ROCm backend. + required: false + default: "" + +outputs: + cache-hit: + description: Whether an exact Windows native ABI cache entry was restored. + value: ${{ steps.restore.outputs.cache-hit }} + cache-primary-key: + description: Exact primary key used for restore and a later cache save. + value: ${{ steps.restore.outputs.cache-primary-key }} + +runs: + using: composite + steps: + - name: Validate Windows ABI cache identity + id: identity + shell: pwsh + env: + INPUT_BACKEND: ${{ inputs.backend }} + INPUT_BUILD_DIR: ${{ inputs.build_dir }} + INPUT_ARCHITECTURE_SET: ${{ inputs.architecture_set }} + INPUT_CUDA_TOOLCHAIN_VERSION: ${{ inputs.cuda_toolchain_version }} + INPUT_VULKAN_TOOLCHAIN_VERSION: ${{ inputs.vulkan_toolchain_version }} + INPUT_ROCM_TOOLCHAIN_VERSION: ${{ inputs.rocm_toolchain_version }} + CACHE_INPUT_HASH: ${{ hashFiles('.github/actions/restore-windows-abi-cache/action.yml', '.github/actions/prepare-native-runtime-input/action.yml', '.github/actions/setup-windows-rocm-sdk/action.yml', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} + run: | + $ErrorActionPreference = "Stop" + + function Normalize-KeySegment { + param( + [string]$Value, + [string]$Name, + [bool]$AllowEmpty = $false + ) + + $segment = "$Value".Trim() + if ($segment.Contains("`r") -or $segment.Contains("`n")) { + throw "$Name must be a single-line cache-key segment" + } + $segment = ($segment -replace '[,;:\s/\\]+', '_').Trim("_") + if (-not $segment) { + if ($AllowEmpty) { + return "" + } + throw "$Name must not be empty" + } + if ($segment -notmatch '^[A-Za-z0-9][A-Za-z0-9._+-]*$') { + throw "$Name contains unsupported cache-key characters: $segment" + } + return $segment + } + + $backend = "$env:INPUT_BACKEND".Trim().ToLowerInvariant() + if ($backend -notin @("cpu", "cuda", "rocm", "vulkan")) { + throw "unsupported Windows ABI cache backend: $backend" + } + + $buildDir = "$env:INPUT_BUILD_DIR".Trim() + if (-not $buildDir) { + throw "build_dir must not be empty" + } + if ( + $buildDir.Contains("`r") -or + $buildDir.Contains("`n") -or + $buildDir.Contains("*") -or + $buildDir.Contains("?") -or + $buildDir.Contains("[") -or + $buildDir.Contains("]") + ) { + throw "build_dir must name one exact directory without newlines or glob characters" + } + if (-not $env:GITHUB_WORKSPACE) { + throw "GITHUB_WORKSPACE must be set" + } + + $workspace = [System.IO.Path]::GetFullPath($env:GITHUB_WORKSPACE).TrimEnd( + [System.IO.Path]::DirectorySeparatorChar, + [System.IO.Path]::AltDirectorySeparatorChar + ) + $resolvedBuildDir = if ([System.IO.Path]::IsPathRooted($buildDir)) { + [System.IO.Path]::GetFullPath($buildDir) + } else { + [System.IO.Path]::GetFullPath((Join-Path $workspace $buildDir)) + } + $workspacePrefix = "$workspace$([System.IO.Path]::DirectorySeparatorChar)" + if ( + $resolvedBuildDir -eq $workspace -or + -not $resolvedBuildDir.StartsWith( + $workspacePrefix, + [System.StringComparison]::OrdinalIgnoreCase + ) + ) { + throw "build_dir must resolve inside GITHUB_WORKSPACE: $resolvedBuildDir" + } + $llamaWorktree = [System.IO.Path]::GetFullPath( + (Join-Path $workspace ".deps\llama.cpp") + ).TrimEnd( + [System.IO.Path]::DirectorySeparatorChar, + [System.IO.Path]::AltDirectorySeparatorChar + ) + $llamaWorktreePrefix = ( + "$llamaWorktree$([System.IO.Path]::DirectorySeparatorChar)" + ) + if ( + $resolvedBuildDir -eq $llamaWorktree -or + $resolvedBuildDir.StartsWith( + $llamaWorktreePrefix, + [System.StringComparison]::OrdinalIgnoreCase + ) + ) { + throw ( + "build_dir must remain outside the replaceable llama.cpp " + + "worktree: $resolvedBuildDir" + ) + } + + $architectureSet = Normalize-KeySegment ` + -Value "$env:INPUT_ARCHITECTURE_SET" ` + -Name "architecture_set" ` + -AllowEmpty $true + if ($backend -in @("cuda", "rocm") -and -not $architectureSet) { + throw "architecture_set is required for the $backend backend" + } + + $toolchain = switch ($backend) { + "cpu" { + "cpu" + } + "cuda" { + $version = Normalize-KeySegment ` + -Value "$env:INPUT_CUDA_TOOLCHAIN_VERSION" ` + -Name "cuda_toolchain_version" + "cuda-$version-Jimver-v0.2.35" + } + "vulkan" { + $version = Normalize-KeySegment ` + -Value "$env:INPUT_VULKAN_TOOLCHAIN_VERSION" ` + -Name "vulkan_toolchain_version" + "vulkan-$version-jakoch-v1.5.2" + } + "rocm" { + $version = Normalize-KeySegment ` + -Value "$env:INPUT_ROCM_TOOLCHAIN_VERSION" ` + -Name "rocm_toolchain_version" + "rocm-$version" + } + } + + $inputHash = "$env:CACHE_INPUT_HASH".Trim().ToLowerInvariant() + if ($inputHash -notmatch '^[0-9a-f]{64}$') { + throw "Windows ABI cache input hash is missing or invalid" + } + + New-Item -ItemType Directory -Force -Path $resolvedBuildDir | Out-Null + $cacheKey = "mesh-llm-windows-2022-skippy-abi-$backend-$architectureSet-$toolchain-$inputHash" + @( + "build-dir=$resolvedBuildDir" + "cache-key=$cacheKey" + ) | Add-Content -Path $env:GITHUB_OUTPUT + + - name: Restore exact Windows ABI cache + id: restore + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: ${{ steps.identity.outputs.build-dir }} + key: ${{ steps.identity.outputs.cache-key }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42d9c5999e..b61d16f297 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,10 +42,9 @@ jobs: rust: ${{ steps.filter.outputs.rust }} ui: ${{ steps.filter.outputs.ui }} benchmarks: ${{ steps.filter.outputs.benchmarks }} - sdk: ${{ steps.filter.outputs.sdk }} docker: ${{ steps.filter.outputs.docker }} - windows_cpu: ${{ steps.filter.outputs.windows_cpu }} - windows_gpu: ${{ steps.filter.outputs.windows_gpu }} + windows_cpu: ${{ steps.compute.outputs.windows_cpu_build_required }} + windows_gpu: ${{ steps.compute.outputs.windows_gpu_build_required }} docs: ${{ steps.filter.outputs.docs }} affected_crates: ${{ steps.compute.outputs.affected_crates }} test_crates: ${{ steps.compute.outputs.test_crates }} @@ -106,42 +105,6 @@ jobs: - 'crates/llama-spec-bench/**' - 'crates/mesh-llm-gpu-bench/**' - '.github/workflows/ci.yml' - sdk: - - 'crates/mesh-llm-api-client/**' - - 'crates/mesh-llm-api-server/**' - - 'crates/mesh-llm-config/**' - - 'crates/mesh-llm-commands/**' - - 'crates/mesh-llm-events/**' - - 'crates/mesh-llm-hardware-profile/**' - - 'crates/mesh-llm-runtime-install/**' - - 'crates/mesh-llm-sdk/**' - - 'crates/mesh-llm-cli/**' - - 'crates/mesh-llm-embedded-runtime/**' - - 'crates/mesh-llm-tui/**' - - 'crates/mesh-llm-console-server/**' - - 'crates/mesh-llm-ffi/**' - - 'crates/mesh-llm-nodejs/**' - - 'crates/mesh-client/**' - - 'crates/mesh-llm-identity/**' - - 'crates/mesh-llm-native-runtime/**' - - 'crates/mesh-llm-protocol/**' - - 'crates/mesh-llm-routing/**' - - 'crates/mesh-llm-types/**' - - 'sdk/**' - - 'Package.swift' - - 'scripts/ci-rust-sdk-smoke.sh' - - 'scripts/ci-prepare-native-runtime.sh' - - 'scripts/ci-install-native-runtime.sh' - - 'scripts/package-sdk-console-assets.sh' - - 'scripts/verify-sdk-console-assets.sh' - - 'scripts/ci-kotlin-sdk-smoke.sh' - - 'scripts/ci-swift-sdk-smoke.sh' - - 'scripts/ci-sdk-fixture.sh' - - 'scripts/prepare-swift-package-release.sh' - - 'scripts/verify-swift-package-manifest.sh' - - 'scripts/verify-swift-privacy-manifest.sh' - - 'scripts/verify-swift-release-artifact.sh' - - '.github/workflows/ci.yml' docker: - '.dockerignore' - 'docker/**' @@ -150,36 +113,6 @@ jobs: - 'Cargo.toml' - 'Cargo.lock' - '.github/workflows/docker.yml' - windows_cpu: - - 'crates/mesh-llm-release-footer/**' - - 'crates/mesh-llm-nodejs/**' - - 'crates/skippy-ffi/**' - - 'scripts/build-windows.ps1' - - 'third_party/llama.cpp/**' - - 'Cargo.toml' - - 'Cargo.lock' - - 'Justfile' - - '.github/cache-version.txt' - - '.github/actions/prepare-windows-host-input/**' - - '.github/workflows/ci.yml' - - '.github/workflows/windows-warm-caches.yml' - windows_gpu: - - 'crates/skippy-ffi/**' - - 'scripts/build-windows.ps1' - - 'scripts/install-windows-sdk.ps1' - - 'scripts/package-native-runtime.sh' - - 'scripts/verify-native-runtime-package.sh' - - 'scripts/windows-native-runtime-deps.py' - - 'scripts/tests/test_windows_native_runtime_deps.py' - - 'third_party/llama.cpp/**' - - 'Justfile' - - '.github/cache-version.txt' - - '.github/actions/prepare-windows-host-input/**' - - '.github/actions/setup-windows-rocm-sdk/**' - - '.github/actions/compute-changes/action.yml' - - '.github/workflows/ci.yml' - - '.github/workflows/windows-warm-caches.yml' - docs: - 'docs/**' - '**.md' @@ -189,8 +122,8 @@ jobs: uses: ./.github/actions/compute-changes with: event_name: ${{ github.event_name }} - base_sha: '' - head_sha: '' + base_sha: ${{ github.event.before || '' }} + head_sha: ${{ github.sha }} - name: Compute UI dist cache key id: ui_key run: | @@ -327,19 +260,11 @@ jobs: if-no-files-found: error retention-days: 1 - - name: Show sccache stats - if: ${{ always() }} - run: | - if ! command -v sccache >/dev/null 2>&1; then - echo "sccache not available; stats skipped" - exit 0 - fi - - sccache --show-stats || true - requests="$(sccache --show-stats 2>/dev/null | awk '/Compile requests/ { print $3; exit }')" - if [ "${requests:-0}" = "0" ]; then - echo "::warning::sccache reported zero compile requests; check RUSTC_WRAPPER wiring if this was not a fully reused target cache." - fi + - name: Capture sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-main-linux-host-${{ github.run_attempt }} linux_cpu_runtime_input: needs: changes @@ -390,9 +315,11 @@ jobs: if-no-files-found: error retention-days: 1 - - name: Show sccache stats - if: ${{ always() }} - run: sccache --show-stats || true + - name: Capture sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-main-linux-cpu-runtime-${{ github.run_attempt }} linux_cpu_artifact: needs: [changes, linux_host_input, linux_cpu_runtime_input] @@ -571,9 +498,11 @@ jobs: cargo test -p "$crate" echo "::endgroup::" done - - name: Show sccache stats - if: ${{ always() }} - run: sccache --show-stats || true + - name: Capture sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-main-rust-crate-tests-${{ matrix.batch.idx }}-${{ github.run_attempt }} linux_test_groups: needs: [changes, linux_static_abi_input] @@ -677,19 +606,11 @@ jobs: path: ${{ runner.temp }}/skippy-ci-smoke-models key: ${{ steps.skippy_smoke_model_cache.outputs.cache-primary-key }} - - name: Show sccache stats - if: ${{ always() }} - run: | - if ! command -v sccache >/dev/null 2>&1; then - echo "sccache not available; stats skipped" - exit 0 - fi - - sccache --show-stats || true - requests="$(sccache --show-stats 2>/dev/null | awk '/Compile requests/ { print $3; exit }')" - if [ "${requests:-0}" = "0" ]; then - echo "::warning::sccache reported zero compile requests; check RUSTC_WRAPPER wiring if this was not a fully reused target cache." - fi + - name: Capture sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-main-linux-tests-${{ matrix.group }}-${{ github.run_attempt }} hf_download_smoke: needs: changes @@ -819,7 +740,7 @@ jobs: swift_sdk_smoke: needs: [changes, macos_cpu_artifact, macos_unit_tests] - if: ${{ !cancelled() && needs.macos_cpu_artifact.result == 'success' && (needs.macos_unit_tests.result == 'success' || needs.macos_unit_tests.result == 'skipped') && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.sdk == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ !cancelled() && needs.macos_cpu_artifact.result == 'success' && (needs.macos_unit_tests.result == 'success' || needs.macos_unit_tests.result == 'skipped') && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} permissions: contents: read uses: ./.github/workflows/sdk-smoke.yml @@ -1066,10 +987,10 @@ jobs: fi done - linux_cuda: - needs: [changes, linux_host_input] + linux_cuda_runtime_input: + needs: changes if: ${{ needs.changes.outputs.docs_only != 'true' }} - name: Linux CUDA compatibility + name: Linux CUDA compatibility runtime runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read @@ -1101,12 +1022,6 @@ jobs: with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - - name: Download immutable host input - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - name: ci-linux-host-input - path: host-input - - name: Prepare Linux CUDA native runtime uses: ./.github/actions/prepare-native-runtime-input with: @@ -1114,6 +1029,48 @@ jobs: target: x86_64-unknown-linux-gnu output_dir: runtime-input + - name: Upload immutable CUDA runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-linux-cuda-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + linux_cuda_product: + needs: [changes, linux_host_input, linux_cuda_runtime_input] + if: ${{ needs.linux_host_input.result == 'success' && needs.linux_cuda_runtime_input.result == 'success' }} + name: Linux CUDA compatibility + runs-on: ${{ needs.changes.outputs.runner_4 }} + permissions: + contents: read + packages: read + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Verify prebuilt CI environment + run: verify-runner-image public + - name: Download immutable host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-linux-host-input + path: host-input + - name: Download immutable CUDA runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-linux-cuda-runtime-input + path: runtime-input - name: Compose CUDA product id: compose uses: ./.github/actions/compose-product-input @@ -1131,10 +1088,10 @@ jobs: if-no-files-found: error retention-days: 1 - linux_rocm: - needs: [changes, linux_host_input] + linux_rocm_runtime_input: + needs: changes if: ${{ needs.changes.outputs.docs_only != 'true' }} - name: Linux ROCm slim + name: Linux ROCm slim runtime runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read @@ -1161,12 +1118,6 @@ jobs: with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - - name: Download immutable host input - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - name: ci-linux-host-input - path: host-input - - name: Prepare Linux ROCm native runtime uses: ./.github/actions/prepare-native-runtime-input with: @@ -1174,6 +1125,48 @@ jobs: target: x86_64-unknown-linux-gnu output_dir: runtime-input + - name: Upload immutable ROCm runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-linux-rocm-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + linux_rocm_product: + needs: [changes, linux_host_input, linux_rocm_runtime_input] + if: ${{ needs.linux_host_input.result == 'success' && needs.linux_rocm_runtime_input.result == 'success' }} + name: Linux ROCm slim + runs-on: ${{ needs.changes.outputs.runner_4 }} + permissions: + contents: read + packages: read + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Verify prebuilt CI environment + run: verify-runner-image public + - name: Download immutable host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-linux-host-input + path: host-input + - name: Download immutable ROCm runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-linux-rocm-runtime-input + path: runtime-input - name: Compose ROCm product id: compose uses: ./.github/actions/compose-product-input @@ -1191,10 +1184,10 @@ jobs: if-no-files-found: error retention-days: 1 - linux_vulkan: - needs: [changes, linux_host_input] + linux_vulkan_runtime_input: + needs: changes if: ${{ needs.changes.outputs.docs_only != 'true' }} - name: Linux Vulkan + name: Linux Vulkan runtime runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read @@ -1219,12 +1212,6 @@ jobs: with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - - name: Download immutable host input - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - name: ci-linux-host-input - path: host-input - - name: Prepare Linux Vulkan native runtime uses: ./.github/actions/prepare-native-runtime-input with: @@ -1232,6 +1219,48 @@ jobs: target: x86_64-unknown-linux-gnu output_dir: runtime-input + - name: Upload immutable Vulkan runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ci-linux-vulkan-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + linux_vulkan_product: + needs: [changes, linux_host_input, linux_vulkan_runtime_input] + if: ${{ needs.linux_host_input.result == 'success' && needs.linux_vulkan_runtime_input.result == 'success' }} + name: Linux Vulkan + runs-on: ${{ needs.changes.outputs.runner_4 }} + permissions: + contents: read + packages: read + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Verify prebuilt CI environment + run: verify-runner-image public + - name: Download immutable host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-linux-host-input + path: host-input + - name: Download immutable Vulkan runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ci-linux-vulkan-runtime-input + path: runtime-input - name: Compose Vulkan product id: compose uses: ./.github/actions/compose-product-input @@ -1289,7 +1318,7 @@ jobs: contents: read env: LLAMA_STAGE_BACKEND: ${{ matrix.backend }} - LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-${{ matrix.backend }} + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/windows/build-stage-abi-${{ matrix.backend }} RUSTC_WRAPPER: sccache WINDOWS_CUDA_VERSION: ${{ vars.CUDA_VERSION || '12.6.3' }} WINDOWS_VULKAN_SDK_VERSION: ${{ vars.VULKAN_SDK_VERSION || '1.4.328.1' }} @@ -1326,6 +1355,15 @@ jobs: prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} shared-key: windows-runtime-${{ matrix.backend }} save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Restore exact Windows GPU ABI cache + uses: ./.github/actions/restore-windows-abi-cache + with: + backend: ${{ matrix.backend }} + build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + architecture_set: ${{ matrix.cuda_architectures || matrix.rocm_architectures || '' }} + cuda_toolchain_version: ${{ env.WINDOWS_CUDA_VERSION }} + vulkan_toolchain_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} + rocm_toolchain_version: ${{ env.ROCM_HIP_SDK_FILENAME }} - name: Install CUDA toolkit if: ${{ matrix.backend == 'cuda' }} uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 @@ -1441,7 +1479,7 @@ jobs: runs-on: windows-2022 env: LLAMA_STAGE_BACKEND: cpu - LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-cpu + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/windows/build-stage-abi-cpu RUSTC_WRAPPER: sccache steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 @@ -1459,6 +1497,11 @@ jobs: prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} shared-key: windows-runtime-cpu save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Restore exact Windows CPU ABI cache + uses: ./.github/actions/restore-windows-abi-cache + with: + backend: cpu + build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} - name: Initialize MSVC for CPU runtime uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: diff --git a/.github/workflows/llama-upstream-canary.yml b/.github/workflows/llama-upstream-canary.yml index 283a3279f0..5fbe65d309 100644 --- a/.github/workflows/llama-upstream-canary.yml +++ b/.github/workflows/llama-upstream-canary.yml @@ -125,5 +125,5 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" - name: Show sccache stats - if: always() + if: ${{ !cancelled() }} run: sccache --show-stats || true diff --git a/.github/workflows/nightly-stability-run.yml b/.github/workflows/nightly-stability-run.yml index d7f159d1b4..d2d4e7274d 100644 --- a/.github/workflows/nightly-stability-run.yml +++ b/.github/workflows/nightly-stability-run.yml @@ -132,7 +132,7 @@ jobs: scripts/qa-nightly-stability.py "${args[@]}" - name: Publish run summary - if: ${{ always() && steps.preflight.outputs.run == 'true' }} + if: ${{ !cancelled() && steps.preflight.outputs.run == 'true' }} shell: bash run: | set -euo pipefail @@ -147,7 +147,7 @@ jobs: fi - name: Upload stability evidence - if: ${{ always() && steps.preflight.outputs.run == 'true' }} + if: ${{ !cancelled() && steps.preflight.outputs.run == 'true' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: nightly-stability-${{ github.run_number }}-${{ github.sha }} diff --git a/.github/workflows/pr_builds.yml b/.github/workflows/pr_builds.yml index 890e4366a0..9c0e1b3667 100644 --- a/.github/workflows/pr_builds.yml +++ b/.github/workflows/pr_builds.yml @@ -43,7 +43,6 @@ jobs: rust: ${{ steps.filter.outputs.rust }} ui: ${{ steps.filter.outputs.ui }} benchmarks: ${{ steps.filter.outputs.benchmarks }} - sdk: ${{ steps.filter.outputs.sdk }} nodejs_release: ${{ steps.filter.outputs.nodejs_release }} windows_cpu: ${{ steps.compute.outputs.windows_cpu_build_required }} windows_gpu: ${{ steps.compute.outputs.windows_gpu_build_required }} @@ -106,48 +105,8 @@ jobs: - 'crates/skippy-bench/**' - 'crates/llama-spec-bench/**' - 'crates/mesh-llm-gpu-bench/**' - sdk: - - 'crates/mesh-llm-api-client/**' - - 'crates/mesh-llm-api-server/**' - - 'crates/mesh-llm-config/**' - - 'crates/mesh-llm-commands/**' - - 'crates/mesh-llm-events/**' - - 'crates/mesh-llm-hardware-profile/**' - - 'crates/mesh-llm-runtime-install/**' - - 'crates/mesh-llm-sdk/**' - - 'crates/mesh-llm-cli/**' - - 'crates/mesh-llm-embedded-runtime/**' - - 'crates/mesh-llm-tui/**' - - 'crates/mesh-llm-console-server/**' - - 'crates/mesh-llm-ffi/**' - - 'crates/mesh-llm-nodejs/**' - - 'crates/mesh-client/**' - - 'crates/mesh-llm-identity/**' - - 'crates/mesh-llm-native-runtime/**' - - 'crates/mesh-llm-protocol/**' - - 'crates/mesh-llm-routing/**' - - 'crates/mesh-llm-types/**' - - 'sdk/**' - - 'Package.swift' - - 'scripts/ci-rust-sdk-smoke.sh' - - 'scripts/ci-prepare-native-runtime.sh' - - 'scripts/ci-install-native-runtime.sh' - - 'scripts/package-sdk-console-assets.sh' - - 'scripts/verify-sdk-console-assets.sh' - - 'scripts/ci-kotlin-sdk-smoke.sh' - - 'scripts/ci-swift-sdk-smoke.sh' - - 'scripts/ci-sdk-fixture.sh' nodejs_release: - 'crates/mesh-llm-nodejs/**' - windows_cpu: - - 'crates/mesh-llm-release-footer/**' - - 'crates/mesh-llm-nodejs/**' - - 'crates/skippy-ffi/**' - - 'scripts/build-windows.ps1' - - 'third_party/llama.cpp/**' - - 'Cargo.toml' - - 'Cargo.lock' - - '.github/cache-version.txt' docs: - 'docs/**' - '**.md' @@ -284,9 +243,11 @@ jobs: if-no-files-found: error retention-days: 1 - - name: Show sccache stats - if: ${{ always() }} - run: sccache --show-stats || true + - name: Capture sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-pr-linux-host-${{ github.run_attempt }} linux_cpu_runtime_input: needs: changes @@ -332,9 +293,11 @@ jobs: if-no-files-found: error retention-days: 1 - - name: Show sccache stats - if: ${{ always() }} - run: sccache --show-stats || true + - name: Capture sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-pr-linux-cpu-runtime-${{ github.run_attempt }} linux_cpu_artifact: needs: [changes, linux_host_input, linux_cpu_runtime_input] @@ -382,48 +345,25 @@ jobs: if-no-files-found: error retention-days: 1 - linux_targets: - needs: [changes, linux_host_input] + linux_cuda_runtime_input: + needs: changes if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} - name: Linux ${{ matrix.name }} + name: Linux CUDA slim runtime runs-on: ${{ needs.changes.outputs.runner_8 }} container: - image: ${{ matrix.container }} + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} defaults: run: shell: bash - strategy: - fail-fast: false - matrix: - include: - - name: CUDA slim - backend: cuda - container: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd - build_dir: .deps/llama-build/build-stage-abi-dynamic-cuda-sm86 - cuda_architectures: "86" - cuda_version: "12.9.2" - cuda_major: "12" - - name: ROCm slim - backend: rocm - container: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:6b88ca9371ada2c507d6e36b71f0e0538fee378c6a5e2b39c17249b4b7e5088a - build_dir: .deps/llama-build/build-stage-abi-dynamic-rocm-gfx1100 - amdgpu_targets: gfx1100 - cmake_prefix_path: /opt/rocm - - name: Vulkan - backend: vulkan - container: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:ce55fed5c680cd3184b5d4770d9a77c43a702687690906e5753efd2cea27ed80 - build_dir: .deps/llama-build/build-stage-abi-dynamic-vulkan env: - LLAMA_STAGE_BACKEND: ${{ matrix.backend }} - LLAMA_STAGE_BUILD_DIR: ${{ matrix.build_dir }} - LLAMA_STAGE_CUDA_ARCHITECTURES: ${{ matrix.cuda_architectures }} - LLAMA_STAGE_AMDGPU_TARGETS: ${{ matrix.amdgpu_targets }} - MESH_CUDA_VERSION: ${{ matrix.cuda_version }} - MESH_LLM_CUDA_TOOLKIT_MAJOR: ${{ matrix.cuda_major }} - CMAKE_PREFIX_PATH: ${{ matrix.cmake_prefix_path }} + LLAMA_STAGE_BACKEND: cuda + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-dynamic-cuda-sm86 + LLAMA_STAGE_CUDA_ARCHITECTURES: "86" + MESH_CUDA_VERSION: "12.9.2" + MESH_LLM_CUDA_TOOLKIT_MAJOR: "12" MESH_LLM_REQUIRE_SCCACHE: "1" SCCACHE_GHA_ENABLED: "false" GGML_CUDA_NO_VMM: "1" @@ -434,47 +374,268 @@ jobs: persist-credentials: false - name: Verify prebuilt backend environment - run: verify-runner-image public "${{ matrix.backend }}" + run: verify-runner-image public cuda - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} + - name: Prepare CUDA native runtime + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: cuda + target: x86_64-unknown-linux-gnu + output_dir: runtime-input + + - name: Upload immutable CUDA runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-linux-cuda-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + linux_cuda_product: + needs: [changes, linux_host_input, linux_cuda_runtime_input] + if: ${{ needs.linux_host_input.result == 'success' && needs.linux_cuda_runtime_input.result == 'success' }} + name: Linux CUDA slim + runs-on: ${{ needs.changes.outputs.runner_4 }} + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Verify prebuilt CI environment + run: verify-runner-image public - name: Download immutable host input uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: pr-linux-host-input path: host-input + - name: Download immutable CUDA runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-linux-cuda-runtime-input + path: runtime-input + - name: Compose CUDA product from immutable inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + backend: cuda + + - name: Upload composed CUDA product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-linux-cuda-product + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 + + linux_rocm_runtime_input: + needs: changes + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Linux ROCm slim runtime + runs-on: ${{ needs.changes.outputs.runner_8 }} + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:6b88ca9371ada2c507d6e36b71f0e0538fee378c6a5e2b39c17249b4b7e5088a + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + env: + LLAMA_STAGE_BACKEND: rocm + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-dynamic-rocm-gfx1100 + LLAMA_STAGE_AMDGPU_TARGETS: gfx1100 + CMAKE_PREFIX_PATH: /opt/rocm + MESH_LLM_REQUIRE_SCCACHE: "1" + SCCACHE_GHA_ENABLED: "false" + GGML_CUDA_NO_VMM: "1" + LLAMA_STAGE_SKIP_NCCL: "1" + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - name: Verify prebuilt backend environment + run: verify-runner-image public rocm + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} + + - name: Prepare ROCm native runtime + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: rocm + target: x86_64-unknown-linux-gnu + output_dir: runtime-input + + - name: Upload immutable ROCm runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-linux-rocm-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + linux_rocm_product: + needs: [changes, linux_host_input, linux_rocm_runtime_input] + if: ${{ needs.linux_host_input.result == 'success' && needs.linux_rocm_runtime_input.result == 'success' }} + name: Linux ROCm slim + runs-on: ${{ needs.changes.outputs.runner_4 }} + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Verify prebuilt CI environment + run: verify-runner-image public + - name: Download immutable host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-linux-host-input + path: host-input + - name: Download immutable ROCm runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-linux-rocm-runtime-input + path: runtime-input + - name: Compose ROCm product from immutable inputs + id: compose + uses: ./.github/actions/compose-product-input + with: + host_input_dir: host-input + runtime_input_dir: runtime-input + output_dir: product-input + backend: rocm + + - name: Upload composed ROCm product + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-linux-rocm-product + path: ${{ steps.compose.outputs.archive_path }} + if-no-files-found: error + retention-days: 1 + + linux_vulkan_runtime_input: + needs: changes + if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + name: Linux Vulkan runtime + runs-on: ${{ needs.changes.outputs.runner_8 }} + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:ce55fed5c680cd3184b5d4770d9a77c43a702687690906e5753efd2cea27ed80 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + env: + LLAMA_STAGE_BACKEND: vulkan + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-dynamic-vulkan + MESH_LLM_REQUIRE_SCCACHE: "1" + SCCACHE_GHA_ENABLED: "false" + GGML_CUDA_NO_VMM: "1" + LLAMA_STAGE_SKIP_NCCL: "1" + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - name: Verify prebuilt backend environment + run: verify-runner-image public vulkan + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - name: Ensure Vulkan ABI cache directory - if: matrix.backend == 'vulkan' run: mkdir -p "$LLAMA_STAGE_BUILD_DIR" - name: Cache Vulkan ABI build - if: matrix.backend == 'vulkan' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ env.LLAMA_STAGE_BUILD_DIR }} key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-vulkan-${{ hashFiles('scripts/build-linux.sh', 'scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - - name: Prepare native runtime + - name: Prepare Vulkan native runtime uses: ./.github/actions/prepare-native-runtime-input with: - backend: ${{ matrix.backend }} + backend: vulkan target: x86_64-unknown-linux-gnu output_dir: runtime-input - - name: Compose backend product from immutable inputs + - name: Upload immutable Vulkan runtime input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: pr-linux-vulkan-runtime-input + path: | + runtime-input/*.tar.gz + runtime-input/*.sha256 + if-no-files-found: error + retention-days: 1 + + linux_vulkan_product: + needs: [changes, linux_host_input, linux_vulkan_runtime_input] + if: ${{ needs.linux_host_input.result == 'success' && needs.linux_vulkan_runtime_input.result == 'success' }} + name: Linux Vulkan + runs-on: ${{ needs.changes.outputs.runner_4 }} + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Verify prebuilt CI environment + run: verify-runner-image public + - name: Download immutable host input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-linux-host-input + path: host-input + - name: Download immutable Vulkan runtime input + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: pr-linux-vulkan-runtime-input + path: runtime-input + - name: Compose Vulkan product from immutable inputs id: compose uses: ./.github/actions/compose-product-input with: host_input_dir: host-input runtime_input_dir: runtime-input output_dir: product-input - backend: ${{ matrix.backend }} + backend: vulkan - - name: Upload composed backend product + - name: Upload composed Vulkan product uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: - name: pr-linux-${{ matrix.backend }}-product + name: pr-linux-vulkan-product path: ${{ steps.compose.outputs.archive_path }} if-no-files-found: error retention-days: 1 @@ -606,9 +767,11 @@ jobs: cargo test -p "$crate" echo "::endgroup::" done - - name: Show sccache stats - if: ${{ always() }} - run: sccache --show-stats || true + - name: Capture sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-pr-rust-crate-tests-${{ matrix.batch.idx }}-${{ github.run_attempt }} linux_test_groups: needs: [changes, linux_static_abi_input] @@ -707,19 +870,11 @@ jobs: path: ${{ runner.temp }}/skippy-ci-smoke-models key: ${{ steps.skippy_smoke_model_cache.outputs.cache-primary-key }} - - name: Show sccache stats - if: ${{ always() }} - run: | - if ! command -v sccache >/dev/null 2>&1; then - echo "sccache not available; stats skipped" - exit 0 - fi - - sccache --show-stats || true - requests="$(sccache --show-stats 2>/dev/null | awk '/Compile requests/ { print $3; exit }')" - if [ "${requests:-0}" = "0" ]; then - echo "::warning::sccache reported zero compile requests; check RUSTC_WRAPPER wiring if this was not a fully reused target cache." - fi + - name: Capture sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-pr-linux-tests-${{ matrix.group }}-${{ github.run_attempt }} linux_public_mesh_admission: needs: [changes, linux_cpu_artifact] @@ -1176,7 +1331,7 @@ jobs: runs-on: windows-2022 env: LLAMA_STAGE_BACKEND: cpu - LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-cpu + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/windows/build-stage-abi-cpu RUSTC_WRAPPER: sccache steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 @@ -1195,6 +1350,11 @@ jobs: prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} shared-key: windows-runtime-cpu save-if: "false" + - name: Restore exact Windows CPU ABI cache + uses: ./.github/actions/restore-windows-abi-cache + with: + backend: cpu + build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} - name: Initialize MSVC for CPU runtime uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: @@ -1234,7 +1394,7 @@ jobs: backend: vulkan env: LLAMA_STAGE_BACKEND: ${{ matrix.backend }} - LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-${{ matrix.backend }} + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/windows/build-stage-abi-${{ matrix.backend }} RUSTC_WRAPPER: sccache WINDOWS_CUDA_VERSION: ${{ vars.CUDA_VERSION || '12.6.3' }} WINDOWS_VULKAN_SDK_VERSION: ${{ vars.VULKAN_SDK_VERSION || '1.4.328.1' }} @@ -1260,6 +1420,15 @@ jobs: prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} shared-key: windows-runtime-${{ matrix.backend }} save-if: "false" + - name: Restore exact Windows GPU ABI cache + uses: ./.github/actions/restore-windows-abi-cache + with: + backend: ${{ matrix.backend }} + build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + architecture_set: ${{ matrix.cuda_architectures || matrix.rocm_architectures || '' }} + cuda_toolchain_version: ${{ env.WINDOWS_CUDA_VERSION }} + vulkan_toolchain_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} + rocm_toolchain_version: ${{ env.ROCM_HIP_SDK_FILENAME }} - name: Install CUDA toolkit if: ${{ matrix.backend == 'cuda' }} uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 diff --git a/.github/workflows/pr_cleanup.yml b/.github/workflows/pr_cleanup.yml index 92bd24e05b..41ae069823 100644 --- a/.github/workflows/pr_cleanup.yml +++ b/.github/workflows/pr_cleanup.yml @@ -445,7 +445,7 @@ jobs: } - name: Upload cache deletion result - if: ${{ always() }} + if: ${{ !cancelled() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: pr-cache-cleanup-result-${{ github.event.inputs.pr_number || github.event.pull_request.number }}-${{ matrix.shard.number }} @@ -456,7 +456,7 @@ jobs: delete_pr_artifacts: name: Delete PR artifacts needs: delete_pr_caches - if: ${{ always() }} + if: ${{ !cancelled() }} runs-on: ubuntu-24.04 permissions: actions: write @@ -602,7 +602,7 @@ jobs: - plan_cache_cleanup - delete_pr_caches - delete_pr_artifacts - if: ${{ always() }} + if: ${{ !cancelled() }} runs-on: ubuntu-24.04 permissions: actions: read diff --git a/.github/workflows/pr_quality.yml b/.github/workflows/pr_quality.yml index 1bd71a4586..f284c90e7c 100644 --- a/.github/workflows/pr_quality.yml +++ b/.github/workflows/pr_quality.yml @@ -67,8 +67,8 @@ jobs: id: compute with: event_name: ${{ github.event_name }} - base_sha: ${{ github.event.pull_request.base.sha || '' }} - head_sha: ${{ github.event.pull_request.head.sha || '' }} + base_sha: ${{ github.event.pull_request.base.sha || github.event.before || '' }} + head_sha: ${{ github.event.pull_request.head.sha || github.sha }} ci-consistency: name: CI consistency @@ -90,6 +90,22 @@ jobs: - name: Check publish crate-chain drift run: cargo run -p xtask -- repo-consistency publish-crates + ci-contract: + name: CI contract + needs: changes + runs-on: ${{ needs.changes.outputs.runner_4 }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # actionlint + with: + tool: actionlint@1.7.12 + - name: Validate GitHub Actions + run: actionlint -config-file .github/actionlint.yaml + - name: Test CI, packaging, and SDK contracts + run: python3 -m unittest discover -s scripts/tests -p 'test_*.py' + rust-fmt: needs: changes if: needs.changes.outputs.rust_changed == 'true' @@ -156,14 +172,11 @@ jobs: fi cargo clippy -p "$CRATE" --all-targets -- -D warnings done - - name: Show sccache stats - if: ${{ always() }} - run: | - sccache --show-stats || true - requests="$(sccache --show-stats 2>/dev/null | awk '/Compile requests/ { print $3; exit }')" - if [ "${requests:-0}" = "0" ]; then - echo "::warning::sccache reported zero compile requests; check RUSTC_WRAPPER wiring if this was not a fully reused target cache." - fi + - name: Capture sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-pr-quality-clippy-${{ matrix.batch.idx }}-${{ github.run_attempt }} ui-quality: needs: changes @@ -231,14 +244,15 @@ jobs: EOF summary: - needs: [changes, ci-consistency, rust-fmt, rust-clippy, ui-quality, cli-docs-sync] - if: always() + needs: [changes, ci-consistency, ci-contract, rust-fmt, rust-clippy, ui-quality, cli-docs-sync] + if: ${{ !cancelled() }} runs-on: ubuntu-24.04 steps: - name: Check quality gate env: CHANGES: ${{ needs.changes.result }} CONSISTENCY: ${{ needs.ci-consistency.result }} + CONTRACT: ${{ needs.ci-contract.result }} FMT: ${{ needs.rust-fmt.result }} CLIPPY: ${{ needs.rust-clippy.result }} UI: ${{ needs.ui-quality.result }} @@ -251,6 +265,7 @@ jobs: echo "| --- | --- |" printf '| %s | %s |\n' "changes" "$CHANGES" printf '| %s | %s |\n' "ci-consistency" "$CONSISTENCY" + printf '| %s | %s |\n' "ci-contract" "$CONTRACT" printf '| %s | %s |\n' "rust-fmt" "$FMT" printf '| %s | %s |\n' "rust-clippy" "$CLIPPY" printf '| %s | %s |\n' "ui-quality" "$UI" @@ -258,7 +273,7 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" FAILED=false - for STATUS in "$CHANGES" "$CONSISTENCY" "$FMT" "$CLIPPY" "$UI" "$CLI_DOCS"; do + for STATUS in "$CHANGES" "$CONSISTENCY" "$CONTRACT" "$FMT" "$CLIPPY" "$UI" "$CLI_DOCS"; do if [[ "$STATUS" == "failure" || "$STATUS" == "cancelled" ]]; then FAILED=true fi diff --git a/.github/workflows/pr_website.yml b/.github/workflows/pr_website.yml index 1f0b9b7e1e..515fff2487 100644 --- a/.github/workflows/pr_website.yml +++ b/.github/workflows/pr_website.yml @@ -74,7 +74,7 @@ jobs: summary: needs: [changes, website-build] - if: always() + if: ${{ !cancelled() }} runs-on: ubuntu-24.04 steps: - name: Check website gate diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e097bffa4e..7358e3f6da 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,17 +67,28 @@ jobs: runner_8: ${{ steps.runners.outputs.runner_8 }} runner_16: ${{ steps.runners.outputs.runner_16 }} steps: + - name: Require the trusted release ref + shell: bash + run: | + set -euo pipefail + if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" && "$GITHUB_REF" != "refs/heads/main" ]]; then + echo "Manual releases must be dispatched from refs/heads/main, not $GITHUB_REF." >&2 + exit 1 + fi - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - name: Select trusted Linux runners - id: runners - uses: ./.github/actions/select-ci-runners - with: - event_name: ${{ github.event_name }} - ref: ${{ github.ref }} - depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} - manual_use_depot: ${{ inputs.use_depot == true }} + fetch-depth: 0 + - name: Require release tags to point at main history + if: github.event_name == 'push' + shell: bash + run: | + set -euo pipefail + git fetch --no-tags origin refs/heads/main:refs/remotes/origin/main + if ! git merge-base --is-ancestor "$GITHUB_SHA" refs/remotes/origin/main; then + echo "Release tag $GITHUB_REF must point at a commit reachable from main." >&2 + exit 1 + fi - id: meta shell: bash env: @@ -109,6 +120,25 @@ jobs: echo "canary=$INPUT_CANARY" echo "force_hosted_runners=$INPUT_FORCE_HOSTED_RUNNERS" } >> "$GITHUB_OUTPUT" + - name: Reject an existing manual release tag + if: ${{ github.event_name == 'workflow_dispatch' && inputs.canary != true }} + shell: bash + env: + RELEASE_TAG: ${{ steps.meta.outputs.tag }} + run: | + set -euo pipefail + if git ls-remote --exit-code --tags origin "refs/tags/$RELEASE_TAG" >/dev/null 2>&1; then + echo "Manual release tag already exists and is immutable: $RELEASE_TAG" >&2 + exit 1 + fi + - name: Select trusted Linux runners + id: runners + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + ref: ${{ github.ref }} + depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + manual_use_depot: ${{ inputs.use_depot == true }} build: name: Build immutable host ${{ matrix.name }} @@ -882,6 +912,9 @@ jobs: if: ${{ needs.compose_linux_arm64_cpu.result == 'success' }} runs-on: ${{ fromJson(vars.USE_SELF_HOSTED == 'true' && needs.metadata.outputs.force_hosted_runners != 'true' && '["self-hosted","Linux","ARM64"]' || '["ubuntu-24.04-arm"]') }} steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-linux-arm64 @@ -891,18 +924,16 @@ jobs: EXPECTED_VERSION: ${{ needs.metadata.outputs.version }} run: | set -euo pipefail - cd release-linux-arm64 - tarball="mesh-llm-aarch64-unknown-linux-gnu.tar.gz" + artifact_dir="release-linux-arm64" + tarball="$artifact_dir/mesh-llm-aarch64-unknown-linux-gnu.tar.gz" if [[ ! -f "$tarball" ]]; then echo "missing expected ARM64 release archive: $tarball" >&2 - find . -maxdepth 1 -type f -print | sort >&2 + find "$artifact_dir" -maxdepth 1 -type f -print | sort >&2 exit 1 fi - if command -v sha256sum >/dev/null 2>&1 && [[ -f "$tarball.sha256" ]]; then - sha256sum -c "$tarball.sha256" - fi + python3 scripts/verify-checksum-sidecar.py "$tarball" mkdir -p smoke - tar -xzf "$tarball" -C smoke + python3 scripts/safe-extract-tar.py "$tarball" smoke binary="smoke/mesh-bundle/mesh-llm" version_output="$("$binary" --version)" actual_version="$(awk '{print $NF}' <<<"$version_output")" @@ -1408,6 +1439,7 @@ jobs: runs-on: windows-2022 env: LLAMA_STAGE_BACKEND: cpu + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/windows/build-stage-abi-cpu MESH_NATIVE_RUNTIME_TARGET: x86_64-pc-windows-msvc steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 @@ -1418,6 +1450,11 @@ jobs: python-version: '3.x' - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - name: Restore exact Windows CPU ABI cache + uses: ./.github/actions/restore-windows-abi-cache + with: + backend: cpu + build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} - name: Initialize MSVC for native runtime uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: @@ -1451,6 +1488,8 @@ jobs: env: ROCM_HIP_SDK_FILENAME: AMD-Software-PRO-Edition-25.Q3-WinSvr2022-For-HIP.exe WINDOWS_CUDA_VERSION: ${{ vars.CUDA_VERSION || '12.9.2' }} + WINDOWS_VULKAN_SDK_VERSION: ${{ vars.VULKAN_SDK_VERSION || '1.4.328.1' }} + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/windows/build-stage-abi-${{ matrix.backend }} MESH_NATIVE_RUNTIME_TARGET: x86_64-pc-windows-msvc strategy: fail-fast: false @@ -1482,6 +1521,15 @@ jobs: python-version: '3.x' - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - name: Restore exact Windows GPU ABI cache + uses: ./.github/actions/restore-windows-abi-cache + with: + backend: ${{ matrix.backend }} + build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + architecture_set: ${{ matrix.cuda_architectures || matrix.rocm_architectures || '' }} + cuda_toolchain_version: ${{ env.WINDOWS_CUDA_VERSION }} + vulkan_toolchain_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} + rocm_toolchain_version: ${{ env.ROCM_HIP_SDK_FILENAME }} - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' shell: bash @@ -1515,18 +1563,34 @@ jobs: throw "CUDA_PATH was not configured by Jimver/cuda-toolkit." } & nvcc --version + - name: Install Vulkan SDK + if: ${{ matrix.backend == 'vulkan' }} + uses: jakoch/install-vulkan-sdk-action@3c53c378c9bfbb2ea122a1cc164a837d4004c871 # v1.5.2 + with: + vulkan_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} + cache: true + stripdown: true + - name: Verify Vulkan SDK + if: ${{ matrix.backend == 'vulkan' }} + shell: pwsh + run: | + if (-not $env:VULKAN_SDK -or -not (Test-Path $env:VULKAN_SDK)) { + throw "VULKAN_SDK was not configured by jakoch/install-vulkan-sdk-action." + } + $glslc = Join-Path $env:VULKAN_SDK "Bin\glslc.exe" + if (-not (Test-Path $glslc)) { + throw "glslc.exe was not found at $glslc" + } + & $glslc --version + - name: Install ROCm HIP SDK + if: ${{ matrix.backend == 'rocm' }} + uses: ./.github/actions/setup-windows-rocm-sdk + with: + rocm-hip-sdk-filename: ${{ env.ROCM_HIP_SDK_FILENAME }} - name: Initialize MSVC for native runtime uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: x64 - - name: Install backend SDK - if: ${{ matrix.backend != 'cuda' }} - shell: pwsh - run: | - .\scripts\install-windows-sdk.ps1 ` - -Backend "${{ matrix.backend }}" ` - -RocmHipSdkFilename "$env:ROCM_HIP_SDK_FILENAME" ` - -InstallerCacheDir "$HOME\sdk-installer-cache" - name: Package native runtime env: LLAMA_STAGE_BACKEND: ${{ matrix.backend }} @@ -1574,14 +1638,16 @@ jobs: - compose_windows_gpu - build_native_runtime_windows_cpu - build_native_runtime_windows_gpu - if: ${{ always() && needs.metadata.result == 'success' && needs.metadata.outputs.canary != 'true' && needs.compose_cpu_products.result == 'success' && needs.inference_smoke_tests.result == 'success' && needs.build_native_sdk_runtime.result == 'success' && needs.build_native_runtime.result == 'success' && (needs.build_native_runtime_linux_aarch64_cuda.result == 'success' || needs.build_native_runtime_linux_aarch64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_cuda.result == 'success' || needs.build_native_runtime_linux_x86_64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_rocm.result == 'success' || needs.build_native_runtime_linux_x86_64_rocm.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_vulkan.result == 'success' || needs.build_native_runtime_linux_x86_64_vulkan.result == 'skipped') && needs.build_swift_sdk_artifact.result == 'success' && needs.compose_linux_arm64_cpu.result == 'success' && needs.smoke_linux_arm64_artifact.result == 'success' && (needs.compose_linux_aarch64_cuda.result == 'success' || needs.compose_linux_aarch64_cuda.result == 'skipped') && (needs.compose_linux_cuda.result == 'success' || needs.compose_linux_cuda.result == 'skipped') && (needs.compose_linux_rocm.result == 'success' || needs.compose_linux_rocm.result == 'skipped') && (needs.compose_linux_vulkan.result == 'success' || needs.compose_linux_vulkan.result == 'skipped') && needs.compose_windows_cpu.result == 'success' && (needs.compose_windows_gpu.result == 'success' || needs.compose_windows_gpu.result == 'skipped') && needs.build_native_runtime_windows_cpu.result == 'success' && (needs.build_native_runtime_windows_gpu.result == 'success' || needs.build_native_runtime_windows_gpu.result == 'skipped') }} + if: ${{ !cancelled() && needs.metadata.result == 'success' && needs.metadata.outputs.canary != 'true' && needs.compose_cpu_products.result == 'success' && needs.inference_smoke_tests.result == 'success' && needs.build_native_sdk_runtime.result == 'success' && needs.build_native_runtime.result == 'success' && (needs.build_native_runtime_linux_aarch64_cuda.result == 'success' || needs.build_native_runtime_linux_aarch64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_cuda.result == 'success' || needs.build_native_runtime_linux_x86_64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_rocm.result == 'success' || needs.build_native_runtime_linux_x86_64_rocm.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_vulkan.result == 'success' || needs.build_native_runtime_linux_x86_64_vulkan.result == 'skipped') && needs.build_swift_sdk_artifact.result == 'success' && needs.compose_linux_arm64_cpu.result == 'success' && needs.smoke_linux_arm64_artifact.result == 'success' && (needs.compose_linux_aarch64_cuda.result == 'success' || needs.compose_linux_aarch64_cuda.result == 'skipped') && (needs.compose_linux_cuda.result == 'success' || needs.compose_linux_cuda.result == 'skipped') && (needs.compose_linux_rocm.result == 'success' || needs.compose_linux_rocm.result == 'skipped') && (needs.compose_linux_vulkan.result == 'success' || needs.compose_linux_vulkan.result == 'skipped') && needs.compose_windows_cpu.result == 'success' && (needs.compose_windows_gpu.result == 'success' || needs.compose_windows_gpu.result == 'skipped') && needs.build_native_runtime_windows_cpu.result == 'success' && (needs.build_native_runtime_windows_gpu.result == 'success' || needs.build_native_runtime_windows_gpu.result == 'skipped') }} runs-on: ubuntu-24.04 permissions: contents: write steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: - persist-credentials: true + # Archive verification and manifest generation run before any push + # credential is exposed to a shell. + persist-credentials: false fetch-depth: 0 - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 @@ -1656,15 +1722,15 @@ jobs: name: swift-package-manifest path: generated-swift-manifest - - name: Create dispatched release tag + - name: Prepare dispatched release tag if: github.event_name == 'workflow_dispatch' env: RELEASE_TAG: ${{ needs.metadata.outputs.tag }} run: | set -euo pipefail if git ls-remote --exit-code --tags origin "refs/tags/$RELEASE_TAG" >/dev/null 2>&1; then - echo "Release tag already exists: $RELEASE_TAG; reusing it" - exit 0 + echo "Release tag already exists and cannot be reused: $RELEASE_TAG" >&2 + exit 1 fi scripts/release-version.sh "$RELEASE_TAG" cp generated-swift-manifest/Package.swift Package.swift @@ -1688,14 +1754,23 @@ jobs: git commit -m "$RELEASE_TAG: prepare release source" fi git tag "$RELEASE_TAG" - git push origin "refs/tags/$RELEASE_TAG" + + - name: Push dispatched release tag + if: github.event_name == 'workflow_dispatch' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_TAG: ${{ needs.metadata.outputs.tag }} + run: | + set -euo pipefail + release_remote="https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git push "$release_remote" "refs/tags/$RELEASE_TAG" - name: Publish GitHub release uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 with: tag_name: ${{ needs.metadata.outputs.tag }} prerelease: ${{ needs.metadata.outputs.prerelease }} - overwrite_files: true + overwrite_files: false files: release-artifacts/* dispatch_packaging_release: diff --git a/.github/workflows/windows-warm-caches.yml b/.github/workflows/windows-warm-caches.yml index d9186d79be..0cab14e9c8 100644 --- a/.github/workflows/windows-warm-caches.yml +++ b/.github/workflows/windows-warm-caches.yml @@ -6,11 +6,17 @@ on: paths: - 'third_party/llama.cpp/upstream.txt' - 'third_party/llama.cpp/patches/**' - - 'scripts/build-windows.ps1' - - 'scripts/install-windows-sdk.ps1' - - 'Justfile' + - 'scripts/build-llama.sh' + - 'scripts/prepare-llama.sh' + - 'scripts/package-native-runtime.sh' + - 'scripts/verify-native-runtime-package.sh' + - 'scripts/verify-checksum-sidecar.py' + - 'scripts/safe-extract-tar.py' + - 'scripts/windows-native-runtime-deps.py' - '.github/cache-version.txt' + - '.github/actions/prepare-native-runtime-input/action.yml' - '.github/actions/setup-windows-rocm-sdk/action.yml' + - '.github/actions/restore-windows-abi-cache/action.yml' - '.github/workflows/ci.yml' - '.github/workflows/pr_builds.yml' - '.github/workflows/windows-warm-caches.yml' @@ -43,52 +49,50 @@ jobs: runs-on: windows-2022 env: LLAMA_STAGE_BACKEND: cpu - LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-cpu - MESH_LLM_SKIP_UI: "1" - MESH_LLM_REQUIRE_SCCACHE: "1" - RUSTC_WRAPPER: sccache + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/windows/build-stage-abi-cpu steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just - - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 with: - workspaces: . -> target - prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} - shared-key: windows-cpu - save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Prepare UI placeholder - shell: pwsh - run: | - New-Item -ItemType Directory -Force -Path crates/mesh-llm-ui/dist | Out-Null - '' | Set-Content -Path crates/mesh-llm-ui/dist/index.html -Encoding utf8 - - name: Ensure ABI cache directory - shell: pwsh - run: New-Item -ItemType Directory -Force -Path $env:LLAMA_STAGE_BUILD_DIR | Out-Null + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.x' + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - name: Restore Windows CPU ABI cache id: llama_cache - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + uses: ./.github/actions/restore-windows-abi-cache with: - path: .deps/llama.cpp/build-stage-abi-cpu - key: ${{ env.CACHE_NAMESPACE }}-windows-2022-skippy-abi-cpu--cpu-${{ hashFiles('scripts/build-windows.ps1', 'scripts/install-windows-sdk.ps1', '.github/actions/setup-windows-rocm-sdk/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - - name: Build Windows CPU ABI cache + backend: cpu + build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + - name: Initialize MSVC for CPU runtime if: steps.llama_cache.outputs.cache-hit != 'true' - shell: pwsh - run: just release-build-windows + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + with: + arch: x64 + - name: Build and verify canonical Windows CPU runtime input + if: steps.llama_cache.outputs.cache-hit != 'true' + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: cpu + target: x86_64-pc-windows-msvc + output_dir: cache-runtime-input/cpu - name: Verify Windows CPU ABI cache shell: pwsh run: | - $libs = Get-ChildItem -Path $env:LLAMA_STAGE_BUILD_DIR -Recurse -File -Filter *.lib -ErrorAction SilentlyContinue | Select-Object -First 20 - if (-not $libs) { - throw "No static libraries were found under $env:LLAMA_STAGE_BUILD_DIR." + $stamp = Join-Path $env:LLAMA_STAGE_BUILD_DIR ".mesh-llm-build-stamp" + if (-not (Test-Path $stamp -PathType Leaf)) { + throw "Canonical build stamp was not found: $stamp" + } + $libraries = Get-ChildItem -Path $env:LLAMA_STAGE_BUILD_DIR -Recurse -File -Filter *.dll -ErrorAction SilentlyContinue | Select-Object -First 20 + if (-not $libraries) { + throw "No dynamic runtime libraries were found under $env:LLAMA_STAGE_BUILD_DIR." } - $libs | ForEach-Object { Write-Host $_.FullName } + $libraries | ForEach-Object { Write-Host $_.FullName } - name: Save Windows CPU ABI cache if: steps.llama_cache.outputs.cache-hit != 'true' uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: .deps/llama.cpp/build-stage-abi-cpu + path: ${{ env.LLAMA_STAGE_BUILD_DIR }} key: ${{ steps.llama_cache.outputs.cache-primary-key }} warm_windows_gpu: @@ -96,50 +100,38 @@ jobs: if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} runs-on: windows-2022 env: - LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-${{ matrix.backend }} - MESH_LLM_SKIP_UI: "1" - RUSTC_WRAPPER: sccache + LLAMA_STAGE_BUILD_DIR: .deps/llama-build/windows/build-stage-abi-${{ matrix.backend }} strategy: fail-fast: false matrix: include: - name: CUDA backend: cuda - build_recipe: release-build-cuda-windows build_args: "75" - name: ROCm backend: rocm - build_recipe: release-build-rocm-windows build_args: "gfx1100" - name: Vulkan backend: vulkan - build_recipe: release-build-vulkan-windows build_args: "" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just - - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 with: - workspaces: . -> target - prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} - shared-key: windows-${{ matrix.backend }} - save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Prepare UI placeholder - shell: pwsh - run: | - New-Item -ItemType Directory -Force -Path crates/mesh-llm-ui/dist | Out-Null - '' | Set-Content -Path crates/mesh-llm-ui/dist/index.html -Encoding utf8 - - name: Ensure ABI cache directory - shell: pwsh - run: New-Item -ItemType Directory -Force -Path $env:LLAMA_STAGE_BUILD_DIR | Out-Null + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.x' + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - name: Restore Windows GPU ABI cache id: llama_cache - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + uses: ./.github/actions/restore-windows-abi-cache with: - path: .deps/llama.cpp/build-stage-abi-${{ matrix.backend }} - key: ${{ env.CACHE_NAMESPACE }}-windows-2022-skippy-abi-${{ matrix.backend }}-${{ matrix.build_args }}-${{ matrix.backend == 'cuda' && format('cuda-{0}-Jimver-v0.2.35', env.WINDOWS_CUDA_VERSION) || matrix.backend == 'vulkan' && format('vulkan-{0}-jakoch-v1.5.2', env.WINDOWS_VULKAN_SDK_VERSION) || format('rocm-{0}', env.ROCM_HIP_SDK_FILENAME) }}-${{ hashFiles('scripts/build-windows.ps1', 'scripts/install-windows-sdk.ps1', '.github/actions/setup-windows-rocm-sdk/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} + backend: ${{ matrix.backend }} + build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + architecture_set: ${{ matrix.build_args }} + cuda_toolchain_version: ${{ env.WINDOWS_CUDA_VERSION }} + vulkan_toolchain_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} + rocm_toolchain_version: ${{ env.ROCM_HIP_SDK_FILENAME }} - name: Install CUDA toolkit if: ${{ matrix.backend == 'cuda' && steps.llama_cache.outputs.cache-hit != 'true' }} uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 @@ -192,30 +184,38 @@ jobs: uses: ./.github/actions/setup-windows-rocm-sdk with: rocm-hip-sdk-filename: ${{ env.ROCM_HIP_SDK_FILENAME }} - - name: Build Windows GPU ABI cache + - name: Initialize MSVC for native runtime + if: steps.llama_cache.outputs.cache-hit != 'true' + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + with: + arch: x64 + - name: Build and verify canonical Windows GPU runtime input if: steps.llama_cache.outputs.cache-hit != 'true' - shell: pwsh env: - MESH_LLM_REQUIRE_SCCACHE: "1" - run: | - if ("${{ matrix.build_args }}" -ne "") { - just ${{ matrix.build_recipe }} "${{ matrix.build_args }}" - } else { - just ${{ matrix.build_recipe }} - } + LLAMA_STAGE_CUDA_ARCHITECTURES: ${{ matrix.backend == 'cuda' && matrix.build_args || '' }} + LLAMA_STAGE_AMDGPU_TARGETS: ${{ matrix.backend == 'rocm' && matrix.build_args || '' }} + uses: ./.github/actions/prepare-native-runtime-input + with: + backend: ${{ matrix.backend }} + target: x86_64-pc-windows-msvc + output_dir: cache-runtime-input/${{ matrix.backend }} - name: Verify Windows GPU ABI cache shell: pwsh run: | - $libs = Get-ChildItem -Path $env:LLAMA_STAGE_BUILD_DIR -Recurse -File -Filter *.lib -ErrorAction SilentlyContinue | Select-Object -First 20 - if (-not $libs) { - throw "No static libraries were found under $env:LLAMA_STAGE_BUILD_DIR." + $stamp = Join-Path $env:LLAMA_STAGE_BUILD_DIR ".mesh-llm-build-stamp" + if (-not (Test-Path $stamp -PathType Leaf)) { + throw "Canonical build stamp was not found: $stamp" + } + $libraries = Get-ChildItem -Path $env:LLAMA_STAGE_BUILD_DIR -Recurse -File -Filter *.dll -ErrorAction SilentlyContinue | Select-Object -First 20 + if (-not $libraries) { + throw "No dynamic runtime libraries were found under $env:LLAMA_STAGE_BUILD_DIR." } - $libs | ForEach-Object { Write-Host $_.FullName } + $libraries | ForEach-Object { Write-Host $_.FullName } - name: Save Windows GPU ABI cache if: steps.llama_cache.outputs.cache-hit != 'true' uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: .deps/llama.cpp/build-stage-abi-${{ matrix.backend }} + path: ${{ env.LLAMA_STAGE_BUILD_DIR }} key: ${{ steps.llama_cache.outputs.cache-primary-key }} prune_windows_abi_caches: @@ -223,7 +223,7 @@ jobs: needs: - warm_windows_cpu - warm_windows_gpu - if: ${{ always() && github.ref == 'refs/heads/main' }} + if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} runs-on: ubuntu-24.04 steps: - name: Prune old Windows ABI caches diff --git a/ci/DEPOT_MIGRATION.md b/ci/DEPOT_MIGRATION.md index 78e33a314c..1178031ccc 100644 --- a/ci/DEPOT_MIGRATION.md +++ b/ci/DEPOT_MIGRATION.md @@ -9,7 +9,8 @@ main, and release products to drift. Use [`scripts/collect-ci-metrics.py`](../scripts/collect-ci-metrics.py) and the methodology in [`METRICS.md`](METRICS.md) for every before/after comparison. -The initial mixed-change-class baseline is: +The initial mixed-change-class baseline is recorded in +[`metrics/2026-07-29-pr-builds-baseline.json`](metrics/2026-07-29-pr-builds-baseline.json): | Workflow/cohort | Sample | p50 wall | p95 wall | Maximum | | --- | ---: | ---: | ---: | ---: | @@ -75,6 +76,10 @@ The shared implementation primitives are: runtime archive and run the release-grade runtime verifier. - `.github/actions/compose-product-input`: checksum and verify producer inputs, compose product-v2 without compiling, and run client readiness. +- `.github/actions/restore-smoke-inputs`: safely extract a composed product, + revalidate its manifest and bytes, and stage that exact host/runtime pair. +- `.github/actions/capture-sccache-stats`: retain per-job JSON counters for + offline aggregation with `scripts/summarize-sccache-stats.py`. `scripts/build-host.sh` is the only Unix host builder. `scripts/build-release.sh` is a compatibility wrapper. Backend recipes and @@ -279,7 +284,8 @@ The next image revision should: 3. publish a `public-compose` image with only Bash, Python standard library, tar/coreutils, runtime libraries, and artifact verifiers. Composition jobs must never pull a multi-gigabyte backend SDK; -4. remove the duplicate Actions runner installation from self-hosted images; +4. retain the Actions runner only in `self-hosted-*` overlays; keep it out of + public builder and composition images; 5. make one content-addressed architecture base feed every backend overlay, and move source-revision provenance after dependency-warm layers; 6. build each architecture once, verify that exact staged digest, and assemble @@ -327,14 +333,15 @@ image digest, and change class: ```bash python3 scripts/collect-ci-metrics.py \ --repo Mesh-LLM/mesh-llm \ - --workflow pr_builds.yml \ - --event pull_request \ - --limit 30 \ + --workflow ci.yml \ + --branch main \ + --event push \ + --limit 5 \ --label provider=depot \ --label runner=depot-ubuntu-24.04-8 \ - --raw-out /tmp/pr-depot-runs.json \ - --json-out /tmp/pr-depot-metrics.json \ - --markdown-out /tmp/pr-depot-metrics.md + --raw-out /tmp/main-depot-runs.json \ + --json-out /tmp/main-depot-metrics.json \ + --markdown-out /tmp/main-depot-metrics.md ``` Rollout sequence: diff --git a/ci/METRICS.md b/ci/METRICS.md index 0c771ad418..91218d468e 100644 --- a/ci/METRICS.md +++ b/ci/METRICS.md @@ -76,3 +76,93 @@ in separate cohorts, and record the runner provider/image revision with `--label`. Compare both wall time and queue time: a faster compiler does not explain provider-capacity delays, and a shorter routed workflow is not evidence that an unchanged build became faster. + +## Migration baseline and targets + +The pre-migration snapshot was collected on 2026-07-29 from the 20 successful +`pull_request` runs of `pr_builds.yml` recorded in the +[normalized baseline report](metrics/2026-07-29-pr-builds-baseline.json). It +includes the legacy workflow graph and its historical runner mix, so it is +historical workload-mix context, not a provider comparison or a controlled +before/after cohort. + +| Cohort | Samples | p50 | p90 | p95 | Max | +| --- | ---: | ---: | ---: | ---: | ---: | +| PR Builds historical workload mix before product-v2 graph cleanup | 20 | 33m 12s | 45m 21s | 55m 33s | 1h 9m 1s | + +The snapshot mixes different routed workloads: 13 runs executed 31 jobs, four +executed four jobs, and one each executed 12, 19, and 21 jobs. Three head SHAs +appear twice. Depot acceptance comparisons must group exact run IDs by change +class and executed-job graph, use the same sample size, and state whether +repeated SHAs are retained or deduplicated. + +The slowest job families in that historical snapshot were Windows CUDA +(42m 9s p95), Windows ROCm (39m 23s), Windows CPU (32m 6s), Swift SDK smoke +(27m 12s), and Linux ROCm (25m 55s). These values identify legacy hotspots; +they are not a controlled before/after cohort. A single warm run is not +sufficient to replace them. + +Migration success targets: + +| Cohort | Target | +| --- | --- | +| Typical affected-Rust PR | p50 under 10m and p95 under 20m | +| PR native-backend build | p95 under 30m | +| Trusted main CI | p95 under 45m | +| Rust compilation cache | at least 80% hit rate on comparable warm runs | +| Artifact consumers | zero host, runtime, or ABI rebuilds | + +`collect-ci-metrics.py` measures timing and runner queue only; it does not +measure sccache. Compile jobs retain machine-readable +`sccache --show-stats --stats-format json` evidence separately. Zero the +counters immediately before the measured compilation and define aggregate hit +rate as: + +```text +sum(cache_hits.counts) / + (sum(cache_hits.counts) + sum(cache_misses.counts)) +``` + +Compare the warm member of same-SHA, same-provider, same-runner-size, and +same-image cold/warm pairs. The 80% row is an unmeasured rollout gate until +those JSON artifacts have been retained and aggregated; human-readable log +output alone is not acceptance evidence. + +After downloading the selected jobs' `sccache-*` artifacts into one directory, +evaluate the gate offline: + +```bash +python3 scripts/summarize-sccache-stats.py \ + --minimum-hit-rate 0.80 \ + /tmp/sccache-evidence +``` + +For this migration, a run is capacity-contaminated when executed-job runner +queue p95 is at least five minutes or its terminal job waits at least five +minutes for a runner. Such a run can validate correctness and artifact reuse, +but it is excluded from provider performance acceptance. + +The first composable-graph quality observation is +[run 30486038630](https://github.com/Mesh-LLM/mesh-llm/actions/runs/30486038630): +27m 20s wall time, while the three clippy rows executed for 7m 4s–8m 43s. +Individual job queues reached 14m 47s, so this run is recorded as +capacity-contaminated and is not evidence of a compile-time regression. + +The first green composable-graph build observation is +[run 30486038843](https://github.com/Mesh-LLM/mesh-llm/actions/runs/30486038843). +Its 36 executed jobs took 1h 7m 34s wall time, but the median job executed for +only 3m 53s while waiting 10m 33s for a runner. Job execution p95 was 17m 9s; +runner-queue p95 was 21m 52s. The terminal Kotlin SDK smoke waited 18m 4s and +then executed for 11m 55s. Queue delay, rather than product composition, was +the dominant wall-time constraint: the nine Linux, macOS, and Windows +composition action steps each took 10s–70s and rebuilt neither the host nor the +runtime. This single capacity-contaminated observation validates artifact reuse +but does not replace the multi-run baseline. It also predates the final split +of the Linux GPU matrix into independent runtime producers and thin composers. + +| Phase | Change class | Provider / runner | Samples | p50 | p90 | p95 | Notes | +| --- | --- | --- | ---: | ---: | ---: | ---: | --- | +| Product-v2 PR graph | full CI refactor | hosted mix | 1 | 1h 7m 34s | 1h 7m 34s | 1h 7m 34s | Green; queue-contaminated; composition 10s–70s | +| Depot canary cold | trusted main canary | Depot | pending | — | — | — | Restricted workflow allowlist | +| Depot canary warm | trusted main canary | Depot | pending | — | — | — | Same SHA, runner size, and image | +| Main after rollout | full main | mixed | pending | — | — | — | Five comparable green runs minimum | diff --git a/ci/ci.md b/ci/ci.md index 66568d71c6..f4692e006a 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -76,7 +76,12 @@ flowchart TD StaticABI["linux_static_abi_input\none immutable CPU llama ABI"] RustCrateTests["rust_crate_tests matrix\nmetadata-derived crate suites"] LinuxTests["linux_test_groups matrix\nprotocol · Skippy smoke"] - LinuxTargets["linux_targets matrix\nruntime-only CUDA / ROCm / Vulkan\ncompose with shared host"] + LinuxCUDARuntime["linux_cuda_runtime_input\none CUDA runtime"] + LinuxCUDAProduct["linux_cuda_product\ncompose host + CUDA runtime"] + LinuxROCmRuntime["linux_rocm_runtime_input\none ROCm runtime"] + LinuxROCmProduct["linux_rocm_product\ncompose host + ROCm runtime"] + LinuxVulkanRuntime["linux_vulkan_runtime_input\none Vulkan runtime"] + LinuxVulkanProduct["linux_vulkan_product\ncompose host + Vulkan runtime"] WindowsChecks["windows_checks\nlightweight broad-Rust signal"] WindowsHost["windows_host_input\none immutable debug host"] WindowsCPURuntime["windows_cpu_runtime_input\none CPU runtime"] @@ -111,8 +116,15 @@ flowchart TD MacHost --> MacCPU MacRuntime --> MacCPU Affected --> MacTests - LinuxHost --> LinuxTargets - Backend --> LinuxTargets + Backend --> LinuxCUDARuntime + Backend --> LinuxROCmRuntime + Backend --> LinuxVulkanRuntime + LinuxHost --> LinuxCUDAProduct + LinuxCUDARuntime --> LinuxCUDAProduct + LinuxHost --> LinuxROCmProduct + LinuxROCmRuntime --> LinuxROCmProduct + LinuxHost --> LinuxVulkanProduct + LinuxVulkanRuntime --> LinuxVulkanProduct Affected --> WindowsChecks WindowsCPU --> WindowsHost WindowsCPU --> WindowsCPURuntime @@ -169,10 +181,13 @@ flowchart TD smokes consume the staged runtime instead of compiling a private replacement. - Main builds immutable Linux, macOS, and Windows release hosts independently from their CPU, Metal, CUDA, ROCm, and Vulkan runtimes. Composition-only jobs - verify and combine those exact producer inputs. Linux and Windows backend - rows build only their runtime and reuse the platform host. Each product - requires `--version`, `runtime list`, and client readiness without a driver - stub. GPU availability remains separate hardware qualification. + verify and combine those exact producer inputs. Each Linux GPU backend has + its own runtime producer and matching thin composer, so a finished backend + never waits for unrelated runtime rows before its product is ready. Linux + and Windows backend rows build only their runtime and reuse the platform + host. Each product requires `--version`, `runtime list`, and client readiness + without a driver stub. GPU availability remains separate hardware + qualification. - `.github/actions/prepare-host-input`, `.github/actions/prepare-windows-host-input`, `.github/actions/prepare-native-runtime-input`, and @@ -215,10 +230,12 @@ flowchart TD protocol compatibility and Skippy smoke remain separate integration rows. Linux host/CPU-runtime and macOS host/Metal-runtime producers run independently, and their product composers never compile. Linux backend rows - consume the same immutable host artifact and build only their selected - runtime. Windows follows the same graph: one debug neutral host, independent - CPU/CUDA/ROCm/Vulkan runtime inputs, and composition-only products. - Unsupported macOS CUDA, ROCm, and Vulkan rows are omitted. + are split into one independent CUDA, ROCm, or Vulkan runtime producer plus + one matching `runner_4` composition-only product job. They consume the same + immutable host without a matrix-wide fan-in barrier. Windows follows the + same graph: one debug neutral host, independent CPU/CUDA/ROCm/Vulkan runtime + inputs, and composition-only products. Unsupported macOS CUDA, ROCm, and + Vulkan rows are omitted. - Product readiness starts a local mDNS client and never depends on the mutable public mesh. The public `client --auto` admission probe is manual-only, so an external peer outage cannot block a pull request or release. @@ -282,8 +299,8 @@ and published at `ghcr.io/mesh-llm/mesh-llm-cuda-runner`. Every image is built from the same core toolchain and selects an execution environment independently from its backend SDK: -- `public--*` runs as a job-level `container:` on an Ubuntu GitHub-hosted - or legacy container-capable self-hosted runner. +- `public--*` runs as a job-level `container:` on an Ubuntu + GitHub-hosted, Depot-managed, or legacy container-capable self-hosted runner. - `self-hosted--*` adds the Actions runner and is used directly as an ARC pod image. Jobs targeting an ARC scale-set label must not wrap that pod in a second job container. @@ -461,14 +478,17 @@ Use these checks when reviewing PR CI wall-clock regressions: - **Duplicate work count**: smoke jobs should consume uploaded Linux/macOS binaries through `.github/actions/restore-smoke-inputs`; they should not build `mesh-llm` or patched llama.cpp again. -- **Prewarmed ABI cache hit ratio**: Windows ABI cache keys in PR Builds must - match the trusted `windows-warm-caches.yml` keys. Check +- **Prewarmed ABI cache hit ratio**: Windows runtime producers in PR, main, and + release use `.github/actions/restore-windows-abi-cache`, the same exact key + contract as `windows-warm-caches.yml`. Architecture sets and backend + toolchain versions are part of the identity; there are no broad restore + prefixes. Check `gh cache list --branch main --limit 100` for - `mesh-llm-windows-2025-skippy-abi-*` entries before + `mesh-llm-windows-2022-skippy-abi-*` entries before treating a slow Windows miss as expected. - **Runner routing**: platform-specific work should run on its native runner - class (Blacksmith Windows 2025 for Windows ABI products, Blacksmith macOS for Swift/Metal, Linux - for Linux backends) and omit unsupported combinations. + class (Windows for Windows ABI products, macOS for Swift/Metal, Linux for + Linux backends) and omit unsupported combinations. For canonical agent-facing CI rules, start with `.agents/skills/manage-ci/SKILL.md`. The scoped `.github/AGENTS.md` file routes diff --git a/ci/metrics/2026-07-29-pr-builds-baseline.json b/ci/metrics/2026-07-29-pr-builds-baseline.json new file mode 100644 index 0000000000..f74d6c39e1 --- /dev/null +++ b/ci/metrics/2026-07-29-pr-builds-baseline.json @@ -0,0 +1,1747 @@ +{ + "benchmark_labels": { + "cohort": "historical-pre-product-v2-workload-mix", + "collected_at": "2026-07-29T16:25:58Z" + }, + "definitions": { + "job_duration_seconds": "job started_at to completed_at", + "job_queue_seconds": "job created_at to started_at; unavailable without job created_at", + "job_start_delay_seconds": "first-attempt workflow created_at to job started_at; includes dependency wait; reruns excluded", + "terminal_job": "last non-skipped job to finish; a critical-path candidate", + "workflow_queue_seconds": "first-attempt run created_at to started_at; reruns excluded", + "workflow_wall_seconds": "first-attempt run created_at to updated_at; reruns excluded" + }, + "generated_at": "2026-07-29T21:57:43.409412Z", + "jobs": { + "by_name": [ + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 3145.0, + "mean": 1476.714, + "min": 66.0, + "p50": 1742.5, + "p90": 2121.4, + "p95": 2528.8 + }, + "name": "Windows CUDA", + "queue_seconds": { + "count": 14, + "max": 414.0, + "mean": 76.714, + "min": 2.0, + "p50": 4.0, + "p90": 296.1, + "p95": 355.5 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 493.0, + "mean": 129.571, + "min": 16.0, + "p50": 70.5, + "p90": 365.4, + "p95": 428.0 + }, + "terminal_count": 2 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 2684.0, + "mean": 1321.714, + "min": 62.0, + "p50": 1499.5, + "p90": 2103.6, + "p95": 2362.9 + }, + "name": "Windows ROCm", + "queue_seconds": { + "count": 14, + "max": 548.0, + "mean": 84.786, + "min": 2.0, + "p50": 4.5, + "p90": 367.4, + "p95": 503.15 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 618.0, + "mean": 137.643, + "min": 16.0, + "p50": 68.0, + "p90": 431.9, + "p95": 572.5 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 2490.0, + "mean": 1413.429, + "min": 1025.0, + "p50": 1372.0, + "p90": 1617.2, + "p95": 1925.8 + }, + "name": "Windows CPU", + "queue_seconds": { + "count": 14, + "max": 345.0, + "mean": 48.571, + "min": 2.0, + "p50": 5.0, + "p90": 162.8, + "p95": 256.6 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 414.0, + "mean": 101.429, + "min": 16.0, + "p50": 61.5, + "p90": 235.5, + "p95": 326.25 + }, + "terminal_count": 1 + }, + { + "conclusions": { + "success": 15 + }, + "duration_seconds": { + "count": 15, + "max": 1640.0, + "mean": 1409.4, + "min": 1202.0, + "p50": 1406.0, + "p90": 1616.0, + "p95": 1631.6 + }, + "name": "swift_sdk_smoke / swift SDK Smoke", + "queue_seconds": { + "count": 15, + "max": 385.0, + "mean": 78.733, + "min": 3.0, + "p50": 8.0, + "p90": 230.0, + "p95": 284.9 + }, + "sample_count": 15, + "start_delay_seconds": { + "count": 15, + "max": 1631.0, + "mean": 640.133, + "min": 205.0, + "p50": 559.0, + "p90": 934.6, + "p95": 1155.0 + }, + "terminal_count": 8 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 1570.0, + "mean": 1459.0, + "min": 1194.0, + "p50": 1469.0, + "p90": 1545.8, + "p95": 1555.05 + }, + "name": "Linux ROCm slim", + "queue_seconds": { + "count": 14, + "max": 1187.0, + "mean": 98.643, + "min": 2.0, + "p50": 9.0, + "p90": 73.7, + "p95": 473.3 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 1257.0, + "mean": 151.5, + "min": 18.0, + "p50": 71.5, + "p90": 105.8, + "p95": 509.5 + }, + "terminal_count": 1 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 1591.0, + "mean": 987.143, + "min": 72.0, + "p50": 1233.5, + "p90": 1462.3, + "p95": 1516.9 + }, + "name": "Windows Vulkan", + "queue_seconds": { + "count": 14, + "max": 304.0, + "mean": 57.643, + "min": 2.0, + "p50": 7.0, + "p90": 226.1, + "p95": 287.1 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 374.0, + "mean": 110.5, + "min": 18.0, + "p50": 73.0, + "p90": 302.1, + "p95": 362.95 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 1104.0, + "mean": 855.143, + "min": 445.0, + "p50": 900.5, + "p90": 1072.3, + "p95": 1100.75 + }, + "name": "Rust crate tests (0)", + "queue_seconds": { + "count": 14, + "max": 1186.0, + "mean": 161.071, + "min": 2.0, + "p50": 3.5, + "p90": 645.8, + "p95": 966.3 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 1256.0, + "mean": 212.786, + "min": 15.0, + "p50": 66.0, + "p90": 698.3, + "p95": 1035.65 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 1101.0, + "mean": 764.214, + "min": 387.0, + "p50": 768.0, + "p90": 1019.3, + "p95": 1084.75 + }, + "name": "Rust crate tests (1)", + "queue_seconds": { + "count": 14, + "max": 892.0, + "mean": 121.857, + "min": 2.0, + "p50": 3.0, + "p90": 534.7, + "p95": 802.3 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 961.0, + "mean": 173.571, + "min": 16.0, + "p50": 62.5, + "p90": 602.3, + "p95": 871.95 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 1111.0, + "mean": 906.429, + "min": 659.0, + "p50": 980.0, + "p90": 1023.4, + "p95": 1058.35 + }, + "name": "Linux Vulkan", + "queue_seconds": { + "count": 14, + "max": 925.0, + "mean": 135.357, + "min": 2.0, + "p50": 3.0, + "p90": 536.3, + "p95": 744.3 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 994.0, + "mean": 188.214, + "min": 16.0, + "p50": 65.0, + "p90": 609.0, + "p95": 813.95 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 15 + }, + "duration_seconds": { + "count": 15, + "max": 1053.0, + "mean": 972.2, + "min": 832.0, + "p50": 1001.0, + "p90": 1024.2, + "p95": 1036.2 + }, + "name": "kotlin_sdk_smoke / kotlin SDK Smoke", + "queue_seconds": { + "count": 15, + "max": 1146.0, + "mean": 234.267, + "min": 1.0, + "p50": 65.0, + "p90": 616.0, + "p95": 894.0 + }, + "sample_count": 15, + "start_delay_seconds": { + "count": 15, + "max": 1663.0, + "mean": 754.933, + "min": 281.0, + "p50": 650.0, + "p90": 1301.6, + "p95": 1428.5 + }, + "terminal_count": 2 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 867.0, + "mean": 841.571, + "min": 822.0, + "p50": 836.5, + "p90": 864.6, + "p95": 867.0 + }, + "name": "Linux CUDA slim", + "queue_seconds": { + "count": 14, + "max": 3214.0, + "mean": 425.571, + "min": 10.0, + "p50": 36.0, + "p90": 1103.0, + "p95": 1943.25 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 3284.0, + "mean": 478.429, + "min": 26.0, + "p50": 102.5, + "p90": 1175.0, + "p95": 2012.6 + }, + "terminal_count": 1 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 858.0, + "mean": 635.429, + "min": 263.0, + "p50": 667.5, + "p90": 823.2, + "p95": 848.25 + }, + "name": "Rust crate tests (3)", + "queue_seconds": { + "count": 14, + "max": 104.0, + "mean": 22.857, + "min": 2.0, + "p50": 7.5, + "p90": 82.3, + "p95": 99.45 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 173.0, + "mean": 74.571, + "min": 16.0, + "p50": 66.0, + "p90": 115.9, + "p95": 137.25 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 859.0, + "mean": 642.5, + "min": 362.0, + "p50": 666.5, + "p90": 820.9, + "p95": 835.6 + }, + "name": "Rust crate tests (2)", + "queue_seconds": { + "count": 14, + "max": 1250.0, + "mean": 154.643, + "min": 2.0, + "p50": 6.0, + "p90": 422.8, + "p95": 720.9 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 1320.0, + "mean": 206.357, + "min": 22.0, + "p50": 62.5, + "p90": 494.8, + "p95": 790.25 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 16 + }, + "duration_seconds": { + "count": 16, + "max": 860.0, + "mean": 378.438, + "min": 177.0, + "p50": 328.0, + "p90": 591.0, + "p95": 734.75 + }, + "name": "macOS CPU", + "queue_seconds": { + "count": 16, + "max": 1190.0, + "mean": 126.188, + "min": 2.0, + "p50": 9.0, + "p90": 359.5, + "p95": 659.75 + }, + "sample_count": 16, + "start_delay_seconds": { + "count": 16, + "max": 1260.0, + "mean": 178.375, + "min": 17.0, + "p50": 67.5, + "p90": 406.0, + "p95": 736.5 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 668.0, + "mean": 509.143, + "min": 354.0, + "p50": 518.0, + "p90": 636.4, + "p95": 667.35 + }, + "name": "macOS unit tests", + "queue_seconds": { + "count": 14, + "max": 914.0, + "mean": 176.071, + "min": 2.0, + "p50": 6.5, + "p90": 671.0, + "p95": 832.1 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 1617.0, + "mean": 745.714, + "min": 198.0, + "p50": 730.0, + "p90": 1290.1, + "p95": 1436.3 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 15 + }, + "duration_seconds": { + "count": 15, + "max": 596.0, + "mean": 537.733, + "min": 470.0, + "p50": 542.0, + "p90": 564.2, + "p95": 574.3 + }, + "name": "rust_sdk_smoke / rust SDK Smoke", + "queue_seconds": { + "count": 15, + "max": 1237.0, + "mean": 300.733, + "min": 1.0, + "p50": 21.0, + "p90": 990.2, + "p95": 1161.4 + }, + "sample_count": 15, + "start_delay_seconds": { + "count": 15, + "max": 1779.0, + "mean": 821.4, + "min": 280.0, + "p50": 552.0, + "p90": 1614.0, + "p95": 1685.9 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 16 + }, + "duration_seconds": { + "count": 16, + "max": 843.0, + "mean": 394.75, + "min": 238.0, + "p50": 431.5, + "p90": 464.5, + "p95": 564.75 + }, + "name": "Linux CPU", + "queue_seconds": { + "count": 16, + "max": 450.0, + "mean": 59.5, + "min": 2.0, + "p50": 6.0, + "p90": 218.5, + "p95": 401.25 + }, + "sample_count": 16, + "start_delay_seconds": { + "count": 16, + "max": 529.0, + "mean": 111.688, + "min": 16.0, + "p50": 62.5, + "p90": 288.0, + "p95": 473.5 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 347.0, + "mean": 297.286, + "min": 97.0, + "p50": 313.5, + "p90": 331.7, + "p95": 337.25 + }, + "name": "inference_smoke_tests / Skippy Inference Smoke Tests", + "queue_seconds": { + "count": 14, + "max": 797.0, + "mean": 169.857, + "min": 2.0, + "p50": 20.5, + "p90": 589.6, + "p95": 718.35 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 1339.0, + "mean": 705.286, + "min": 287.0, + "p50": 602.5, + "p90": 1248.9, + "p95": 1309.1 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 16 + }, + "duration_seconds": { + "count": 16, + "max": 380.0, + "mean": 230.188, + "min": 33.0, + "p50": 233.5, + "p90": 267.5, + "p95": 302.0 + }, + "name": "Linux client-auto boot test", + "queue_seconds": { + "count": 16, + "max": 2752.0, + "mean": 392.812, + "min": 2.0, + "p50": 8.0, + "p90": 973.5, + "p95": 1554.25 + }, + "sample_count": 16, + "start_delay_seconds": { + "count": 16, + "max": 3050.0, + "mean": 899.562, + "min": 286.0, + "p50": 663.5, + "p90": 1582.5, + "p95": 2035.25 + }, + "terminal_count": 1 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 290.0, + "mean": 236.643, + "min": 69.0, + "p50": 243.0, + "p90": 282.1, + "p95": 285.45 + }, + "name": "two_node_split_smoke / Scripted Binary Smoke", + "queue_seconds": { + "count": 14, + "max": 762.0, + "mean": 110.5, + "min": 1.0, + "p50": 33.0, + "p90": 194.2, + "p95": 401.9 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 1304.0, + "mean": 645.929, + "min": 286.0, + "p50": 546.0, + "p90": 970.5, + "p95": 1111.6 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 298.0, + "mean": 240.857, + "min": 67.0, + "p50": 249.0, + "p90": 272.9, + "p95": 283.05 + }, + "name": "two_node_client_serving_smoke / Scripted Binary Smoke", + "queue_seconds": { + "count": 14, + "max": 1159.0, + "mean": 184.857, + "min": 2.0, + "p50": 37.0, + "p90": 396.9, + "p95": 666.95 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 1676.0, + "mean": 720.286, + "min": 286.0, + "p50": 623.5, + "p90": 983.7, + "p95": 1241.8 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 16 + }, + "duration_seconds": { + "count": 16, + "max": 262.0, + "mean": 220.625, + "min": 115.0, + "p50": 225.0, + "p90": 255.5, + "p95": 258.25 + }, + "name": "Linux tests (protocol)", + "queue_seconds": { + "count": 16, + "max": 2635.0, + "mean": 419.875, + "min": 1.0, + "p50": 8.0, + "p90": 1129.5, + "p95": 1567.75 + }, + "sample_count": 16, + "start_delay_seconds": { + "count": 16, + "max": 2933.0, + "mean": 926.625, + "min": 286.0, + "p50": 626.0, + "p90": 1659.0, + "p95": 2048.75 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 16 + }, + "duration_seconds": { + "count": 16, + "max": 195.0, + "mean": 157.5, + "min": 139.0, + "p50": 156.5, + "p90": 167.0, + "p95": 175.5 + }, + "name": "Linux tests (skippy-smoke)", + "queue_seconds": { + "count": 16, + "max": 1767.0, + "mean": 368.562, + "min": 2.0, + "p50": 38.5, + "p90": 1079.5, + "p95": 1264.5 + }, + "sample_count": 16, + "start_delay_seconds": { + "count": 16, + "max": 2065.0, + "mean": 875.312, + "min": 286.0, + "p50": 729.0, + "p90": 1609.0, + "p95": 1726.75 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 20 + }, + "duration_seconds": { + "count": 20, + "max": 169.0, + "mean": 156.4, + "min": 153.0, + "p50": 155.5, + "p90": 158.1, + "p95": 159.5 + }, + "name": "Runner image / ARC arm64", + "queue_seconds": { + "count": 20, + "max": 10.0, + "mean": 8.8, + "min": 8.0, + "p50": 9.0, + "p90": 9.1, + "p95": 10.0 + }, + "sample_count": 20, + "start_delay_seconds": { + "count": 20, + "max": 90.0, + "mean": 62.45, + "min": 21.0, + "p50": 66.5, + "p90": 88.1, + "p95": 89.05 + }, + "terminal_count": 3 + }, + { + "conclusions": { + "success": 20 + }, + "duration_seconds": { + "count": 20, + "max": 114.0, + "mean": 106.0, + "min": 91.0, + "p50": 108.5, + "p90": 112.1, + "p95": 113.05 + }, + "name": "Runner image / GitHub-hosted public", + "queue_seconds": { + "count": 20, + "max": 565.0, + "mean": 41.0, + "min": 2.0, + "p50": 7.5, + "p90": 24.8, + "p95": 178.35 + }, + "sample_count": 20, + "start_delay_seconds": { + "count": 20, + "max": 635.0, + "mean": 94.65, + "min": 16.0, + "p50": 62.5, + "p90": 104.7, + "p95": 256.9 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 67.0, + "mean": 46.0, + "min": 33.0, + "p50": 45.0, + "p90": 60.7, + "p95": 65.05 + }, + "name": "HuggingFace download smoke / HuggingFace download smoke", + "queue_seconds": { + "count": 14, + "max": 587.0, + "mean": 56.643, + "min": 3.0, + "p50": 8.5, + "p90": 84.1, + "p95": 274.35 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 657.0, + "mean": 108.357, + "min": 19.0, + "p50": 71.5, + "p90": 111.0, + "p95": 307.95 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 20 + }, + "duration_seconds": { + "count": 20, + "max": 65.0, + "mean": 40.55, + "min": 11.0, + "p50": 46.0, + "p90": 56.1, + "p95": 57.4 + }, + "name": "changes", + "queue_seconds": { + "count": 20, + "max": 48.0, + "mean": 7.6, + "min": 1.0, + "p50": 2.0, + "p90": 20.6, + "p95": 27.1 + }, + "sample_count": 20, + "start_delay_seconds": { + "count": 20, + "max": 54.0, + "mean": 12.95, + "min": 2.0, + "p50": 9.0, + "p90": 23.3, + "p95": 27.4 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 20 + }, + "duration_seconds": { + "count": 20, + "max": 17.0, + "mean": 14.35, + "min": 12.0, + "p50": 14.0, + "p90": 16.0, + "p95": 16.05 + }, + "name": "Runner image / ARC amd64", + "queue_seconds": { + "count": 20, + "max": 2531.0, + "mean": 780.6, + "min": 10.0, + "p50": 169.0, + "p90": 2157.9, + "p95": 2355.25 + }, + "sample_count": 20, + "start_delay_seconds": { + "count": 20, + "max": 2585.0, + "mean": 834.25, + "min": 23.0, + "p50": 238.5, + "p90": 2227.0, + "p95": 2424.45 + }, + "terminal_count": 1 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 7.0, + "mean": 3.857, + "min": 2.0, + "p50": 3.5, + "p90": 5.0, + "p95": 5.7 + }, + "name": "macOS ROCm", + "queue_seconds": { + "count": 14, + "max": 918.0, + "mean": 166.214, + "min": 2.0, + "p50": 10.0, + "p90": 660.0, + "p95": 758.1 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 988.0, + "mean": 219.071, + "min": 17.0, + "p50": 72.0, + "p90": 696.5, + "p95": 801.45 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 5.0, + "mean": 3.857, + "min": 2.0, + "p50": 4.0, + "p90": 5.0, + "p95": 5.0 + }, + "name": "macOS CUDA", + "queue_seconds": { + "count": 14, + "max": 932.0, + "mean": 74.429, + "min": 2.0, + "p50": 3.5, + "p90": 38.0, + "p95": 356.75 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 1002.0, + "mean": 127.286, + "min": 16.0, + "p50": 66.0, + "p90": 106.4, + "p95": 426.1 + }, + "terminal_count": 0 + }, + { + "conclusions": { + "success": 14 + }, + "duration_seconds": { + "count": 14, + "max": 5.0, + "mean": 3.857, + "min": 2.0, + "p50": 4.0, + "p90": 5.0, + "p95": 5.0 + }, + "name": "macOS Vulkan", + "queue_seconds": { + "count": 14, + "max": 878.0, + "mean": 72.143, + "min": 3.0, + "p50": 4.0, + "p90": 56.9, + "p95": 357.35 + }, + "sample_count": 14, + "start_delay_seconds": { + "count": 14, + "max": 947.0, + "mean": 125.0, + "min": 16.0, + "p50": 68.5, + "p90": 90.4, + "p95": 390.6 + }, + "terminal_count": 0 + } + ], + "critical_finish_candidates": [ + { + "name": "swift_sdk_smoke / swift SDK Smoke", + "share": 0.4, + "terminal_count": 8 + }, + { + "name": "Runner image / ARC arm64", + "share": 0.15, + "terminal_count": 3 + }, + { + "name": "Windows CUDA", + "share": 0.1, + "terminal_count": 2 + }, + { + "name": "kotlin_sdk_smoke / kotlin SDK Smoke", + "share": 0.1, + "terminal_count": 2 + }, + { + "name": "Runner image / ARC amd64", + "share": 0.05, + "terminal_count": 1 + }, + { + "name": "Windows CPU", + "share": 0.05, + "terminal_count": 1 + }, + { + "name": "Linux client-auto boot test", + "share": 0.05, + "terminal_count": 1 + }, + { + "name": "Linux ROCm slim", + "share": 0.05, + "terminal_count": 1 + }, + { + "name": "Linux CUDA slim", + "share": 0.05, + "terminal_count": 1 + } + ], + "duration_seconds": { + "count": 471, + "max": 3145.0, + "mean": 529.229, + "min": 2.0, + "p50": 298.0, + "p90": 1400.0, + "p95": 1558.5 + }, + "queue_seconds": { + "count": 471, + "max": 3214.0, + "mean": 172.856, + "min": 1.0, + "p50": 8.0, + "p90": 647.0, + "p95": 989.5 + }, + "sample_count": 471, + "slowest_observations": [ + { + "conclusion": "success", + "duration_seconds": 3145.0, + "job_id": 90226220305, + "job_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30344045951/job/90226220305", + "name": "Windows CUDA", + "queue_seconds": 231.0, + "run_attempt": 1, + "run_id": 30344045951, + "run_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30344045951", + "runner_labels": [ + "windows-2022" + ], + "start_delay_seconds": 301.0 + }, + { + "conclusion": "success", + "duration_seconds": 2684.0, + "job_id": 90226220362, + "job_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30344045951/job/90226220362", + "name": "Windows ROCm", + "queue_seconds": 548.0, + "run_attempt": 1, + "run_id": 30344045951, + "run_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30344045951", + "runner_labels": [ + "windows-2022" + ], + "start_delay_seconds": 618.0 + }, + { + "conclusion": "success", + "duration_seconds": 2490.0, + "job_id": 90522879634, + "job_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30435682397/job/90522879634", + "name": "Windows CPU", + "queue_seconds": 3.0, + "run_attempt": 1, + "run_id": 30435682397, + "run_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30435682397", + "runner_labels": [ + "windows-2022" + ], + "start_delay_seconds": 38.0 + }, + { + "conclusion": "success", + "duration_seconds": 2197.0, + "job_id": 90522879591, + "job_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30435682397/job/90522879591", + "name": "Windows CUDA", + "queue_seconds": 3.0, + "run_attempt": 1, + "run_id": 30435682397, + "run_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30435682397", + "runner_labels": [ + "windows-2022" + ], + "start_delay_seconds": 38.0 + }, + { + "conclusion": "success", + "duration_seconds": 2190.0, + "job_id": 90522879567, + "job_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30435682397/job/90522879567", + "name": "Windows ROCm", + "queue_seconds": 3.0, + "run_attempt": 1, + "run_id": 30435682397, + "run_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30435682397", + "runner_labels": [ + "windows-2022" + ], + "start_delay_seconds": 38.0 + }, + { + "conclusion": "success", + "duration_seconds": 1945.0, + "job_id": 90465318251, + "job_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30416902770/job/90465318251", + "name": "Windows CUDA", + "queue_seconds": 63.0, + "run_attempt": 1, + "run_id": 30416902770, + "run_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30416902770", + "runner_labels": [ + "windows-2022" + ], + "start_delay_seconds": 77.0 + }, + { + "conclusion": "success", + "duration_seconds": 1902.0, + "job_id": 90409838809, + "job_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30399212634/job/90409838809", + "name": "Windows ROCm", + "queue_seconds": 107.0, + "run_attempt": 1, + "run_id": 30399212634, + "run_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30399212634", + "runner_labels": [ + "windows-2022" + ], + "start_delay_seconds": 161.0 + }, + { + "conclusion": "success", + "duration_seconds": 1895.0, + "job_id": 90409838754, + "job_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30399212634/job/90409838754", + "name": "Windows CUDA", + "queue_seconds": 2.0, + "run_attempt": 1, + "run_id": 30399212634, + "run_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30399212634", + "runner_labels": [ + "windows-2022" + ], + "start_delay_seconds": 56.0 + }, + { + "conclusion": "success", + "duration_seconds": 1838.0, + "job_id": 90472166252, + "job_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30419140040/job/90472166252", + "name": "Windows CUDA", + "queue_seconds": 2.0, + "run_attempt": 1, + "run_id": 30419140040, + "run_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30419140040", + "runner_labels": [ + "windows-2022" + ], + "start_delay_seconds": 50.0 + }, + { + "conclusion": "success", + "duration_seconds": 1832.0, + "job_id": 90240122010, + "job_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30348390498/job/90240122010", + "name": "Windows ROCm", + "queue_seconds": 4.0, + "run_attempt": 1, + "run_id": 30348390498, + "run_url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30348390498", + "runner_labels": [ + "windows-2022" + ], + "start_delay_seconds": 71.0 + } + ], + "start_delay_seconds": { + "count": 471, + "max": 3284.0, + "mean": 374.223, + "min": 2.0, + "p50": 88.0, + "p90": 994.0, + "p95": 1403.5 + } + }, + "runs": [ + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 4, + "head_branch": "codex/release-composed-smoke-artifact", + "head_sha": "eda9c348ebcfcd9a73d834c8eabc7784e4e0f1c2", + "id": 30470003625, + "longest_job": "Runner image / ARC arm64", + "longest_job_seconds": 156.0, + "queue_seconds": 0.0, + "skipped_job_count": 17, + "terminal_job": "Runner image / ARC arm64", + "title": "Fix release inference smoke artifact composition", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30470003625", + "wall_seconds": 235.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 4, + "head_branch": "codex/release-composed-smoke-artifact", + "head_sha": "eda9c348ebcfcd9a73d834c8eabc7784e4e0f1c2", + "id": 30469571715, + "longest_job": "Runner image / ARC arm64", + "longest_job_seconds": 155.0, + "queue_seconds": 0.0, + "skipped_job_count": 17, + "terminal_job": "Runner image / ARC arm64", + "title": "Fix release inference smoke artifact composition", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30469571715", + "wall_seconds": 215.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 4, + "head_branch": "codex/release-host-import-python", + "head_sha": "e7a7dbeff2eca7c3f16fb972c49e64bff5050236", + "id": 30468256564, + "longest_job": "Runner image / ARC arm64", + "longest_job_seconds": 154.0, + "queue_seconds": 0.0, + "skipped_job_count": 17, + "terminal_job": "Runner image / ARC amd64", + "title": "Fix release host dependency verifier invocation", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30468256564", + "wall_seconds": 400.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "codex/build-unification-foundation", + "head_sha": "c594ab435576e84a13cda19ff8101a11b8351b66", + "id": 30435682397, + "longest_job": "Windows CPU", + "longest_job_seconds": 2490.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "Windows CPU", + "title": "task: unify release hosts, native runtimes, and product bundles", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30435682397", + "wall_seconds": 2531.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "feat/laguna-certification", + "head_sha": "6ee85338063f3968285777064360b0cce31bfb97", + "id": 30419140040, + "longest_job": "Windows CUDA", + "longest_job_seconds": 1838.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "Windows CUDA", + "title": "feat(skippy): add Poolside Laguna staged-runtime candidate", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30419140040", + "wall_seconds": 1889.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "codex/inkling-q2-skippy", + "head_sha": "f6f5adad42c9abd4113108882d0b47cd00df3c08", + "id": 30416902770, + "longest_job": "Windows CUDA", + "longest_job_seconds": 1945.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "swift_sdk_smoke / swift SDK Smoke", + "title": "Add Inkling Q2 staged runtime and certify text split serving", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30416902770", + "wall_seconds": 2223.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "fix/skippy-hybrid-verify-recovery", + "head_sha": "0abff1a25f336f9135e53843097dec42b931191b", + "id": 30416288428, + "longest_job": "Windows CUDA", + "longest_job_seconds": 1675.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "Windows CUDA", + "title": "skippy: recover hybrid speculative windows and orphaned lanes", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30416288428", + "wall_seconds": 1757.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "agent/refresh-llama-patch-queue", + "head_sha": "a32fc98e06f94cbcb127f6e0d0e166c2e225f1b8", + "id": 30406445718, + "longest_job": "Linux ROCm slim", + "longest_job_seconds": 1448.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "swift_sdk_smoke / swift SDK Smoke", + "title": "Refresh llama.cpp upstream patch queue", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30406445718", + "wall_seconds": 1789.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "agent/refresh-llama-patch-queue", + "head_sha": "61e67fe992c949b06470d6713abaaade7b691d53", + "id": 30403101741, + "longest_job": "Windows CUDA", + "longest_job_seconds": 1650.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "swift_sdk_smoke / swift SDK Smoke", + "title": "Refresh llama.cpp upstream patch queue", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30403101741", + "wall_seconds": 2101.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 19, + "head_branch": "revert-1099-agent/refresh-llama-patch-queue", + "head_sha": "b0e2a21e8c6ae932341d81c9e2b0c2aeb680c0ef", + "id": 30399212634, + "longest_job": "Windows ROCm", + "longest_job_seconds": 1902.0, + "queue_seconds": 0.0, + "skipped_job_count": 10, + "terminal_job": "Linux client-auto boot test", + "title": "Revert \"Refresh llama.cpp upstream patch queue\"", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30399212634", + "wall_seconds": 3290.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "codex/inkling-q2-skippy", + "head_sha": "19653da20721d287b8856dc56eca097d3868fd83", + "id": 30397484523, + "longest_job": "Linux ROCm slim", + "longest_job_seconds": 1442.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "Linux ROCm slim", + "title": "Add Inkling Q2 staged runtime and certify text split serving", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30397484523", + "wall_seconds": 1461.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 21, + "head_branch": "chore/code-quality-review", + "head_sha": "3770a3ef5955f9f25a781a1615ea6cdc4c71f141", + "id": 30388836882, + "longest_job": "swift_sdk_smoke / swift SDK Smoke", + "longest_job_seconds": 1544.0, + "queue_seconds": 0.0, + "skipped_job_count": 4, + "terminal_job": "swift_sdk_smoke / swift SDK Smoke", + "title": "chore(code-quality): generalized code-quality refactoring", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30388836882", + "wall_seconds": 1778.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "codex/inkling-q2-skippy", + "head_sha": "19653da20721d287b8856dc56eca097d3868fd83", + "id": 30387256890, + "longest_job": "Windows CUDA", + "longest_job_seconds": 1817.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "swift_sdk_smoke / swift SDK Smoke", + "title": "Add Inkling Q2 staged runtime and certify text split serving", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30387256890", + "wall_seconds": 2086.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 4, + "head_branch": "codex/canonical-homebrew-tap", + "head_sha": "66f946e4e70d187ec73d84b9a6e9cd2dda14e251", + "id": 30387136356, + "longest_job": "Runner image / ARC arm64", + "longest_job_seconds": 159.0, + "queue_seconds": 0.0, + "skipped_job_count": 17, + "terminal_job": "Runner image / ARC arm64", + "title": "Document canonical Homebrew tap", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30387136356", + "wall_seconds": 227.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 12, + "head_branch": "codex/rename-node-sdk-npm-scope", + "head_sha": "da0b68c53b0906aaa68ab5f9bc9cff16f02a6d37", + "id": 30380850230, + "longest_job": "swift_sdk_smoke / swift SDK Smoke", + "longest_job_seconds": 1449.0, + "queue_seconds": 0.0, + "skipped_job_count": 10, + "terminal_job": "swift_sdk_smoke / swift SDK Smoke", + "title": "Rename the Node SDK npm package scope", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30380850230", + "wall_seconds": 1898.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "fix/skippy-hybrid-verify-recovery", + "head_sha": "1d3ee4a065d820603d5343abcefbde96182ea005", + "id": 30351493597, + "longest_job": "Linux ROCm slim", + "longest_job_seconds": 1518.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "swift_sdk_smoke / swift SDK Smoke", + "title": "skippy: recover hybrid speculative windows and orphaned lanes", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30351493597", + "wall_seconds": 2210.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "feat/laguna-certification", + "head_sha": "28e66d1562cd0f5c6bcd3f7c6d064cfd703252c7", + "id": 30351368311, + "longest_job": "Linux ROCm slim", + "longest_job_seconds": 1532.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "swift_sdk_smoke / swift SDK Smoke", + "title": "feat(skippy): add Poolside Laguna staged-runtime candidate", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30351368311", + "wall_seconds": 2096.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "feat/laguna-certification", + "head_sha": "181360e00b51a7cdf5e08e739fe83fd8eda94358", + "id": 30348404840, + "longest_job": "Windows CUDA", + "longest_job_seconds": 1810.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "kotlin_sdk_smoke / kotlin SDK Smoke", + "title": "feat(skippy): add Poolside Laguna staged-runtime candidate", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30348404840", + "wall_seconds": 2330.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "fix/skippy-hybrid-verify-recovery", + "head_sha": "1d3ee4a065d820603d5343abcefbde96182ea005", + "id": 30348390498, + "longest_job": "Windows ROCm", + "longest_job_seconds": 1832.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "kotlin_sdk_smoke / kotlin SDK Smoke", + "title": "skippy: recover hybrid speculative windows and orphaned lanes", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30348390498", + "wall_seconds": 2658.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + }, + { + "attempt": 1, + "conclusion": "success", + "event": "pull_request", + "executed_job_count": 31, + "head_branch": "codex/inkling-q2-skippy", + "head_sha": "976c30152492b96b7181f431a47864215bf7627e", + "id": 30344045951, + "longest_job": "Windows CUDA", + "longest_job_seconds": 3145.0, + "queue_seconds": 0.0, + "skipped_job_count": 1, + "terminal_job": "Linux CUDA slim", + "title": "Add Inkling Q2 staged runtime and certify text split serving", + "url": "https://github.com/Mesh-LLM/mesh-llm/actions/runs/30344045951", + "wall_seconds": 4141.0, + "workflow": "PR Builds", + "workflow_timing_excluded": false + } + ], + "schema_version": 1, + "selection": { + "included_run_count": 20, + "requested_status": "success", + "seen_run_count": 20, + "skipped_runs": {}, + "workflow_timing_excluded_reruns": 0 + }, + "source": { + "description": "Mesh-LLM/mesh-llm:30470003625,30469571715,30468256564,30435682397,30419140040,30416902770,30416288428,30406445718,30403101741,30399212634,30397484523,30388836882,30387256890,30387136356,30380850230,30351493597,30351368311,30348404840,30348390498,30344045951", + "kind": "github", + "repository": "Mesh-LLM/mesh-llm", + "workflow": null + }, + "workflow": { + "queue_seconds": { + "count": 20, + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0 + }, + "wall_seconds": { + "count": 20, + "max": 4141.0, + "mean": 1865.75, + "min": 215.0, + "p50": 1992.0, + "p90": 2721.2, + "p95": 3332.55 + } + } +} diff --git a/scripts/build-windows.ps1 b/scripts/build-windows.ps1 index 02c3fb447a..f1f5f0e645 100755 --- a/scripts/build-windows.ps1 +++ b/scripts/build-windows.ps1 @@ -1176,7 +1176,10 @@ Invoke-InRepo { $env:LLAMA_STAGE_AMDGPU_TARGETS = $RocmArch } $profileDir = if ($buildProfile -eq "release") { "release" } else { "debug" } - $runtimeOut = Join-Path (Join-Path (Join-Path $repoRoot "target") $profileDir) "native-runtimes" + # Invoke-InRepo makes this shell-relative path portable across Git Bash and + # WSL. Passing a native `D:\...` path to GNU tar makes it parse `D:` as a + # remote host and fail after the expensive ABI build has already completed. + $runtimeOut = "target/$profileDir/native-runtimes" Invoke-NativeCommand "bash" @( (Join-Path $scriptDir "package-native-runtime.sh"), "--backend", $backendName, diff --git a/scripts/ci-compose-product-input.sh b/scripts/ci-compose-product-input.sh index 2fcc2797ea..f7d3d0b305 100755 --- a/scripts/ci-compose-product-input.sh +++ b/scripts/ci-compose-product-input.sh @@ -57,16 +57,6 @@ require_nonempty_file() { fi } -require_checksum_match() { - local label="$1" - local expected="$2" - local actual="$3" - if [[ "$actual" != "$expected" ]]; then - echo "$label checksum mismatch: expected $expected, got $actual" >&2 - exit 1 - fi -} - canonical_paths=() while IFS= read -r path; do canonical_paths+=("$(to_shell_path "$path")") @@ -153,13 +143,7 @@ require_file "immutable host" "$host" chmod +x "$host" require_nonempty_file "host import report" "$host_imports" require_nonempty_file "host checksum" "$host_checksum" -expected_host_checksum="$(awk 'NR == 1 {print $1}' "$host_checksum")" -if command -v sha256sum >/dev/null 2>&1; then - actual_host_checksum="$(sha256sum "$host" | awk '{print $1}')" -else - actual_host_checksum="$(shasum -a 256 "$host" | awk '{print $1}')" -fi -require_checksum_match "immutable host" "$expected_host_checksum" "$actual_host_checksum" +"$python_bin" scripts/verify-checksum-sidecar.py "$host" if [[ -n "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" ]]; then attestation_verifier="${INPUT_ATTESTATION_VERIFIER:-$host_input_dir/release-attestation-verifier}" @@ -169,16 +153,8 @@ if [[ -n "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" ]]; then "$INPUT_ATTESTATION_PUBLIC_KEY_FILE" require_file "release attestation verifier" "$attestation_verifier" require_nonempty_file "release attestation verifier checksum" "$verifier_checksum" - expected_verifier_checksum="$(awk 'NR == 1 {print $1}' "$verifier_checksum")" - if command -v sha256sum >/dev/null 2>&1; then - actual_verifier_checksum="$(sha256sum "$attestation_verifier" | awk '{print $1}')" - else - actual_verifier_checksum="$(shasum -a 256 "$attestation_verifier" | awk '{print $1}')" - fi - require_checksum_match \ - "release attestation verifier" \ - "$expected_verifier_checksum" \ - "$actual_verifier_checksum" + "$python_bin" scripts/verify-checksum-sidecar.py \ + "$attestation_verifier" chmod +x "$attestation_verifier" "$attestation_verifier" release-attestation inspect \ --binary "$host" \ @@ -199,13 +175,28 @@ runtime_archives=() while IFS= read -r archive; do runtime_archives+=("$archive") done < <(find "$runtime_input_dir" -type f -name '*.tar.gz' -print) +runtime_sidecars=() +while IFS= read -r sidecar; do + runtime_sidecars+=("$sidecar") +done < <(find "$runtime_input_dir" -type f -name '*.tar.gz.sha256' -print) if [[ "${#runtime_archives[@]}" -gt 1 ]]; then echo "expected at most one runtime archive; found ${#runtime_archives[@]}" >&2 exit 1 elif [[ "${#runtime_archives[@]}" -eq 1 ]]; then + expected_sidecar="${runtime_archives[0]}.sha256" + if [[ "${#runtime_sidecars[@]}" -ne 1 || "${runtime_sidecars[0]}" != "$expected_sidecar" ]]; then + echo "expected exactly one checksum sidecar for ${runtime_archives[0]}; found ${#runtime_sidecars[@]}" >&2 + exit 1 + fi scripts/verify-native-runtime-package.sh "${runtime_archives[0]}" - tar -xzf "${runtime_archives[0]}" -C "$output_dir/native-runtimes" + "$python_bin" scripts/safe-extract-tar.py \ + "${runtime_archives[0]}" \ + "$output_dir/native-runtimes" else + if [[ "${#runtime_sidecars[@]}" -ne 0 ]]; then + echo "runtime checksum sidecar exists without a runtime archive" >&2 + exit 1 + fi runtime_dirs=() while IFS= read -r manifest; do runtime_dirs+=("$(dirname "$manifest")") diff --git a/scripts/generate-native-runtime-release-manifest.sh b/scripts/generate-native-runtime-release-manifest.sh index 2f50bc5c14..d5d88c454f 100755 --- a/scripts/generate-native-runtime-release-manifest.sh +++ b/scripts/generate-native-runtime-release-manifest.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" OUT="" REPO="${GITHUB_REPOSITORY:-Mesh-LLM/mesh-llm}" TAG="${RELEASE_TAG:-}" @@ -12,8 +13,9 @@ usage() { Usage: scripts/generate-native-runtime-release-manifest.sh --tag TAG --out FILE [--repo OWNER/REPO] [...] Generates native-runtimes.json for a GitHub release from packaged native -runtime artifacts. Each artifact archive must contain a manifest.json with the -native runtime resolver fields emitted by package-native-runtime.sh. +runtime artifacts. Each artifact archive must have its canonical .sha256 +sidecar and contain a manifest.json with the native runtime resolver fields +emitted by package-native-runtime.sh. EOF } @@ -59,17 +61,37 @@ if [[ -z "$TMP_ROOT" ]]; then TMP_ROOT="$(mktemp -d)" fi -python3 - "$OUT" "$REPO" "$TAG" "$TMP_ROOT" "$@" <<'PY' +for archive in "$@"; do + "$SCRIPT_DIR/verify-native-runtime-package.sh" --portable "$archive" +done + +python3 - \ + "$OUT" \ + "$REPO" \ + "$TAG" \ + "$TMP_ROOT" \ + "$SCRIPT_DIR/safe-extract-tar.py" \ + "$@" <<'PY' import hashlib import json import os +import subprocess import sys -import tarfile -out, repo, tag, tmp_root, *archives = sys.argv[1:] +( + out, + repo, + tag, + tmp_root, + safe_extractor, + *archives, +) = sys.argv[1:] artifacts = [] mesh_version = None skippy_abi = None +release_version = tag[1:] if tag.startswith("v") else tag +if not release_version: + raise SystemExit("release tag must contain a version") required = { "id", @@ -78,17 +100,31 @@ required = { "platform", "backend", "libraries", + "files", } -for archive in archives: + +def sha256_file(path): + digest = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +for index, archive in enumerate(archives): archive = os.path.abspath(archive) - with open(archive, "rb") as fh: - archive_sha256 = hashlib.sha256(fh.read()).hexdigest() + archive_sha256 = sha256_file(archive) - extract_dir = os.path.join(tmp_root, os.path.basename(archive).replace(os.sep, "_")) - os.makedirs(extract_dir, exist_ok=True) - with tarfile.open(archive, "r:gz") as tar: - tar.extractall(extract_dir) + extract_dir = os.path.join(tmp_root, f"archive-{index}") + extraction_result = subprocess.run( + [sys.executable, safe_extractor, archive, extract_dir], + check=False, + ) + if extraction_result.returncode != 0: + raise SystemExit( + f"unsafe or invalid native runtime archive: {archive}" + ) manifest_paths = [] for root, _, files in os.walk(extract_dir): @@ -106,11 +142,23 @@ for archive in archives: if missing: raise SystemExit(f"{archive} is missing native runtime field(s): {', '.join(missing)}") + runtime_version = runtime["mesh_version"] + normalized_runtime_version = ( + runtime_version[1:] + if runtime_version.startswith("v") + else runtime_version + ) + if normalized_runtime_version != release_version: + raise SystemExit( + f"{archive} mesh_version {runtime_version} does not match " + f"release tag {tag}" + ) + if mesh_version is None: - mesh_version = runtime["mesh_version"] - elif runtime["mesh_version"] != mesh_version: + mesh_version = runtime_version + elif runtime_version != mesh_version: raise SystemExit( - f"mixed mesh versions in native runtime artifacts: {runtime['mesh_version']} != {mesh_version}" + f"mixed mesh versions in native runtime artifacts: {runtime_version} != {mesh_version}" ) if skippy_abi is None: skippy_abi = runtime["skippy_abi"] diff --git a/scripts/package-native-sdk.sh b/scripts/package-native-sdk.sh index a3cefe3746..3c989ae9a0 100755 --- a/scripts/package-native-sdk.sh +++ b/scripts/package-native-sdk.sh @@ -340,7 +340,6 @@ manifest = { "llama_upstream_sha": "$upstream_sha" or None, "llama_patched_sha": "$patched_sha" or None, "llama_patch_digest": "$patch_digest" or None, - "llama_build_dir": os.path.abspath("$LLAMA_STAGE_BUILD_DIR"), "cuda_architectures": os.environ.get("LLAMA_STAGE_CUDA_ARCHITECTURES") or os.environ.get("SKIPPY_CUDA_ARCHITECTURES"), "amdgpu_targets": os.environ.get("LLAMA_STAGE_AMDGPU_TARGETS") or os.environ.get("SKIPPY_AMDGPU_TARGETS"), "features": [ diff --git a/scripts/safe-extract-tar.py b/scripts/safe-extract-tar.py new file mode 100755 index 0000000000..52e38eb3e5 --- /dev/null +++ b/scripts/safe-extract-tar.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Extract a tar archive without permitting writes outside the destination.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import tarfile + + +WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:") + + +def normalized_parts( + raw_name: str, + *, + label: str, + allow_root: bool = False, +) -> tuple[str, ...]: + if not raw_name or "\x00" in raw_name or "\\" in raw_name: + raise ValueError(f"unsafe {label}: {raw_name!r}") + if raw_name.startswith("/") or WINDOWS_DRIVE.match(raw_name): + raise ValueError(f"absolute {label} is not allowed: {raw_name}") + parts = tuple(part for part in PurePosixPath(raw_name).parts if part != ".") + if not parts: + if allow_root: + return () + raise ValueError(f"empty {label} is not allowed: {raw_name!r}") + if any(part in ("", "..") for part in parts): + raise ValueError(f"traversing {label} is not allowed: {raw_name}") + return parts + + +def destination_path(root: Path, parts: tuple[str, ...]) -> Path: + return root.joinpath(*parts) + + +def validate_members( + archive: tarfile.TarFile, +) -> list[tuple[tarfile.TarInfo, tuple[str, ...]]]: + validated: list[tuple[tarfile.TarInfo, tuple[str, ...]]] = [] + seen: set[tuple[str, ...]] = set() + for member in archive.getmembers(): + parts = normalized_parts( + member.name, + label="archive member path", + allow_root=member.isdir(), + ) + if parts in seen: + raise ValueError(f"duplicate archive member path: {member.name}") + seen.add(parts) + if not parts: + continue + if not ( + member.isdir() + or member.isreg() + or member.issym() + or member.islnk() + ): + raise ValueError( + f"unsupported archive member type for {member.name}: " + f"{member.type!r}" + ) + if member.issym() or member.islnk(): + link_parts = normalized_parts( + member.linkname, + label=f"link target for {member.name}", + ) + target_parts = ( + (*parts[:-1], *link_parts) if member.issym() else link_parts + ) + normalized_target: list[str] = [] + for part in target_parts: + if part == "..": + if not normalized_target: + raise ValueError( + f"archive link escapes destination: {member.name}" + ) + normalized_target.pop() + elif part != ".": + normalized_target.append(part) + if not normalized_target: + raise ValueError( + f"archive link has an empty target: {member.name}" + ) + validated.append((member, parts)) + return validated + + +def apply_mode(path: Path, member: tarfile.TarInfo) -> None: + if os.name != "nt": + path.chmod(member.mode & 0o777) + + +def safe_extract(archive_path: Path, destination: Path) -> None: + destination.mkdir(parents=True, exist_ok=True) + if destination.is_symlink(): + raise ValueError( + f"extraction destination cannot be a symlink: {destination}" + ) + root = destination.resolve(strict=True) + if any(root.iterdir()): + raise ValueError(f"extraction destination must be empty: {root}") + with tarfile.open(archive_path, mode="r:*") as archive: + validated = validate_members(archive) + + directories = [ + (member, parts) + for member, parts in validated + if member.isdir() + ] + regular_files = [ + (member, parts) + for member, parts in validated + if member.isreg() + ] + links = [ + (member, parts) + for member, parts in validated + if member.issym() or member.islnk() + ] + + for member, parts in directories: + destination_path(root, parts).mkdir(parents=True, exist_ok=True) + + for member, parts in regular_files: + output = destination_path(root, parts) + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists() or output.is_symlink(): + raise ValueError( + f"archive member would overwrite an existing path: " + f"{member.name}" + ) + source = archive.extractfile(member) + if source is None: + raise ValueError( + f"archive member has no file payload: {member.name}" + ) + with source, output.open("xb") as handle: + shutil.copyfileobj(source, handle) + apply_mode(output, member) + + for member, parts in links: + output = destination_path(root, parts) + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists() or output.is_symlink(): + raise ValueError( + f"archive link would overwrite an existing path: " + f"{member.name}" + ) + if member.issym(): + os.symlink(member.linkname, output) + else: + link_parts = normalized_parts( + member.linkname, + label=f"hard-link target for {member.name}", + ) + target = destination_path(root, link_parts) + if not target.is_file() or target.is_symlink(): + raise ValueError( + f"hard-link target is not a regular extracted file: " + f"{member.linkname}" + ) + os.link(target, output) + + for member, parts in reversed(directories): + apply_mode(destination_path(root, parts), member) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("archive", type=Path) + parser.add_argument("destination", type=Path) + args = parser.parse_args() + try: + safe_extract(args.archive, args.destination) + except (OSError, tarfile.TarError, ValueError) as error: + parser.exit(1, f"unsafe or invalid tar archive: {error}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/summarize-sccache-stats.py b/scripts/summarize-sccache-stats.py new file mode 100644 index 0000000000..bede58d7b5 --- /dev/null +++ b/scripts/summarize-sccache-stats.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Aggregate downloaded sccache JSON evidence without network access.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys +from typing import Any + + +class SummaryError(RuntimeError): + """Raised when downloaded evidence cannot be summarized safely.""" + + +def hit_rate(value: str) -> float: + try: + parsed = float(value) + except ValueError as error: + raise argparse.ArgumentTypeError( + "minimum hit rate must be a number between 0 and 1", + ) from error + if not 0 <= parsed <= 1: + raise argparse.ArgumentTypeError( + "minimum hit rate must be a number between 0 and 1", + ) + return parsed + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Aggregate cache hits and misses from downloaded " + "sccache-stats*.json evidence." + ), + ) + parser.add_argument( + "paths", + nargs="+", + type=Path, + help="Evidence JSON files or directories to scan recursively.", + ) + parser.add_argument( + "--minimum-hit-rate", + type=hit_rate, + help="Fail unless aggregate hits / (hits + misses) meets this ratio.", + ) + parser.add_argument( + "--format", + choices=("text", "json"), + default="text", + help="Output format (default: text).", + ) + return parser.parse_args() + + +def discover_evidence(paths: list[Path]) -> list[Path]: + evidence: set[Path] = set() + for path in paths: + if path.is_file(): + evidence.add(path.resolve()) + elif path.is_dir(): + evidence.update( + candidate.resolve() + for candidate in path.rglob("sccache-stats*.json") + if candidate.is_file() + ) + else: + raise SummaryError(f"evidence path does not exist: {path}") + if not evidence: + raise SummaryError("no sccache-stats*.json evidence files were found") + return sorted(evidence) + + +def sum_count_tree(value: Any, field: str) -> int: + if isinstance(value, bool): + raise SummaryError(f"{field} contains a boolean") + if isinstance(value, int): + if value < 0: + raise SummaryError(f"{field} contains a negative counter") + return value + if isinstance(value, dict): + return sum( + sum_count_tree(child, f"{field}.{name}") + for name, child in value.items() + ) + raise SummaryError(f"{field} must contain only counter maps and integers") + + +def read_count(path: Path, payload: Any, name: str) -> int: + if not isinstance(payload, dict): + raise SummaryError(f"{path}: JSON root must be an object") + stats = payload.get("stats") + if not isinstance(stats, dict): + raise SummaryError(f"{path}: stats must be an object") + count_map = stats.get(name) + if not isinstance(count_map, dict): + raise SummaryError(f"{path}: stats.{name} must be an object") + counts = count_map.get("counts") + if not isinstance(counts, dict): + raise SummaryError(f"{path}: stats.{name}.counts must be an object") + return sum_count_tree(counts, f"{path}: stats.{name}.counts") + + +def aggregate(paths: list[Path]) -> tuple[int, int]: + hits = 0 + misses = 0 + for path in paths: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise SummaryError(f"{path}: unable to read valid JSON: {error}") from error + hits += read_count(path, payload, "cache_hits") + misses += read_count(path, payload, "cache_misses") + return hits, misses + + +def render_text(summary: dict[str, Any]) -> str: + rate = summary["hit_rate"] + rate_text = "n/a" if rate is None else f"{rate:.2%}" + lines = [ + "Sccache cache-hit summary", + f"Evidence files: {summary['file_count']}", + f"Cache hits: {summary['cache_hits']}", + f"Cache misses: {summary['cache_misses']}", + f"Hit rate: {rate_text}", + ] + minimum = summary["minimum_hit_rate"] + if minimum is not None: + outcome = "PASS" if summary["passed"] else "FAIL" + lines.append(f"Minimum hit rate: {minimum:.2%} ({outcome})") + return "\n".join(lines) + + +def main() -> int: + arguments = parse_args() + try: + evidence = discover_evidence(arguments.paths) + hits, misses = aggregate(evidence) + except SummaryError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + requests = hits + misses + rate = hits / requests if requests else None + passed = ( + arguments.minimum_hit_rate is None + or (rate is not None and rate >= arguments.minimum_hit_rate) + ) + summary = { + "file_count": len(evidence), + "cache_hits": hits, + "cache_misses": misses, + "cache_requests": requests, + "hit_rate": rate, + "minimum_hit_rate": arguments.minimum_hit_rate, + "passed": passed, + } + if arguments.format == "json": + print(json.dumps(summary, sort_keys=True)) + else: + print(render_text(summary)) + return 0 if passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_build_windows.py b/scripts/tests/test_build_windows.py index d491435bb2..2e17fa49b1 100644 --- a/scripts/tests/test_build_windows.py +++ b/scripts/tests/test_build_windows.py @@ -8,6 +8,11 @@ ROOT: Final = Path(__file__).resolve().parents[2] SCRIPT: Final = ROOT / "scripts" / "build-windows.ps1" PR_BUILDS: Final = ROOT / ".github" / "workflows" / "pr_builds.yml" +CI_WORKFLOW: Final = ROOT / ".github" / "workflows" / "ci.yml" +RELEASE_WORKFLOW: Final = ROOT / ".github" / "workflows" / "release.yml" +WINDOWS_WARM_CACHES: Final = ( + ROOT / ".github" / "workflows" / "windows-warm-caches.yml" +) class BuildWindowsScriptTests(unittest.TestCase): @@ -29,9 +34,107 @@ def test_native_runtime_package_out_path_is_git_bash_safe(self) -> None: package_call = package_call[: package_call.index("\n )")] self.assertIn('"--out", $runtimeOut', package_call) - self.assertIn('Join-Path (Join-Path (Join-Path $repoRoot "target") $profileDir) "native-runtimes"', script) + self.assertIn( + '$runtimeOut = "target/$profileDir/native-runtimes"', + script, + ) + self.assertNotIn( + 'Join-Path (Join-Path (Join-Path $repoRoot "target")', + script, + ) self.assertNotIn('"--out", (Join-Path $repoRoot', package_call) + def test_cache_warmers_use_the_canonical_runtime_action(self) -> None: + justfile = (ROOT / "Justfile").read_text(encoding="utf-8") + script = SCRIPT.read_text(encoding="utf-8") + workflow = WINDOWS_WARM_CACHES.read_text(encoding="utf-8") + + self.assertEqual( + workflow.count( + "uses: ./.github/actions/prepare-native-runtime-input", + ), + 2, + ) + self.assertEqual(workflow.count(".mesh-llm-build-stamp"), 2) + self.assertNotIn("run: just ", workflow) + self.assertNotIn("build-windows.ps1", workflow) + self.assertNotIn("release-runtime-abi-", justfile) + self.assertNotIn("[switch]$AbiOnly", script) + + def test_every_windows_runtime_graph_consumes_the_shared_abi_cache( + self, + ) -> None: + workflows = { + "pr": PR_BUILDS, + "main": CI_WORKFLOW, + "release": RELEASE_WORKFLOW, + "warmer": WINDOWS_WARM_CACHES, + } + + for name, path in workflows.items(): + with self.subTest(workflow=name): + workflow = path.read_text(encoding="utf-8") + self.assertEqual( + workflow.count( + "uses: ./.github/actions/restore-windows-abi-cache", + ), + 2, + ) + + def test_windows_abi_caches_live_outside_the_llama_worktree(self) -> None: + workflows = { + "pr": PR_BUILDS, + "main": CI_WORKFLOW, + "release": RELEASE_WORKFLOW, + "warmer": WINDOWS_WARM_CACHES, + } + unsafe_cpu = ( + "LLAMA_STAGE_BUILD_DIR: " + ".deps/llama.cpp/build-stage-abi-cpu" + ) + unsafe_matrix = ( + "LLAMA_STAGE_BUILD_DIR: " + ".deps/llama.cpp/build-stage-abi-${{ matrix.backend }}" + ) + safe_prefix = ( + "LLAMA_STAGE_BUILD_DIR: " + ".deps/llama-build/windows/build-stage-abi-" + ) + + for name, path in workflows.items(): + with self.subTest(workflow=name): + workflow = path.read_text(encoding="utf-8") + self.assertEqual(workflow.count(safe_prefix), 2) + self.assertNotIn(unsafe_cpu, workflow) + self.assertNotIn(unsafe_matrix, workflow) + + script = SCRIPT.read_text(encoding="utf-8") + self.assertIn( + "$buildDir = if ($env:LLAMA_STAGE_BUILD_DIR)", + script, + ) + self.assertIn( + 'Join-Path $repoRoot ".deps\\llama-build"', + script, + ) + + def test_release_windows_gpu_setup_matches_pr_and_main(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + start = workflow.index(" build_native_runtime_windows_gpu:") + end = workflow.index("\n publish:", start) + windows_gpu = workflow[start:end] + + self.assertIn("uses: Jimver/cuda-toolkit@", windows_gpu) + self.assertIn( + "uses: jakoch/install-vulkan-sdk-action@", + windows_gpu, + ) + self.assertIn( + "uses: ./.github/actions/setup-windows-rocm-sdk", + windows_gpu, + ) + self.assertNotIn("install-windows-sdk.ps1", windows_gpu) + def test_all_normal_profiles_build_a_dynamic_host_and_adjacent_runtime(self) -> None: script = SCRIPT.read_text(encoding="utf-8") diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index ea1c11c2d2..01df99100d 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -5,6 +5,7 @@ import os from pathlib import Path import re +import shutil import subprocess import tarfile import tempfile @@ -48,6 +49,39 @@ def test_external_actions_have_sha_and_release_provenance(self) -> None: ): self.assertRegex(value, exact_pin) + def test_workflow_status_gates_do_not_resist_cancellation(self) -> None: + workflow_files = sorted( + (ROOT / ".github" / "workflows").glob("*.yml"), + ) + + for path in workflow_files: + with self.subTest(path=path.relative_to(ROOT)): + self.assertNotIn( + "always()", + path.read_text(encoding="utf-8"), + ) + + def test_pr_quality_requires_ci_contract_validation(self) -> None: + workflow = ( + ROOT / ".github" / "workflows" / "pr_quality.yml" + ).read_text(encoding="utf-8") + contract_start = workflow.index(" ci-contract:") + contract_end = workflow.index("\n rust-fmt:", contract_start) + contract = workflow[contract_start:contract_end] + summary = workflow[workflow.index(" summary:") :] + + self.assertIn("actionlint@1.7.12", contract) + self.assertIn( + "actionlint -config-file .github/actionlint.yaml", + contract, + ) + self.assertIn( + "python3 -m unittest discover -s scripts/tests -p 'test_*.py'", + contract, + ) + self.assertIn("ci-contract", summary) + self.assertIn("needs.ci-contract.result", summary) + def write_fake_product_inputs( self, workspace: Path, @@ -76,7 +110,7 @@ def write_fake_product_inputs( encoding="utf-8", ) - runtime_id = "meshllm-native-runtime-test-x86_64-cpu" + runtime_id = "meshllm-native-runtime-darwin-x86_64-cpu" runtime = runtime_input / runtime_id (runtime / "lib").mkdir(parents=True) (runtime / "tools").mkdir() @@ -91,10 +125,17 @@ def write_fake_product_inputs( "runtime": { "id": runtime_id, "mesh_version": "1.2.3", - "skippy_abi": {"major": 1, "minor": 0, "patch": 0}, - "platform": {"os": "test", "arch": "x86_64"}, + "skippy_abi": "1.0.0", + "platform": { + "os": "macos", + "arch": "x86_64", + "target": "x86_64-apple-darwin", + }, "backend": {"kind": "cpu"}, "libraries": ["lib/libmesh_fake.a"], + "files": { + "lib/libmesh_fake.a": library_digest, + }, "tools": {"tools/mesh-runtime-bench": tool_digest}, }, "build": { @@ -109,31 +150,98 @@ def write_fake_product_inputs( ) return host_input, runtime_input + def write_noncanonical_sidecar( + self, + artifact: Path, + mode: str, + ) -> None: + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + if mode == "wrong-name": + contents = f"{digest} unexpected-name\n" + elif mode == "multiline": + contents = ( + f"{digest} {artifact.name}\n" + f"{digest} {artifact.name}\n" + ) + else: + raise ValueError(f"unsupported sidecar mode: {mode}") + artifact.with_name(f"{artifact.name}.sha256").write_text( + contents, + encoding="utf-8", + ) + def run_product_composer( self, workspace: Path, *, host_version: str = "1.2.3", + runtime_archive: str | None = None, + host_sidecar: str | None = None, + attestation_sidecar: str | None = None, ) -> subprocess.CompletedProcess[str]: host_input, runtime_input = self.write_fake_product_inputs( workspace, host_version=host_version, ) + if host_sidecar is not None: + self.write_noncanonical_sidecar( + host_input / "mesh-llm", + host_sidecar, + ) + if runtime_archive is not None: + runtime_dir = next( + path + for path in runtime_input.iterdir() + if path.is_dir() + ) + archive = runtime_input / f"{runtime_dir.name}.tar.gz" + with tarfile.open(archive, "w:gz") as bundle: + bundle.add(runtime_dir, arcname=runtime_dir.name) + shutil.rmtree(runtime_dir) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + if runtime_archive != "missing": + sidecar_digest = ( + "0" * 64 + if runtime_archive == "corrupt" + else digest + ) + archive.with_name(f"{archive.name}.sha256").write_text( + f"{sidecar_digest} {archive.name}\n", + encoding="utf-8", + ) + if runtime_archive == "duplicate": + (runtime_input / "unrelated.tar.gz.sha256").write_text( + f"{digest} unrelated.tar.gz\n", + encoding="utf-8", + ) + environment = { + **os.environ, + "GITHUB_WORKSPACE": str(workspace), + "GITHUB_OUTPUT": str(workspace / "github-output"), + "INPUT_HOST_INPUT_DIR": str(host_input), + "INPUT_RUNTIME_INPUT_DIR": str(runtime_input), + "INPUT_OUTPUT_DIR": str(workspace / "product-input"), + "INPUT_BACKEND": "cpu", + "INPUT_VERSION": "1.2.3", + "INPUT_BINARY_NAME": "mesh-llm", + "INPUT_READINESS_SMOKE": "false", + } + if attestation_sidecar is not None: + verifier = host_input / "release-attestation-verifier" + verifier.write_bytes(b"test verifier") + self.write_noncanonical_sidecar( + verifier, + attestation_sidecar, + ) + public_key = workspace / "release-attestation-public-key.json" + public_key.write_text("{}\n", encoding="utf-8") + environment["INPUT_ATTESTATION_PUBLIC_KEY_FILE"] = str( + public_key, + ) return subprocess.run( [str(COMPOSE_SCRIPT)], cwd=ROOT, - env={ - **os.environ, - "GITHUB_WORKSPACE": str(workspace), - "GITHUB_OUTPUT": str(workspace / "github-output"), - "INPUT_HOST_INPUT_DIR": str(host_input), - "INPUT_RUNTIME_INPUT_DIR": str(runtime_input), - "INPUT_OUTPUT_DIR": str(workspace / "product-input"), - "INPUT_BACKEND": "cpu", - "INPUT_VERSION": "1.2.3", - "INPUT_BINARY_NAME": "mesh-llm", - "INPUT_READINESS_SMOKE": "false", - }, + env=environment, check=False, capture_output=True, text=True, @@ -242,6 +350,17 @@ def test_windows_routes_cover_every_shared_product_primitive(self) -> None: self.assertIn("^crates/mesh-llm-release-footer/", cpu_routing) self.assertNotIn("^crates/mesh-llm-release-footer/", gpu_routing) + self.assertIn("package-release", cpu_routing) + self.assertIn("package-release", gpu_routing) + for workflow in ( + "ci", + "pr_builds", + "release", + "windows-warm-caches", + ): + with self.subTest(workflow=workflow): + self.assertIn(workflow, cpu_routing) + self.assertIn(workflow, gpu_routing) for primitive in ( "prepare-windows-host-input", @@ -249,6 +368,8 @@ def test_windows_routes_cover_every_shared_product_primitive(self) -> None: "compose-product-input", "package-native-runtime", "verify-native-runtime-package", + "verify-checksum-sidecar", + "safe-extract-tar", "compose-product-bundle", "ci-compose-product-input", "ci-client-readiness-smoke", @@ -256,6 +377,178 @@ def test_windows_routes_cover_every_shared_product_primitive(self) -> None: with self.subTest(primitive=primitive): self.assertIn(primitive, routing) + def test_windows_abi_cache_action_keys_every_compatibility_boundary( + self, + ) -> None: + action = self.read_action("restore-windows-abi-cache") + + for action_input in ( + "backend:", + "build_dir:", + "architecture_set:", + "cuda_toolchain_version:", + "vulkan_toolchain_version:", + "rocm_toolchain_version:", + ): + with self.subTest(action_input=action_input): + self.assertIn(action_input, action) + + self.assertIn( + '$backend -notin @("cpu", "cuda", "rocm", "vulkan")', + action, + ) + self.assertIn( + '$backend -in @("cuda", "rocm") -and -not $architectureSet', + action, + ) + self.assertIn( + "build_dir must resolve inside GITHUB_WORKSPACE", + action, + ) + self.assertIn( + "build_dir must remain outside the replaceable llama.cpp ", + action, + ) + self.assertIn( + "worktree: $resolvedBuildDir", + action, + ) + for toolchain_boundary in ( + "cuda-$version-Jimver-v0.2.35", + "vulkan-$version-jakoch-v1.5.2", + "rocm-$version", + ): + with self.subTest(toolchain_boundary=toolchain_boundary): + self.assertIn(toolchain_boundary, action) + + expected_hash = ( + "${{ hashFiles(" + "'.github/actions/restore-windows-abi-cache/action.yml', " + "'.github/actions/prepare-native-runtime-input/action.yml', " + "'.github/actions/setup-windows-rocm-sdk/action.yml', " + "'scripts/build-llama.sh', 'scripts/prepare-llama.sh', " + "'scripts/package-native-runtime.sh', " + "'third_party/llama.cpp/upstream.txt', " + "'third_party/llama.cpp/patches/**', " + "'.github/cache-version.txt') }}" + ) + self.assertIn(expected_hash, action) + self.assertIn( + '"mesh-llm-windows-2022-skippy-abi-' + '$backend-$architectureSet-$toolchain-$inputHash"', + action, + ) + self.assertIn( + "actions/cache/restore@" + "caa296126883cff596d87d8935842f9db880ef25 # v5.1.0", + action, + ) + self.assertNotIn("restore-keys:", action) + self.assertIn( + "value: ${{ steps.restore.outputs.cache-hit }}", + action, + ) + self.assertIn( + "value: ${{ steps.restore.outputs.cache-primary-key }}", + action, + ) + + def test_push_routing_diffs_the_complete_event_range(self) -> None: + action = self.read_action("compute-changes") + push_start = action.index( + 'elif [[ "${{ inputs.event_name }}" == "push" ]]', + ) + push_end = action.index( + 'elif [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]', + push_start, + ) + push = action[push_start:push_end] + + self.assertIn('base_sha="${{ inputs.base_sha }}"', push) + self.assertIn('head_sha="${{ inputs.head_sha }}"', push) + self.assertIn('git diff --name-only "$base_sha" "$head_sha"', push) + self.assertIn('"$base_sha" =~ ^0+$', push) + self.assertIn('"__force_all__"', push) + self.assertNotIn("HEAD^", action) + self.assertIn('if [[ "$FORCE_ALL" == "true" ]]', action) + force_windows = action[ + action.index('if [[ "$FORCE_ALL" == "true" ]]') + : action.index( + "# SDK smokes are consumer tests", + ) + ] + self.assertIn('WINDOWS_CPU_BUILD_REQUIRED="true"', force_windows) + self.assertIn('WINDOWS_GPU_BUILD_REQUIRED="true"', force_windows) + + def test_runner_contract_routing_covers_cache_evidence_actions( + self, + ) -> None: + action = self.read_action("compute-changes") + routing = action[ + action.index("RUNNER_CONTRACT_INPUTS=") + : action.index("# Determine docs_only") + ] + + for local_action in ( + "capture-sccache-stats", + "configure-sccache-gha", + "select-ci-runners", + ): + with self.subTest(local_action=local_action): + self.assertIn(local_action, routing) + + def test_justfile_release_primitives_route_backend_builds(self) -> None: + action = self.read_action("compute-changes") + match = re.search( + r"function is_backend_recipe\(name\).*?" + r"return name ~ /\^\((.*?)\)\$/", + action, + re.DOTALL, + ) + self.assertIsNotNone(match) + recipe_names = set(match.group(1).split("|")) + + for recipe in ( + "release-host-build", + "release-runtime-build", + "release-host-build-windows", + ): + with self.subTest(recipe=recipe): + self.assertIn(recipe, recipe_names) + + def test_sdk_routing_covers_every_direct_smoke_script(self) -> None: + action = self.read_action("compute-changes") + match = re.search( + r"DIRECT_SDK_INPUTS=.*?grep -E '([^']+)'", + action, + ) + self.assertIsNotNone(match) + direct_sdk_pattern = re.compile(match.group(1)) + self.assertRegex( + ".github/actions/restore-smoke-inputs/action.yml", + direct_sdk_pattern, + ) + smoke_scripts = ( + ROOT / "scripts" / "ci-rust-sdk-smoke.sh", + ROOT / "scripts" / "ci-kotlin-sdk-smoke.sh", + ROOT / "scripts" / "ci-swift-sdk-smoke.sh", + ) + + direct_calls: set[str] = set() + for smoke_script in smoke_scripts: + direct_calls.update( + f"scripts/{name}" + for name in re.findall( + r"(?m)^\s*(?:retry_transient\s+)?" + r"scripts/([A-Za-z0-9_.-]+\.sh)", + smoke_script.read_text(encoding="utf-8"), + ) + ) + + for script in sorted(direct_calls): + with self.subTest(script=script): + self.assertRegex(script, direct_sdk_pattern) + def test_runtime_action_never_builds_the_host(self) -> None: action = self.read_action("prepare-native-runtime-input") @@ -273,6 +566,8 @@ def test_product_action_only_composes_verified_inputs(self) -> None: script = COMPOSE_SCRIPT.read_text(encoding="utf-8") self.assertIn("scripts/compose-product-bundle.py", script) self.assertIn("scripts/verify-native-runtime-package.sh", script) + self.assertIn("scripts/verify-checksum-sidecar.py", script) + self.assertIn("scripts/safe-extract-tar.py", script) self.assertIn("scripts/ci-client-readiness-smoke.sh", script) self.assertIn('archive_path="$product_dir.tar.gz"', script) self.assertIn('tar -C "$product_dir" -czf "$archive_path" .', script) @@ -330,6 +625,72 @@ def test_product_composer_rejects_host_version_drift(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn("composed host version mismatch", result.stderr) + def test_product_composer_accepts_one_checksums_runtime_archive(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + result = self.run_product_composer( + Path(temp_dir), + runtime_archive="valid", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + + def test_product_composer_requires_exact_runtime_archive_sidecar( + self, + ) -> None: + for mode in ("missing", "duplicate"): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as temp_dir: + result = self.run_product_composer( + Path(temp_dir), + runtime_archive=mode, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("expected exactly one checksum sidecar", result.stderr) + + def test_product_composer_rejects_corrupt_runtime_archive_sidecar( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + result = self.run_product_composer( + Path(temp_dir), + runtime_archive="corrupt", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("archive checksum mismatch", result.stderr) + + def test_product_composer_rejects_noncanonical_host_sidecar(self) -> None: + expected_errors = { + "wrong-name": "checksum sidecar names", + "multiline": "exactly one canonical line", + } + for mode, expected_error in expected_errors.items(): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as temp_dir: + result = self.run_product_composer( + Path(temp_dir), + host_sidecar=mode, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected_error, result.stderr) + + def test_product_composer_rejects_noncanonical_verifier_sidecar( + self, + ) -> None: + expected_errors = { + "wrong-name": "checksum sidecar names", + "multiline": "exactly one canonical line", + } + for mode, expected_error in expected_errors.items(): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as temp_dir: + result = self.run_product_composer( + Path(temp_dir), + attestation_sidecar=mode, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected_error, result.stderr) + def test_release_attestation_is_verified_without_compiling_in_composer( self, ) -> None: @@ -345,8 +706,8 @@ def test_release_attestation_is_verified_without_compiling_in_composer( product_script, ) self.assertIn( - '"$expected_verifier_checksum" \\\n' - ' "$actual_verifier_checksum"', + '"$python_bin" scripts/verify-checksum-sidecar.py \\\n' + ' "$attestation_verifier"', product_script, ) @@ -354,7 +715,18 @@ def test_smoke_restore_rechecks_the_archived_product(self) -> None: action = self.read_action("restore-smoke-inputs") self.assertIn("expected exactly one composed product archive", action) - self.assertIn("tar -xzf", action) + self.assertIn("scripts/safe-extract-tar.py", action) + self.assertNotIn("tar -xzf", action) + self.assertIn("product host path must be", action) + self.assertIn( + "product runtime must be one direct child of native-runtimes", + action, + ) + self.assertIn("product top-level contents are not canonical", action) + self.assertIn( + "product must contain exactly its manifest-selected runtime", + action, + ) self.assertIn("scripts/verify-native-runtime-package.sh", action) self.assertIn("--check", action) diff --git a/scripts/tests/test_ci_prepare_native_runtime.py b/scripts/tests/test_ci_prepare_native_runtime.py index baf049a0b8..286c0ee3f6 100644 --- a/scripts/tests/test_ci_prepare_native_runtime.py +++ b/scripts/tests/test_ci_prepare_native_runtime.py @@ -39,6 +39,21 @@ def host_arch() -> str: return machine +def host_target() -> str: + targets = { + ("macos", "aarch64"): "aarch64-apple-darwin", + ("macos", "x86_64"): "x86_64-apple-darwin", + ("linux", "aarch64"): "aarch64-unknown-linux-gnu", + ("linux", "x86_64"): "x86_64-unknown-linux-gnu", + ("windows", "x86_64"): "x86_64-pc-windows-msvc", + } + return targets[(host_os(), host_arch())] + + +def accelerated_backend() -> str: + return "metal" if host_os() == "macos" else "vulkan" + + def make_executable(path: Path) -> None: path.chmod(path.stat().st_mode | stat.S_IXUSR) @@ -72,11 +87,19 @@ def write_runtime( "id": runtime_id, "mesh_version": "0.72.1", "skippy_abi": skippy_abi or current_skippy_abi(), - "platform": {"os": host_os(), "arch": host_arch()}, + "platform": { + "os": host_os(), + "arch": host_arch(), + "target": host_target(), + }, "backend": {"kind": backend}, "libraries": ["lib/runtime.bin"], "files": {"lib/runtime.bin": digest}, - } + }, + "build": { + "primary_library": "lib/runtime.bin", + "library_sha256": digest, + }, } (artifact / "manifest.json").write_text( json.dumps(manifest), encoding="utf-8" @@ -167,9 +190,14 @@ def test_reuses_sole_compatible_product_backend_before_cpu_fallback(self) -> Non product = Path(directory) / "product" runtime = self.write_runtime( product / "native-runtimes", - backend="metal", + backend=accelerated_backend(), + ) + binary = self.write_fake_binary( + product, + runtime.name, + accelerated_backend(), + True, ) - binary = self.write_fake_binary(product, runtime.name, "metal", True) out = Path(directory) / "fallback" result = self.run_script(out, binary) diff --git a/scripts/tests/test_ci_workflow_artifacts.py b/scripts/tests/test_ci_workflow_artifacts.py index 00ab002807..526af14111 100644 --- a/scripts/tests/test_ci_workflow_artifacts.py +++ b/scripts/tests/test_ci_workflow_artifacts.py @@ -48,6 +48,25 @@ def test_arc_runner_contract_is_trusted_main_only(self) -> None: self.assertNotIn("runner: mesh-llm-amd64", pr_workflow) self.assertNotIn("runner: mesh-llm-arm64", pr_workflow) + def test_main_uses_event_range_and_centralized_platform_routes(self) -> None: + changes = job_section(self.workflow, "changes") + + self.assertIn("base_sha: ${{ github.event.before || '' }}", changes) + self.assertIn("head_sha: ${{ github.sha }}", changes) + self.assertIn( + "windows_cpu: ${{ steps.compute.outputs.windows_cpu_build_required }}", + changes, + ) + self.assertIn( + "windows_gpu: ${{ steps.compute.outputs.windows_gpu_build_required }}", + changes, + ) + self.assertNotIn("steps.filter.outputs.windows_cpu", changes) + self.assertNotIn("steps.filter.outputs.windows_gpu", changes) + swift = job_section(self.workflow, "swift_sdk_smoke") + self.assertIn("needs.changes.outputs.sdk_smoke_required == 'true'", swift) + self.assertNotIn("needs.changes.outputs.sdk == 'true'", swift) + def test_cpu_runtime_is_an_independent_producer(self) -> None: runtime = job_section(self.workflow, "linux_cpu_runtime_input") @@ -80,26 +99,100 @@ def test_cpu_product_only_composes_immutable_inputs(self) -> None: self.assertNotIn("scripts/package-native-runtime.sh", product) self.assertNotIn("configure-sccache-gha", product) - def test_gpu_products_reuse_the_neutral_host(self) -> None: - artifacts = { - "linux_cuda": "ci-linux-cuda-product", - "linux_rocm": "ci-linux-rocm-product", - "linux_vulkan": "ci-linux-vulkan-product", + def test_gpu_runtime_inputs_are_independent_producers(self) -> None: + expected = { + "cuda": ( + "sha256:c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd", + 'LLAMA_STAGE_CUDA_ARCHITECTURES: "75;80;86;87;89;90"', + ), + "rocm": ( + "sha256:0e13e5d2d2c121df265ff6c69be81e468989e09f81d6b7ff049b110cc0bb0d2b", + "LLAMA_STAGE_AMDGPU_TARGETS: gfx1100", + ), + "vulkan": ( + "sha256:ce55fed5c680cd3184b5d4770d9a77c43a702687690906e5753efd2cea27ed80", + "build-stage-abi-dynamic-vulkan", + ), } - for job_name, product_artifact in artifacts.items(): - with self.subTest(job=job_name): - job = job_section(self.workflow, job_name) - self.assertIn("needs: [changes, linux_host_input]", job) - self.assertIn("name: ci-linux-host-input", job) + for backend, (image, backend_env) in expected.items(): + with self.subTest(backend=backend): + runtime = job_section( + self.workflow, + f"linux_{backend}_runtime_input", + ) + self.assertIn("needs: changes", runtime) + self.assertIn( + "if: ${{ needs.changes.outputs.docs_only != 'true' }}", + runtime, + ) + self.assertIn( + "runs-on: ${{ needs.changes.outputs.runner_8 }}", + runtime, + ) + self.assertIn(image, runtime) + self.assertIn(backend_env, runtime) self.assertIn( "uses: ./.github/actions/prepare-native-runtime-input", - job, + runtime, ) - self.assertIn("uses: ./.github/actions/compose-product-input", job) - self.assertIn(f"name: {product_artifact}", job) - self.assertNotIn("prepare-host-input", job) - self.assertNotIn("scripts/build-host.sh", job) + self.assertIn(f"backend: {backend}", runtime) + self.assertIn( + f"name: ci-linux-{backend}-runtime-input", + runtime, + ) + self.assertIn("runtime-input/*.tar.gz", runtime) + self.assertIn("runtime-input/*.sha256", runtime) + self.assertNotIn("linux_host_input", runtime) + self.assertNotIn("name: ci-linux-host-input", runtime) + self.assertNotIn("prepare-host-input", runtime) + self.assertNotIn("compose-product-input", runtime) + + for fused_job in ("linux_cuda", "linux_rocm", "linux_vulkan"): + self.assertNotIn(f"\n {fused_job}:\n", self.workflow) + + def test_gpu_products_reuse_exact_immutable_inputs(self) -> None: + neutral_image = ( + "sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d" + ) + + for backend in ("cuda", "rocm", "vulkan"): + with self.subTest(backend=backend): + product = job_section( + self.workflow, + f"linux_{backend}_product", + ) + self.assertIn( + "needs: [changes, linux_host_input, " + f"linux_{backend}_runtime_input]", + product, + ) + self.assertIn( + "needs.linux_host_input.result == 'success' " + f"&& needs.linux_{backend}_runtime_input.result == 'success'", + product, + ) + self.assertIn( + "runs-on: ${{ needs.changes.outputs.runner_4 }}", + product, + ) + self.assertIn(neutral_image, product) + self.assertIn("name: ci-linux-host-input", product) + self.assertIn( + f"name: ci-linux-{backend}-runtime-input", + product, + ) + self.assertIn( + "uses: ./.github/actions/compose-product-input", + product, + ) + self.assertIn(f"backend: {backend}", product) + self.assertIn(f"name: ci-linux-{backend}-product", product) + self.assertNotIn("prepare-host-input", product) + self.assertNotIn("prepare-native-runtime-input", product) + self.assertNotIn("configure-sccache-gha", product) + self.assertNotIn("LLAMA_STAGE_BUILD_DIR", product) + self.assertNotIn("matrix.", product) def test_linux_tests_share_one_static_abi_producer(self) -> None: producer = job_section(self.workflow, "linux_static_abi_input") diff --git a/scripts/tests/test_generate_native_runtime_release_manifest.py b/scripts/tests/test_generate_native_runtime_release_manifest.py index 39f636fd97..20027c26d9 100644 --- a/scripts/tests/test_generate_native_runtime_release_manifest.py +++ b/scripts/tests/test_generate_native_runtime_release_manifest.py @@ -1,3 +1,5 @@ +import hashlib +import io import json import pathlib import subprocess @@ -13,49 +15,87 @@ class GenerateNativeRuntimeReleaseManifestTests(unittest.TestCase): + def create_archive(self, root: pathlib.Path) -> pathlib.Path: + package_dir = root / "meshllm-native-runtime-linux-aarch64-cpu" + package_dir.mkdir() + library = package_dir / "lib" / "runtime.bin" + library.parent.mkdir() + library.write_bytes(b"test native runtime") + library_digest = hashlib.sha256(library.read_bytes()).hexdigest() + (package_dir / "manifest.json").write_text( + json.dumps( + { + "runtime": { + "id": "meshllm-native-runtime-linux-aarch64-cpu", + "mesh_version": "0.68.0", + "skippy_abi": "0.1.25", + "platform": { + "os": "linux", + "arch": "aarch64", + "target": "aarch64-unknown-linux-gnu", + }, + "backend": {"kind": "cpu"}, + "rank": 0, + "libraries": ["lib/runtime.bin"], + "files": { + "lib/runtime.bin": library_digest, + }, + }, + "build": { + "primary_library": "lib/runtime.bin", + "library_sha256": library_digest, + }, + } + ), + encoding="utf-8", + ) + archive = root / "meshllm-native-runtime-linux-aarch64-cpu.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + tar.add(package_dir, arcname=package_dir.name) + return archive + + def write_sidecar( + self, + archive: pathlib.Path, + *, + digest: str | None = None, + ) -> None: + actual = hashlib.sha256(archive.read_bytes()).hexdigest() + archive.with_name(f"{archive.name}.sha256").write_text( + f"{digest or actual} {archive.name}\n", + encoding="utf-8", + ) + + def run_generator( + self, + archive: pathlib.Path, + out: pathlib.Path, + *, + tag: str = "v0.68.0", + ) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + str(SCRIPT), + "--tag", + tag, + "--out", + str(out), + str(archive), + ], + check=False, + text=True, + capture_output=True, + ) + def test_generated_manifest_is_single_valid_json_document(self): with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) - package_dir = root / "meshllm-native-runtime-linux-aarch64-cpu" - package_dir.mkdir() - (package_dir / "manifest.json").write_text( - json.dumps( - { - "runtime": { - "id": "meshllm-native-runtime-linux-aarch64-cpu", - "mesh_version": "0.68.0", - "skippy_abi": "0.1.25", - "platform": { - "os": "linux", - "arch": "aarch64", - "target": "aarch64-unknown-linux-gnu", - }, - "backend": {"kind": "cpu"}, - "rank": 0, - "libraries": ["lib/libllama.so"], - } - } - ), - encoding="utf-8", - ) - archive = root / "meshllm-native-runtime-linux-aarch64-cpu.tar.gz" - with tarfile.open(archive, "w:gz") as tar: - tar.add(package_dir, arcname=package_dir.name) + archive = self.create_archive(root) + self.write_sidecar(archive) out = root / "native-runtimes.json" - subprocess.run( - [ - str(SCRIPT), - "--tag", - "v0.68.0", - "--out", - str(out), - str(archive), - ], - check=True, - text=True, - capture_output=True, - ) + result = self.run_generator(archive, out) + self.assertEqual(result.returncode, 0, result.stderr) with out.open(encoding="utf-8") as handle: manifest = json.load(handle) @@ -63,6 +103,90 @@ def test_generated_manifest_is_single_valid_json_document(self): self.assertEqual(manifest["mesh_version"], "0.68.0") self.assertEqual(len(manifest["artifacts"]), 1) + def test_requires_valid_canonical_sidecar(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + archive = self.create_archive(root) + out = root / "native-runtimes.json" + + missing = self.run_generator(archive, out) + self.assertNotEqual(missing.returncode, 0) + self.assertIn("sidecar is missing or empty", missing.stderr) + self.assertFalse(out.exists()) + + self.write_sidecar(archive, digest="0" * 64) + corrupt = self.run_generator(archive, out) + self.assertNotEqual(corrupt.returncode, 0) + self.assertIn("archive checksum mismatch", corrupt.stderr) + self.assertFalse(out.exists()) + + def test_rejects_traversing_archive_member(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + archive = self.create_archive(root) + + package_dir = root / "meshllm-native-runtime-linux-aarch64-cpu" + with tarfile.open(archive, "w:gz") as tar: + tar.add(package_dir, arcname=package_dir.name) + escaping = tarfile.TarInfo("../escaped") + payload = b"must not escape" + escaping.size = len(payload) + tar.addfile(escaping, io.BytesIO(payload)) + self.write_sidecar(archive) + + out = root / "native-runtimes.json" + result = self.run_generator(archive, out) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("unsafe or invalid tar archive", result.stderr) + self.assertFalse((root / "escaped").exists()) + self.assertFalse(out.exists()) + + def test_rejects_runtime_version_that_does_not_match_tag(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + archive = self.create_archive(root) + self.write_sidecar(archive) + + out = root / "native-runtimes.json" + result = self.run_generator( + archive, + out, + tag="v0.69.0-rc1", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "mesh_version 0.68.0 does not match " + "release tag v0.69.0-rc1", + result.stderr, + ) + self.assertFalse(out.exists()) + + def test_rejects_sibling_payload_outside_artifact_directory(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + archive = self.create_archive(root) + package_dir = root / "meshllm-native-runtime-linux-aarch64-cpu" + + with tarfile.open(archive, "w:gz") as tar: + tar.add(package_dir, arcname=package_dir.name) + sibling = tarfile.TarInfo("unexpected.txt") + payload = b"not part of the runtime artifact" + sibling.size = len(payload) + tar.addfile(sibling, io.BytesIO(payload)) + self.write_sidecar(archive) + + out = root / "native-runtimes.json" + result = self.run_generator(archive, out) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "expected archive to contain one top-level artifact directory", + result.stderr, + ) + self.assertFalse(out.exists()) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_native_artifact_verifiers.py b/scripts/tests/test_native_artifact_verifiers.py new file mode 100644 index 0000000000..69b39885e8 --- /dev/null +++ b/scripts/tests/test_native_artifact_verifiers.py @@ -0,0 +1,531 @@ +from __future__ import annotations + +import hashlib +import io +import json +import os +from pathlib import Path +import shutil +import subprocess +import tarfile +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +RUNTIME_VERIFIER = ROOT / "scripts" / "verify-native-runtime-package.sh" +SDK_VERIFIER = ROOT / "scripts" / "verify-native-sdk-package.sh" + + +def bash_executable() -> str: + if os.name != "nt": + return shutil.which("bash") or "bash" + git = shutil.which("git") + if git: + candidate = Path(git).parent.parent / "bin" / "bash.exe" + if candidate.is_file(): + return str(candidate) + raise RuntimeError("Git Bash is required for artifact verifier tests") + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +class NativeArtifactVerifierTests(unittest.TestCase): + def run_verifier( + self, + verifier: Path, + artifact: Path, + *options: str, + ) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + bash_executable(), + verifier.as_posix(), + *options, + artifact.as_posix(), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + def write_runtime_artifact(self, root: Path) -> tuple[Path, dict]: + artifact = root / "meshllm-native-runtime-darwin-x86_64-cpu" + library = artifact / "lib" / "llama.bin" + tool = artifact / "tools" / "probe" + library.parent.mkdir(parents=True) + tool.parent.mkdir(parents=True) + library.write_bytes(b"runtime library") + tool.write_bytes(b"runtime tool") + tool.chmod(0o755) + manifest = { + "runtime": { + "id": artifact.name, + "mesh_version": "0.75.0", + "skippy_abi": "0.1.32", + "platform": { + "os": "macos", + "arch": "x86_64", + "target": "x86_64-apple-darwin", + }, + "backend": {"kind": "cpu"}, + "libraries": ["lib/llama.bin"], + "files": {"lib/llama.bin": sha256(library)}, + "tools": {"tools/probe": sha256(tool)}, + }, + "build": { + "primary_library": "lib/llama.bin", + "library_sha256": sha256(library), + }, + } + self.write_manifest(artifact, manifest) + return artifact, manifest + + def write_sdk_artifact(self, root: Path) -> tuple[Path, dict]: + artifact = root / "meshllm-native-linux-x86_64-cpu" + library = artifact / "lib" / "libmesh_llm_ffi.so" + uniffi_library = artifact / "lib" / "libmesh_llm_uniffi.so" + library.parent.mkdir(parents=True) + library.write_bytes(b"native SDK library") + uniffi_library.write_bytes(library.read_bytes()) + manifest = { + "schema_version": 1, + "artifact_id": artifact.name, + "native_runtime_id": artifact.name, + "sdk_version": "0.75.0", + "mesh_version": "0.75.0", + "target_triple": "x86_64-unknown-linux-gnu", + "platform": "linux-x86_64", + "os": "linux", + "arch": "x86_64", + "backend": "cpu", + "flavor": "cpu", + "library": "lib/libmesh_llm_ffi.so", + "library_paths": ["lib/libmesh_llm_ffi.so"], + "uniffi_library": "lib/libmesh_llm_uniffi.so", + "library_sha256": sha256(library), + "requirements": [], + "features": [ + "mesh-inference", + "model-management", + "local-serving", + "chat", + "responses", + ], + } + self.write_manifest(artifact, manifest) + return artifact, manifest + + def write_manifest(self, artifact: Path, manifest: dict) -> None: + (artifact / "manifest.json").write_text( + json.dumps(manifest), + encoding="utf-8", + ) + + def archive_artifact( + self, + artifact: Path, + archive: Path, + *, + sibling_payload: bool = False, + ) -> None: + with tarfile.open(archive, "w:gz") as bundle: + bundle.add(artifact, arcname=artifact.name) + if sibling_payload: + payload = b"unexpected sibling" + sibling = tarfile.TarInfo("unexpected.txt") + sibling.size = len(payload) + bundle.addfile(sibling, io.BytesIO(payload)) + archive.with_name(f"{archive.name}.sha256").write_text( + f"{sha256(archive)} {archive.name}\n", + encoding="utf-8", + ) + + def test_runtime_rejects_every_unsafe_manifest_path(self) -> None: + def replace_library(manifest: dict, path: str) -> None: + runtime = manifest["runtime"] + runtime["libraries"] = [path] + runtime["files"] = {path: "0" * 64} + manifest["build"]["primary_library"] = path + manifest["build"]["library_sha256"] = "0" * 64 + + def replace_primary(manifest: dict, path: str) -> None: + runtime = manifest["runtime"] + runtime["libraries"].append(path) + runtime["files"][path] = "0" * 64 + manifest["build"]["primary_library"] = path + manifest["build"]["library_sha256"] = "0" * 64 + + cases = { + "library traversal": lambda manifest: replace_library( + manifest, + "../outside.bin", + ), + "file traversal": lambda manifest: manifest["runtime"][ + "files" + ].update( + {"../outside.bin": "0" * 64} + ), + "tool traversal": lambda manifest: manifest["runtime"].update( + {"tools": {"../outside.bin": "0" * 64}} + ), + "primary traversal": lambda manifest: replace_primary( + manifest, + "../outside.bin", + ), + "backslash": lambda manifest: replace_library( + manifest, + r"lib\llama.bin", + ), + "drive": lambda manifest: replace_primary( + manifest, + "C:/outside.bin", + ), + } + for name, mutate in cases.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifact, manifest = self.write_runtime_artifact(root) + (root / "outside.bin").write_bytes(b"outside") + mutate(manifest) + self.write_manifest(artifact, manifest) + + result = self.run_verifier(RUNTIME_VERIFIER, artifact) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("artifact", result.stdout + result.stderr) + + def test_sdk_rejects_every_unsafe_manifest_path(self) -> None: + def mutate_primary(manifest: dict) -> None: + manifest["library"] = "../outside.so" + manifest["library_paths"] = ["../outside.so"] + + cases = { + "library traversal": mutate_primary, + "library_paths traversal": lambda manifest: manifest[ + "library_paths" + ].append("../outside.so"), + "uniffi traversal": lambda manifest: manifest.update( + {"uniffi_library": "../outside.so"} + ), + "backslash": lambda manifest: manifest["library_paths"].append( + r"lib\outside.so" + ), + "drive": lambda manifest: manifest.update( + {"uniffi_library": "C:/outside.so"} + ), + } + for name, mutate in cases.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifact, manifest = self.write_sdk_artifact(root) + (root / "outside.so").write_bytes(b"outside") + mutate(manifest) + self.write_manifest(artifact, manifest) + + result = self.run_verifier(SDK_VERIFIER, artifact) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("artifact", result.stdout + result.stderr) + + def test_runtime_rejects_non_string_platform_discriminator(self) -> None: + with tempfile.TemporaryDirectory() as directory: + artifact, manifest = self.write_runtime_artifact(Path(directory)) + manifest["runtime"]["platform"]["os"] = ["macos"] + self.write_manifest(artifact, manifest) + + result = self.run_verifier(RUNTIME_VERIFIER, artifact) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "runtime platform must declare os and arch", + result.stderr, + ) + + def test_runtime_requires_complete_library_checksum_contract(self) -> None: + cases = { + "missing files": ( + lambda manifest: manifest["runtime"].pop("files"), + "missing runtime manifest field", + ), + "missing library checksum": ( + lambda manifest: manifest["runtime"].update( + {"files": {}} + ), + "checksum", + ), + "missing primary": ( + lambda manifest: manifest["build"].pop( + "primary_library" + ), + "build.primary_library", + ), + "missing primary checksum": ( + lambda manifest: manifest["build"].pop( + "library_sha256" + ), + "build.library_sha256", + ), + } + for name, (mutate, expected_error) in cases.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + artifact, manifest = self.write_runtime_artifact( + Path(directory), + ) + mutate(manifest) + self.write_manifest(artifact, manifest) + + result = self.run_verifier( + RUNTIME_VERIFIER, + artifact, + "--portable", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected_error, result.stderr) + + def test_runtime_rejects_target_and_backend_platform_mismatch( + self, + ) -> None: + cases = { + "target": lambda manifest: manifest["runtime"][ + "platform" + ].update({"target": "aarch64-unknown-linux-gnu"}), + "backend": lambda manifest: ( + manifest["runtime"].update( + {"backend": {"kind": "metal"}}, + ), + manifest["runtime"].update( + { + "platform": { + "os": "linux", + "arch": "x86_64", + "target": "x86_64-unknown-linux-gnu", + } + }, + ), + ), + } + for name, mutate in cases.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + artifact, manifest = self.write_runtime_artifact( + Path(directory), + ) + mutate(manifest) + self.write_manifest(artifact, manifest) + + result = self.run_verifier( + RUNTIME_VERIFIER, + artifact, + "--portable", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "do not match target" + if name == "target" + else "unsupported on", + result.stderr, + ) + + def test_portable_runtime_verification_skips_host_binary_probes( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifact, manifest = self.write_runtime_artifact(root) + old_library = artifact / "lib" / "llama.bin" + library = artifact / "lib" / "libllama.so" + old_library.rename(library) + library_digest = sha256(library) + runtime = manifest["runtime"] + runtime["platform"] = { + "os": "linux", + "arch": "x86_64", + "target": "x86_64-unknown-linux-gnu", + } + runtime["libraries"] = ["lib/libllama.so"] + runtime["files"] = {"lib/libllama.so": library_digest} + manifest["build"]["primary_library"] = "lib/libllama.so" + manifest["build"]["library_sha256"] = library_digest + self.write_manifest(artifact, manifest) + + result = self.run_verifier( + RUNTIME_VERIFIER, + artifact, + "--portable", + ) + + self.assertEqual( + result.returncode, + 0, + result.stdout + result.stderr, + ) + self.assertIn( + "verified portable native runtime artifact", + result.stdout, + ) + + def test_sdk_rejects_unknown_target_backend_and_flavor(self) -> None: + cases = { + "target": lambda manifest: manifest.update( + {"target_triple": "mystery-vendor-platform"} + ), + "backend": lambda manifest: manifest.update( + {"backend": "made-up"} + ), + "flavor": lambda manifest: manifest.update( + {"flavor": "made-up"} + ), + } + for name, mutate in cases.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + artifact, manifest = self.write_sdk_artifact( + Path(directory), + ) + mutate(manifest) + if name == "flavor": + renamed = artifact.with_name( + "meshllm-native-linux-x86_64-made-up", + ) + artifact.rename(renamed) + artifact = renamed + manifest["artifact_id"] = artifact.name + manifest["native_runtime_id"] = artifact.name + self.write_manifest(artifact, manifest) + + result = self.run_verifier(SDK_VERIFIER, artifact) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "unsupported" if name != "flavor" else "flavor", + result.stderr, + ) + + def test_sdk_manifest_omits_runner_local_build_directory(self) -> None: + packager = ( + ROOT / "scripts" / "package-native-sdk.sh" + ).read_text(encoding="utf-8") + + self.assertNotIn('"llama_build_dir"', packager) + self.assertNotIn( + 'os.path.abspath("$LLAMA_STAGE_BUILD_DIR")', + packager, + ) + + @unittest.skipIf(os.name == "nt", "symlink creation is restricted on Windows") + def test_resolved_manifest_paths_cannot_escape_artifact(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + outside = root / "outside.bin" + outside.write_bytes(b"outside") + runtime, runtime_manifest = self.write_runtime_artifact(root) + runtime_link = runtime / "lib" / "escape.bin" + runtime_link.symlink_to(outside) + runtime_manifest["runtime"]["libraries"] = ["lib/escape.bin"] + self.write_manifest(runtime, runtime_manifest) + + runtime_result = self.run_verifier(RUNTIME_VERIFIER, runtime) + + self.assertNotEqual(runtime_result.returncode, 0) + self.assertIn("resolves outside", runtime_result.stderr) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + outside = root / "outside.so" + outside.write_bytes(b"outside") + sdk, sdk_manifest = self.write_sdk_artifact(root) + sdk_link = sdk / "lib" / "escape.so" + sdk_link.symlink_to(outside) + sdk_manifest["library_paths"].append("lib/escape.so") + self.write_manifest(sdk, sdk_manifest) + + sdk_result = self.run_verifier(SDK_VERIFIER, sdk) + + self.assertNotEqual(sdk_result.returncode, 0) + self.assertIn("resolves outside", sdk_result.stderr) + + def test_archive_wrappers_accept_one_artifact_directory(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cases = ( + ( + RUNTIME_VERIFIER, + self.write_runtime_artifact(root)[0], + root / "runtime.tar.gz", + ), + ( + SDK_VERIFIER, + self.write_sdk_artifact(root)[0], + root / "sdk.tar.gz", + ), + ) + for verifier, artifact, archive in cases: + with self.subTest(verifier=verifier.name): + self.archive_artifact(artifact, archive) + result = self.run_verifier(verifier, archive) + self.assertEqual( + result.returncode, + 0, + result.stdout + result.stderr, + ) + + def test_archive_wrappers_reject_sibling_top_level_payload(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cases = ( + ( + RUNTIME_VERIFIER, + self.write_runtime_artifact(root)[0], + root / "runtime.tar.gz", + ), + ( + SDK_VERIFIER, + self.write_sdk_artifact(root)[0], + root / "sdk.tar.gz", + ), + ) + for verifier, artifact, archive in cases: + with self.subTest(verifier=verifier.name): + self.archive_artifact( + artifact, + archive, + sibling_payload=True, + ) + result = self.run_verifier(verifier, archive) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "one top-level artifact directory", + result.stdout + result.stderr, + ) + + def test_archive_wrappers_reject_single_top_level_file(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for verifier in (RUNTIME_VERIFIER, SDK_VERIFIER): + with self.subTest(verifier=verifier.name): + archive = root / f"{verifier.stem}.tar.gz" + with tarfile.open(archive, "w:gz") as bundle: + payload = b"not an artifact directory" + member = tarfile.TarInfo("payload") + member.size = len(payload) + bundle.addfile(member, io.BytesIO(payload)) + archive.with_name(f"{archive.name}.sha256").write_text( + f"{sha256(archive)} {archive.name}\n", + encoding="utf-8", + ) + + result = self.run_verifier(verifier, archive) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "one top-level artifact directory", + result.stdout + result.stderr, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_pr_workflow_artifacts.py b/scripts/tests/test_pr_workflow_artifacts.py index 531c5efead..817a469e65 100644 --- a/scripts/tests/test_pr_workflow_artifacts.py +++ b/scripts/tests/test_pr_workflow_artifacts.py @@ -37,13 +37,42 @@ def setUpClass(cls) -> None: cls.cpu_product = job_section( cls.workflow, "linux_cpu_artifact", - "linux_targets", - ) - cls.backend_products = job_section( - cls.workflow, - "linux_targets", - "rust_crate_tests", + "linux_cuda_runtime_input", ) + cls.backend_runtimes = { + "cuda": job_section( + cls.workflow, + "linux_cuda_runtime_input", + "linux_cuda_product", + ), + "rocm": job_section( + cls.workflow, + "linux_rocm_runtime_input", + "linux_rocm_product", + ), + "vulkan": job_section( + cls.workflow, + "linux_vulkan_runtime_input", + "linux_vulkan_product", + ), + } + cls.backend_products = { + "cuda": job_section( + cls.workflow, + "linux_cuda_product", + "linux_rocm_runtime_input", + ), + "rocm": job_section( + cls.workflow, + "linux_rocm_product", + "linux_vulkan_runtime_input", + ), + "vulkan": job_section( + cls.workflow, + "linux_vulkan_product", + "linux_static_abi_input", + ), + } cls.macos_host = job_section( cls.workflow, "macos_host_input", @@ -102,12 +131,13 @@ def test_host_profile_covers_every_backend_product_route(self) -> None: ) self.assertIn("&& 'release' || 'debug'", self.host) - self.assertIn( - "github.event_name == 'workflow_dispatch' " - "|| needs.changes.outputs.backend_changed == 'true' " - "|| needs.changes.outputs.benchmarks == 'true'", - self.backend_products, - ) + for runtime in self.backend_runtimes.values(): + self.assertIn( + "github.event_name == 'workflow_dispatch' " + "|| needs.changes.outputs.backend_changed == 'true' " + "|| needs.changes.outputs.benchmarks == 'true'", + runtime, + ) def test_cpu_runtime_only_runs_for_cpu_product_consumers(self) -> None: condition = ( @@ -134,17 +164,100 @@ def test_cpu_product_uses_matching_immutable_inputs(self) -> None: self.cpu_product, ) - def test_backend_products_reuse_the_same_host_artifact(self) -> None: - self.assertIn("needs: [changes, linux_host_input]", self.backend_products) - self.assertIn("name: pr-linux-host-input", self.backend_products) - self.assertIn("path: host-input", self.backend_products) - self.assertNotIn("pr-linux-release-host-input", self.workflow) - self.assertIn("output_dir: product-input", self.backend_products) - self.assertIn( - "path: ${{ steps.compose.outputs.archive_path }}", - self.backend_products, + def test_backend_runtime_inputs_are_independent_producers(self) -> None: + artifacts = { + "cuda": "pr-linux-cuda-runtime-input", + "rocm": "pr-linux-rocm-runtime-input", + "vulkan": "pr-linux-vulkan-runtime-input", + } + expected = { + "cuda": ( + "sha256:c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd", + 'LLAMA_STAGE_CUDA_ARCHITECTURES: "86"', + ), + "rocm": ( + "sha256:6b88ca9371ada2c507d6e36b71f0e0538fee378c6a5e2b39c17249b4b7e5088a", + "LLAMA_STAGE_AMDGPU_TARGETS: gfx1100", + ), + "vulkan": ( + "sha256:ce55fed5c680cd3184b5d4770d9a77c43a702687690906e5753efd2cea27ed80", + "build-stage-abi-dynamic-vulkan", + ), + } + + self.assertNotIn(" linux_targets:", self.workflow) + for backend, runtime in self.backend_runtimes.items(): + with self.subTest(backend=backend): + self.assertIn("needs: changes", runtime) + self.assertIn( + "runs-on: ${{ needs.changes.outputs.runner_8 }}", + runtime, + ) + self.assertIn(expected[backend][0], runtime) + self.assertIn(expected[backend][1], runtime) + self.assertIn( + "uses: ./.github/actions/prepare-native-runtime-input", + runtime, + ) + self.assertIn(f"backend: {backend}", runtime) + self.assertIn(f"name: {artifacts[backend]}", runtime) + self.assertIn("runtime-input/*.tar.gz", runtime) + self.assertIn("runtime-input/*.sha256", runtime) + self.assertNotIn("linux_host_input", runtime) + self.assertNotIn("name: pr-linux-host-input", runtime) + self.assertNotIn("prepare-host-input", runtime) + self.assertNotIn("compose-product-input", runtime) + + self.assertIn("Cache Vulkan ABI build", self.backend_runtimes["vulkan"]) + self.assertNotIn("Cache Vulkan ABI build", self.backend_runtimes["cuda"]) + self.assertNotIn("Cache Vulkan ABI build", self.backend_runtimes["rocm"]) + + def test_backend_products_reuse_exact_immutable_inputs(self) -> None: + neutral_image = ( + "sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d" ) + self.assertNotIn("pr-linux-release-host-input", self.workflow) + for backend, product in self.backend_products.items(): + with self.subTest(backend=backend): + self.assertIn( + "needs: [changes, linux_host_input, " + f"linux_{backend}_runtime_input]", + product, + ) + self.assertIn( + "needs.linux_host_input.result == 'success' " + f"&& needs.linux_{backend}_runtime_input.result == 'success'", + product, + ) + self.assertIn( + "runs-on: ${{ needs.changes.outputs.runner_4 }}", + product, + ) + self.assertIn(neutral_image, product) + self.assertIn("name: pr-linux-host-input", product) + self.assertIn("path: host-input", product) + self.assertIn( + f"name: pr-linux-{backend}-runtime-input", + product, + ) + self.assertIn("path: runtime-input", product) + self.assertIn( + "uses: ./.github/actions/compose-product-input", + product, + ) + self.assertIn(f"backend: {backend}", product) + self.assertIn("output_dir: product-input", product) + self.assertIn(f"name: pr-linux-{backend}-product", product) + self.assertIn( + "path: ${{ steps.compose.outputs.archive_path }}", + product, + ) + self.assertNotIn("prepare-native-runtime-input", product) + self.assertNotIn("configure-sccache-gha", product) + self.assertNotIn("LLAMA_STAGE_BUILD_DIR", product) + self.assertNotIn("matrix.", product) + def test_cuda_runtime_uses_the_production_multiarch_image(self) -> None: self.assertIn( "sha256:c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd", diff --git a/scripts/tests/test_release_workflow_artifacts.py b/scripts/tests/test_release_workflow_artifacts.py index 16da4b75ba..c2bcf1c310 100644 --- a/scripts/tests/test_release_workflow_artifacts.py +++ b/scripts/tests/test_release_workflow_artifacts.py @@ -15,6 +15,36 @@ def job_block(workflow: str, job_name: str, next_job_name: str) -> str: class ReleaseWorkflowArtifactTests(unittest.TestCase): + def test_release_entrypoint_rejects_untrusted_refs(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + metadata = job_block(workflow, "metadata", "build") + + manual_guard = metadata.index("Require the trusted release ref") + checkout = metadata.index("uses: actions/checkout@") + selector = metadata.index( + "uses: ./.github/actions/select-ci-runners", + ) + self.assertLess(manual_guard, checkout) + self.assertLess(checkout, selector) + self.assertIn( + '"$GITHUB_EVENT_NAME" == "workflow_dispatch"', + metadata, + ) + self.assertIn( + '"$GITHUB_REF" != "refs/heads/main"', + metadata, + ) + self.assertIn( + 'git merge-base --is-ancestor "$GITHUB_SHA" ' + "refs/remotes/origin/main", + metadata, + ) + self.assertIn("Reject an existing manual release tag", metadata) + self.assertIn( + "Manual release tag already exists and is immutable", + metadata, + ) + def test_release_depot_policy_is_main_ref_only_and_selected_once( self, ) -> None: @@ -134,6 +164,87 @@ def test_release_permissions_are_least_privilege(self) -> None: ) self.assertNotIn("packages: write", publish) + def test_publish_fan_in_stops_when_release_is_cancelled(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + publish = job_block( + workflow, + "publish", + "dispatch_packaging_release", + ) + + self.assertIn("if: ${{ !cancelled()", publish) + self.assertNotIn("always()", publish) + + def test_release_assets_and_manual_tags_are_immutable(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + publish = job_block( + workflow, + "publish", + "dispatch_packaging_release", + ) + + self.assertIn("Release tag already exists and cannot be reused", publish) + self.assertNotIn("reusing it", publish) + self.assertIn("overwrite_files: false", publish) + self.assertIn("persist-credentials: false", publish) + self.assertNotIn("persist-credentials: true", publish) + self.assertIn( + 'git push "$release_remote" "refs/tags/$RELEASE_TAG"', + publish, + ) + self.assertNotIn( + 'git push origin "refs/tags/$RELEASE_TAG"', + publish, + ) + + def test_release_push_token_is_isolated_to_the_push_step(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + publish = job_block( + workflow, + "publish", + "dispatch_packaging_release", + ) + generate_start = publish.index( + "- name: Generate native runtime release manifest", + ) + prepare_start = publish.index( + "- name: Prepare dispatched release tag", + ) + push_start = publish.index( + "- name: Push dispatched release tag", + ) + release_start = publish.index( + "- name: Publish GitHub release", + ) + generate = publish[generate_start:prepare_start] + prepare = publish[prepare_start:push_start] + push = publish[push_start:release_start] + + token_binding = "GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}" + self.assertNotIn(token_binding, generate) + self.assertNotIn(token_binding, prepare) + self.assertIn(token_binding, push) + self.assertEqual(publish.count(token_binding), 1) + self.assertNotIn("release_remote=", prepare) + self.assertNotIn("git push", prepare) + self.assertIn( + 'git push "$release_remote" "refs/tags/$RELEASE_TAG"', + push, + ) + + def test_arm64_smoke_requires_integrity_and_safe_extraction(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + smoke = job_block( + workflow, + "smoke_linux_arm64_artifact", + "compose_linux_aarch64_cuda", + ) + + self.assertIn("scripts/verify-checksum-sidecar.py", smoke) + self.assertIn("scripts/safe-extract-tar.py", smoke) + self.assertNotIn("tar -xzf", smoke) + self.assertNotIn("command -v sha256sum", smoke) + def test_native_sdk_assets_are_staged_flat_for_publishing(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") producer = job_block( diff --git a/scripts/tests/test_safe_extract_tar.py b/scripts/tests/test_safe_extract_tar.py new file mode 100644 index 0000000000..fa6b786fbf --- /dev/null +++ b/scripts/tests/test_safe_extract_tar.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import hashlib +import io +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +EXTRACTOR = ROOT / "scripts" / "safe-extract-tar.py" +CHECKSUM_VERIFIER = ROOT / "scripts" / "verify-checksum-sidecar.py" + + +class SafeExtractTarTests(unittest.TestCase): + def run_extract( + self, + archive: Path, + destination: Path, + ) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["python3", str(EXTRACTOR), str(archive), str(destination)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + def write_archive( + self, + archive: Path, + members: list[tuple[tarfile.TarInfo, bytes | None]], + ) -> None: + with tarfile.open(archive, "w:gz") as bundle: + for member, payload in members: + bundle.addfile( + member, + io.BytesIO(payload) if payload is not None else None, + ) + + def test_extracts_regular_files_with_executable_mode(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + archive = root / "runtime.tar.gz" + directory = tarfile.TarInfo("runtime") + directory.type = tarfile.DIRTYPE + directory.mode = 0o755 + tool = tarfile.TarInfo("runtime/tool") + payload = b"#!/bin/sh\nexit 0\n" + tool.size = len(payload) + tool.mode = 0o755 + self.write_archive( + archive, + [(directory, None), (tool, payload)], + ) + + output = root / "output" + result = self.run_extract(archive, output) + + self.assertEqual(result.returncode, 0, result.stderr) + extracted = output / "runtime" / "tool" + self.assertEqual(extracted.read_bytes(), payload) + self.assertNotEqual(extracted.stat().st_mode & 0o111, 0) + + def test_accepts_the_root_directory_member_emitted_by_tar_dot( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + archive = root / "product.tar.gz" + archive_root = tarfile.TarInfo(".") + archive_root.type = tarfile.DIRTYPE + payload = b"product" + product = tarfile.TarInfo("./mesh-llm") + product.size = len(payload) + product.mode = 0o755 + self.write_archive( + archive, + [(archive_root, None), (product, payload)], + ) + + output = root / "output" + result = self.run_extract(archive, output) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((output / "mesh-llm").read_bytes(), payload) + + def test_rejects_traversal_absolute_and_device_members(self) -> None: + cases: list[tuple[str, tarfile.TarInfo]] = [] + traversal = tarfile.TarInfo("../escape") + traversal.size = 1 + cases.append(("traversal", traversal)) + absolute = tarfile.TarInfo("/absolute") + absolute.size = 1 + cases.append(("absolute", absolute)) + device = tarfile.TarInfo("runtime/device") + device.type = tarfile.CHRTYPE + cases.append(("device", device)) + + for name, member in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + archive = root / "unsafe.tar.gz" + payload = b"x" if member.isreg() else None + self.write_archive(archive, [(member, payload)]) + + result = self.run_extract(archive, root / "output") + + self.assertNotEqual(result.returncode, 0) + self.assertIn("unsafe or invalid tar archive", result.stderr) + + def test_rejects_escaping_link_target(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + archive = root / "unsafe-link.tar.gz" + link = tarfile.TarInfo("runtime/link") + link.type = tarfile.SYMTYPE + link.linkname = "../../escape" + self.write_archive(archive, [(link, None)]) + + result = self.run_extract(archive, root / "output") + + self.assertNotEqual(result.returncode, 0) + self.assertIn("unsafe or invalid tar archive", result.stderr) + + def test_rejects_nonempty_destination_with_preexisting_symlink( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + archive = root / "runtime.tar.gz" + payload = b"must stay inside the extraction root" + member = tarfile.TarInfo("redirect/payload") + member.size = len(payload) + self.write_archive(archive, [(member, payload)]) + + output = root / "output" + output.mkdir() + outside = root / "outside" + outside.mkdir() + (output / "redirect").symlink_to(outside, target_is_directory=True) + + result = self.run_extract(archive, output) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("destination must be empty", result.stderr) + self.assertFalse((outside / "payload").exists()) + + +class ChecksumSidecarTests(unittest.TestCase): + def run_verify(self, artifact: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["python3", str(CHECKSUM_VERIFIER), str(artifact)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + def test_requires_exact_canonical_sidecar(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + artifact = Path(temp_dir) / "runtime.tar.gz" + artifact.write_bytes(b"runtime") + + missing = self.run_verify(artifact) + self.assertNotEqual(missing.returncode, 0) + self.assertIn("sidecar is missing or empty", missing.stderr) + + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + sidecar = artifact.with_name(f"{artifact.name}.sha256") + sidecar.write_text( + f"{digest} {artifact.name}\n", + encoding="utf-8", + ) + valid = self.run_verify(artifact) + self.assertEqual(valid.returncode, 0, valid.stderr) + + sidecar.write_text( + f"{digest} wrong-name.tar.gz\n", + encoding="utf-8", + ) + wrong_name = self.run_verify(artifact) + self.assertNotEqual(wrong_name.returncode, 0) + self.assertIn("checksum sidecar names", wrong_name.stderr) + + sidecar.write_text( + f"{digest} {artifact.name}\n{digest} {artifact.name}\n", + encoding="utf-8", + ) + duplicate = self.run_verify(artifact) + self.assertNotEqual(duplicate.returncode, 0) + self.assertIn("exactly one canonical line", duplicate.stderr) + + def test_rejects_noncanonical_checksum_separators(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + artifact = Path(temp_dir) / "runtime.tar.gz" + artifact.write_bytes(b"runtime") + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + sidecar = artifact.with_name(f"{artifact.name}.sha256") + + for separator in (" ", "\t", " *"): + with self.subTest(separator=repr(separator)): + sidecar.write_text( + f"{digest}{separator}{artifact.name}\n", + encoding="utf-8", + ) + result = self.run_verify(artifact) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "must use ' ' format", + result.stderr, + ) + + def test_rejects_surrounding_whitespace_and_blank_lines(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + artifact = Path(temp_dir) / "runtime.tar.gz" + artifact.write_bytes(b"runtime") + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + sidecar = artifact.with_name(f"{artifact.name}.sha256") + + sidecars = ( + f" {digest} {artifact.name}\n", + f"{digest} {artifact.name} \n", + f"\n{digest} {artifact.name}\n", + f"{digest} {artifact.name}\n\n", + f"{digest.upper()} {artifact.name}\n", + ) + for contents in sidecars: + with self.subTest(contents=repr(contents)): + sidecar.write_text(contents, encoding="utf-8") + result = self.run_verify(artifact) + self.assertNotEqual(result.returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_sccache_evidence.py b/scripts/tests/test_sccache_evidence.py new file mode 100644 index 0000000000..908f5f324d --- /dev/null +++ b/scripts/tests/test_sccache_evidence.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +ACTION_DIR = ROOT / ".github" / "actions" / "capture-sccache-stats" +CAPTURE = ACTION_DIR / "capture.py" +SUMMARY = ROOT / "scripts" / "summarize-sccache-stats.py" +CONFIGURE_ACTION = ( + ROOT / ".github" / "actions" / "configure-sccache-gha" / "action.yml" +) +WORKFLOWS = { + "pr-builds": ROOT / ".github" / "workflows" / "pr_builds.yml", + "pr-quality": ROOT / ".github" / "workflows" / "pr_quality.yml", + "main": ROOT / ".github" / "workflows" / "ci.yml", +} + + +def valid_payload(*, compile_requests: int = 12) -> dict[str, object]: + return { + "stats": { + "compile_requests": compile_requests, + "requests_executed": 10, + "compilations": 4, + "cache_writes": 3, + "cache_read_errors": 0, + "cache_write_errors": 0, + "cache_hits": { + "counts": {"Rust": 6}, + "adv_counts": {}, + }, + "cache_misses": { + "counts": {"Rust": 4}, + "adv_counts": {}, + }, + "cache_errors": { + "counts": {}, + "adv_counts": {}, + }, + }, + "version": "test", + } + + +class SccacheEvidenceTests(unittest.TestCase): + def run_capture( + self, + payload: dict[str, object], + *, + artifact_name: str = "sccache-test-1", + ) -> tuple[subprocess.CompletedProcess[str], Path, Path]: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + fake_sccache = root / "sccache" + fake_sccache.write_text( + "#!/usr/bin/env python3\n" + "import os\n" + "import sys\n" + "if sys.argv[1:] == ['--show-stats']:\n" + " print('Compile requests 12')\n" + "elif sys.argv[1:] == ['--show-stats', '--stats-format', 'json']:\n" + " print(os.environ['FAKE_SCCACHE_JSON'])\n" + "else:\n" + " raise SystemExit(2)\n", + encoding="utf-8", + ) + fake_sccache.chmod( + fake_sccache.stat().st_mode | stat.S_IXUSR, + ) + stats_file = root / "evidence" / "sccache-stats.json" + github_output = root / "github-output" + result = subprocess.run( + [ + sys.executable, + str(CAPTURE), + "--artifact-name", + artifact_name, + "--output", + str(stats_file), + "--github-output", + str(github_output), + ], + env={ + **os.environ, + "PATH": f"{root}{os.pathsep}{os.environ['PATH']}", + "FAKE_SCCACHE_JSON": json.dumps(payload), + }, + check=False, + capture_output=True, + text=True, + ) + return result, stats_file, github_output + + def test_capture_emits_human_stats_and_machine_readable_counters(self) -> None: + payload = valid_payload() + result, stats_file, github_output = self.run_capture(payload) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Human-readable sccache statistics", result.stdout) + self.assertIn("Compile requests 12", result.stdout) + self.assertEqual(json.loads(stats_file.read_text()), payload) + outputs = github_output.read_text(encoding="utf-8") + self.assertIn("compile_requests=12", outputs) + self.assertIn("requests_executed=10", outputs) + self.assertIn("cache_hits=6", outputs) + self.assertIn("cache_misses=4", outputs) + + def test_zero_compile_requests_warns_but_remains_valid_evidence(self) -> None: + result, stats_file, _ = self.run_capture( + valid_payload(compile_requests=0), + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(stats_file.is_file()) + self.assertIn("::warning title=sccache reported zero compile requests", result.stdout) + + def test_missing_or_invalid_counter_rejects_evidence(self) -> None: + payload = valid_payload() + stats = payload["stats"] + self.assertIsInstance(stats, dict) + del stats["cache_misses"] + + result, stats_file, _ = self.run_capture(payload) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(stats_file.exists()) + self.assertIn("stats.cache_misses must be an object", result.stderr) + + def test_artifact_name_cannot_escape_the_evidence_namespace(self) -> None: + result, stats_file, _ = self.run_capture( + valid_payload(), + artifact_name="../sccache-test", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(stats_file.exists()) + self.assertIn("artifact name must contain only", result.stderr) + + def test_composite_action_uploads_fourteen_day_json_evidence(self) -> None: + action = (ACTION_DIR / "action.yml").read_text(encoding="utf-8") + capture = CAPTURE.read_text(encoding="utf-8") + + self.assertIn("artifact_name:", action) + self.assertIn("actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f", action) + self.assertIn("retention-days: 14", action) + self.assertIn("if-no-files-found: error", action) + self.assertIn('"--show-stats", "--stats-format", "json"', capture) + self.assertIn("REQUIRED_COUNTERS", capture) + self.assertIn("REQUIRED_COUNT_MAPS", capture) + + def test_configure_action_resets_each_successful_server_route(self) -> None: + configure = CONFIGURE_ACTION.read_text(encoding="utf-8") + + self.assertIn("['--zero-stats']", configure) + self.assertEqual(configure.count("await resetStatistics("), 6) + + def test_instrumented_workflows_use_unique_evidence_artifacts(self) -> None: + expected_names = { + "pr-builds": ( + "sccache-pr-linux-host-${{ github.run_attempt }}", + "sccache-pr-linux-cpu-runtime-${{ github.run_attempt }}", + "sccache-pr-rust-crate-tests-${{ matrix.batch.idx }}-${{ github.run_attempt }}", + "sccache-pr-linux-tests-${{ matrix.group }}-${{ github.run_attempt }}", + ), + "pr-quality": ( + "sccache-pr-quality-clippy-${{ matrix.batch.idx }}-${{ github.run_attempt }}", + ), + "main": ( + "sccache-main-linux-host-${{ github.run_attempt }}", + "sccache-main-linux-cpu-runtime-${{ github.run_attempt }}", + "sccache-main-rust-crate-tests-${{ matrix.batch.idx }}-${{ github.run_attempt }}", + "sccache-main-linux-tests-${{ matrix.group }}-${{ github.run_attempt }}", + ), + } + + for workflow_name, path in WORKFLOWS.items(): + workflow = path.read_text(encoding="utf-8") + with self.subTest(workflow=workflow_name): + self.assertNotIn("Show sccache stats", workflow) + self.assertEqual( + workflow.count("uses: ./.github/actions/capture-sccache-stats"), + len(expected_names[workflow_name]), + ) + for artifact_name in expected_names[workflow_name]: + self.assertIn(f"artifact_name: {artifact_name}", workflow) + + +class SccacheStatsSummaryTests(unittest.TestCase): + def write_evidence( + self, + path: Path, + *, + hits: int, + misses: int, + advanced_hits: int = 0, + ) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "stats": { + "cache_hits": { + "counts": {"Rust": hits}, + "adv_counts": {"Rust": advanced_hits}, + }, + "cache_misses": { + "counts": {"Rust": misses}, + "adv_counts": {}, + }, + }, + }, + ), + encoding="utf-8", + ) + + def run_summary( + self, + evidence: Path, + *, + minimum: str | None = None, + ) -> subprocess.CompletedProcess[str]: + command = [ + sys.executable, + str(SUMMARY), + "--format", + "json", + ] + if minimum is not None: + command.extend(["--minimum-hit-rate", minimum]) + command.append(str(evidence)) + return subprocess.run( + command, + check=False, + capture_output=True, + text=True, + ) + + def test_offline_summary_aggregates_counts_without_advanced_duplicates(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + evidence = Path(temporary) + self.write_evidence( + evidence / "job-a" / "sccache-stats.json", + hits=60, + misses=10, + advanced_hits=60, + ) + self.write_evidence( + evidence / "job-b" / "sccache-stats-warm.json", + hits=20, + misses=10, + advanced_hits=20, + ) + + result = self.run_summary(evidence, minimum="0.80") + + self.assertEqual(result.returncode, 0, result.stderr) + summary = json.loads(result.stdout) + self.assertEqual(summary["file_count"], 2) + self.assertEqual(summary["cache_hits"], 80) + self.assertEqual(summary["cache_misses"], 20) + self.assertEqual(summary["hit_rate"], 0.8) + self.assertTrue(summary["passed"]) + + def test_offline_summary_fails_a_missed_hit_rate_gate(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + evidence = Path(temporary) / "sccache-stats.json" + self.write_evidence(evidence, hits=79, misses=21) + + result = self.run_summary(evidence, minimum="0.80") + + self.assertEqual(result.returncode, 1) + summary = json.loads(result.stdout) + self.assertEqual(summary["hit_rate"], 0.79) + self.assertFalse(summary["passed"]) + + def test_offline_summary_rejects_invalid_count_evidence(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + evidence = Path(temporary) / "sccache-stats.json" + self.write_evidence(evidence, hits=-1, misses=1) + + result = self.run_summary(evidence) + + self.assertEqual(result.returncode, 1) + self.assertIn("negative counter", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_windows_native_runtime_deps.py b/scripts/tests/test_windows_native_runtime_deps.py index 90358bf2ac..7ee63fbf31 100644 --- a/scripts/tests/test_windows_native_runtime_deps.py +++ b/scripts/tests/test_windows_native_runtime_deps.py @@ -1,3 +1,4 @@ +import hashlib import importlib.util import json import os @@ -113,19 +114,36 @@ def test_package_verifier_accepts_closed_windows_dependency_graph(self): write_pe(lib_dir / "ggml-vulkan.dll", ["vulkan-1.dll", "libstdc++-6.dll"]) write_pe(lib_dir / "libstdc++-6.dll", ["KERNEL32.dll"]) write_pe(lib_dir / "llama.dll", ["ggml-vulkan.dll"]) + libraries = [ + "lib/ggml-vulkan.dll", + "lib/libstdc++-6.dll", + "lib/llama.dll", + ] + files = { + path: hashlib.sha256( + (artifact / path).read_bytes() + ).hexdigest() + for path in libraries + } manifest = { "runtime": { "id": artifact.name, "mesh_version": "0.72.1", "skippy_abi": "0.1.32", - "platform": {"os": "windows", "arch": "x86_64"}, + "platform": { + "os": "windows", + "arch": "x86_64", + "target": "x86_64-pc-windows-msvc", + }, "backend": {"kind": "vulkan"}, - "libraries": [ - "lib/ggml-vulkan.dll", - "lib/libstdc++-6.dll", - "lib/llama.dll", - ], - } + "libraries": libraries, + "files": files, + "tools": {}, + }, + "build": { + "primary_library": "lib/llama.dll", + "library_sha256": files["lib/llama.dll"], + }, } (artifact / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") @@ -147,22 +165,40 @@ def test_package_verifier_rejects_present_falsy_files_and_tools(self): with tempfile.TemporaryDirectory() as directory: artifact = ( pathlib.Path(directory) - / "meshllm-native-runtime-test-x86_64-cpu" + / "meshllm-native-runtime-darwin-x86_64-cpu" ) lib_dir = artifact / "lib" lib_dir.mkdir(parents=True) (lib_dir / "llama.bin").write_bytes(b"runtime") + digest = hashlib.sha256( + (lib_dir / "llama.bin").read_bytes() + ).hexdigest() runtime = { "id": artifact.name, "mesh_version": "0.72.1", "skippy_abi": "0.1.32", - "platform": {"os": "test", "arch": "x86_64"}, + "platform": { + "os": "macos", + "arch": "x86_64", + "target": "x86_64-apple-darwin", + }, "backend": {"kind": "cpu"}, "libraries": ["lib/llama.bin"], - field: value, + "files": {"lib/llama.bin": digest}, + "tools": {}, } + runtime[field] = value (artifact / "manifest.json").write_text( - json.dumps({"runtime": runtime}), encoding="utf-8" + json.dumps( + { + "runtime": runtime, + "build": { + "primary_library": "lib/llama.bin", + "library_sha256": digest, + }, + } + ), + encoding="utf-8", ) result = subprocess.run( diff --git a/scripts/verify-checksum-sidecar.py b/scripts/verify-checksum-sidecar.py new file mode 100755 index 0000000000..d304b9764f --- /dev/null +++ b/scripts/verify-checksum-sidecar.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Require and verify the canonical SHA-256 sidecar for one artifact.""" + +from __future__ import annotations + +import argparse +import hashlib +from pathlib import Path +import re + + +SIDECAR_LINE = re.compile( + r"([0-9a-f]{64}) {2}([^/\\\r\n]+)", +) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify(artifact: Path) -> None: + sidecar = artifact.with_name(f"{artifact.name}.sha256") + if not sidecar.is_file() or sidecar.stat().st_size == 0: + raise ValueError( + f"archive checksum sidecar is missing or empty: {sidecar}" + ) + lines = sidecar.read_text(encoding="utf-8").splitlines() + if len(lines) != 1: + raise ValueError( + "checksum sidecar must contain exactly one canonical line" + ) + match = SIDECAR_LINE.fullmatch(lines[0]) + if match is None: + raise ValueError( + "checksum sidecar must use ' ' format" + ) + expected, recorded_name = match.groups() + if recorded_name != artifact.name: + raise ValueError( + f"checksum sidecar names {recorded_name!r}, " + f"expected {artifact.name!r}" + ) + actual = sha256_file(artifact) + if actual != expected.lower(): + raise ValueError( + f"archive checksum mismatch: {artifact}\n" + f" expected: {expected.lower()}\n" + f" actual: {actual}" + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("artifact", type=Path) + args = parser.parse_args() + try: + verify(args.artifact) + except (OSError, UnicodeError, ValueError) as error: + parser.exit(1, f"{error}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify-native-runtime-package.sh b/scripts/verify-native-runtime-package.sh index 3ca4ff6dbd..572e6d285b 100755 --- a/scripts/verify-native-runtime-package.sh +++ b/scripts/verify-native-runtime-package.sh @@ -3,11 +3,14 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TMP_ROOT="" +ARTIFACT_DIR_RESULT="" +ARTIFACT_INDEX=0 +PORTABLE=0 trap 'rm -rf "$TMP_ROOT"' EXIT usage() { cat >&2 <<'EOF' -Usage: scripts/verify-native-runtime-package.sh [...] +Usage: scripts/verify-native-runtime-package.sh [--portable] [...] Verifies MeshLLM native runtime artifacts: - manifest schema and resolver fields @@ -16,19 +19,12 @@ Verifies MeshLLM native runtime artifacts: - library_sha256 matches the primary library - Linux shared-library RUNPATH/RPATH is relocatable and resolves packaged deps - Windows non-system DLL imports are present in the artifact - - archive checksum sidecar when present -EOF -} + - required archive checksum sidecar + - archive paths and links cannot escape the extraction directory -sha256_file() { - if command -v shasum >/dev/null 2>&1; then - shasum -a 256 "$1" | awk '{print $1}' - elif command -v sha256sum >/dev/null 2>&1; then - sha256sum "$1" | awk '{print $1}' - else - echo "shasum or sha256sum is required" >&2 - exit 1 - fi +--portable validates integrity, archive shape, manifest schema, paths, and +checksums without running host-specific binary dependency probes. +EOF } python_bin() { @@ -46,25 +42,13 @@ python_bin() { verify_sidecar_checksum() { local archive="$1" - local sidecar="$archive.sha256" - if [[ ! -f "$sidecar" ]]; then - return 0 - fi - local expected actual - expected="$(awk '{print $1}' "$sidecar")" - actual="$(sha256_file "$archive")" - if [[ "$expected" != "$actual" ]]; then - echo "archive checksum mismatch: $archive" >&2 - echo " expected: $expected" >&2 - echo " actual: $actual" >&2 - exit 1 - fi + "$(python_bin)" "$SCRIPT_DIR/verify-checksum-sidecar.py" "$archive" } artifact_dir_for_input() { local input="$1" if [[ -d "$input" ]]; then - printf '%s\n' "$input" + ARTIFACT_DIR_RESULT="$input" return 0 fi case "$input" in @@ -74,21 +58,28 @@ artifact_dir_for_input() { exit 1 ;; esac - verify_sidecar_checksum "$input" + verify_sidecar_checksum "$input" || return 1 if [[ -z "$TMP_ROOT" ]]; then TMP_ROOT="$(mktemp -d)" fi local extract_dir - extract_dir="$TMP_ROOT/$(basename "$input" | tr -cd 'A-Za-z0-9_.-')" + extract_dir="$TMP_ROOT/artifact-$ARTIFACT_INDEX" + ARTIFACT_INDEX=$((ARTIFACT_INDEX + 1)) mkdir -p "$extract_dir" - tar -C "$extract_dir" -xzf "$input" - local count - count="$(find "$extract_dir" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" - if [[ "$count" != "1" ]]; then + "$(python_bin)" "$SCRIPT_DIR/safe-extract-tar.py" "$input" "$extract_dir" || + return 1 + local count entry + count="$( + find "$extract_dir" -mindepth 1 -maxdepth 1 -print | + wc -l | + tr -d ' ' + )" + entry="$(find "$extract_dir" -mindepth 1 -maxdepth 1 -print -quit)" + if [[ "$count" != "1" || ! -d "$entry" || -L "$entry" ]]; then echo "expected archive to contain one top-level artifact directory: $input" >&2 - exit 1 + return 1 fi - find "$extract_dir" -mindepth 1 -maxdepth 1 -type d -print -quit + ARTIFACT_DIR_RESULT="$entry" } verify_artifact_dir() { @@ -102,9 +93,48 @@ verify_artifact_dir() { import hashlib import json import os +from pathlib import Path, PurePosixPath +import re import sys artifact_dir, manifest_path = sys.argv[1:3] +artifact_root = Path(artifact_dir).resolve(strict=True) +windows_drive = re.compile(r"^[A-Za-z]:") + + +def artifact_file(label, raw_path): + if not isinstance(raw_path, str) or not raw_path or "\x00" in raw_path: + raise SystemExit(f"{label} path must be a non-empty string") + if "\\" in raw_path: + raise SystemExit( + f"{label} path must use forward slashes inside the artifact: " + f"{raw_path}" + ) + rel_path = PurePosixPath(raw_path) + if ( + rel_path.is_absolute() + or windows_drive.match(raw_path) + or ".." in rel_path.parts + ): + raise SystemExit( + f"{label} path must be relative inside the artifact: {raw_path}" + ) + candidate = artifact_root.joinpath(*rel_path.parts) + try: + resolved = candidate.resolve(strict=True) + except OSError: + raise SystemExit(f"missing {label}: {candidate}") from None + try: + resolved.relative_to(artifact_root) + except ValueError: + raise SystemExit( + f"{label} path resolves outside the artifact: {raw_path}" + ) from None + if not resolved.is_file(): + raise SystemExit(f"missing {label}: {candidate}") + return resolved + + with open(manifest_path, encoding="utf-8") as fh: manifest = json.load(fh) @@ -118,58 +148,150 @@ required = { "platform", "backend", "libraries", + "files", } missing = sorted(required - runtime.keys()) if missing: raise SystemExit(f"missing runtime manifest field(s): {', '.join(missing)}") +for field in ("id", "mesh_version", "skippy_abi"): + if not isinstance(runtime[field], str) or not runtime[field]: + raise SystemExit(f"runtime {field} must be a non-empty string") if os.path.basename(os.path.normpath(artifact_dir)) != runtime["id"]: raise SystemExit("artifact directory name must match runtime id") if not isinstance(runtime["libraries"], list) or not runtime["libraries"]: raise SystemExit("runtime libraries must be a non-empty list") platform = runtime["platform"] -if not isinstance(platform, dict) or not platform.get("os") or not platform.get("arch"): +if not isinstance(platform, dict): + raise SystemExit("runtime platform must be an object") +runtime_os = platform.get("os") +runtime_arch = platform.get("arch") +if not isinstance(runtime_os, str) or not runtime_os: raise SystemExit("runtime platform must declare os and arch") +if runtime_os not in {"linux", "macos", "windows"}: + raise SystemExit(f"unsupported runtime platform os: {runtime_os!r}") +if not isinstance(runtime_arch, str) or not runtime_arch: + raise SystemExit("runtime platform must declare os and arch") +if runtime_arch not in {"x86_64", "aarch64", "arm"}: + raise SystemExit(f"unsupported runtime platform arch: {runtime_arch!r}") +target = platform.get("target") +if not isinstance(target, str) or not target: + raise SystemExit("runtime platform target must be a non-empty string") +target_contracts = { + "aarch64-apple-darwin": ("macos", "aarch64"), + "x86_64-apple-darwin": ("macos", "x86_64"), + "x86_64-unknown-linux-gnu": ("linux", "x86_64"), + "aarch64-unknown-linux-gnu": ("linux", "aarch64"), + "armv7-unknown-linux-gnueabihf": ("linux", "arm"), + "aarch64-linux-android": ("linux", "aarch64"), + "armv7-linux-androideabi": ("linux", "arm"), + "x86_64-linux-android": ("linux", "x86_64"), + "x86_64-pc-windows-msvc": ("windows", "x86_64"), +} +target_contract = target_contracts.get(target) +if target_contract is None: + raise SystemExit(f"unsupported runtime platform target: {target}") +if (runtime_os, runtime_arch) != target_contract: + raise SystemExit( + "runtime os/arch do not match target: " + f"{runtime_os}/{runtime_arch} != " + f"{target_contract[0]}/{target_contract[1]}" + ) backend = runtime["backend"] -if not isinstance(backend, dict) or not backend.get("kind"): +if not isinstance(backend, dict): + raise SystemExit("runtime backend must be an object") +backend_kind = backend.get("kind") +if not isinstance(backend_kind, str) or not backend_kind: raise SystemExit("runtime backend must declare kind") +if backend_kind not in {"cpu", "metal", "cuda", "rocm", "vulkan"}: + raise SystemExit(f"unsupported runtime backend kind: {backend_kind!r}") +backend_operating_systems = { + "cpu": {"linux", "macos", "windows"}, + "metal": {"macos"}, + "cuda": {"linux", "windows"}, + "rocm": {"linux", "windows"}, + "vulkan": {"linux", "windows"}, +} +if runtime_os not in backend_operating_systems[backend_kind]: + raise SystemExit( + f"runtime backend {backend_kind} is unsupported on {runtime_os}" + ) for rel_path in runtime["libraries"]: - if os.path.isabs(rel_path) or ".." in rel_path.split(os.sep): - raise SystemExit(f"library path must be relative inside the artifact: {rel_path}") - path = os.path.join(artifact_dir, rel_path) - if not os.path.isfile(path): - raise SystemExit(f"missing library: {path}") + artifact_file("library", rel_path) -files = runtime["files"] if "files" in runtime else {} +files = runtime["files"] tools = runtime["tools"] if "tools" in runtime else {} if not isinstance(files, dict) or not isinstance(tools, dict): raise SystemExit("runtime files and tools must be checksum maps") +if not files: + raise SystemExit("runtime files must be a non-empty checksum map") +missing_library_checksums = [ + rel_path + for rel_path in runtime["libraries"] + if rel_path not in files +] +if missing_library_checksums: + raise SystemExit( + "runtime libraries are missing file checksums: " + + ", ".join(missing_library_checksums) + ) +sha256_pattern = re.compile(r"^[0-9a-f]{64}$") for kind, checksums in (("file", files), ("tool", tools)): for rel_path, expected in checksums.items(): - if os.path.isabs(rel_path) or ".." in rel_path.split(os.sep): - raise SystemExit(f"{kind} path must be relative inside the artifact: {rel_path}") - path = os.path.join(artifact_dir, rel_path) - if not os.path.isfile(path): - raise SystemExit(f"missing {kind}: {path}") + path = artifact_file(kind, rel_path) + if not isinstance(expected, str) or not sha256_pattern.fullmatch( + expected + ): + raise SystemExit( + f"{kind} checksum must be a canonical SHA-256 for {rel_path}" + ) with open(path, "rb") as fh: actual = hashlib.sha256(fh.read()).hexdigest() - if actual != expected.removeprefix("sha256:").lower(): + if actual != expected: raise SystemExit(f"{kind} checksum mismatch for {rel_path}") - if kind == "tool" and os.name != "nt" and not os.access(path, os.X_OK): + if ( + kind == "tool" + and runtime_os != "windows" + and os.name != "nt" + and not os.access(path, os.X_OK) + ): raise SystemExit(f"runtime tool is not executable: {rel_path}") -build = manifest.get("build") or {} +build = manifest.get("build") +if not isinstance(build, dict): + raise SystemExit("runtime build metadata must be an object") library_sha256 = build.get("library_sha256") -primary_library = build.get("primary_library") or runtime["libraries"][0] -if library_sha256: - primary = os.path.join(artifact_dir, primary_library) - with open(primary, "rb") as fh: - actual = hashlib.sha256(fh.read()).hexdigest() - if actual != library_sha256: - raise SystemExit( - f"library_sha256 mismatch for {primary_library}: {actual} != {library_sha256}" - ) +primary_library = build.get("primary_library") +if not isinstance(primary_library, str) or not primary_library: + raise SystemExit("build.primary_library must be a non-empty string") +if primary_library not in runtime["libraries"]: + raise SystemExit( + "build.primary_library must be declared in runtime.libraries" + ) +if not isinstance(library_sha256, str) or not sha256_pattern.fullmatch( + library_sha256 +): + raise SystemExit( + "build.library_sha256 must be a canonical SHA-256" + ) +if files[primary_library] != library_sha256: + raise SystemExit( + "build.library_sha256 must match runtime.files for " + f"{primary_library}" + ) +primary = artifact_file("primary library", primary_library) +with open(primary, "rb") as fh: + actual = hashlib.sha256(fh.read()).hexdigest() +if actual != library_sha256: + raise SystemExit( + f"library_sha256 mismatch for {primary_library}: " + f"{actual} != {library_sha256}" + ) PY + if [[ "$PORTABLE" == "1" ]]; then + echo "verified portable native runtime artifact: $artifact_dir" + return 0 + fi verify_macos_runtime_paths "$artifact_dir" "$manifest" verify_linux_runtime_paths "$artifact_dir" "$manifest" verify_windows_runtime_dependencies "$artifact_dir" "$manifest" @@ -352,12 +474,33 @@ for rel_path in [*libraries, *tools]: PY } +while [[ "$#" -gt 0 ]]; do + case "$1" in + --portable) + PORTABLE=1 + shift + ;; + --) + shift + break + ;; + -*) + echo "unknown argument: $1" >&2 + usage + exit 1 + ;; + *) + break + ;; + esac +done + if [[ "$#" -lt 1 ]]; then usage exit 1 fi for input in "$@"; do - artifact_dir="$(artifact_dir_for_input "$input")" - verify_artifact_dir "$artifact_dir" + artifact_dir_for_input "$input" + verify_artifact_dir "$ARTIFACT_DIR_RESULT" done diff --git a/scripts/verify-native-sdk-package.sh b/scripts/verify-native-sdk-package.sh index 266370e9db..1d2191fc37 100755 --- a/scripts/verify-native-sdk-package.sh +++ b/scripts/verify-native-sdk-package.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TMP_ROOT="" +ARTIFACT_DIR_RESULT="" +ARTIFACT_INDEX=0 trap 'rm -rf "$TMP_ROOT"' EXIT usage() { @@ -9,7 +12,8 @@ usage() { Usage: scripts/verify-native-sdk-package.sh [...] Verifies MeshLLM native SDK runtime artifacts: - - archive checksum sidecar when present + - required archive checksum sidecar + - archive paths and links cannot escape the extraction directory - manifest schema and required fields - artifact directory name matches manifest artifact_id - native library exists @@ -18,41 +22,16 @@ Verifies MeshLLM native SDK runtime artifacts: EOF } -sha256_file() { - if command -v shasum >/dev/null 2>&1; then - shasum -a 256 "$1" | awk '{print $1}' - elif command -v sha256sum >/dev/null 2>&1; then - sha256sum "$1" | awk '{print $1}' - else - echo "shasum or sha256sum is required" >&2 - exit 1 - fi -} - verify_sidecar_checksum() { local archive="$1" - local sidecar="$archive.sha256" - - if [[ ! -f "$sidecar" ]]; then - return 0 - fi - - local expected actual - expected="$(awk '{print $1}' "$sidecar")" - actual="$(sha256_file "$archive")" - if [[ "$expected" != "$actual" ]]; then - echo "archive checksum mismatch: $archive" >&2 - echo " expected: $expected" >&2 - echo " actual: $actual" >&2 - exit 1 - fi + python3 "$SCRIPT_DIR/verify-checksum-sidecar.py" "$archive" } artifact_dir_for_input() { local input="$1" if [[ -d "$input" ]]; then - printf '%s\n' "$input" + ARTIFACT_DIR_RESULT="$input" return 0 fi @@ -64,25 +43,32 @@ artifact_dir_for_input() { ;; esac - verify_sidecar_checksum "$input" + verify_sidecar_checksum "$input" || return 1 if [[ -z "$TMP_ROOT" ]]; then TMP_ROOT="$(mktemp -d)" fi local extract_dir - extract_dir="$TMP_ROOT/$(basename "$input" | tr -cd 'A-Za-z0-9_.-')" + extract_dir="$TMP_ROOT/artifact-$ARTIFACT_INDEX" + ARTIFACT_INDEX=$((ARTIFACT_INDEX + 1)) mkdir -p "$extract_dir" - tar -C "$extract_dir" -xzf "$input" - - local count - count="$(find "$extract_dir" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" - if [[ "$count" != "1" ]]; then + python3 "$SCRIPT_DIR/safe-extract-tar.py" "$input" "$extract_dir" || + return 1 + + local count entry + count="$( + find "$extract_dir" -mindepth 1 -maxdepth 1 -print | + wc -l | + tr -d ' ' + )" + entry="$(find "$extract_dir" -mindepth 1 -maxdepth 1 -print -quit)" + if [[ "$count" != "1" || ! -d "$entry" || -L "$entry" ]]; then echo "expected archive to contain one top-level artifact directory: $input" >&2 - exit 1 + return 1 fi - find "$extract_dir" -mindepth 1 -maxdepth 1 -type d -print -quit + ARTIFACT_DIR_RESULT="$entry" } verify_artifact_dir() { @@ -97,10 +83,48 @@ verify_artifact_dir() { python3 - "$artifact_dir" "$manifest" <<'PY' import hashlib import json -import os +from pathlib import Path, PurePosixPath +import re import sys artifact_dir, manifest_path = sys.argv[1:3] +artifact_root = Path(artifact_dir).resolve(strict=True) +windows_drive = re.compile(r"^[A-Za-z]:") + + +def artifact_file(label, raw_path): + if not isinstance(raw_path, str) or not raw_path or "\x00" in raw_path: + raise SystemExit(f"{label} path must be a non-empty string") + if "\\" in raw_path: + raise SystemExit( + f"{label} path must use forward slashes inside the artifact: " + f"{raw_path}" + ) + rel_path = PurePosixPath(raw_path) + if ( + rel_path.is_absolute() + or windows_drive.match(raw_path) + or ".." in rel_path.parts + ): + raise SystemExit( + f"{label} must be a relative path inside the artifact: {raw_path}" + ) + candidate = artifact_root.joinpath(*rel_path.parts) + try: + resolved = candidate.resolve(strict=True) + except OSError: + raise SystemExit(f"missing {label}: {candidate}") from None + try: + resolved.relative_to(artifact_root) + except ValueError: + raise SystemExit( + f"{label} path resolves outside the artifact: {raw_path}" + ) from None + if not resolved.is_file(): + raise SystemExit(f"missing {label}: {candidate}") + return resolved + + with open(manifest_path, encoding="utf-8") as fh: manifest = json.load(fh) @@ -129,6 +153,24 @@ if missing: if manifest["schema_version"] != 1: raise SystemExit(f"unsupported schema_version: {manifest['schema_version']!r}") +string_fields = ( + "artifact_id", + "native_runtime_id", + "sdk_version", + "mesh_version", + "target_triple", + "platform", + "os", + "arch", + "backend", + "flavor", + "library", + "library_sha256", +) +for field in string_fields: + if not isinstance(manifest[field], str) or not manifest[field]: + raise SystemExit(f"{field} must be a non-empty string") + expected_artifact_id = f"meshllm-native-{manifest['platform']}-{manifest['flavor']}" if manifest["artifact_id"] != expected_artifact_id: raise SystemExit( @@ -143,48 +185,66 @@ if manifest["mesh_version"] != manifest["sdk_version"]: f"mesh_version must match sdk_version: {manifest['mesh_version']} != {manifest['sdk_version']}" ) -expected_os = { - "aarch64-apple-darwin": "macos", - "x86_64-apple-darwin": "macos", - "x86_64-unknown-linux-gnu": "linux", - "aarch64-unknown-linux-gnu": "linux", - "aarch64-linux-android": "linux", - "armv7-linux-androideabi": "linux", - "x86_64-linux-android": "linux", - "x86_64-pc-windows-msvc": "windows", -}.get(manifest["target_triple"]) -expected_arch = { - "aarch64-apple-darwin": "aarch64", - "x86_64-apple-darwin": "x86_64", - "x86_64-unknown-linux-gnu": "x86_64", - "aarch64-unknown-linux-gnu": "aarch64", - "aarch64-linux-android": "aarch64", - "armv7-linux-androideabi": "arm", - "x86_64-linux-android": "x86_64", - "x86_64-pc-windows-msvc": "x86_64", -}.get(manifest["target_triple"]) -if expected_os and manifest["os"] != expected_os: +target_contracts = { + "aarch64-apple-darwin": ("darwin-aarch64", "macos", "aarch64"), + "x86_64-apple-darwin": ("darwin-x86_64", "macos", "x86_64"), + "x86_64-unknown-linux-gnu": ("linux-x86_64", "linux", "x86_64"), + "aarch64-unknown-linux-gnu": ("linux-aarch64", "linux", "aarch64"), + "aarch64-linux-android": ("android-arm64-v8a", "linux", "aarch64"), + "armv7-linux-androideabi": ("android-armeabi-v7a", "linux", "arm"), + "x86_64-linux-android": ("android-x86_64", "linux", "x86_64"), + "x86_64-pc-windows-msvc": ("windows-x86_64", "windows", "x86_64"), +} +target_contract = target_contracts.get(manifest["target_triple"]) +if target_contract is None: + raise SystemExit( + f"unsupported target_triple: {manifest['target_triple']}" + ) +expected_platform, expected_os, expected_arch = target_contract +if manifest["platform"] != expected_platform: + raise SystemExit( + "platform does not match target_triple: " + f"{manifest['platform']} != {expected_platform}" + ) +if manifest["os"] != expected_os: raise SystemExit(f"os does not match target_triple: {manifest['os']} != {expected_os}") -if expected_arch and manifest["arch"] != expected_arch: +if manifest["arch"] != expected_arch: raise SystemExit(f"arch does not match target_triple: {manifest['arch']} != {expected_arch}") -dir_name = os.path.basename(os.path.normpath(artifact_dir)) +flavor_for_backend = { + "cpu": "cpu", + "metal": "metal", + "cuda": "cuda", + "cuda-blackwell": "cuda-blackwell", + "rocm": "rocm", + "hip": "rocm", + "vulkan": "vulkan", +} +expected_flavor = flavor_for_backend.get(manifest["backend"]) +if expected_flavor is None: + raise SystemExit(f"unsupported native SDK backend: {manifest['backend']}") +if manifest["flavor"] != expected_flavor: + raise SystemExit( + "flavor does not match backend: " + f"{manifest['flavor']} != {expected_flavor}" + ) + +dir_name = Path(artifact_dir).name if dir_name != manifest["artifact_id"]: raise SystemExit(f"artifact directory name does not match artifact_id: {dir_name} != {manifest['artifact_id']}") library = manifest["library"] -if library not in manifest["library_paths"]: +library_paths = manifest["library_paths"] +if not isinstance(library_paths, list) or not library_paths: + raise SystemExit("library_paths must be a non-empty list") +if library not in library_paths: raise SystemExit("library_paths must include the primary library") if not isinstance(manifest["requirements"], list): raise SystemExit("requirements must be a list") -for key, rel_path in (("library", library),): - if os.path.isabs(rel_path) or ".." in rel_path.split(os.sep): - raise SystemExit(f"{key} must be a relative path inside the artifact: {rel_path}") - path = os.path.join(artifact_dir, rel_path) - if not os.path.isfile(path): - raise SystemExit(f"missing {key}: {path}") - -library_path = os.path.join(artifact_dir, library) +for rel_path in library_paths: + artifact_file("library_paths entry", rel_path) + +library_path = artifact_file("library", library) with open(library_path, "rb") as fh: actual = hashlib.sha256(fh.read()).hexdigest() if actual != manifest["library_sha256"]: @@ -194,13 +254,7 @@ if actual != manifest["library_sha256"]: legacy_uniffi_library = manifest.get("uniffi_library") if legacy_uniffi_library: - if os.path.isabs(legacy_uniffi_library) or ".." in legacy_uniffi_library.split(os.sep): - raise SystemExit( - f"uniffi_library must be a relative path inside the artifact: {legacy_uniffi_library}" - ) - legacy_path = os.path.join(artifact_dir, legacy_uniffi_library) - if not os.path.isfile(legacy_path): - raise SystemExit(f"missing uniffi_library: {legacy_path}") + legacy_path = artifact_file("uniffi_library", legacy_uniffi_library) with open(legacy_path, "rb") as fh: legacy_actual = hashlib.sha256(fh.read()).hexdigest() if legacy_actual != actual: @@ -208,13 +262,17 @@ if legacy_uniffi_library: f"uniffi_library checksum mismatch: {legacy_actual} != {actual}" ) +if not isinstance(manifest["features"], list) or not all( + isinstance(feature, str) and feature for feature in manifest["features"] +): + raise SystemExit("features must be a list of non-empty strings") features = set(manifest["features"]) for feature in ("mesh-inference", "model-management", "local-serving", "chat", "responses"): if feature not in features: raise SystemExit(f"missing feature marker: {feature}") platform = manifest["platform"] -library_name = os.path.basename(library) +library_name = PurePosixPath(library).name if platform.startswith("darwin-") and not library_name.endswith(".dylib"): raise SystemExit(f"darwin artifact must contain a dylib: {library_name}") if (platform.startswith("linux-") or platform.startswith("android-")) and not library_name.endswith(".so"): @@ -232,6 +290,6 @@ if [[ "$#" -lt 1 ]]; then fi for input in "$@"; do - artifact_dir="$(artifact_dir_for_input "$input")" - verify_artifact_dir "$artifact_dir" + artifact_dir_for_input "$input" + verify_artifact_dir "$ARTIFACT_DIR_RESULT" done From d7c3a6a3909091aae3ba30f7861be5b64d63820c Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 18:20:02 -0400 Subject: [PATCH 08/23] fix(ci): install actionlint from verified release --- .github/actions/install-actionlint/action.yml | 58 +++++++++++++++++++ .github/workflows/pr_quality.yml | 4 +- scripts/tests/test_ci_artifact_actions.py | 24 +++++++- 3 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 .github/actions/install-actionlint/action.yml diff --git a/.github/actions/install-actionlint/action.yml b/.github/actions/install-actionlint/action.yml new file mode 100644 index 0000000000..29018426d1 --- /dev/null +++ b/.github/actions/install-actionlint/action.yml @@ -0,0 +1,58 @@ +name: Install pinned actionlint +description: Install the repository-pinned actionlint release after verifying its official archive checksum. + +runs: + using: composite + steps: + - name: Install actionlint 1.7.12 + shell: bash + env: + ACTIONLINT_VERSION: "1.7.12" + run: | + set -euo pipefail + + case "$(uname -s)/$(uname -m)" in + Linux/x86_64) + platform="linux_amd64" + checksum="8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8" + ;; + Linux/aarch64 | Linux/arm64) + platform="linux_arm64" + checksum="325e971b6ba9bfa504672e29be93c24981eeb1c07576d730e9f7c8805afff0c6" + ;; + *) + echo "unsupported actionlint installer platform: $(uname -s)/$(uname -m)" >&2 + exit 1 + ;; + esac + + archive_name="actionlint_${ACTIONLINT_VERSION}_${platform}.tar.gz" + archive="$RUNNER_TEMP/$archive_name" + install_dir="$RUNNER_TEMP/actionlint-${ACTIONLINT_VERSION}-${platform}" + curl \ + --fail \ + --location \ + --retry 5 \ + --retry-all-errors \ + --show-error \ + --silent \ + "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/${archive_name}" \ + --output "$archive" + python3 - "$archive" "$checksum" <<'PY' + import hashlib + from pathlib import Path + import sys + + archive, expected = Path(sys.argv[1]), sys.argv[2] + actual = hashlib.sha256(archive.read_bytes()).hexdigest() + if actual != expected: + raise SystemExit( + f"actionlint archive checksum mismatch: {actual} != {expected}" + ) + PY + + mkdir -p "$install_dir" + python3 scripts/safe-extract-tar.py "$archive" "$install_dir" + test -x "$install_dir/actionlint" + "$install_dir/actionlint" -version + printf '%s\n' "$install_dir" >> "$GITHUB_PATH" diff --git a/.github/workflows/pr_quality.yml b/.github/workflows/pr_quality.yml index f284c90e7c..0de9fed23c 100644 --- a/.github/workflows/pr_quality.yml +++ b/.github/workflows/pr_quality.yml @@ -98,9 +98,7 @@ jobs: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false - - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # actionlint - with: - tool: actionlint@1.7.12 + - uses: ./.github/actions/install-actionlint - name: Validate GitHub Actions run: actionlint -config-file .github/actionlint.yaml - name: Test CI, packaging, and SDK contracts diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index 01df99100d..dd25b0a807 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -70,7 +70,11 @@ def test_pr_quality_requires_ci_contract_validation(self) -> None: contract = workflow[contract_start:contract_end] summary = workflow[workflow.index(" summary:") :] - self.assertIn("actionlint@1.7.12", contract) + self.assertIn( + "uses: ./.github/actions/install-actionlint", + contract, + ) + self.assertNotIn("tool: actionlint@", contract) self.assertIn( "actionlint -config-file .github/actionlint.yaml", contract, @@ -82,6 +86,24 @@ def test_pr_quality_requires_ci_contract_validation(self) -> None: self.assertIn("ci-contract", summary) self.assertIn("needs.ci-contract.result", summary) + def test_actionlint_installer_verifies_pinned_release_archives( + self, + ) -> None: + action = self.read_action("install-actionlint") + + self.assertIn('ACTIONLINT_VERSION: "1.7.12"', action) + self.assertIn( + "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8", + action, + ) + self.assertIn( + "325e971b6ba9bfa504672e29be93c24981eeb1c07576d730e9f7c8805afff0c6", + action, + ) + self.assertIn("actionlint archive checksum mismatch", action) + self.assertIn("scripts/safe-extract-tar.py", action) + self.assertNotIn("tar -x", action) + def write_fake_product_inputs( self, workspace: Path, From 52f2f0febc3c193625797c35689ef051f37b71ac Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 18:25:59 -0400 Subject: [PATCH 09/23] fix(installer): normalize runtime digest paths --- install.ps1 | 8 ++++++-- scripts/tests/test_install_ps1.py | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/install.ps1 b/install.ps1 index 3fa9a6d902..9d43f5a6e0 100644 --- a/install.ps1 +++ b/install.ps1 @@ -285,10 +285,14 @@ function Get-DeterministicTreeSha256 { $hasher = [System.Security.Cryptography.SHA256]::Create() try { - $root = (Resolve-Path -LiteralPath $Path).ProviderPath.TrimEnd([char]'\') + $pathSeparators = [char[]]@( + [System.IO.Path]::DirectorySeparatorChar, + [System.IO.Path]::AltDirectorySeparatorChar + ) + $root = (Resolve-Path -LiteralPath $Path).ProviderPath.TrimEnd($pathSeparators) $filesByRelativePath = @{} foreach ($file in Get-ChildItem -LiteralPath $Path -Recurse -File) { - $relative = $file.FullName.Substring($root.Length).TrimStart([char]'\') -replace '\\', '/' + $relative = $file.FullName.Substring($root.Length).TrimStart($pathSeparators) -replace '\\', '/' $filesByRelativePath[$relative] = $file.FullName } [string[]]$relativePaths = @($filesByRelativePath.Keys) diff --git a/scripts/tests/test_install_ps1.py b/scripts/tests/test_install_ps1.py index c619d16143..ed9c391e70 100644 --- a/scripts/tests/test_install_ps1.py +++ b/scripts/tests/test_install_ps1.py @@ -64,6 +64,13 @@ def test_script_validates_product_bundle_before_mutation(self) -> None: self.assertIn("Installing supported legacy MeshLLM", contents) self.assertIn("requires product-manifest.json and native-runtimes", contents) + def test_tree_digest_normalizes_platform_path_separators(self) -> None: + contents = SCRIPT.read_text(encoding="utf-8") + + self.assertIn("[System.IO.Path]::DirectorySeparatorChar", contents) + self.assertIn("[System.IO.Path]::AltDirectorySeparatorChar", contents) + self.assertIn("TrimStart($pathSeparators)", contents) + def test_script_stages_replacement_and_removes_stale_host_imports(self) -> None: contents = SCRIPT.read_text(encoding="utf-8") From 953073582843c7234fef19da427a96ce8ae0f2c6 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 18:33:15 -0400 Subject: [PATCH 10/23] fix(ci): align installer contract output --- scripts/tests/test_install_ps1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/test_install_ps1.py b/scripts/tests/test_install_ps1.py index ed9c391e70..37d993e36b 100644 --- a/scripts/tests/test_install_ps1.py +++ b/scripts/tests/test_install_ps1.py @@ -118,7 +118,7 @@ def test_interactive_install_runs_setup_and_warns_for_legacy_flavor(self) -> Non self.assertEqual(result.returncode, 0, self._combined_output(result)) self.assertEqual(self._read_calls(calls), ["--version", "setup"]) - self.assertIn("Installing Windows x64 host binary", result.stdout) + self.assertIn("Installing Windows x64 MeshLLM product bundle", result.stdout) self.assertIn("Ignoring legacy -Flavor 'cuda'", self._combined_output(result)) def test_noninteractive_install_prints_setup_command(self) -> None: From 76987f69074f9fab8d791a9c0d96489bcf662e4f Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 18:33:15 -0400 Subject: [PATCH 11/23] docs(ci): ground runner image migration plan --- ci/DEPOT_MIGRATION.md | 64 ++++++++++++++++++++++++++++--------------- ci/METRICS.md | 32 ++++++++++++++++++++++ ci/ci.md | 28 +++++++++++++++++++ 3 files changed, 102 insertions(+), 22 deletions(-) diff --git a/ci/DEPOT_MIGRATION.md b/ci/DEPOT_MIGRATION.md index 1178031ccc..91abf03f48 100644 --- a/ci/DEPOT_MIGRATION.md +++ b/ci/DEPOT_MIGRATION.md @@ -276,30 +276,50 @@ Relevant Depot documentation: Runner images own stable tools and backend SDKs, not commit-specific products. The next image revision should: -1. publish a `public-rust-host` image with Rust/Cargo/sccache, host libraries, - CMake/Ninja, lld, and only Cargo dependency warming; -2. publish `public-native-{cpu,cuda,rocm,vulkan}` images with the C/C++ SDK and - packaging tools but no Rust registry, Node, pnpm, UI, website, or Python +1. build the UI once in a Node-capable producer and upload it before any + Node-free host role starts; `public-rust-host` consumes those prepared UI + bytes and contains Rust/Cargo/sccache, host libraries, CMake/Ninja, lld, and + only Cargo dependency warming; +2. publish `public-native-cpu` with the CPU C/C++ toolchain and packaging tools, + and separate `public-native-{cuda,rocm,vulkan}` roles with only the matching + GPU SDK. None of these roles owns Rust, Node, pnpm, UI, website, or Python application dependencies; -3. publish a `public-compose` image with only Bash, Python standard library, - tar/coreutils, runtime libraries, and artifact verifiers. Composition jobs - must never pull a multi-gigabyte backend SDK; -4. retain the Actions runner only in `self-hosted-*` overlays; keep it out of - public builder and composition images; -5. make one content-addressed architecture base feed every backend overlay, +3. publish `public-compose` with only Bash, Python standard library, + tar/coreutils, required runtime libraries, and artifact verifiers. + Composition jobs must not carry a backend compiler or SDK; +4. give every role its own verifier that asserts both required capabilities and + forbidden tools/layers; +5. canary a pinned JavaScript action inside every public role as a job container + on both GitHub-hosted and trusted Depot runners. This proves the Actions + Node-external mount contract independently of whether the image ships Node; +6. make one content-addressed architecture base feed every backend overlay, and move source-revision provenance after dependency-warm layers; -6. build each architecture once, verify that exact staged digest, and assemble - manifests/tags without rebuilding; -7. gate updates on compressed size and cold-pull measurements. - -The current pinned CPU image is about 1.53 GB compressed, while the latest -candidate is about 1.92 GB (+25.5%). Do not bump that candidate until it beats -the current image's size/cold-start contract. Existing cold initialization -medians were approximately 80s CPU, 45–49s CUDA, 124–155s ROCm, and 78–81s -Vulkan. Initial acceptance targets are at most 1.0 GB for `public-rust-host`, -500 MB for a CPU native builder, 250 MB and 20s cold-pull p50 for -`public-compose`, at least 1 GB removed from each backend image, and image -publication reduced from 39 minutes to 25 minutes. +7. build each architecture/role once, push it under an immutable staging + digest, run the role verifier and canaries against that exact digest, then + promote only the verified digest into multi-architecture manifests and + human-facing tags. Promotion must not invoke another image build; +8. add `self-hosted-*` Actions-runner/device overlays only after all public + roles pass. Keep the runner agent out of public builder and composition + images; +9. gate updates on retained compressed-size and controlled cold-pull + measurements. + +The latest measured publication evidence is +[runner-images run 30248081255](https://github.com/Mesh-LLM/mesh-llm-runner-images/actions/runs/30248081255). +It completed 55 jobs in 39m 15s. Its slowest initial `Build and verify test +image` step took 14m 25s, then the later publication pass rebuilt the public +ROCm 7.2 AMD64 image in an 18m 03s `Build and push architecture image by +digest` step. This demonstrates duplicate test/publish image construction; it +does not measure image size, cold-pull time, or cache effectiveness. + +No retained audit evidence currently substantiates the previously cited +1.53 GB/1.92 GB compressed sizes or backend cold-initialization medians, so +those values are not migration baselines. The following are provisional design +budgets, not measurements: at most 1.0 GB for `public-rust-host`, 500 MB for a +CPU native builder, 250 MB and 20s cold-pull p50 for `public-compose`, at least +1 GB removed from each backend image, and publication under 25 minutes. Record +per-platform compressed bytes and controlled cold-pull samples before enforcing +or revising any of these gates. GHCR remains canonical. After the split, a trusted Depot canary may compare a Depot Registry pull-through reference and the containerd layer store against diff --git a/ci/METRICS.md b/ci/METRICS.md index 91218d468e..d980d145fb 100644 --- a/ci/METRICS.md +++ b/ci/METRICS.md @@ -160,6 +160,38 @@ runtime. This single capacity-contaminated observation validates artifact reuse but does not replace the multi-run baseline. It also predates the final split of the Linux GPU matrix into independent runtime producers and thin composers. +## Runner-image publication observation + +The latest read-only runner-image observation is +[run 30248081255](https://github.com/Mesh-LLM/mesh-llm-runner-images/actions/runs/30248081255) +at source `890cdc6a1472028a67f7013baa29e29be57e6529`. GitHub's run and job +timestamps provide the following measured evidence: + +| Observation | Measured value | +| --- | ---: | +| Workflow wall time | 39m 15s | +| Completed jobs | 55 | +| Slowest initial `Build and verify test image` step | 14m 25s | +| Later public ROCm 7.2 AMD64 `Build and push architecture image by digest` step | 18m 03s | + +The initial test matrix and later publish matrix both build architecture +images. The second timing is therefore evidence of duplicate image +construction on the publication path. It is not evidence of a cold pull, +compressed image size, or the amount of reusable cache. + +The runner-image migration must measure one explicit lifecycle: + +```text +build once -> stage immutable digest -> verify exact digest -> promote digest +``` + +Record the role, platform, backend, source SHA, staged digest, build duration, +verification duration, promotion duration, compressed bytes, provider, and +runner class. Measure cold pull only in a controlled fresh-worker cohort and +retain the raw observations. Until those records exist, the proposed role-size, +cold-pull, and publication-time thresholds in +[`DEPOT_MIGRATION.md`](DEPOT_MIGRATION.md) are design budgets, not baselines. + | Phase | Change class | Provider / runner | Samples | p50 | p90 | p95 | Notes | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | | Product-v2 PR graph | full CI refactor | hosted mix | 1 | 1h 7m 34s | 1h 7m 34s | 1h 7m 34s | Green; queue-contaminated; composition 10s–70s | diff --git a/ci/ci.md b/ci/ci.md index f4692e006a..25fa58b2bb 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -313,6 +313,34 @@ compatibility `self-hosted-*` manifest preserves the deployed K3s topology by selecting CUDA 12 on AMD64 and CPU on ARM64. New consumers should use an explicit backend image instead of relying on those aliases. +The current image family is a compatibility contract, not the final +role-isolated topology. The planned split has these prerequisites: + +- a Node-capable UI producer uploads prepared UI assets before the Node-free + `public-rust-host` role starts; +- `public-native-cpu` and the + `public-native-{cuda,rocm,vulkan}` roles own only their matching native + toolchain and packaging surface; +- `public-compose` owns artifact extraction, verification, and composition, + without a compiler or backend SDK; +- every role has a role-specific verifier that checks required capabilities + and forbidden dependency overlap; +- a pinned JavaScript action is canaried in every public role on both + GitHub-hosted and trusted Depot runners, proving the Actions Node-external + contract even for Node-free images; +- `self-hosted-*` runner/device overlays are added and verified last. + +Runner-image publication will follow one immutable chain: +`build once -> stage digest -> verify that exact digest -> promote digest`. +Manifest assembly and human-facing tags consume verified digests and must not +rebuild an architecture image. The latest measured compatibility-image +[run 30248081255](https://github.com/Mesh-LLM/mesh-llm-runner-images/actions/runs/30248081255) +took 39m 15s across 55 jobs; its slowest test build step was 14m 25s and a +later second public ROCm 7.2 AMD64 publication build took 18m 03s. That run +demonstrates duplicate construction, but it did not retain authoritative +compressed-size or controlled cold-pull evidence. Role-size and pull-time +thresholds remain proposed rollout gates until measured. + Production workflows and Flux resources must pin the multi-architecture OCI digest, using `ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:`. Timestamp, source-revision, and `*-latest` tags are discovery or evaluation inputs only; From e947f8baf7bf171fdc695c0b4196e33bb06a45d0 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 18:43:35 -0400 Subject: [PATCH 12/23] ci: route trusted ARM lanes through Depot selector --- .github/actionlint.yaml | 5 ++- .github/actions/select-ci-runners/action.yml | 24 +++++++++++ .github/workflows/ci.yml | 3 -- .github/workflows/depot-canary.yml | 14 +++++++ .github/workflows/release.yml | 21 ++++++---- ci/DEPOT_MIGRATION.md | 3 +- ci/ci.md | 6 ++- scripts/tests/test_ci_artifact_actions.py | 17 ++++++++ scripts/tests/test_depot_canary_workflow.py | 4 ++ .../tests/test_release_workflow_artifacts.py | 40 +++++++++++++++++++ 10 files changed, 123 insertions(+), 14 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 04f45ef4d7..5acc1db7cf 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -3,7 +3,6 @@ self-hosted-runner: - self-hosted - Linux - X64 - - ARM64 - amd64 - gpu-nvidia - mesh-llm-amd64 @@ -12,3 +11,7 @@ self-hosted-runner: - depot-ubuntu-24.04-4 - depot-ubuntu-24.04-8 - depot-ubuntu-24.04-16 + - depot-ubuntu-24.04-arm + - depot-ubuntu-24.04-arm-4 + - depot-ubuntu-24.04-arm-8 + - depot-ubuntu-24.04-arm-16 diff --git a/.github/actions/select-ci-runners/action.yml b/.github/actions/select-ci-runners/action.yml index 57f9ba98ab..b8633b56fc 100644 --- a/.github/actions/select-ci-runners/action.yml +++ b/.github/actions/select-ci-runners/action.yml @@ -36,6 +36,18 @@ outputs: runner_16: description: Sixteen-vCPU Linux runner label for measured high-parallelism builds. value: ${{ steps.select.outputs.runner_16 }} + runner_arm: + description: Default two-vCPU Linux ARM64 runner label. + value: ${{ steps.select.outputs.runner_arm }} + runner_arm_4: + description: Four-vCPU Linux ARM64 runner label. + value: ${{ steps.select.outputs.runner_arm_4 }} + runner_arm_8: + description: Eight-vCPU Linux ARM64 runner label. + value: ${{ steps.select.outputs.runner_arm_8 }} + runner_arm_16: + description: Sixteen-vCPU Linux ARM64 runner label for measured high-parallelism builds. + value: ${{ steps.select.outputs.runner_arm_16 }} runs: using: composite @@ -96,11 +108,19 @@ runs: runner_4=depot-ubuntu-24.04-4 runner_8=depot-ubuntu-24.04-8 runner_16=depot-ubuntu-24.04-16 + runner_arm=depot-ubuntu-24.04-arm + runner_arm_4=depot-ubuntu-24.04-arm-4 + runner_arm_8=depot-ubuntu-24.04-arm-8 + runner_arm_16=depot-ubuntu-24.04-arm-16 else runner=ubuntu-24.04 runner_4=ubuntu-24.04 runner_8=ubuntu-24.04 runner_16=ubuntu-24.04 + runner_arm=ubuntu-24.04-arm + runner_arm_4=ubuntu-24.04-arm + runner_arm_8=ubuntu-24.04-arm + runner_arm_16=ubuntu-24.04-arm fi { @@ -110,4 +130,8 @@ runs: echo "runner_4=$runner_4" echo "runner_8=$runner_8" echo "runner_16=$runner_16" + echo "runner_arm=$runner_arm" + echo "runner_arm_4=$runner_arm_4" + echo "runner_arm_8=$runner_arm_8" + echo "runner_arm_16=$runner_arm_16" } >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b61d16f297..7fac77c9b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,5 @@ name: CI -# Set USE_SELF_HOSTED=true to route CUDA CI to the dedicated NVIDIA runner; -# unset or false uses GitHub-hosted runners. - on: workflow_dispatch: inputs: diff --git a/.github/workflows/depot-canary.yml b/.github/workflows/depot-canary.yml index f7e75772dc..9eef1e82d3 100644 --- a/.github/workflows/depot-canary.yml +++ b/.github/workflows/depot-canary.yml @@ -20,14 +20,28 @@ jobs: - depot-ubuntu-24.04-4 - depot-ubuntu-24.04-8 - depot-ubuntu-24.04-16 + - depot-ubuntu-24.04-arm + - depot-ubuntu-24.04-arm-8 runs-on: ${{ matrix.runner }} timeout-minutes: 10 steps: - name: Verify ephemeral runner resources shell: bash + env: + CANARY_RUNNER_LABEL: ${{ matrix.runner }} run: | set -euo pipefail + expected_arch=x86_64 + if [[ "$CANARY_RUNNER_LABEL" == *-arm* ]]; then + expected_arch=aarch64 + fi + actual_arch="$(uname -m)" + if [[ "$actual_arch" != "$expected_arch" ]]; then + echo "runner architecture mismatch: expected $expected_arch, got $actual_arch" >&2 + exit 1 + fi + { echo "### ${{ matrix.runner }}" echo diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7358e3f6da..56ceb3b385 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ name: Release -# Set USE_SELF_HOSTED=true to route x86_64 CUDA release jobs and ARM64 smoke -# validation to self-hosted runners; unset or false uses GitHub-hosted runners. +# Set USE_SELF_HOSTED=true to route x86_64 CUDA release jobs to the dedicated +# GPU runner; unset or false uses GitHub-hosted runners. on: push: @@ -23,7 +23,7 @@ on: default: false type: boolean force_hosted_runners: - description: Route self-hosted release lanes (x86_64 CUDA, ARM64 smoke) to GitHub-hosted runners for this run only + description: Route the self-hosted x86_64 CUDA release lane to GitHub-hosted runners for this run only required: true default: false type: boolean @@ -66,6 +66,10 @@ jobs: runner_4: ${{ steps.runners.outputs.runner_4 }} runner_8: ${{ steps.runners.outputs.runner_8 }} runner_16: ${{ steps.runners.outputs.runner_16 }} + runner_arm: ${{ steps.runners.outputs.runner_arm }} + runner_arm_4: ${{ steps.runners.outputs.runner_arm_4 }} + runner_arm_8: ${{ steps.runners.outputs.runner_arm_8 }} + runner_arm_16: ${{ steps.runners.outputs.runner_arm_16 }} steps: - name: Require the trusted release ref shell: bash @@ -314,13 +318,14 @@ jobs: mesh_binary_target: target/release/mesh-llm cache_key_prefix: release- release_tag: ${{ needs.metadata.outputs.tag }} + runs_on: ${{ toJson(needs.metadata.outputs.runner_8) }} secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} build_native_sdk_runtime: name: Build native SDK runtime ${{ matrix.name }} needs: metadata - runs-on: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && needs.metadata.outputs.runner_8 || matrix.os }} + runs-on: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && needs.metadata.outputs.runner_8 || matrix.target == 'aarch64-unknown-linux-gnu' && needs.metadata.outputs.runner_arm_8 || matrix.os }} strategy: fail-fast: false matrix: @@ -423,7 +428,7 @@ jobs: build_native_runtime: name: Build native runtime ${{ matrix.name }} needs: metadata - runs-on: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && needs.metadata.outputs.runner_8 || matrix.os }} + runs-on: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && needs.metadata.outputs.runner_8 || matrix.target == 'aarch64-unknown-linux-gnu' && needs.metadata.outputs.runner_arm_8 || matrix.os }} strategy: fail-fast: false matrix: @@ -800,6 +805,8 @@ jobs: build_linux_arm64: name: Build immutable host Linux ARM64 needs: metadata + # Host attestation consumes the signing key. Keep this producer on GitHub + # until compilation and signing are split into separate jobs. runs-on: ubuntu-24.04-arm env: LLAMA_STAGE_BACKEND: cpu @@ -854,7 +861,7 @@ jobs: compose_linux_arm64_cpu: name: Compose Linux ARM64 CPU product needs: [metadata, build_linux_arm64, build_native_runtime] - runs-on: ubuntu-24.04-arm + runs-on: ${{ needs.metadata.outputs.runner_arm_4 }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -910,7 +917,7 @@ jobs: name: Smoke Linux ARM64 artifact needs: [metadata, compose_linux_arm64_cpu] if: ${{ needs.compose_linux_arm64_cpu.result == 'success' }} - runs-on: ${{ fromJson(vars.USE_SELF_HOSTED == 'true' && needs.metadata.outputs.force_hosted_runners != 'true' && '["self-hosted","Linux","ARM64"]' || '["ubuntu-24.04-arm"]') }} + runs-on: ${{ needs.metadata.outputs.runner_arm }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: diff --git a/ci/DEPOT_MIGRATION.md b/ci/DEPOT_MIGRATION.md index 91abf03f48..16834152d3 100644 --- a/ci/DEPOT_MIGRATION.md +++ b/ci/DEPOT_MIGRATION.md @@ -187,7 +187,8 @@ Activation prerequisites: 3. Only after both restrictions are saved, enable public repositories for the `Default` group. Depot-managed ephemeral runners register in that group. 4. Dispatch `depot-canary.yml` from `refs/heads/main` twice. Verify all four - runner sizes and a cold-to-warm cache hit without printing credentials. + Intel runner sizes, both ARM runner sizes, their reported architectures, + and a cold-to-warm cache hit without printing credentials. 5. Dispatch the canary from a feature ref, prove that it cannot acquire a Depot runner, and cancel that exact queued run. 6. Add exact default-branch workflow refs only as their phase starts. Reusable diff --git a/ci/ci.md b/ci/ci.md index 25fa58b2bb..262cbe91dc 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -430,8 +430,10 @@ because PR workflow and local-action files are themselves PR-controlled. Trusted main/release jobs use `DEPOT_RUNNERS_ENABLED`, and a trusted main-ref manual dispatch can use `use_depot=true` for a bounded canary. The selector requires `refs/heads/main`; tag pushes and feature refs fall back to hosted -runners. Depot-managed runners register in the organization `Default` runner -group. Before enabling public access, restrict that group to +runners. It emits both Intel and ARM64 labels from the same trust decision, so +release CPU producers, composers, and smokes do not bypass the policy. +Depot-managed runners register in the organization `Default` runner group. +Before enabling public access, restrict that group to `Mesh-LLM/mesh-llm` and exact default-branch workflow refs, beginning with `depot-canary.yml@refs/heads/main`. The existing `mesh-llm` runner group owns the dedicated GPU scale sets and is not the Depot group. diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index dd25b0a807..04c47e5c1f 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -890,6 +890,7 @@ def test_runner_selection_never_routes_pull_requests_to_depot(self) -> None: self.assertIn("depot_enabled=false", default_case) self.assertNotIn("depot_enabled=true", default_case) self.assertIn("depot-ubuntu-24.04-16", action) + self.assertIn("depot-ubuntu-24.04-arm-16", action) cases = ( ("pull_request", "refs/pull/12/merge", "true", "true", "false", "ubuntu-24.04"), @@ -913,6 +914,22 @@ def test_runner_selection_never_routes_pull_requests_to_depot(self) -> None: self.assertEqual(outputs["depot_enabled"], enabled) self.assertEqual(outputs["allow_depot_remote_cache"], enabled) self.assertEqual(outputs["runner"], runner) + expected_arm = ( + "depot-ubuntu-24.04-arm" + if enabled == "true" + else "ubuntu-24.04-arm" + ) + self.assertEqual(outputs["runner_arm"], expected_arm) + for size in ("4", "8", "16"): + expected_sized_arm = ( + f"depot-ubuntu-24.04-arm-{size}" + if enabled == "true" + else "ubuntu-24.04-arm" + ) + self.assertEqual( + outputs[f"runner_arm_{size}"], + expected_sized_arm, + ) def test_pr_caches_rely_on_github_ref_scoping_while_depot_is_blocked( self, diff --git a/scripts/tests/test_depot_canary_workflow.py b/scripts/tests/test_depot_canary_workflow.py index 5c9bf9f8fb..6f74287c00 100644 --- a/scripts/tests/test_depot_canary_workflow.py +++ b/scripts/tests/test_depot_canary_workflow.py @@ -23,9 +23,13 @@ def test_canary_covers_measured_depot_sizes(self) -> None: "depot-ubuntu-24.04-4", "depot-ubuntu-24.04-8", "depot-ubuntu-24.04-16", + "depot-ubuntu-24.04-arm", + "depot-ubuntu-24.04-arm-8", ): with self.subTest(runner=runner): self.assertIn(f"- {runner}", self.workflow) + self.assertIn("expected_arch=aarch64", self.workflow) + self.assertIn('actual_arch="$(uname -m)"', self.workflow) def test_canary_uses_a_pinned_cache_action_without_printing_tokens( self, diff --git a/scripts/tests/test_release_workflow_artifacts.py b/scripts/tests/test_release_workflow_artifacts.py index c2bcf1c310..62738112db 100644 --- a/scripts/tests/test_release_workflow_artifacts.py +++ b/scripts/tests/test_release_workflow_artifacts.py @@ -85,6 +85,21 @@ def test_release_routes_only_initial_non_secret_linux_lanes( "build_native_runtime", "build_native_runtime_linux_aarch64_cuda", ) + arm_host = job_block( + workflow, + "build_linux_arm64", + "compose_linux_arm64_cpu", + ) + arm_compose = job_block( + workflow, + "compose_linux_arm64_cpu", + "smoke_linux_arm64_artifact", + ) + arm_smoke = job_block( + workflow, + "smoke_linux_arm64_artifact", + "compose_linux_aarch64_cuda", + ) rocm = job_block( workflow, "build_native_runtime_linux_x86_64_rocm", @@ -112,6 +127,10 @@ def test_release_routes_only_initial_non_secret_linux_lanes( "needs.metadata.outputs.runner_8", producer, ) + self.assertIn( + "needs.metadata.outputs.runner_arm_8", + producer, + ) for producer in (rocm, vulkan): self.assertIn( "runs-on: ${{ needs.metadata.outputs.runner_16 }}", @@ -124,6 +143,18 @@ def test_release_routes_only_initial_non_secret_linux_lanes( ) self.assertIn("runs-on: ubuntu-24.04", publish) self.assertNotIn("needs.metadata.outputs.runner", publish) + self.assertIn("RELEASE_ATTESTATION_SIGNING_KEY", arm_host) + self.assertIn("runs-on: ubuntu-24.04-arm", arm_host) + self.assertNotIn("needs.metadata.outputs.runner_arm", arm_host) + self.assertIn( + "runs-on: ${{ needs.metadata.outputs.runner_arm_4 }}", + arm_compose, + ) + self.assertIn( + "runs-on: ${{ needs.metadata.outputs.runner_arm }}", + arm_smoke, + ) + self.assertNotIn("USE_SELF_HOSTED", arm_smoke) def test_inference_smoke_consumes_composed_product(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") @@ -142,6 +173,15 @@ def test_inference_smoke_consumes_composed_product(self) -> None: "path: ${{ steps.compose.outputs.archive_path }}", workflow, ) + inference = job_block( + workflow, + "inference_smoke_tests", + "build_native_sdk_runtime", + ) + self.assertIn( + "runs_on: ${{ toJson(needs.metadata.outputs.runner_8) }}", + inference, + ) def test_release_permissions_are_least_privilege(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") From 5cf40376d5e337769f081104d17049838798045d Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 18:48:11 -0400 Subject: [PATCH 13/23] ci: enable remote sccache for fast lanes --- .github/workflows/ci.yml | 8 -------- .github/workflows/pr_builds.yml | 8 -------- .github/workflows/pr_quality.yml | 1 - scripts/tests/test_ci_artifact_actions.py | 9 +++++++++ 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fac77c9b9..5eacbca459 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,7 +177,6 @@ jobs: MESH_LLM_SKIP_UI: "1" MESH_LLM_REQUIRE_SCCACHE: "1" RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -283,7 +282,6 @@ jobs: LLAMA_STAGE_BACKEND: cpu LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-dynamic-cpu MESH_LLM_REQUIRE_SCCACHE: "1" - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -388,7 +386,6 @@ jobs: LLAMA_STAGE_BACKEND: cpu LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static MESH_LLM_REQUIRE_SCCACHE: "1" - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -451,7 +448,6 @@ jobs: MESH_LLM_SKIP_UI: "1" RUSTFLAGS: "-C link-arg=-fuse-ld=lld" RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -529,7 +525,6 @@ jobs: MESH_LLM_SKIP_UI: "1" MESH_LLM_REQUIRE_SCCACHE: "1" RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -1008,7 +1003,6 @@ jobs: GGML_CUDA_NO_VMM: "1" LLAMA_STAGE_SKIP_NCCL: "1" MESH_LLM_REQUIRE_SCCACHE: "1" - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -1104,7 +1098,6 @@ jobs: LLAMA_STAGE_AMDGPU_TARGETS: gfx1100 CMAKE_PREFIX_PATH: /opt/rocm MESH_LLM_REQUIRE_SCCACHE: "1" - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -1198,7 +1191,6 @@ jobs: LLAMA_STAGE_BACKEND: vulkan LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-dynamic-vulkan MESH_LLM_REQUIRE_SCCACHE: "1" - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: diff --git a/.github/workflows/pr_builds.yml b/.github/workflows/pr_builds.yml index 9c0e1b3667..92652b0f5c 100644 --- a/.github/workflows/pr_builds.yml +++ b/.github/workflows/pr_builds.yml @@ -165,7 +165,6 @@ jobs: MESH_LLM_SKIP_UI: "1" MESH_LLM_REQUIRE_SCCACHE: "1" RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 @@ -266,7 +265,6 @@ jobs: LLAMA_STAGE_BACKEND: cpu LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-dynamic-cpu MESH_LLM_REQUIRE_SCCACHE: "1" - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -365,7 +363,6 @@ jobs: MESH_CUDA_VERSION: "12.9.2" MESH_LLM_CUDA_TOOLKIT_MAJOR: "12" MESH_LLM_REQUIRE_SCCACHE: "1" - SCCACHE_GHA_ENABLED: "false" GGML_CUDA_NO_VMM: "1" LLAMA_STAGE_SKIP_NCCL: "1" steps: @@ -461,7 +458,6 @@ jobs: LLAMA_STAGE_AMDGPU_TARGETS: gfx1100 CMAKE_PREFIX_PATH: /opt/rocm MESH_LLM_REQUIRE_SCCACHE: "1" - SCCACHE_GHA_ENABLED: "false" GGML_CUDA_NO_VMM: "1" LLAMA_STAGE_SKIP_NCCL: "1" steps: @@ -555,7 +551,6 @@ jobs: LLAMA_STAGE_BACKEND: vulkan LLAMA_STAGE_BUILD_DIR: .deps/llama-build/build-stage-abi-dynamic-vulkan MESH_LLM_REQUIRE_SCCACHE: "1" - SCCACHE_GHA_ENABLED: "false" GGML_CUDA_NO_VMM: "1" LLAMA_STAGE_SKIP_NCCL: "1" steps: @@ -660,7 +655,6 @@ jobs: LLAMA_STAGE_BACKEND: cpu LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static MESH_LLM_REQUIRE_SCCACHE: "1" - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -723,7 +717,6 @@ jobs: MESH_LLM_SKIP_UI: "1" RUSTFLAGS: "-C link-arg=-fuse-ld=lld" RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -798,7 +791,6 @@ jobs: MESH_LLM_SKIP_UI: "1" MESH_LLM_REQUIRE_SCCACHE: "1" RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 diff --git a/.github/workflows/pr_quality.yml b/.github/workflows/pr_quality.yml index 0de9fed23c..5a934e594a 100644 --- a/.github/workflows/pr_quality.yml +++ b/.github/workflows/pr_quality.yml @@ -136,7 +136,6 @@ jobs: env: RUSTFLAGS: "-C link-arg=-fuse-ld=lld" RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "false" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - name: Verify prebuilt CI environment diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index 04c47e5c1f..3fd78f8302 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -942,11 +942,20 @@ def test_pr_caches_rely_on_github_ref_scoping_while_depot_is_blocked( self.assertIn("CACHE_NAMESPACE: mesh-llm", workflow) self.assertNotIn("CACHE_NAMESPACE: mesh-llm-pr", workflow) self.assertNotIn("'mesh-llm-pr'", workflow) + self.assertNotIn( + 'SCCACHE_GHA_ENABLED: "false"', + workflow, + ) self.assertIn( "save-if: ${{ github.ref == 'refs/heads/main' }}", workflow, ) + main = ( + ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + self.assertNotIn('SCCACHE_GHA_ENABLED: "false"', main) + quality = ( ROOT / ".github" / "workflows" / "pr_quality.yml" ).read_text(encoding="utf-8") From 130b07b56bf7644eefe06c13c764aa6c4817e455 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 19:20:00 -0400 Subject: [PATCH 14/23] fix(ci): await remote sccache writes --- .../actions/configure-sccache-gha/action.yml | 14 ++++++-- scripts/tests/test_sccache_evidence.py | 34 ++++++++++++++----- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/.github/actions/configure-sccache-gha/action.yml b/.github/actions/configure-sccache-gha/action.yml index 2e278886c8..20350b5eeb 100644 --- a/.github/actions/configure-sccache-gha/action.yml +++ b/.github/actions/configure-sccache-gha/action.yml @@ -86,7 +86,13 @@ runs: core.exportVariable('SCCACHE_GHA_ENABLED', 'false'); core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk,webdav'); core.exportVariable('SCCACHE_IGNORE_SERVER_IO_ERROR', '1'); - core.exportVariable('SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'ignore'); + // sccache's ignore/l0 policies dispatch slower-level writes in the + // background. Ephemeral CI jobs can exit before those uploads + // finish, leaving a permanently partial remote cache. The all + // policy waits for every configured level before the compiler + // request completes; cache failures remain visible in the + // captured counters. + core.exportVariable('SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'all'); await exec.exec('sccache', ['--stop-server'], { ignoreReturnCode: true, }); @@ -215,7 +221,11 @@ runs: core.exportVariable('ACTIONS_RUNTIME_TOKEN', runtimeToken); core.exportVariable('SCCACHE_IGNORE_SERVER_IO_ERROR', '1'); core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk,gha'); - core.exportVariable('SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'ignore'); + // Wait for the GitHub Actions tier to finish each write. With the + // asynchronous policies, the runner can terminate while L1 uploads + // are still pending and the next identical run observes avoidable + // misses. + core.exportVariable('SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'all'); await exec.exec('sccache', ['--stop-server'], { ignoreReturnCode: true, diff --git a/scripts/tests/test_sccache_evidence.py b/scripts/tests/test_sccache_evidence.py index 908f5f324d..417f089a7c 100644 --- a/scripts/tests/test_sccache_evidence.py +++ b/scripts/tests/test_sccache_evidence.py @@ -62,15 +62,15 @@ def run_capture( root = Path(temporary.name) fake_sccache = root / "sccache" fake_sccache.write_text( - "#!/usr/bin/env python3\n" - "import os\n" - "import sys\n" - "if sys.argv[1:] == ['--show-stats']:\n" - " print('Compile requests 12')\n" - "elif sys.argv[1:] == ['--show-stats', '--stats-format', 'json']:\n" - " print(os.environ['FAKE_SCCACHE_JSON'])\n" - "else:\n" - " raise SystemExit(2)\n", + "#!/bin/sh\n" + "if [ \"$#\" -eq 1 ] && [ \"$1\" = \"--show-stats\" ]; then\n" + " printf '%s\\n' 'Compile requests 12'\n" + "elif [ \"$#\" -eq 3 ] && [ \"$1\" = \"--show-stats\" ] " + "&& [ \"$2\" = \"--stats-format\" ] && [ \"$3\" = \"json\" ]; then\n" + " printf '%s\\n' \"$FAKE_SCCACHE_JSON\"\n" + "else\n" + " exit 2\n" + "fi\n", encoding="utf-8", ) fake_sccache.chmod( @@ -163,6 +163,22 @@ def test_configure_action_resets_each_successful_server_route(self) -> None: self.assertIn("['--zero-stats']", configure) self.assertEqual(configure.count("await resetStatistics("), 6) + def test_remote_multilevel_writes_finish_before_ephemeral_job_exit(self) -> None: + configure = CONFIGURE_ACTION.read_text(encoding="utf-8") + + self.assertEqual( + configure.count( + "core.exportVariable(" + "'SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'all'" + ")", + ), + 2, + ) + self.assertNotIn( + "'SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'ignore'", + configure, + ) + def test_instrumented_workflows_use_unique_evidence_artifacts(self) -> None: expected_names = { "pr-builds": ( From 099d54b086e79f210674b3f4a8b96d1ec2f1646e Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 19:25:29 -0400 Subject: [PATCH 15/23] fix(ci): align sccache policy contract --- tools/xtask/src/workflow_checks.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/xtask/src/workflow_checks.rs b/tools/xtask/src/workflow_checks.rs index c4995a8ae1..cdc014b16a 100644 --- a/tools/xtask/src/workflow_checks.rs +++ b/tools/xtask/src/workflow_checks.rs @@ -525,8 +525,8 @@ fn check_release_container_contracts( "sccache GHA action disk-only fallback", ), ( - "core.exportVariable('SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'ignore')", - "sccache GHA action write fallback", + "core.exportVariable('SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'all')", + "sccache GHA action synchronous remote writes", ), ("['--start-server']", "sccache GHA action server start"), ("['--stop-server']", "sccache GHA action server stop"), @@ -851,7 +851,7 @@ process.env.SCCACHE_WEBDAV_ENDPOINT process.env.DEPOT_CACHE_TOKEN core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk,webdav') core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk') -core.exportVariable('SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'ignore') +core.exportVariable('SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY', 'all') ['--start-server'] ['--stop-server'] "#; From af944315b144d14b860bf8bc47eb47862c78ad18 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 21:56:03 -0400 Subject: [PATCH 16/23] refactor(ci): reuse typed SDK and static ABI inputs --- .agents/skills/manage-ci/SKILL.md | 57 +- .../manage-ci/references/current-inventory.md | 104 ++- .../actions/capture-sccache-stats/capture.py | 57 +- .github/actions/compute-changes/action.yml | 4 +- .../prepare-native-sdk-input/action.yml | 221 ++++++ .../prepare-static-abi-input/action.yml | 273 ++++++++ .github/workflows/ci.yml | 137 ++-- .github/workflows/docker-precheck.yml | 2 + .github/workflows/hf-download-smoke.yml | 13 +- .github/workflows/native-sdk-artifact.yml | 332 +++++++++ .github/workflows/nightly-stability-run.yml | 10 +- .github/workflows/nightly-stability.yml | 5 - .github/workflows/pr_builds.yml | 366 ++++++---- .github/workflows/pr_quality.yml | 9 + .github/workflows/pr_website.yml | 3 + .github/workflows/release.yml | 205 ++---- .github/workflows/scripted-binary-smoke.yml | 12 +- .github/workflows/sdk-smoke.yml | 192 +++++- .github/workflows/smoke.yml | 12 +- .github/workflows/static-abi-artifact.yml | 187 +++++ .github/workflows/swift-sdk-artifact.yml | 227 ++++++ RELEASE.md | 42 +- ci/DEPOT_MIGRATION.md | 185 +++-- ci/ci.md | 140 +++- docs/AGENTS.md | 5 +- docs/SDK.md | 39 +- docs/design/TESTING.md | 7 +- scripts/build-llama.sh | 89 ++- scripts/ci-kotlin-sdk-smoke.sh | 57 +- scripts/ci-swift-sdk-smoke.sh | 43 +- scripts/package-native-sdk.sh | 31 +- scripts/plan-pr-build-jobs.py | 246 +++++++ scripts/restore-native-sdk-input.sh | 101 +++ scripts/restore-static-abi-input.sh | 156 +++++ scripts/safe-extract-zip.py | 161 +++++ scripts/tests/test_ci_artifact_actions.py | 652 ++++++++++++++++++ scripts/tests/test_ci_workflow_artifacts.py | 97 ++- .../tests/test_native_artifact_verifiers.py | 151 +++- scripts/tests/test_plan_pr_build_jobs.py | 225 ++++++ scripts/tests/test_pr_builds_summary.py | 113 +++ scripts/tests/test_pr_workflow_artifacts.py | 175 +++-- .../tests/test_release_workflow_artifacts.py | 124 +++- .../test_reusable_workflow_runner_trust.py | 143 ++++ scripts/tests/test_safe_extract_zip.py | 161 +++++ scripts/tests/test_sccache_evidence.py | 105 ++- scripts/tests/test_static_abi_artifacts.py | 338 +++++++++ .../tests/test_verify_swift_xcframework.py | 205 ++++++ scripts/verify-static-abi-build-stamp.py | 101 +++ scripts/verify-swift-release-artifact.sh | 78 +-- scripts/verify-swift-xcframework.py | 223 ++++++ .../scripts/build-host-macos-xcframework.sh | 2 +- tools/xtask/src/workflow_checks.rs | 252 ++++++- 52 files changed, 6142 insertions(+), 733 deletions(-) create mode 100644 .github/actions/prepare-native-sdk-input/action.yml create mode 100644 .github/actions/prepare-static-abi-input/action.yml create mode 100644 .github/workflows/native-sdk-artifact.yml create mode 100644 .github/workflows/static-abi-artifact.yml create mode 100644 .github/workflows/swift-sdk-artifact.yml create mode 100644 scripts/plan-pr-build-jobs.py create mode 100755 scripts/restore-native-sdk-input.sh create mode 100755 scripts/restore-static-abi-input.sh create mode 100755 scripts/safe-extract-zip.py create mode 100644 scripts/tests/test_plan_pr_build_jobs.py create mode 100644 scripts/tests/test_pr_builds_summary.py create mode 100644 scripts/tests/test_reusable_workflow_runner_trust.py create mode 100644 scripts/tests/test_safe_extract_zip.py create mode 100644 scripts/tests/test_static_abi_artifacts.py create mode 100644 scripts/tests/test_verify_swift_xcframework.py create mode 100644 scripts/verify-static-abi-build-stamp.py create mode 100755 scripts/verify-swift-xcframework.py diff --git a/.agents/skills/manage-ci/SKILL.md b/.agents/skills/manage-ci/SKILL.md index 4bbbed0faf..fdfc634357 100644 --- a/.agents/skills/manage-ci/SKILL.md +++ b/.agents/skills/manage-ci/SKILL.md @@ -106,8 +106,17 @@ update the skill resources in the same change. expectations together. Do not add new crates to `plan-test-batches.sh`; its metadata-derived membership and default weight handle them automatically. - If a consumer downloads an artifact, its producer must be reachable in the - same workflow graph under every matching condition. Use `needs` and explicit - result checks; do not rely on job ordering by file position. + same workflow graph under every matching condition. Use `needs` with normal + dependency-success semantics, or an explicit result check when status-aware + continuation is intentional; do not rely on job ordering by file position. +- Give each PR entry workflow one stable, non-matrix summary job suitable for + branch protection. Conditional top-level jobs and the summary must consume + the same checked-in required-job plan so route conditions cannot drift. The + summary must directly need every other top-level job, use + `if: ${{ !cancelled() }}` rather than `always()`, require unconditional + routing/planning jobs to succeed, and permit `skipped` only when that job is + absent from the plan. Reject required skips, failure, cancellation, unknown + results, duplicate plan entries, and required IDs outside the needs graph. - Set `strategy.fail-fast: false` when every platform/backend result is useful. Use fail-fast only when later matrix results would be redundant or unsafe. @@ -175,6 +184,21 @@ update the skill resources in the same change. fallback for tags and every other ref. Current `pull_request` workflows must always select GitHub-hosted runners; `DEPOT_PR_RUNNERS_ENABLED` is ignored. +- A default-branch-pinned reusable workflow that directly allocates Depot must + not accept a caller-supplied runner label or Depot-cache permission. Derive + both inside the protected workflow from the exact repository, event, main + ref, `DEPOT_RUNNERS_ENABLED` gate or event-owned main-dispatch canary flag, + target architecture, and a validated bounded runner-size input. Pull + requests, tags, feature refs, external repositories, and unsupported targets + must never resolve to a Depot label; the cache permission must be the output + of the same decision. +- Reusable smoke workflows that receive `HF_TOKEN` must remain fixed to + GitHub-hosted runners during the Depot rollout. They must not accept raw + `runs_on` JSON or any other caller-provided label that can resolve to Depot or + a dedicated runner group. Multi-platform smoke/producer APIs may expose only + bounded GitHub-hosted labels and must fail closed before running checked-out + source. Pull-request callers must not pass `HF_TOKEN`; use public fixtures and + merge-ref-scoped caches for untrusted PR validation. - Treat a checked-out repository-local selector as defense in depth, never as the PR runner trust boundary: pull requests can modify both their workflow and local action code. Before any PR uses Depot, automatic cache authority must be @@ -311,6 +335,35 @@ update the skill resources in the same change. model/artifact restore blocks in consumers. SDK smokes consume the runtime adjacent to their staged producer binary and must fail rather than silently compiling a replacement in CI. +- Build Swift XCFramework inputs through the typed + `swift-sdk-artifact.yml` reusable producer. Pull-request validation uses its + `host-only` mode, while main and release use `full`; Swift smoke consumers + download and verify both the immutable XCFramework and generated + `mesh_ffi.swift` artifacts and must not invoke Cargo, llama.cpp builds, + native-SDK packaging, or either XCFramework build script. Producer and smoke + are fixed to `macos-15`; the native cache includes an explicit macOS/Xcode + epoch, Rust uses `RUSTC_WRAPPER=sccache`, and the producer retains + mode/run-attempt-unique sccache evidence. Main and tag producers must fail on + tracked-binding drift, while a dispatched release must copy the producer + binding into its tag commit. +- Build native SDK runtime inputs through the typed + `native-sdk-artifact.yml` reusable producer. Pull-request, main, and release + callers select an explicit target, backend, and Cargo profile; Kotlin smoke + downloads and verifies that immutable producer artifact and must not invoke + Cargo, llama.cpp preparation/builds, or native-SDK packaging. Release callers + use the same producer with release-asset staging enabled so archive, checksum, + and native runtime crate names remain identical to the published contract. +- Build Linux static llama ABI inputs through the typed + `static-abi-artifact.yml` reusable producer. Its artifact must carry an exact + target/backend manifest, pinned build-image/toolchain epoch, build-stamp + checksum, archive checksum, the full llama/common/mtmd/ggml static link + closure, and only the canonical minimal `build-stage-abi-static` link tree. + Cache and artifact payloads must contain the same path-normalized archive, + not CMake's producer-local build graph. Crate tests and native SDK producers + restore it with `restore-static-abi-input.sh`; + never extract it with raw `tar`, relabel it across architectures, or rebuild + the same CPU ABI in a downstream consumer. Native SDK reuse must call the + verification-only prebuilt path with build.rs auto-build disabled. - Never put credentials, local absolute paths, private endpoints, or secret material into cache/artifact content or workflow summaries. diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 2441822539..575ca522c3 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -9,7 +9,7 @@ the commands at the end before operational changes. | Workflow | Trigger | Ownership | | --- | --- | --- | | `pr_quality.yml` | PR, main push, dispatch | Formatting, affected-crate Clippy, UI quality, CLI/docs synchronization, quality summary | -| `pr_builds.yml` | PR, dispatch | Cross-platform build/test matrices, native backends, artifact producers, integration/smoke consumers | +| `pr_builds.yml` | PR, dispatch | Cross-platform build/test matrices, native backends, artifact producers, integration/smoke consumers, stable aggregate summary | | `pr_website.yml` | PR, dispatch | Public website build canary and summary | | `pr_cleanup.yml` | PR close via `pull_request_target`, dispatch | Positively matched PR cache/artifact cleanup only; never executes PR code | | `pr_auto_assign.yml` | PR lifecycle via `pull_request_target` | PR metadata assignment only; never executes PR code | @@ -18,7 +18,10 @@ the commands at the end before operational changes. | `docker-precheck.yml` | Reusable call | Shared Docker validation precheck | | `smoke.yml` | Reusable call | Artifact-based inference/OpenAI/split smoke | | `scripted-binary-smoke.yml` | Reusable call | Artifact-based scripted/two-node smoke | -| `sdk-smoke.yml` | Reusable call | Native, Kotlin, and Swift SDK smoke | +| `sdk-smoke.yml` | Reusable call | Artifact-based Rust, Kotlin, and Swift SDK smoke | +| `native-sdk-artifact.yml` | Reusable call | Typed target/backend/profile native SDK producer with protected runner/cache policy | +| `static-abi-artifact.yml` | Reusable call | Typed target/backend static llama ABI producer with protected runner/cache policy | +| `swift-sdk-artifact.yml` | Reusable call | Typed host-only/full Swift XCFramework producer | | `hf-download-smoke.yml` | Reusable call | Hugging Face download smoke | | `nightly-stability.yml` | Schedule, dispatch | Nightly operator entry point | | `nightly-stability-run.yml` | Reusable call | Stability probes and evidence | @@ -60,14 +63,21 @@ CPU or Metal runtimes, then upload complete product-v2 trees from composition-only jobs. Linux CUDA, ROCm, and Vulkan each use an independent runtime producer plus a thin composer that downloads the same immutable Linux host; no backend waits on a matrix-wide fan-in. SDK consumers reuse the -producer's adjacent runtime and fail if CI would silently rebuild it. Windows -likewise builds one immutable -release-profile host, independent CPU/CUDA/ROCm/Vulkan runtime inputs, and -composition-only products. Broad main Rust changes exercise the Windows CPU -product; Windows GPU products remain limited to GPU/backend inputs or manual -dispatch. Every composed backend product requires `runtime list` plus -no-driver client readiness; hosted GPU rows neither inject a driver stub nor -skip startup because no device is present. +producer's adjacent runtime and fail if CI would silently rebuild it. Kotlin +additionally downloads the verified native SDK runtime built by +`native-sdk-artifact.yml` after that producer restores the shared +`linux_static_abi_input`; it runs in parallel with the Linux product (debug on +PR, release on main). Release nests one `static-abi-artifact.yml` producer per +Linux native target through the same native-SDK workflow. Swift downloads an +immutable XCFramework and exact generated `mesh_ffi.swift` from the shared +`swift-sdk-artifact.yml` producer: PR uses `host-only`, while main and release +use exhaustive `full` mode, all on `macos-15`. Windows likewise builds one +immutable release-profile host, independent CPU/CUDA/ROCm/Vulkan runtime +inputs, and composition-only products. Broad main Rust changes exercise the +Windows CPU product; Windows GPU products remain limited to GPU/backend inputs +or manual dispatch. Every composed backend product requires `runtime list` +plus no-driver client readiness; hosted GPU rows neither inject a driver stub +nor skip startup because no device is present. `pr_builds.yml` uses the same split producer/composer shape for Linux CPU/GPU and macOS Metal products while retaining debug-profile hosts for lightweight @@ -75,6 +85,15 @@ PR iteration. Windows broad-Rust validation stays at lightweight Cargo checks; the debug host plus CPU or GPU runtime/product graph runs only for its platform/backend input or manual dispatch. Unsupported macOS CUDA, ROCm, and Vulkan combinations are omitted rather than emitted as no-op jobs. +`scripts/plan-pr-build-jobs.py` converts the central change signals into one +ordered `required_jobs_json` list. Every conditional PR Builds job routes on +membership in that list and retains normal dependency-success behavior through +`needs`. Its static `PR Builds Summary` job directly needs every other +top-level job and consumes the same plan. It accepts a skipped result only for +an unplanned job and rejects required skips, failures, cancellations, unknown +results, duplicate plan entries, and required IDs outside its needs graph, +making that one non-matrix check the workflow's stable branch-protection +target. Local actions: @@ -101,6 +120,15 @@ Local actions: verification, checksum, and verifier artifact. - `.github/actions/prepare-native-runtime-input` owns runtime build/package invocation and the release-grade artifact verifier. +- `.github/actions/prepare-native-sdk-input` owns the native SDK + prepare-llama/build-llama/mesh-llm-ffi/package chain, verifies the exact + target/backend/profile manifest, and stages a flat immutable upload. Release + mode adds the native runtime crate through the same path. +- `.github/actions/prepare-static-abi-input` owns the shared Linux static llama + ABI build/stamp validation and emits a checksummed, target-described ABI v3 + archive containing only the path-normalized static link closure and portable + OpenMP metadata. The reusable workflow caches that archive, not the local + CMake build graph; crate tests and native SDK producers consume it. - `.github/actions/compose-product-input` verifies producer inputs, creates one product-v2 tree without compiling, and runs CLI/client readiness. - `.github/actions/restore-smoke-inputs` owns producer artifact staging and @@ -114,6 +142,9 @@ Local actions: Routing and test-planning scripts: - `scripts/affected-crates.sh` computes affected crates and reverse dependents. +- `scripts/plan-pr-build-jobs.py` maps PR change signals to the single ordered + top-level job plan consumed by both conditional PR Builds jobs and its stable + summary gate. - `scripts/plan-clippy-batches.sh` owns weighted Clippy sharding and retains a checked workspace-member list for fail-open/all-rust planning. - `scripts/plan-test-batches.sh` owns weighted crate-test sharding. It derives @@ -131,7 +162,7 @@ GitHub-hosted labels currently used: - Linux AMD64: `ubuntu-24.04` - Linux ARM64: `ubuntu-24.04-arm` -- macOS: `macos-15` and legacy `macos-latest` +- macOS: pinned `macos-15` - Windows: `windows-2022` Depot labels referenced behind the rollout gate: @@ -148,17 +179,46 @@ use `DEPOT_RUNNERS_ENABLED`; a trusted main-ref manual dispatch can set `use_depot=true`. Hardware-qualified GPU execution is not part of the gate. +The default-branch-selected `native-sdk-artifact.yml` and +`static-abi-artifact.yml` workflows do not accept a runner label or Depot-cache +permission from callers. Each first runs a fixed `ubuntu-24.04` policy job, +validates `runner_size` as `default`, `4`, `8`, or `16`, maps the declared +target to the checked-in AMD64/ARM64 hosted and Depot labels, and grants both +the Depot runner and WebDAV cache only for exact +`Mesh-LLM/mesh-llm` `push`/`workflow_dispatch` calls on +`refs/heads/main` when `DEPOT_RUNNERS_ENABLED == 'true'` or when the immutable +main-dispatch event payload has `use_depot == 'true'`. Pull requests, +`pull_request_target`, tags, feature refs, external repositories, macOS, and a +disabled gate without that authorized canary resolve to a GitHub-hosted runner +with Depot cache permission false. The event-owned manual canary is evaluated +only under the same exact repository/main/dispatch guard and is not a +reusable-workflow input. + The Depot dashboard reports the `Mesh-LLM` GitHub connection active, and GitHub lists both `depot-managed-runners` and `depot-code-access` installations for all organization repositories. GitHub's organization settings show the Depot-managed `Default` group currently allows all workflows but excludes public repositories, while the separate `mesh-llm` group owns the two dedicated -GPU scale sets. Before a canary, restrict `Default` to this repository and exact -default-branch workflow refs, then enable public-repository access. +GPU scale sets. Before a canary, protect `main` with an enforceable +review/status gate, restrict `Default` to this repository and exact +default-branch workflow refs, then enable public-repository access. The +`mesh-llm` GPU group also permits all workflows in a public repository and must +be restricted to protected runner-owning entry points before it is considered a +trusted-only pool. The checked-out local selector is not the security boundary because PRs can modify workflow and local-action files. The Depot runner group must use `restricted_to_workflows=true` and exact default-branch selected-workflow refs. +The initial selected set includes `native-sdk-artifact.yml@refs/heads/main` and +`static-abi-artifact.yml@refs/heads/main` because those reusable workflows +directly allocate eligible Linux runners; a caller-only `ci.yml` entry is not +sufficient. +Credential-bearing `hf-download-smoke.yml`, `smoke.yml`, +`scripted-binary-smoke.yml`, and `sdk-smoke.yml` are deliberately excluded and +fixed to bounded GitHub-hosted labels. `swift-sdk-artifact.yml` is fixed to +GitHub-hosted `macos-15`. No reusable workflow passes caller-provided +runner JSON directly to `runs-on`. PR callers pass no `HF_TOKEN`; trusted +main/release callers may pass it only on the fixed hosted smoke lanes. Automatic Depot Cache still grants repository-scoped cache authority to the whole job, so even a trusted reusable caller cannot safely execute untrusted PR code while that injection is enabled. PRs remain GitHub-hosted. @@ -197,6 +257,23 @@ as `ghcr.io/mesh-llm/mesh-llm-cuda-runner`. The source repository owns: Production consumers must use the multi-architecture manifest digest. Tags are discovery inputs and are mutable absent separately verified registry controls. +Draft runner-images PR +[`#9`](https://github.com/Mesh-LLM/mesh-llm-runner-images/pull/9) changes the +publication control plane without changing those production digests. PRs route +affected families plus a mandatory public CPU AMD64 contract, use BuildKit +cache read-only, and cannot stage or promote. Main pushes stage verified +candidate digests; weekly or explicit manual runs promote a retained cohort. +The reusable family workflow independently derives trusted runner/cache +authority, verifies the requested MeshLLM source revision, uses content-digest +immutable tags, and feeds one serial `latest` cohort reconciliation. Deleted +files are included in affected-family routing. + +Its exhaustive Dockerfile-change PR +[run 30504335079](https://github.com/Mesh-LLM/mesh-llm-runner-images/actions/runs/30504335079) +completed all 20 platform rows in 6m 22s wall / 1h 13m 07s aggregate with no +Depot jobs and no PR cache export. Treat that as validation-path evidence, not +as proof of the trusted stage/promotion path. + The public repository and its GHCR package have independent visibility. Until anonymous pull of the package succeeds, GitHub-hosted container jobs must grant `packages: read` and provide `github.actor`/`secrets.GITHUB_TOKEN` through @@ -273,7 +350,6 @@ All GitHub Actions variables are strings. | `MESH_NIGHTLY_STABILITY_ATTEMPTS` | Attempts per model; fallback `5` | | `MESH_NIGHTLY_STABILITY_AGENT_SMOKES` | Optional agent CLI smoke list | | `MESH_NIGHTLY_STABILITY_TIMEOUT` | Per-probe seconds; fallback `180` | -| `MESH_NIGHTLY_STABILITY_RUNS_ON` | JSON runner label string/array; fallback `"ubuntu-24.04"` | ## Secret names referenced by workflows diff --git a/.github/actions/capture-sccache-stats/capture.py b/.github/actions/capture-sccache-stats/capture.py index d09b2a61ea..194d45d775 100644 --- a/.github/actions/capture-sccache-stats/capture.py +++ b/.github/actions/capture-sccache-stats/capture.py @@ -14,6 +14,8 @@ ARTIFACT_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +EVIDENCE_SCHEMA = "mesh-llm.sccache-stats" +EVIDENCE_SCHEMA_VERSION = 1 REQUIRED_COUNTERS = ( "compile_requests", "requests_executed", @@ -57,15 +59,18 @@ def validate_count_tree(value: Any, field: str) -> int: return value if isinstance(value, dict): return sum( - validate_count_tree(child, f"{field}.{name}") - for name, child in value.items() + validate_count_tree(child, f"{field} entry") + for child in value.values() ) raise EvidenceError( f"sccache JSON field {field} must contain only counter maps and integers", ) -def require_count_map(stats: dict[str, Any], name: str) -> int: +def sanitize_count_map( + stats: dict[str, Any], + name: str, +) -> tuple[dict[str, dict[str, int]], int]: value = stats.get(name) if not isinstance(value, dict): raise EvidenceError(f"sccache JSON field stats.{name} must be an object") @@ -74,11 +79,11 @@ def require_count_map(stats: dict[str, Any], name: str) -> int: raise EvidenceError( f"sccache JSON field stats.{name}.counts must be an object", ) - validate_count_tree(value, f"stats.{name}") - return validate_count_tree(counts, f"stats.{name}.counts") + total = validate_count_tree(counts, f"stats.{name}.counts") + return {"counts": {"total": total}}, total -def validate_stats(payload: Any) -> dict[str, int]: +def sanitize_stats(payload: Any) -> tuple[dict[str, Any], dict[str, int]]: if not isinstance(payload, dict): raise EvidenceError("sccache JSON root must be an object") stats = payload.get("stats") @@ -86,27 +91,32 @@ def validate_stats(payload: Any) -> dict[str, int]: raise EvidenceError("sccache JSON field stats must be an object") counters = {name: require_counter(stats, name) for name in REQUIRED_COUNTERS} - counters.update( - {name: require_count_map(stats, name) for name in REQUIRED_COUNT_MAPS}, - ) - return counters - - -def run_sccache(arguments: list[str], *, capture: bool = False) -> str: + sanitized_stats: dict[str, Any] = dict(counters) + for name in REQUIRED_COUNT_MAPS: + sanitized_map, total = sanitize_count_map(stats, name) + sanitized_stats[name] = sanitized_map + counters[name] = total + evidence = { + "schema": EVIDENCE_SCHEMA, + "schema_version": EVIDENCE_SCHEMA_VERSION, + "stats": sanitized_stats, + } + return evidence, counters + + +def run_sccache(arguments: list[str]) -> str: result = subprocess.run( ["sccache", *arguments], check=False, - capture_output=capture, + capture_output=True, text=True, ) if result.returncode != 0: - detail = result.stderr.strip() if capture else "" - suffix = f": {detail}" if detail else "" raise EvidenceError( f"sccache {' '.join(arguments)} failed with " - f"exit code {result.returncode}{suffix}", + f"exit code {result.returncode}", ) - return result.stdout if capture else "" + return result.stdout def write_github_outputs( @@ -141,25 +151,18 @@ def main() -> int: if shutil.which("sccache") is None: raise EvidenceError("sccache is required to capture build-cache evidence") - print("::group::Human-readable sccache statistics", flush=True) - try: - run_sccache(["--show-stats"]) - finally: - print("::endgroup::", flush=True) - raw_json = run_sccache( ["--show-stats", "--stats-format", "json"], - capture=True, ) try: payload = json.loads(raw_json) except json.JSONDecodeError as error: raise EvidenceError(f"sccache returned invalid JSON: {error}") from error - counters = validate_stats(payload) + evidence, counters = sanitize_stats(payload) arguments.output.parent.mkdir(parents=True, exist_ok=True) arguments.output.write_text( - raw_json.rstrip("\n") + "\n", + json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) stats_file = arguments.output.resolve() diff --git a/.github/actions/compute-changes/action.yml b/.github/actions/compute-changes/action.yml index fa9de33eb4..413dcf71c5 100644 --- a/.github/actions/compute-changes/action.yml +++ b/.github/actions/compute-changes/action.yml @@ -164,7 +164,7 @@ runs: if [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]; then RUNNER_CONTRACT_REQUIRED="true" elif [[ -n "$CHANGED_FILES" ]]; then - RUNNER_CONTRACT_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^\.github/cache-version\.txt$|^\.github/actionlint\.yaml$|^\.github/actions/(capture-sccache-stats|configure-sccache-gha|select-ci-runners)/|^\.github/workflows/(ci|depot-canary|pr_builds|pr_quality|release)\.yml$)' || true) + RUNNER_CONTRACT_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^\.github/cache-version\.txt$|^\.github/actionlint\.yaml$|^\.github/actions/(capture-sccache-stats|configure-sccache-gha|select-ci-runners)/|^\.github/workflows/(ci|depot-canary|native-sdk-artifact|pr_builds|pr_quality|release|sdk-smoke|static-abi-artifact|swift-sdk-artifact)\.yml$)' || true) if [[ -n "$RUNNER_CONTRACT_INPUTS" ]]; then RUNNER_CONTRACT_REQUIRED="true" fi @@ -343,7 +343,7 @@ runs: if [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]; then SDK_SMOKE_REQUIRED="true" elif [[ -n "$CHANGED_FILES" ]]; then - DIRECT_SDK_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^sdk/|^Package\.swift$|^scripts/ci-(rust|kotlin|swift)-sdk-smoke\.sh$|^scripts/ci-prepare-native-runtime\.sh$|^scripts/ci-sdk-fixture\.sh$|^scripts/(check-sdk-contract|package-sdk-console-assets|verify-sdk-console-assets|verify-swift-privacy-manifest|verify-swift-release-artifact|prepare-llama|build-llama)\.sh$|^scripts/(package-native-sdk|package-native-sdk-crate|verify-native-sdk-package|verify-checksum-sidecar|safe-extract-tar)\.(sh|py)$|^\.github/actions/restore-smoke-inputs/|^\.github/workflows/sdk-smoke\.yml$)' || true) + DIRECT_SDK_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^sdk/|^Package\.swift$|^scripts/ci-(rust|kotlin|swift)-sdk-smoke\.sh$|^scripts/ci-prepare-native-runtime\.sh$|^scripts/ci-sdk-fixture\.sh$|^scripts/(check-sdk-contract|package-sdk-console-assets|restore-native-sdk-input|restore-static-abi-input|verify-sdk-console-assets|verify-swift-privacy-manifest|verify-swift-release-artifact|prepare-llama|build-llama)\.sh$|^scripts/(package-native-sdk|package-native-sdk-crate|verify-native-sdk-package|verify-checksum-sidecar|verify-static-abi-build-stamp|safe-extract-(tar|zip)|verify-swift-xcframework)\.(sh|py)$|^\.github/actions/(prepare-native-sdk-input|prepare-static-abi-input|restore-smoke-inputs)/|^\.github/workflows/(native-sdk-artifact|sdk-smoke|static-abi-artifact|swift-sdk-artifact)\.yml$)' || true) if [[ -n "$DIRECT_SDK_INPUTS" ]]; then SDK_SMOKE_REQUIRED="true" elif echo "$AFFECTED_CRATES" | jq -e 'index("mesh-llm-client") or index("mesh-llm-api-client") or index("mesh-llm-api-server") or index("mesh-llm-config") or index("mesh-llm-console-server") or index("mesh-llm-ffi") or index("mesh-llm-native-runtime") or index("mesh-llm-protocol") or index("mesh-llm-routing") or index("mesh-llm-types")' >/dev/null; then diff --git a/.github/actions/prepare-native-sdk-input/action.yml b/.github/actions/prepare-native-sdk-input/action.yml new file mode 100644 index 0000000000..78ed8a0f63 --- /dev/null +++ b/.github/actions/prepare-native-sdk-input/action.yml @@ -0,0 +1,221 @@ +name: Prepare native SDK input +description: Build, package, verify, and stage one immutable MeshLLM native SDK input. + +inputs: + backend: + description: Native SDK backend. + required: true + target: + description: Rust target triple. + required: true + profile: + description: Cargo profile (debug or release). + required: true + output_dir: + description: New relative directory that receives the native SDK package. + required: false + default: native-sdk-output + include_runtime_crate: + description: Include the generated native runtime crate in the flat upload. + required: false + default: "false" + require_prebuilt_static_abi: + description: Fail instead of rebuilding the restored static llama ABI. + required: false + default: "false" + +outputs: + archive_path: + description: Absolute path to the verified native SDK archive. + value: ${{ steps.prepare.outputs.archive_path }} + checksum_path: + description: Absolute path to the archive checksum sidecar. + value: ${{ steps.prepare.outputs.checksum_path }} + artifact_dir: + description: Absolute path to the verified extracted native SDK directory. + value: ${{ steps.prepare.outputs.artifact_dir }} + upload_path: + description: Flat glob containing the exact immutable upload payload. + value: ${{ steps.prepare.outputs.upload_path }} + +runs: + using: composite + steps: + - name: Build and stage immutable native SDK input + id: prepare + shell: bash + env: + INPUT_BACKEND: ${{ inputs.backend }} + INPUT_TARGET: ${{ inputs.target }} + INPUT_PROFILE: ${{ inputs.profile }} + INPUT_OUTPUT_DIR: ${{ inputs.output_dir }} + INPUT_INCLUDE_RUNTIME_CRATE: ${{ inputs.include_runtime_crate }} + INPUT_REQUIRE_PREBUILT_STATIC_ABI: ${{ inputs.require_prebuilt_static_abi }} + run: | + set -euo pipefail + + case "$INPUT_BACKEND" in + cpu|metal|cuda|cuda-blackwell|rocm|hip|vulkan) ;; + *) + echo "unsupported native SDK backend: $INPUT_BACKEND" >&2 + exit 1 + ;; + esac + case "$INPUT_TARGET" in + aarch64-apple-darwin) + expected_os="Darwin" + expected_arch="aarch64" + ;; + x86_64-unknown-linux-gnu) + expected_os="Linux" + expected_arch="x86_64" + ;; + aarch64-unknown-linux-gnu) + expected_os="Linux" + expected_arch="aarch64" + ;; + *) + echo "unsupported native SDK producer target: $INPUT_TARGET" >&2 + exit 1 + ;; + esac + case "$(uname -m)" in + x86_64|amd64) runner_arch="x86_64" ;; + aarch64|arm64) runner_arch="aarch64" ;; + *) runner_arch="$(uname -m)" ;; + esac + if [[ "$(uname -s)" != "$expected_os" || + "$runner_arch" != "$expected_arch" ]]; then + echo "native SDK target/runner mismatch: $INPUT_TARGET on $(uname -s)/$runner_arch" >&2 + exit 1 + fi + case "$INPUT_PROFILE" in + debug|release) ;; + *) + echo "unsupported native SDK Cargo profile: $INPUT_PROFILE" >&2 + exit 1 + ;; + esac + case "$INPUT_INCLUDE_RUNTIME_CRATE" in + true|false) ;; + *) + echo "include_runtime_crate must be true or false" >&2 + exit 1 + ;; + esac + case "$INPUT_REQUIRE_PREBUILT_STATIC_ABI" in + true|false) ;; + *) + echo "require_prebuilt_static_abi must be true or false" >&2 + exit 1 + ;; + esac + if [[ "$INPUT_OUTPUT_DIR" == /* || + "$INPUT_OUTPUT_DIR" == "." || + "$INPUT_OUTPUT_DIR" == *".."* || + ! "$INPUT_OUTPUT_DIR" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]]; then + echo "output_dir must be a safe relative path: $INPUT_OUTPUT_DIR" >&2 + exit 1 + fi + if [[ -e "$INPUT_OUTPUT_DIR" ]]; then + echo "native SDK output directory already exists: $INPUT_OUTPUT_DIR" >&2 + exit 1 + fi + + package_args=( + --build + --backend "$INPUT_BACKEND" + --target "$INPUT_TARGET" + --profile "$INPUT_PROFILE" + --out "$INPUT_OUTPUT_DIR" + ) + if [[ "$INPUT_REQUIRE_PREBUILT_STATIC_ABI" == "true" ]]; then + package_args+=(--require-prebuilt-llama) + fi + scripts/package-native-sdk.sh "${package_args[@]}" + + shopt -s nullglob + archives=("$INPUT_OUTPUT_DIR"/*.tar.gz) + checksums=("$INPUT_OUTPUT_DIR"/*.tar.gz.sha256) + artifact_manifests=("$INPUT_OUTPUT_DIR"/meshllm-native-*/manifest.json) + if [[ "${#archives[@]}" -ne 1 || + "${#checksums[@]}" -ne 1 || + "${#artifact_manifests[@]}" -ne 1 ]]; then + echo "expected one native SDK archive, checksum, and artifact directory" >&2 + exit 1 + fi + + archive_path="${archives[0]}" + checksum_path="${checksums[0]}" + artifact_dir="$(dirname "${artifact_manifests[0]}")" + if [[ "$checksum_path" != "$archive_path.sha256" ]]; then + echo "native SDK checksum sidecar does not match archive" >&2 + exit 1 + fi + + scripts/verify-native-sdk-package.sh "$archive_path" + scripts/verify-native-sdk-package.sh "$artifact_dir" + python3 - \ + "$artifact_dir/manifest.json" \ + "$INPUT_TARGET" \ + "$INPUT_BACKEND" \ + "$INPUT_PROFILE" <<'PY' + import json + import sys + + manifest_path, target, backend, profile = sys.argv[1:] + with open(manifest_path, encoding="utf-8") as handle: + manifest = json.load(handle) + expected = { + "target_triple": target, + "backend": backend, + "cargo_profile": profile, + } + for field, value in expected.items(): + if manifest.get(field) != value: + raise SystemExit( + f"native SDK manifest {field} mismatch: " + f"expected {value!r}, got {manifest.get(field)!r}" + ) + PY + + upload_dir="$INPUT_OUTPUT_DIR-upload" + if [[ -e "$upload_dir" ]]; then + echo "native SDK upload directory already exists: $upload_dir" >&2 + exit 1 + fi + mkdir -p "$upload_dir" + + upload_sources=("$archive_path" "$checksum_path") + if [[ "$INPUT_INCLUDE_RUNTIME_CRATE" == "true" ]]; then + crate_output="$INPUT_OUTPUT_DIR-crates" + scripts/package-native-sdk-crate.sh \ + --out "$crate_output" \ + "$artifact_dir" + runtime_crates=("$crate_output"/*/target/package/*.crate) + if [[ "${#runtime_crates[@]}" -ne 1 ]]; then + echo "expected one native SDK runtime crate" >&2 + exit 1 + fi + upload_sources+=("${runtime_crates[0]}") + fi + + for source in "${upload_sources[@]}"; do + destination="$upload_dir/$(basename "$source")" + if [[ -e "$destination" ]]; then + echo "native SDK release asset basename collision: $destination" >&2 + exit 1 + fi + cp -p "$source" "$destination" + done + + output_root="$(cd "$INPUT_OUTPUT_DIR" && pwd)" + upload_root="$(cd "$upload_dir" && pwd)" + artifact_name="$(basename "$artifact_dir")" + archive_name="$(basename "$archive_path")" + { + echo "archive_path=$output_root/$archive_name" + echo "checksum_path=$output_root/$archive_name.sha256" + echo "artifact_dir=$output_root/$artifact_name" + echo "upload_path=$upload_root/*" + } >> "$GITHUB_OUTPUT" diff --git a/.github/actions/prepare-static-abi-input/action.yml b/.github/actions/prepare-static-abi-input/action.yml new file mode 100644 index 0000000000..e0151ce045 --- /dev/null +++ b/.github/actions/prepare-static-abi-input/action.yml @@ -0,0 +1,273 @@ +name: Prepare static ABI input +description: Build or verify, describe, and archive one immutable static llama ABI input. + +inputs: + backend: + description: Static llama ABI backend. + required: true + target: + description: Native target triple represented by this ABI. + required: true + build: + description: Build the ABI before archiving it. + required: false + default: "true" + +outputs: + archive_path: + description: Absolute path to the static ABI archive. + value: ${{ steps.prepare.outputs.archive_path }} + checksum_path: + description: Absolute path to the archive checksum sidecar. + value: ${{ steps.prepare.outputs.checksum_path }} + upload_path: + description: Flat glob containing the exact immutable upload payload. + value: ${{ steps.prepare.outputs.upload_path }} + +runs: + using: composite + steps: + - name: Prepare immutable static ABI input + id: prepare + shell: bash + env: + INPUT_BACKEND: ${{ inputs.backend }} + INPUT_TARGET: ${{ inputs.target }} + INPUT_BUILD: ${{ inputs.build }} + run: | + set -euo pipefail + + if [[ "$INPUT_BACKEND" != "cpu" ]]; then + echo "unsupported shared static ABI backend: $INPUT_BACKEND" >&2 + exit 1 + fi + case "$INPUT_TARGET" in + x86_64-unknown-linux-gnu) expected_arch="x86_64" ;; + aarch64-unknown-linux-gnu) expected_arch="aarch64" ;; + *) + echo "unsupported shared static ABI target: $INPUT_TARGET" >&2 + exit 1 + ;; + esac + case "$(uname -m)" in + x86_64|amd64) runner_arch="x86_64" ;; + aarch64|arm64) runner_arch="aarch64" ;; + *) runner_arch="$(uname -m)" ;; + esac + if [[ "$runner_arch" != "$expected_arch" ]]; then + echo "static ABI target/runner architecture mismatch: $INPUT_TARGET on $runner_arch" >&2 + exit 1 + fi + case "$INPUT_BUILD" in + true|false) ;; + *) + echo "build must be true or false" >&2 + exit 1 + ;; + esac + toolchain_epoch="${MESH_LLM_LLAMA_TOOLCHAIN_EPOCH:-}" + if [[ -z "$toolchain_epoch" || + ! "$toolchain_epoch" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "MESH_LLM_LLAMA_TOOLCHAIN_EPOCH must identify the pinned build image" >&2 + exit 1 + fi + + expected_build_dir=".deps/llama.cpp/build-stage-abi-static" + if [[ "${LLAMA_STAGE_BUILD_DIR:-}" != "$expected_build_dir" ]]; then + echo "shared static ABI build dir must be $expected_build_dir" >&2 + exit 1 + fi + + scripts/prepare-llama.sh pinned + if [[ "$INPUT_BUILD" == "true" ]]; then + scripts/build-llama.sh + fi + + source_build_dir="$LLAMA_STAGE_BUILD_DIR" + build_stamp="$source_build_dir/.mesh-llm-build-stamp" + patched_sha_file=".deps/llama.cpp/.mesh-llm-patched-sha" + required_archives=( + "src/libllama.a" + "common/libllama-common.a" + "common/libllama-common-base.a" + "ggml/src/libggml.a" + "ggml/src/libggml-base.a" + "tools/mtmd/libmtmd.a" + ) + test -s "$build_stamp" + test -s "$patched_sha_file" + test -s "$source_build_dir/CMakeCache.txt" + for relative in "${required_archives[@]}"; do + if [[ ! -s "$source_build_dir/$relative" ]]; then + echo "static ABI is missing required archive: $relative" >&2 + exit 1 + fi + done + cpu_archive= + for candidate in \ + "ggml/src/libggml-cpu.a" \ + "ggml/src/ggml-cpu/libggml-cpu.a"; do + if [[ -s "$source_build_dir/$candidate" ]]; then + cpu_archive="$candidate" + break + fi + done + if [[ -z "$cpu_archive" ]]; then + echo "static ABI is missing libggml-cpu.a" >&2 + exit 1 + fi + + upload_dir="static-abi-artifact-output" + if [[ -e "$upload_dir" ]]; then + echo "static ABI artifact output already exists: $upload_dir" >&2 + exit 1 + fi + mkdir -p "$upload_dir" + + stage_root="$( + mktemp -d "${RUNNER_TEMP:-/tmp}/mesh-static-abi-stage.XXXXXX" + )" + trap 'rm -rf -- "$stage_root"' EXIT + portable_build_dir="$stage_root/$(basename "$source_build_dir")" + mkdir -p "$portable_build_dir" + + copy_archive() { + local relative="$1" + mkdir -p "$portable_build_dir/$(dirname "$relative")" + cp -p "$source_build_dir/$relative" "$portable_build_dir/$relative" + } + for relative in "${required_archives[@]}"; do + copy_archive "$relative" + done + copy_archive "$cpu_archive" + optional_blas_archive="ggml/src/ggml-blas/libggml-blas.a" + if [[ -s "$source_build_dir/$optional_blas_archive" ]]; then + copy_archive "$optional_blas_archive" + fi + cp -p "$build_stamp" "$portable_build_dir/.mesh-llm-build-stamp" + + # build.rs only consumes this OpenMP subset for a CPU static ABI. + # Excluding the rest removes CMake's producer-local source/build paths + # while retaining the exact native link decision. + python3 - \ + "$source_build_dir/CMakeCache.txt" \ + "$portable_build_dir/CMakeCache.txt" <<'PY' + import re + import sys + + source, destination = sys.argv[1:] + safe_key = re.compile( + r"^(?:" + r"GGML_OPENMP_ENABLED|" + r"OpenMP_(?:C|CXX)_LIB_NAMES|" + r"OpenMP_[A-Za-z0-9_]+_LIBRARY" + r"):[^=]+=" + ) + retained = [ + line + for line in open(source, encoding="utf-8", errors="strict") + if safe_key.match(line) + ] + with open(destination, "w", encoding="utf-8") as handle: + handle.write("# Portable MeshLLM static ABI link metadata\n") + handle.writelines(retained) + PY + + manifest="$portable_build_dir/.mesh-llm-static-abi-input.json" + python3 - \ + "$manifest" \ + "$INPUT_TARGET" \ + "$INPUT_BACKEND" \ + "$(basename "$source_build_dir")" \ + "$toolchain_epoch" \ + "$patched_sha_file" \ + "$portable_build_dir/.mesh-llm-build-stamp" <<'PY' + import hashlib + import json + import sys + + ( + manifest_path, + target, + backend, + build_dir, + toolchain_epoch, + patched_sha_path, + stamp_path, + ) = sys.argv[1:] + with open(patched_sha_path, encoding="utf-8") as handle: + expected_patched_sha = handle.read().strip() + if not expected_patched_sha: + raise SystemExit("prepared llama.cpp patched SHA is empty") + with open(stamp_path, "rb") as handle: + stamp_bytes = handle.read() + stamp_sha256 = hashlib.sha256(stamp_bytes).hexdigest() + manifest = { + "schema_version": 3, + "contract": "mesh-llm-static-abi-v3", + "target_triple": target, + "backend": backend, + "build_directory": build_dir, + "toolchain_epoch": toolchain_epoch, + "build_stamp_sha256": stamp_sha256, + } + with open(manifest_path, "w", encoding="utf-8") as handle: + json.dump(manifest, handle, indent=2, sort_keys=True) + handle.write("\n") + PY + expected_patched_sha="$(tr -d '[:space:]' < "$patched_sha_file")" + python3 scripts/verify-static-abi-build-stamp.py \ + "$portable_build_dir/.mesh-llm-build-stamp" \ + --backend "$INPUT_BACKEND" \ + --link-mode static \ + --stamp-version 3 \ + --toolchain-epoch "$toolchain_epoch" \ + --patched-sha "$expected_patched_sha" + + # Fail closed if compiler output or staged metadata retained the + # producer's checkout/build paths. + python3 - \ + "$portable_build_dir" \ + "$(pwd -P)" \ + "$(cd .deps/llama.cpp && pwd -P)" \ + "$(cd "$source_build_dir" && pwd -P)" <<'PY' + import sys + from pathlib import Path + + stage = Path(sys.argv[1]) + forbidden = tuple( + value.encode() + for value in dict.fromkeys(sys.argv[2:]) + if value + ) + for path in sorted(stage.rglob("*")): + if not path.is_file(): + continue + data = path.read_bytes() + for value in forbidden: + if value in data: + raise SystemExit( + f"portable static ABI retained producer-local path " + f"{value.decode()!r} in {path.relative_to(stage)}" + ) + PY + + archive_path="$upload_dir/mesh-llm-static-abi.tar.gz" + tar -C "$stage_root" \ + -czf "$archive_path" \ + "$(basename "$portable_build_dir")" + if command -v sha256sum >/dev/null 2>&1; then + archive_sha="$(sha256sum "$archive_path" | awk '{print $1}')" + else + archive_sha="$(shasum -a 256 "$archive_path" | awk '{print $1}')" + fi + printf '%s %s\n' "$archive_sha" "$(basename "$archive_path")" \ + > "$archive_path.sha256" + python3 scripts/verify-checksum-sidecar.py "$archive_path" + + upload_root="$(cd "$upload_dir" && pwd)" + { + echo "archive_path=$upload_root/mesh-llm-static-abi.tar.gz" + echo "checksum_path=$upload_root/mesh-llm-static-abi.tar.gz.sha256" + echo "upload_path=$upload_root/*" + } >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5eacbca459..2b8bc005ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ env: CARGO_HTTP_MULTIPLEXING: "false" SCCACHE_GHA_ENABLED: "true" LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static + MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d permissions: contents: read @@ -58,6 +59,7 @@ jobs: runner_contract_required: ${{ steps.compute.outputs.runner_contract_required }} ui_dist_cache_key: ${{ steps.ui_key.outputs.ui_dist_cache_key }} linux_inference_artifact_required: ${{ github.event_name == 'workflow_dispatch' || steps.compute.outputs.inference_artifact_required == 'true' }} + macos_inference_artifact_required: ${{ github.event_name == 'workflow_dispatch' || steps.compute.outputs.inference_artifact_required == 'true' || steps.filter.outputs.benchmarks == 'true' }} depot_enabled: ${{ steps.runners.outputs.depot_enabled }} allow_depot_remote_cache: ${{ steps.runners.outputs.allow_depot_remote_cache }} runner: ${{ steps.runners.outputs.runner }} @@ -368,58 +370,18 @@ jobs: linux_static_abi_input: needs: changes - if: ${{ needs.changes.outputs.docs_only != 'true' && (needs.changes.outputs.test_batches_json != '[]' || needs.changes.outputs.linux_test_groups_json != '[]') }} + if: ${{ needs.changes.outputs.docs_only != 'true' && (needs.changes.outputs.test_batches_json != '[]' || needs.changes.outputs.linux_test_groups_json != '[]' || needs.changes.outputs.sdk_smoke_required == 'true') }} name: Linux static ABI input - runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read packages: read - container: - image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - defaults: - run: - shell: bash - env: - LLAMA_STAGE_BACKEND: cpu - LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static - MESH_LLM_REQUIRE_SCCACHE: "1" - steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - persist-credentials: false - - name: Verify prebuilt CI environment - run: verify-runner-image public - - uses: ./.github/actions/configure-sccache-gha - with: - allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - - name: Cache patched llama.cpp ABI build - id: static_abi_cache - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 - with: - path: .deps/llama.cpp/build-stage-abi-static - key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-cpu-${{ hashFiles('scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - - name: Prepare patched llama.cpp ABI checkout - if: ${{ steps.static_abi_cache.outputs.cache-hit != 'true' }} - run: scripts/prepare-llama.sh pinned - - name: Build patched llama.cpp ABI libraries - if: ${{ steps.static_abi_cache.outputs.cache-hit != 'true' }} - run: scripts/build-llama.sh - - name: Archive immutable static ABI input - run: | - test -d "$LLAMA_STAGE_BUILD_DIR" - tar -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" \ - -czf "$RUNNER_TEMP/mesh-llm-static-abi.tar.gz" \ - "$(basename "$LLAMA_STAGE_BUILD_DIR")" - - name: Upload immutable static ABI input - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: ci-linux-static-abi-input - path: ${{ runner.temp }}/mesh-llm-static-abi.tar.gz - if-no-files-found: error - retention-days: 1 + uses: ./.github/workflows/static-abi-artifact.yml + with: + backend: cpu + target: x86_64-unknown-linux-gnu + artifact_name: ci-linux-static-abi-input + runner_size: '8' + retention_days: 1 rust_crate_tests: needs: [changes, linux_static_abi_input] @@ -476,11 +438,7 @@ jobs: name: ci-linux-static-abi-input path: ${{ runner.temp }}/static-abi-input - name: Restore immutable static ABI input - run: | - mkdir -p "$(dirname "$LLAMA_STAGE_BUILD_DIR")" - tar -xzf "$RUNNER_TEMP/static-abi-input/mesh-llm-static-abi.tar.gz" \ - -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" - test -d "$LLAMA_STAGE_BUILD_DIR" + run: scripts/restore-static-abi-input.sh "$RUNNER_TEMP/static-abi-input" "$LLAMA_STAGE_BUILD_DIR" x86_64-unknown-linux-gnu cpu - name: Run crate tests env: TEST_CRATES: ${{ toJson(matrix.batch.crates) }} @@ -559,11 +517,7 @@ jobs: name: ci-linux-static-abi-input path: ${{ runner.temp }}/static-abi-input - name: Restore immutable static ABI input - run: | - mkdir -p "$(dirname "$LLAMA_STAGE_BUILD_DIR")" - tar -xzf "$RUNNER_TEMP/static-abi-input/mesh-llm-static-abi.tar.gz" \ - -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" - test -d "$LLAMA_STAGE_BUILD_DIR" + run: scripts/restore-static-abi-input.sh "$RUNNER_TEMP/static-abi-input" "$LLAMA_STAGE_BUILD_DIR" x86_64-unknown-linux-gnu cpu - name: Restore Skippy smoke model cache if: ${{ matrix.group == 'skippy-smoke' }} @@ -612,7 +566,6 @@ jobs: contents: read uses: ./.github/workflows/hf-download-smoke.yml with: - runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} timeout_minutes: 15 secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -627,7 +580,6 @@ jobs: artifact_name: ci-linux-inference-binaries mesh_binary_target: target/release/mesh-llm cache_key_prefix: '' - runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -693,7 +645,6 @@ jobs: staged_binary_path: target/release/mesh-llm model_cache_scope: two-node-smoke-model smoke_script: scripts/ci-two-node-client-serving-smoke.sh - runs_on: ${{ toJson(needs.changes.outputs.runner_4) }} timeout_minutes: 20 secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -710,13 +661,30 @@ jobs: artifact_path: ci-artifacts/linux staged_binary_path: target/release/mesh-llm model_cache_scope: sdk-smoke-model - runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} + kotlin_sdk_input: + name: Kotlin native SDK input + needs: [changes, linux_static_abi_input] + if: ${{ needs.linux_static_abi_input.result == 'success' && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + permissions: + contents: read + packages: read + uses: ./.github/workflows/native-sdk-artifact.yml + with: + backend: cpu + target: x86_64-unknown-linux-gnu + profile: release + artifact_name: ci-kotlin-native-sdk-input + static_abi_artifact_name: ci-linux-static-abi-input + static_abi_artifact_path: native-sdk-static-abi-input + runner_size: '8' + retention_days: 1 + kotlin_sdk_smoke: - needs: [changes, linux_cpu_artifact] - if: ${{ needs.linux_cpu_artifact.result == 'success' && needs.changes.outputs.linux_inference_artifact_required == 'true' && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + needs: [changes, linux_cpu_artifact, kotlin_sdk_input] + if: ${{ needs.linux_cpu_artifact.result == 'success' && needs.kotlin_sdk_input.result == 'success' && needs.changes.outputs.linux_inference_artifact_required == 'true' && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} permissions: contents: read uses: ./.github/workflows/sdk-smoke.yml @@ -725,14 +693,30 @@ jobs: artifact_name: ci-linux-inference-binaries artifact_path: ci-artifacts/linux staged_binary_path: target/release/mesh-llm + kotlin_artifact_name: ci-kotlin-native-sdk-input + kotlin_artifact_target: x86_64-unknown-linux-gnu + kotlin_artifact_backend: cpu + kotlin_artifact_profile: release model_cache_scope: sdk-smoke-model - runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} + swift_sdk_input: + name: Swift SDK full input + needs: changes + if: ${{ needs.changes.outputs.macos_inference_artifact_required == 'true' && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + permissions: + contents: read + uses: ./.github/workflows/swift-sdk-artifact.yml + with: + mode: full + artifact_name: ci-swift-sdk-input + retention_days: 1 + timeout_minutes: 90 + swift_sdk_smoke: - needs: [changes, macos_cpu_artifact, macos_unit_tests] - if: ${{ !cancelled() && needs.macos_cpu_artifact.result == 'success' && (needs.macos_unit_tests.result == 'success' || needs.macos_unit_tests.result == 'skipped') && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + needs: [changes, macos_cpu_artifact, swift_sdk_input] + if: ${{ needs.macos_cpu_artifact.result == 'success' && needs.swift_sdk_input.result == 'success' && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} permissions: contents: read uses: ./.github/workflows/sdk-smoke.yml @@ -741,17 +725,18 @@ jobs: artifact_name: ci-macos-inference-binaries artifact_path: ci-artifacts/macos staged_binary_path: target/release/mesh-llm + swift_artifact_name: ci-swift-sdk-input + swift_artifact_mode: full model_cache_scope: sdk-smoke-model - runs_on: '"macos-latest"' - timeout_minutes: 40 + timeout_minutes: 30 secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} macos_host_input: needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.rust == 'true' || needs.changes.outputs.ui == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ needs.changes.outputs.macos_inference_artifact_required == 'true' && needs.changes.outputs.docs_only != 'true' }} name: macOS immutable release host - runs-on: macos-latest + runs-on: macos-15 permissions: contents: read env: @@ -763,7 +748,7 @@ jobs: - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} with: - version: latest + version: 10 - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} with: @@ -844,9 +829,9 @@ jobs: macos_metal_runtime_input: needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.rust == 'true' || needs.changes.outputs.ui == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ needs.changes.outputs.macos_inference_artifact_required == 'true' && needs.changes.outputs.docs_only != 'true' }} name: macOS Metal runtime - runs-on: macos-latest + runs-on: macos-15 permissions: contents: read env: @@ -878,7 +863,7 @@ jobs: needs: [changes, macos_host_input, macos_metal_runtime_input] if: ${{ needs.macos_host_input.result == 'success' && needs.macos_metal_runtime_input.result == 'success' }} name: macOS Metal release product - runs-on: macos-latest + runs-on: macos-15 permissions: contents: read steps: @@ -916,7 +901,7 @@ jobs: needs: changes if: ${{ needs.changes.outputs.docs_only != 'true' && (needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'model-artifact') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-host-runtime') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm')) }} name: macOS unit tests - runs-on: macos-latest + runs-on: macos-15 permissions: contents: read env: diff --git a/.github/workflows/docker-precheck.yml b/.github/workflows/docker-precheck.yml index 3d25c004a4..aebb41d8ea 100644 --- a/.github/workflows/docker-precheck.yml +++ b/.github/workflows/docker-precheck.yml @@ -35,6 +35,8 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Validate target Dockerfile exists run: test -f "${{ inputs.dockerfile_path }}" diff --git a/.github/workflows/hf-download-smoke.yml b/.github/workflows/hf-download-smoke.yml index ee0f211819..2d157f509b 100644 --- a/.github/workflows/hf-download-smoke.yml +++ b/.github/workflows/hf-download-smoke.yml @@ -3,10 +3,6 @@ name: Reusable HuggingFace Download Smoke Tests on: workflow_call: inputs: - runs_on: - required: false - default: '"ubuntu-24.04"' - type: string timeout_minutes: required: false default: 20 @@ -18,16 +14,23 @@ on: env: CACHE_NAMESPACE: mesh-llm +permissions: + contents: read + jobs: hf_download_smoke: name: HuggingFace download smoke - runs-on: ${{ fromJson(inputs.runs_on) }} + # This credential-bearing reusable workflow is intentionally ineligible for + # Depot and self-hosted runner groups. + runs-on: ubuntu-24.04 timeout-minutes: ${{ inputs.timeout_minutes }} env: HF_TOKEN: ${{ secrets.HF_TOKEN }} HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" diff --git a/.github/workflows/native-sdk-artifact.yml b/.github/workflows/native-sdk-artifact.yml new file mode 100644 index 0000000000..7613cc666f --- /dev/null +++ b/.github/workflows/native-sdk-artifact.yml @@ -0,0 +1,332 @@ +name: Reusable Native SDK Artifact + +on: + workflow_call: + inputs: + backend: + description: Native SDK backend. + required: true + type: string + target: + description: Rust target triple. + required: true + type: string + profile: + description: Cargo profile (debug or release). + required: true + type: string + artifact_name: + description: Immutable GitHub artifact name. + required: true + type: string + static_abi_artifact_name: + description: Optional immutable static ABI artifact name to restore. + required: false + default: '' + type: string + static_abi_artifact_path: + description: Directory used to download the optional static ABI input. + required: false + default: native-sdk-static-abi-input + type: string + produce_static_abi: + description: Produce a matching Linux static ABI before native SDK packaging. + required: false + default: false + type: boolean + runner_size: + description: Eligible Linux runner size (default, 4, 8, or 16). + required: false + default: '8' + type: string + include_runtime_crate: + description: Include the release native runtime crate in the flat artifact. + required: false + default: false + type: boolean + release_tag: + description: Optional release tag applied before packaging. + required: false + default: '' + type: string + prepare_release_version: + description: Prepare checked-out sources for release_tag before building. + required: false + default: false + type: boolean + retention_days: + description: Artifact retention in days. + required: false + default: 1 + type: number + timeout_minutes: + description: Producer timeout in minutes. + required: false + default: 60 + type: number + +env: + CARGO_INCREMENTAL: "0" + CARGO_NET_RETRY: "10" + CARGO_HTTP_MULTIPLEXING: "false" + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + +jobs: + runner_policy: + name: Select protected native SDK runner + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: none + outputs: + runner: ${{ steps.policy.outputs.runner }} + allow_depot_remote_cache: ${{ steps.policy.outputs.allow_depot_remote_cache }} + steps: + - name: Derive protected runner policy + id: policy + shell: bash + env: + POLICY_REPOSITORY: ${{ github.repository }} + POLICY_REF: ${{ github.ref }} + POLICY_EVENT_NAME: ${{ github.event_name }} + POLICY_DEPOT_ENABLED: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + POLICY_MANUAL_USE_DEPOT: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.use_depot == 'true' }} + POLICY_TARGET: ${{ inputs.target }} + POLICY_RUNNER_SIZE: ${{ inputs.runner_size }} + run: | + set -euo pipefail + + case "$POLICY_RUNNER_SIZE" in + default|4|8|16) ;; + *) + echo "runner_size must be one of: default, 4, 8, 16" >&2 + exit 1 + ;; + esac + + case "$POLICY_TARGET" in + x86_64-unknown-linux-gnu) + hosted_runner=ubuntu-24.04 + depot_runner=depot-ubuntu-24.04 + ;; + aarch64-unknown-linux-gnu) + hosted_runner=ubuntu-24.04-arm + depot_runner=depot-ubuntu-24.04-arm + ;; + aarch64-apple-darwin) + hosted_runner=macos-15 + depot_runner= + ;; + *) + echo "unsupported native SDK producer target: $POLICY_TARGET" >&2 + exit 1 + ;; + esac + + allow_depot_remote_cache=false + runner=$hosted_runner + if [[ "$POLICY_REPOSITORY" == "Mesh-LLM/mesh-llm" && + "$POLICY_REF" == "refs/heads/main" && + ( "$POLICY_EVENT_NAME" == "push" || + "$POLICY_EVENT_NAME" == "workflow_dispatch" ) && + ( "$POLICY_DEPOT_ENABLED" == "true" || + ( "$POLICY_EVENT_NAME" == "workflow_dispatch" && + "$POLICY_MANUAL_USE_DEPOT" == "true" ) ) && + -n "$depot_runner" ]]; then + suffix= + if [[ "$POLICY_RUNNER_SIZE" != "default" ]]; then + suffix="-$POLICY_RUNNER_SIZE" + fi + runner="${depot_runner}${suffix}" + allow_depot_remote_cache=true + fi + + { + echo "runner=$runner" + echo "allow_depot_remote_cache=$allow_depot_remote_cache" + } >> "$GITHUB_OUTPUT" + + produce_linux_static_abi: + needs: runner_policy + if: ${{ inputs.produce_static_abi && endsWith(inputs.target, '-unknown-linux-gnu') }} + name: Produce matching static ABI + permissions: + contents: read + packages: read + uses: ./.github/workflows/static-abi-artifact.yml + with: + backend: ${{ inputs.backend }} + target: ${{ inputs.target }} + artifact_name: ${{ inputs.static_abi_artifact_name }} + runner_size: ${{ inputs.runner_size }} + retention_days: ${{ inputs.retention_days }} + timeout_minutes: ${{ inputs.timeout_minutes }} + + linux_native_sdk_artifact: + needs: [runner_policy, produce_linux_static_abi] + if: ${{ !cancelled() && needs.runner_policy.result == 'success' && endsWith(inputs.target, '-unknown-linux-gnu') && (needs.produce_linux_static_abi.result == 'success' || (needs.produce_linux_static_abi.result == 'skipped' && !inputs.produce_static_abi)) }} + name: Native SDK input (${{ inputs.target }}, ${{ inputs.backend }}) + runs-on: ${{ needs.runner_policy.outputs.runner }} + timeout-minutes: ${{ inputs.timeout_minutes }} + permissions: + contents: read + packages: read + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ github.token }} + defaults: + run: + shell: bash + env: + LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static + MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - name: Validate release preparation inputs + env: + RELEASE_TAG: ${{ inputs.release_tag }} + PREPARE_RELEASE_VERSION: ${{ inputs.prepare_release_version }} + PRODUCE_STATIC_ABI: ${{ inputs.produce_static_abi }} + STATIC_ABI_ARTIFACT_NAME: ${{ inputs.static_abi_artifact_name }} + run: | + set -euo pipefail + if [[ "$PREPARE_RELEASE_VERSION" == "true" && -z "$RELEASE_TAG" ]]; then + echo "release_tag is required for release preparation" >&2 + exit 1 + fi + if [[ "$PRODUCE_STATIC_ABI" == "true" && + -z "$STATIC_ABI_ARTIFACT_NAME" ]]; then + echo "static_abi_artifact_name is required when producing an ABI" >&2 + exit 1 + fi + + - name: Verify prebuilt native SDK environment + run: verify-runner-image public + + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} + + - name: Prepare patched llama.cpp checkout for ABI reuse + if: ${{ inputs.static_abi_artifact_name != '' }} + run: scripts/prepare-llama.sh pinned + + - name: Download immutable static ABI input + if: ${{ inputs.static_abi_artifact_name != '' }} + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ${{ inputs.static_abi_artifact_name }} + path: ${{ inputs.static_abi_artifact_path }} + + - name: Restore immutable static ABI input + if: ${{ inputs.static_abi_artifact_name != '' }} + env: + STATIC_ABI_ARTIFACT_PATH: ${{ inputs.static_abi_artifact_path }} + STATIC_ABI_TARGET: ${{ inputs.target }} + STATIC_ABI_BACKEND: ${{ inputs.backend }} + run: | + scripts/restore-static-abi-input.sh \ + "$STATIC_ABI_ARTIFACT_PATH" \ + "$LLAMA_STAGE_BUILD_DIR" \ + "$STATIC_ABI_TARGET" \ + "$STATIC_ABI_BACKEND" + + - name: Prepare dispatched release version + if: ${{ inputs.prepare_release_version }} + env: + RELEASE_TAG: ${{ inputs.release_tag }} + run: scripts/release-version.sh "$RELEASE_TAG" + + - name: Prepare immutable native SDK input + id: native-sdk + uses: ./.github/actions/prepare-native-sdk-input + with: + backend: ${{ inputs.backend }} + target: ${{ inputs.target }} + profile: ${{ inputs.profile }} + include_runtime_crate: ${{ inputs.include_runtime_crate }} + require_prebuilt_static_abi: ${{ inputs.static_abi_artifact_name != '' }} + + - name: Capture native SDK sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-native-sdk-${{ inputs.target }}-${{ inputs.backend }}-${{ inputs.profile }}-${{ github.run_attempt }} + + - name: Upload immutable native SDK input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ inputs.artifact_name }} + path: ${{ steps.native-sdk.outputs.upload_path }} + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} + + macos_native_sdk_artifact: + needs: runner_policy + if: ${{ needs.runner_policy.result == 'success' && endsWith(inputs.target, '-apple-darwin') }} + name: Native SDK input (${{ inputs.target }}, ${{ inputs.backend }}) + runs-on: ${{ needs.runner_policy.outputs.runner }} + timeout-minutes: ${{ inputs.timeout_minutes }} + permissions: + contents: read + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - name: Validate release preparation inputs + env: + RELEASE_TAG: ${{ inputs.release_tag }} + PREPARE_RELEASE_VERSION: ${{ inputs.prepare_release_version }} + run: | + set -euo pipefail + if [[ "$PREPARE_RELEASE_VERSION" == "true" && -z "$RELEASE_TAG" ]]; then + echo "release_tag is required for release preparation" >&2 + exit 1 + fi + + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} + + - name: Install macOS native SDK dependencies + run: brew install cmake ninja lld + + - name: Prepare dispatched release version + if: ${{ inputs.prepare_release_version }} + env: + RELEASE_TAG: ${{ inputs.release_tag }} + run: scripts/release-version.sh "$RELEASE_TAG" + + - name: Prepare immutable native SDK input + id: native-sdk + uses: ./.github/actions/prepare-native-sdk-input + with: + backend: ${{ inputs.backend }} + target: ${{ inputs.target }} + profile: ${{ inputs.profile }} + include_runtime_crate: ${{ inputs.include_runtime_crate }} + + - name: Capture native SDK sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-native-sdk-${{ inputs.target }}-${{ inputs.backend }}-${{ inputs.profile }}-${{ github.run_attempt }} + + - name: Upload immutable native SDK input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ inputs.artifact_name }} + path: ${{ steps.native-sdk.outputs.upload_path }} + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} diff --git a/.github/workflows/nightly-stability-run.yml b/.github/workflows/nightly-stability-run.yml index d2d4e7274d..8e9eb41c16 100644 --- a/.github/workflows/nightly-stability-run.yml +++ b/.github/workflows/nightly-stability-run.yml @@ -27,10 +27,6 @@ on: required: false default: "180" type: string - runs_on: - required: false - default: '"ubuntu-24.04"' - type: string output_dir: required: false default: nightly-artifacts/stability/latest @@ -42,7 +38,9 @@ permissions: jobs: stability: name: Run stability harness - runs-on: ${{ fromJson(inputs.runs_on) }} + # This reusable workflow checks out caller content, so it never accepts a + # caller-selected label for a privileged runner group. + runs-on: ubuntu-24.04 timeout-minutes: 120 env: MESH_STABILITY_BASE_URL: ${{ inputs.base_url }} @@ -58,6 +56,8 @@ jobs: steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Preflight configuration id: preflight diff --git a/.github/workflows/nightly-stability.yml b/.github/workflows/nightly-stability.yml index 3185159cfc..9a53742096 100644 --- a/.github/workflows/nightly-stability.yml +++ b/.github/workflows/nightly-stability.yml @@ -29,10 +29,6 @@ on: description: "Request timeout in seconds for each probe." required: false default: "180" - runs_on: - description: "JSON runner label string/array for the reusable workflow." - required: false - default: '"ubuntu-24.04"' output_dir: description: "Evidence output directory. Defaults to nightly-artifacts/stability/." required: false @@ -56,5 +52,4 @@ jobs: agent_smokes: ${{ github.event.inputs.agent_smokes || vars.MESH_NIGHTLY_STABILITY_AGENT_SMOKES || '' }} skip_streaming: ${{ github.event_name == 'workflow_dispatch' && inputs.skip_streaming || false }} timeout: ${{ github.event.inputs.timeout || vars.MESH_NIGHTLY_STABILITY_TIMEOUT || '180' }} - runs_on: ${{ github.event.inputs.runs_on || vars.MESH_NIGHTLY_STABILITY_RUNS_ON || '"ubuntu-24.04"' }} output_dir: ${{ github.event.inputs.output_dir || format('nightly-artifacts/stability/{0}', github.run_id) }} diff --git a/.github/workflows/pr_builds.yml b/.github/workflows/pr_builds.yml index 92652b0f5c..c1367d2727 100644 --- a/.github/workflows/pr_builds.yml +++ b/.github/workflows/pr_builds.yml @@ -32,6 +32,7 @@ env: CARGO_HTTP_MULTIPLEXING: "false" SCCACHE_GHA_ENABLED: "true" LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static + MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d jobs: changes: @@ -70,10 +71,12 @@ jobs: runner_4: ${{ steps.runners.outputs.runner_4 }} runner_8: ${{ steps.runners.outputs.runner_8 }} runner_16: ${{ steps.runners.outputs.runner_16 }} + required_jobs_json: ${{ steps.plan.outputs.required_jobs_json }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: fetch-depth: 0 + persist-credentials: false - name: Select trusted Linux runners id: runners uses: ./.github/actions/select-ci-runners @@ -118,6 +121,38 @@ jobs: event_name: ${{ github.event_name }} base_sha: ${{ github.event.pull_request.base.sha || '' }} head_sha: ${{ github.event.pull_request.head.sha || '' }} + - name: Plan required PR Builds jobs + id: plan + env: + PR_BUILD_PLAN_INPUT: >- + { + "event_name": "${{ github.event_name }}", + "all_rust": ${{ steps.compute.outputs.all_rust == 'true' && 'true' || 'false' }}, + "backend_changed": ${{ steps.compute.outputs.backend_changed == 'true' && 'true' || 'false' }}, + "benchmarks": ${{ steps.filter.outputs.benchmarks == 'true' && 'true' || 'false' }}, + "docs_only": ${{ steps.compute.outputs.docs_only == 'true' && 'true' || 'false' }}, + "inference_artifact_required": ${{ steps.compute.outputs.inference_artifact_required == 'true' && 'true' || 'false' }}, + "linux_test_groups_nonempty": ${{ steps.compute.outputs.linux_test_groups_json != '[]' && 'true' || 'false' }}, + "live_agent_enabled": ${{ (vars.MESH_AGENT_BASE_URL != '' || vars.MESH_OPENCODE_BASE_URL != '') && 'true' || 'false' }}, + "runner_contract_required": ${{ steps.compute.outputs.runner_contract_required == 'true' && 'true' || 'false' }}, + "sdk_smoke_required": ${{ steps.compute.outputs.sdk_smoke_required == 'true' && 'true' || 'false' }}, + "test_batches_nonempty": ${{ steps.compute.outputs.test_batches_json != '[]' && 'true' || 'false' }}, + "windows_cpu_required": ${{ steps.compute.outputs.windows_cpu_build_required == 'true' && 'true' || 'false' }}, + "windows_gpu_required": ${{ steps.compute.outputs.windows_gpu_build_required == 'true' && 'true' || 'false' }}, + "affected_crates": ${{ steps.compute.outputs.affected_crates || '[]' }} + } + run: | + set -euo pipefail + required_jobs_json="$( + python3 scripts/plan-pr-build-jobs.py <<<"$PR_BUILD_PLAN_INPUT" + )" + echo "required_jobs_json=$required_jobs_json" >> "$GITHUB_OUTPUT" + + { + echo "## PR Builds execution plan" + echo + jq -r '.[] | "- `\(.)`"' <<<"$required_jobs_json" + } >> "$GITHUB_STEP_SUMMARY" - name: Compute UI dist cache key id: ui_key run: | @@ -126,7 +161,7 @@ jobs: public_runner_image_contract: needs: changes - if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.runner_contract_required == 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'public_runner_image_contract') }} name: Runner image / GitHub-hosted public runs-on: ${{ needs.changes.outputs.runner_4 }} container: @@ -150,7 +185,7 @@ jobs: linux_host_input: needs: changes - if: ${{ (needs.changes.outputs.linux_inference_artifact_required == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_host_input') }} name: Linux immutable host runs-on: ${{ needs.changes.outputs.runner_8 }} container: @@ -167,6 +202,8 @@ jobs: RUSTC_WRAPPER: sccache steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Verify prebuilt CI environment run: verify-runner-image public @@ -250,7 +287,7 @@ jobs: linux_cpu_runtime_input: needs: changes - if: ${{ needs.changes.outputs.linux_inference_artifact_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_cpu_runtime_input') }} name: Linux CPU runtime runs-on: ${{ needs.changes.outputs.runner_8 }} container: @@ -299,7 +336,7 @@ jobs: linux_cpu_artifact: needs: [changes, linux_host_input, linux_cpu_runtime_input] - if: ${{ needs.linux_host_input.result == 'success' && needs.linux_cpu_runtime_input.result == 'success' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_cpu_artifact') }} name: Linux CPU product runs-on: ${{ needs.changes.outputs.runner_4 }} container: @@ -345,7 +382,7 @@ jobs: linux_cuda_runtime_input: needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_cuda_runtime_input') }} name: Linux CUDA slim runtime runs-on: ${{ needs.changes.outputs.runner_8 }} container: @@ -395,7 +432,7 @@ jobs: linux_cuda_product: needs: [changes, linux_host_input, linux_cuda_runtime_input] - if: ${{ needs.linux_host_input.result == 'success' && needs.linux_cuda_runtime_input.result == 'success' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_cuda_product') }} name: Linux CUDA slim runs-on: ${{ needs.changes.outputs.runner_4 }} container: @@ -441,7 +478,7 @@ jobs: linux_rocm_runtime_input: needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_rocm_runtime_input') }} name: Linux ROCm slim runtime runs-on: ${{ needs.changes.outputs.runner_8 }} container: @@ -490,7 +527,7 @@ jobs: linux_rocm_product: needs: [changes, linux_host_input, linux_rocm_runtime_input] - if: ${{ needs.linux_host_input.result == 'success' && needs.linux_rocm_runtime_input.result == 'success' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_rocm_product') }} name: Linux ROCm slim runs-on: ${{ needs.changes.outputs.runner_4 }} container: @@ -536,7 +573,7 @@ jobs: linux_vulkan_runtime_input: needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend_changed == 'true' || needs.changes.outputs.benchmarks == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_vulkan_runtime_input') }} name: Linux Vulkan runtime runs-on: ${{ needs.changes.outputs.runner_8 }} container: @@ -591,7 +628,7 @@ jobs: linux_vulkan_product: needs: [changes, linux_host_input, linux_vulkan_runtime_input] - if: ${{ needs.linux_host_input.result == 'success' && needs.linux_vulkan_runtime_input.result == 'success' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_vulkan_product') }} name: Linux Vulkan runs-on: ${{ needs.changes.outputs.runner_4 }} container: @@ -637,62 +674,22 @@ jobs: linux_static_abi_input: needs: changes - if: ${{ needs.changes.outputs.docs_only != 'true' && (needs.changes.outputs.test_batches_json != '[]' || needs.changes.outputs.linux_test_groups_json != '[]') }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_static_abi_input') }} name: Linux static ABI input - runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: contents: read packages: read - container: - image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - defaults: - run: - shell: bash - env: - LLAMA_STAGE_BACKEND: cpu - LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static - MESH_LLM_REQUIRE_SCCACHE: "1" - steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - persist-credentials: false - - name: Verify prebuilt CI environment - run: verify-runner-image public - - uses: ./.github/actions/configure-sccache-gha - with: - allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} - - name: Cache patched llama.cpp ABI build - id: static_abi_cache - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 - with: - path: .deps/llama.cpp/build-stage-abi-static - key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-cpu-${{ hashFiles('scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - - name: Prepare patched llama.cpp ABI checkout - if: ${{ steps.static_abi_cache.outputs.cache-hit != 'true' }} - run: scripts/prepare-llama.sh pinned - - name: Build patched llama.cpp ABI libraries - if: ${{ steps.static_abi_cache.outputs.cache-hit != 'true' }} - run: scripts/build-llama.sh - - name: Archive immutable static ABI input - run: | - test -d "$LLAMA_STAGE_BUILD_DIR" - tar -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" \ - -czf "$RUNNER_TEMP/mesh-llm-static-abi.tar.gz" \ - "$(basename "$LLAMA_STAGE_BUILD_DIR")" - - name: Upload immutable static ABI input - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: pr-linux-static-abi-input - path: ${{ runner.temp }}/mesh-llm-static-abi.tar.gz - if-no-files-found: error - retention-days: 1 + uses: ./.github/workflows/static-abi-artifact.yml + with: + backend: cpu + target: x86_64-unknown-linux-gnu + artifact_name: pr-linux-static-abi-input + runner_size: '8' + retention_days: 1 rust_crate_tests: needs: [changes, linux_static_abi_input] - if: ${{ needs.linux_static_abi_input.result == 'success' && needs.changes.outputs.test_batches_json != '[]' && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'rust_crate_tests') }} name: Rust crate tests (${{ matrix.batch.idx }}) runs-on: ${{ needs.changes.outputs.runner_8 }} permissions: @@ -732,7 +729,8 @@ jobs: workspaces: . -> target cache-bin: "false" prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} - shared-key: pr-rust-crate-tests-${{ matrix.batch.idx }} + # Read the already-seeded main shard without allowing PR writes. + shared-key: main-rust-crate-tests-${{ matrix.batch.idx }} save-if: "false" - name: Prepare UI placeholder run: mkdir -p crates/mesh-llm-ui/dist && printf '' > crates/mesh-llm-ui/dist/index.html @@ -745,11 +743,7 @@ jobs: name: pr-linux-static-abi-input path: ${{ runner.temp }}/static-abi-input - name: Restore immutable static ABI input - run: | - mkdir -p "$(dirname "$LLAMA_STAGE_BUILD_DIR")" - tar -xzf "$RUNNER_TEMP/static-abi-input/mesh-llm-static-abi.tar.gz" \ - -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" - test -d "$LLAMA_STAGE_BUILD_DIR" + run: scripts/restore-static-abi-input.sh "$RUNNER_TEMP/static-abi-input" "$LLAMA_STAGE_BUILD_DIR" x86_64-unknown-linux-gnu cpu - name: Run crate tests env: TEST_CRATES: ${{ toJson(matrix.batch.crates) }} @@ -768,7 +762,7 @@ jobs: linux_test_groups: needs: [changes, linux_static_abi_input] - if: ${{ needs.linux_static_abi_input.result == 'success' && needs.changes.outputs.docs_only != 'true' && needs.changes.outputs.linux_test_groups_json != '[]' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_test_groups') }} name: Linux tests (${{ matrix.group }}) runs-on: ${{ needs.changes.outputs.runner_8 }} container: @@ -793,6 +787,8 @@ jobs: RUSTC_WRAPPER: sccache steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Verify prebuilt CI environment run: verify-runner-image public @@ -823,11 +819,7 @@ jobs: name: pr-linux-static-abi-input path: ${{ runner.temp }}/static-abi-input - name: Restore immutable static ABI input - run: | - mkdir -p "$(dirname "$LLAMA_STAGE_BUILD_DIR")" - tar -xzf "$RUNNER_TEMP/static-abi-input/mesh-llm-static-abi.tar.gz" \ - -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")" - test -d "$LLAMA_STAGE_BUILD_DIR" + run: scripts/restore-static-abi-input.sh "$RUNNER_TEMP/static-abi-input" "$LLAMA_STAGE_BUILD_DIR" x86_64-unknown-linux-gnu cpu - name: Restore Skippy smoke model cache if: ${{ matrix.group == 'skippy-smoke' }} @@ -849,8 +841,6 @@ jobs: if: ${{ matrix.group == 'skippy-smoke' }} timeout-minutes: 45 env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} WORK_DIR: ${{ runner.temp }}/skippy-ci-smoke MODEL_DIR: ${{ runner.temp }}/skippy-ci-smoke-models run: scripts/skippy-ci-smoke.sh @@ -870,11 +860,13 @@ jobs: linux_public_mesh_admission: needs: [changes, linux_cpu_artifact] - if: ${{ github.event_name == 'workflow_dispatch' && needs.linux_cpu_artifact.result == 'success' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'linux_public_mesh_admission') }} name: Linux public-mesh admission runs-on: ${{ needs.changes.outputs.runner_4 }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Install client-auto smoke dependencies run: sudo apt-get update && sudo apt-get install -y curl jq python3 - uses: ./.github/actions/restore-smoke-inputs @@ -887,30 +879,24 @@ jobs: hf_download_smoke: needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'model-artifact')) && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'hf_download_smoke') }} name: HuggingFace download smoke uses: ./.github/workflows/hf-download-smoke.yml with: - runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} timeout_minutes: 15 - secrets: - HF_TOKEN: ${{ secrets.HF_TOKEN }} inference_smoke_tests: needs: [changes, linux_cpu_artifact] - if: ${{ needs.linux_cpu_artifact.result == 'success' && needs.changes.outputs.linux_inference_artifact_required == 'true' && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'skippy-server') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'skippy-runtime') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'openai-frontend') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'model-artifact')) && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'inference_smoke_tests') }} uses: ./.github/workflows/smoke.yml with: artifact_name: ci-linux-inference-binaries mesh_binary_target: target/debug/mesh-llm cache_key_prefix: '' - runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} - secrets: - HF_TOKEN: ${{ secrets.HF_TOKEN }} agent_live_smokes: needs: [changes, linux_cpu_artifact] - if: ${{ needs.linux_cpu_artifact.result == 'success' && (vars.MESH_AGENT_BASE_URL != '' || vars.MESH_OPENCODE_BASE_URL != '') && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'openai-frontend') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-client')) && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'agent_live_smokes') }} runs-on: ${{ needs.changes.outputs.runner_4 }} timeout-minutes: 45 env: @@ -927,6 +913,8 @@ jobs: # Keep live-agent smoke on an explicit Node version so CI does not depend # on GitHub-hosted image defaults. - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 24 @@ -956,7 +944,7 @@ jobs: two_node_client_serving_smoke: needs: [changes, linux_cpu_artifact] - if: ${{ needs.linux_cpu_artifact.result == 'success' && needs.changes.outputs.linux_inference_artifact_required == 'true' && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'openai-frontend') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-client')) && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'two_node_client_serving_smoke') }} uses: ./.github/workflows/scripted-binary-smoke.yml with: artifact_name: ci-linux-inference-binaries @@ -964,14 +952,11 @@ jobs: staged_binary_path: target/debug/mesh-llm model_cache_scope: two-node-smoke-model smoke_script: scripts/ci-two-node-client-serving-smoke.sh - runs_on: ${{ toJson(needs.changes.outputs.runner_4) }} timeout_minutes: 20 - secrets: - HF_TOKEN: ${{ secrets.HF_TOKEN }} two_node_split_smoke: needs: [changes, linux_cpu_artifact] - if: ${{ needs.linux_cpu_artifact.result == 'success' && needs.changes.outputs.linux_inference_artifact_required == 'true' && (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'skippy-server') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'skippy-runtime') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'model-artifact')) && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'two_node_split_smoke') }} uses: ./.github/workflows/scripted-binary-smoke.yml with: artifact_name: ci-linux-inference-binaries @@ -979,14 +964,11 @@ jobs: staged_binary_path: target/debug/mesh-llm model_cache_scope: two-node-split-smoke-model smoke_script: scripts/ci-two-node-split-smoke.sh - runs_on: ${{ toJson(needs.changes.outputs.runner_4) }} timeout_minutes: 25 - secrets: - HF_TOKEN: ${{ secrets.HF_TOKEN }} rust_sdk_smoke: needs: [changes, linux_cpu_artifact] - if: ${{ needs.linux_cpu_artifact.result == 'success' && needs.changes.outputs.linux_inference_artifact_required == 'true' && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'rust_sdk_smoke') }} uses: ./.github/workflows/sdk-smoke.yml with: sdk_kind: rust @@ -994,27 +976,56 @@ jobs: artifact_path: ci-artifacts/linux staged_binary_path: target/debug/mesh-llm model_cache_scope: sdk-smoke-model - runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} - secrets: - HF_TOKEN: ${{ secrets.HF_TOKEN }} + + kotlin_sdk_input: + name: Kotlin native SDK input + needs: [changes, linux_static_abi_input] + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'kotlin_sdk_input') }} + permissions: + contents: read + packages: read + uses: ./.github/workflows/native-sdk-artifact.yml + with: + backend: cpu + target: x86_64-unknown-linux-gnu + profile: debug + artifact_name: pr-kotlin-native-sdk-input + static_abi_artifact_name: pr-linux-static-abi-input + static_abi_artifact_path: native-sdk-static-abi-input + runner_size: '8' + retention_days: 1 kotlin_sdk_smoke: - needs: [changes, linux_cpu_artifact] - if: ${{ needs.linux_cpu_artifact.result == 'success' && needs.changes.outputs.linux_inference_artifact_required == 'true' && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + needs: [changes, linux_cpu_artifact, kotlin_sdk_input] + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'kotlin_sdk_smoke') }} uses: ./.github/workflows/sdk-smoke.yml with: sdk_kind: kotlin artifact_name: ci-linux-inference-binaries artifact_path: ci-artifacts/linux staged_binary_path: target/debug/mesh-llm + kotlin_artifact_name: pr-kotlin-native-sdk-input + kotlin_artifact_target: x86_64-unknown-linux-gnu + kotlin_artifact_backend: cpu + kotlin_artifact_profile: debug model_cache_scope: sdk-smoke-model - runs_on: ${{ toJson(needs.changes.outputs.runner_8) }} - secrets: - HF_TOKEN: ${{ secrets.HF_TOKEN }} + + swift_sdk_input: + name: Swift SDK host input + needs: changes + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'swift_sdk_input') }} + permissions: + contents: read + uses: ./.github/workflows/swift-sdk-artifact.yml + with: + mode: host-only + artifact_name: pr-swift-sdk-input + retention_days: 1 + timeout_minutes: 45 macos_host_input: needs: changes - if: ${{ needs.changes.outputs.macos_inference_artifact_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'macos_host_input') }} name: macOS immutable host runs-on: macos-15 steps: @@ -1024,7 +1035,7 @@ jobs: - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} with: - version: latest + version: 10 - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.ui == 'true' }} with: @@ -1088,7 +1099,7 @@ jobs: macos_metal_runtime_input: needs: changes - if: ${{ needs.changes.outputs.macos_inference_artifact_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'macos_metal_runtime_input') }} name: macOS Metal runtime runs-on: macos-15 env: @@ -1125,7 +1136,7 @@ jobs: macos_cpu_artifact: needs: [changes, macos_host_input, macos_metal_runtime_input] - if: ${{ needs.macos_host_input.result == 'success' && needs.macos_metal_runtime_input.result == 'success' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'macos_cpu_artifact') }} name: macOS CPU product runs-on: macos-15 steps: @@ -1159,23 +1170,22 @@ jobs: retention-days: 1 swift_sdk_smoke: - needs: [changes, macos_cpu_artifact, macos_unit_tests] - if: ${{ !cancelled() && needs.macos_cpu_artifact.result == 'success' && (needs.macos_unit_tests.result == 'success' || needs.macos_unit_tests.result == 'skipped') && needs.changes.outputs.macos_inference_artifact_required == 'true' && needs.changes.outputs.sdk_smoke_required == 'true' && needs.changes.outputs.docs_only != 'true' }} + needs: [changes, macos_cpu_artifact, swift_sdk_input] + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'swift_sdk_smoke') }} uses: ./.github/workflows/sdk-smoke.yml with: sdk_kind: swift artifact_name: ci-macos-inference-binaries artifact_path: ci-artifacts/macos staged_binary_path: target/debug/mesh-llm + swift_artifact_name: pr-swift-sdk-input + swift_artifact_mode: host-only model_cache_scope: sdk-smoke-model - runs_on: '"macos-15"' - timeout_minutes: 40 - secrets: - HF_TOKEN: ${{ secrets.HF_TOKEN }} + timeout_minutes: 25 macos_unit_tests: needs: changes - if: ${{ needs.changes.outputs.docs_only != 'true' && (needs.changes.outputs.all_rust == 'true' || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'model-artifact') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm-host-runtime') || contains(fromJson(needs.changes.outputs.affected_crates || '[]'), 'mesh-llm')) }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'macos_unit_tests') }} name: macOS unit tests runs-on: macos-15 env: @@ -1241,7 +1251,7 @@ jobs: windows_checks: needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.all_rust == 'true' || needs.changes.outputs.windows_cpu == 'true' || needs.changes.outputs.windows_gpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'windows_checks') }} name: Windows lightweight checks runs-on: windows-2022 env: @@ -1276,7 +1286,7 @@ jobs: windows_host_input: needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_cpu == 'true' || needs.changes.outputs.windows_gpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'windows_host_input') }} name: Windows immutable debug host runs-on: windows-2022 env: @@ -1318,7 +1328,7 @@ jobs: windows_cpu_runtime_input: needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_cpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'windows_cpu_runtime_input') }} name: Windows CPU runtime input runs-on: windows-2022 env: @@ -1369,7 +1379,7 @@ jobs: windows_gpu_runtime_inputs: needs: changes - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_gpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'windows_gpu_runtime_inputs') }} name: Windows ${{ matrix.name }} runtime input runs-on: windows-2022 strategy: @@ -1490,7 +1500,7 @@ jobs: windows_cpu_product: needs: [changes, windows_host_input, windows_cpu_runtime_input] - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_cpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'windows_cpu_product') }} name: Windows CPU runs-on: windows-2022 steps: @@ -1530,7 +1540,7 @@ jobs: windows_gpu_products: needs: [changes, windows_host_input, windows_gpu_runtime_inputs] - if: ${{ (github.event_name == 'workflow_dispatch' || needs.changes.outputs.windows_gpu == 'true') && needs.changes.outputs.docs_only != 'true' }} + if: ${{ contains(fromJson(needs.changes.outputs.required_jobs_json), 'windows_gpu_products') }} name: Windows ${{ matrix.name }} runs-on: windows-2022 strategy: @@ -1577,3 +1587,125 @@ jobs: path: ${{ steps.compose.outputs.archive_path }} if-no-files-found: error retention-days: 1 + + summary: + name: PR Builds Summary + needs: + - changes + - public_runner_image_contract + - linux_host_input + - linux_cpu_runtime_input + - linux_cpu_artifact + - linux_cuda_runtime_input + - linux_cuda_product + - linux_rocm_runtime_input + - linux_rocm_product + - linux_vulkan_runtime_input + - linux_vulkan_product + - linux_static_abi_input + - rust_crate_tests + - linux_test_groups + - linux_public_mesh_admission + - hf_download_smoke + - inference_smoke_tests + - agent_live_smokes + - two_node_client_serving_smoke + - two_node_split_smoke + - rust_sdk_smoke + - kotlin_sdk_input + - kotlin_sdk_smoke + - swift_sdk_input + - macos_host_input + - macos_metal_runtime_input + - macos_cpu_artifact + - swift_sdk_smoke + - macos_unit_tests + - windows_checks + - windows_host_input + - windows_cpu_runtime_input + - windows_gpu_runtime_inputs + - windows_cpu_product + - windows_gpu_products + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + permissions: {} + timeout-minutes: 5 + steps: + - name: Check PR Builds gate + env: + NEEDS_RESULTS: ${{ toJson(needs) }} + REQUIRED_JOBS: ${{ needs.changes.outputs.required_jobs_json }} + run: | + set -euo pipefail + + if ! jq -e ' + type == "array" + and all(.[]; type == "string" and length > 0) + and (length == (unique | length)) + ' <<<"$REQUIRED_JOBS" >/dev/null; then + echo "ERROR: required_jobs_json is not a unique JSON string array." >&2 + exit 1 + fi + + if ! jq -e --argjson required "$REQUIRED_JOBS" ' + . as $needs + | all( + $required[]; + . as $job + | $needs + | has($job) + ) + ' <<<"$NEEDS_RESULTS" >/dev/null; then + echo "ERROR: required_jobs_json names a job outside the summary needs graph." >&2 + exit 1 + fi + + { + echo "## PR Builds" + echo + echo "| Job | Required | Result |" + echo "| --- | --- | --- |" + jq -r --argjson required "$REQUIRED_JOBS" ' + to_entries + | sort_by(.key)[] + | . as $entry + | ( + if $entry.key == "changes" + or ($required | index($entry.key)) != null + then "yes" + else "no" + end + ) as $is_required + | "| \($entry.key) | \($is_required) | \($entry.value.result) |" + ' <<<"$NEEDS_RESULTS" + } >> "$GITHUB_STEP_SUMMARY" + + if ! jq -e --argjson required "$REQUIRED_JOBS" ' + def accepted: + . as $entry + | $entry.value.result == "success" + or ( + $entry.key != "changes" + and $entry.value.result == "skipped" + and ($required | index($entry.key)) == null + ); + + all(to_entries[]; accepted) + ' <<<"$NEEDS_RESULTS" >/dev/null; then + echo "ERROR: A required PR Builds job did not succeed, or an unplanned job failed." >&2 + jq -r --argjson required "$REQUIRED_JOBS" ' + def accepted: + . as $entry + | $entry.value.result == "success" + or ( + $entry.key != "changes" + and $entry.value.result == "skipped" + and ($required | index($entry.key)) == null + ); + + to_entries[] + | select(accepted | not) + | " \(.key)=\(.value.result)" + ' <<<"$NEEDS_RESULTS" >&2 + exit 1 + fi diff --git a/.github/workflows/pr_quality.yml b/.github/workflows/pr_quality.yml index 5a934e594a..d9cd02bd64 100644 --- a/.github/workflows/pr_quality.yml +++ b/.github/workflows/pr_quality.yml @@ -55,6 +55,7 @@ jobs: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: fetch-depth: 0 + persist-credentials: false - name: Select trusted Linux runners id: runners uses: ./.github/actions/select-ci-runners @@ -77,6 +78,8 @@ jobs: runs-on: ${{ needs.changes.outputs.runner_4 }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: @@ -110,6 +113,8 @@ jobs: runs-on: ${{ needs.changes.outputs.runner_4 }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 with: components: rustfmt @@ -138,6 +143,8 @@ jobs: RUSTC_WRAPPER: sccache steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Verify prebuilt CI environment run: verify-runner-image public - uses: ./.github/actions/configure-sccache-gha @@ -184,6 +191,8 @@ jobs: working-directory: crates/mesh-llm-ui steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 with: version: 10 diff --git a/.github/workflows/pr_website.yml b/.github/workflows/pr_website.yml index 515fff2487..7469310e01 100644 --- a/.github/workflows/pr_website.yml +++ b/.github/workflows/pr_website.yml @@ -24,6 +24,7 @@ jobs: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/actions/compute-changes id: compute with: @@ -40,6 +41,8 @@ jobs: working-directory: website steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version: 24 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 56ceb3b385..c9da391cab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -318,14 +318,15 @@ jobs: mesh_binary_target: target/release/mesh-llm cache_key_prefix: release- release_tag: ${{ needs.metadata.outputs.tag }} - runs_on: ${{ toJson(needs.metadata.outputs.runner_8) }} secrets: HF_TOKEN: ${{ secrets.HF_TOKEN }} build_native_sdk_runtime: name: Build native SDK runtime ${{ matrix.name }} needs: metadata - runs-on: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && needs.metadata.outputs.runner_8 || matrix.target == 'aarch64-unknown-linux-gnu' && needs.metadata.outputs.runner_arm_8 || matrix.os }} + permissions: + contents: read + packages: read strategy: fail-fast: false matrix: @@ -345,85 +346,21 @@ jobs: backend: cpu target: aarch64-unknown-linux-gnu artifact_suffix: linux-aarch64-cpu - env: - LLAMA_STAGE_BACKEND: ${{ matrix.backend }} - MESH_NATIVE_SDK_TARGET: ${{ matrix.target }} - steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - persist-credentials: false - - - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - - - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - - uses: ./.github/actions/configure-sccache-gha - with: - allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} - - - name: Install Linux dependencies - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y build-essential cmake ninja-build pkg-config libssl-dev libdbus-1-dev curl lld - - - name: Install macOS dependencies - if: runner.os == 'macOS' - run: brew install cmake ninja lld - - - name: Prepare dispatched release version - if: github.event_name == 'workflow_dispatch' - env: - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} - run: scripts/release-version.sh "$RELEASE_TAG" - - - name: Package native SDK runtime - run: | - scripts/package-native-sdk.sh \ - --build \ - --backend "${{ matrix.backend }}" \ - --target "${{ matrix.target }}" \ - --out dist/native-sdk - - - name: Verify native SDK runtime artifact - run: scripts/verify-native-sdk-package.sh dist/native-sdk/*.tar.gz - - - name: Package native SDK runtime crate - run: scripts/package-native-sdk-crate.sh --out dist/native-sdk-crates dist/native-sdk/*.tar.gz - - - name: Stage flat native SDK release assets - shell: bash - run: | - set -euo pipefail - shopt -s nullglob - runtime_archives=(dist/native-sdk/*.tar.gz) - runtime_checksums=(dist/native-sdk/*.sha256) - runtime_crates=(dist/native-sdk-crates/*/target/package/*.crate) - if [[ "${#runtime_archives[@]}" -ne 1 || - "${#runtime_checksums[@]}" -ne 1 || - "${#runtime_crates[@]}" -ne 1 ]]; then - echo "expected one native SDK archive, checksum, and crate" >&2 - exit 1 - fi - - staging_dir="release-native-sdk-assets" - rm -rf "$staging_dir" - mkdir -p "$staging_dir" - for source in \ - "${runtime_archives[@]}" \ - "${runtime_checksums[@]}" \ - "${runtime_crates[@]}"; do - destination="$staging_dir/$(basename "$source")" - if [[ -e "$destination" ]]; then - echo "native SDK release asset basename collision: $destination" >&2 - exit 1 - fi - cp -p "$source" "$destination" - done - - - name: Upload native SDK runtime - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: release-native-sdk-${{ matrix.artifact_suffix }} - path: release-native-sdk-assets/* - if-no-files-found: error + uses: ./.github/workflows/native-sdk-artifact.yml + with: + backend: ${{ matrix.backend }} + target: ${{ matrix.target }} + profile: release + artifact_name: release-native-sdk-${{ matrix.artifact_suffix }} + static_abi_artifact_name: ci-release-native-sdk-static-abi-${{ matrix.artifact_suffix }} + static_abi_artifact_path: native-sdk-static-abi-input + produce_static_abi: ${{ endsWith(matrix.target, '-unknown-linux-gnu') }} + runner_size: '8' + include_runtime_crate: true + release_tag: ${{ needs.metadata.outputs.tag }} + prepare_release_version: ${{ github.event_name == 'workflow_dispatch' }} + retention_days: 90 + timeout_minutes: 90 build_native_runtime: name: Build native runtime ${{ matrix.name }} @@ -722,85 +659,18 @@ jobs: build_swift_sdk_artifact: name: Build Swift SDK XCFramework needs: metadata - runs-on: macos-15 - env: - LLAMA_STAGE_BACKEND: metal - steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - with: - persist-credentials: false - - - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 - with: - version: 10 - - - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 - with: - node-version: 24 - cache: pnpm - cache-dependency-path: | - .github/cache-version.txt - crates/mesh-llm-ui/pnpm-lock.yaml - - - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - - - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - - - name: Install macOS dependencies - run: brew install cmake ninja lld - - - name: Prepare dispatched release version - if: github.event_name == 'workflow_dispatch' - env: - RELEASE_TAG: ${{ needs.metadata.outputs.tag }} - run: scripts/release-version.sh "$RELEASE_TAG" - - - name: Prepare Swift console resources - run: | - scripts/package-sdk-console-assets.sh --sdk swift - scripts/verify-sdk-console-assets.sh --sdk swift - - - name: Verify tagged Swift console resources - if: github.event_name != 'workflow_dispatch' - run: git ls-files --error-unmatch sdk/swift/Sources/MeshLLM/Resources/Console/index.html - - - name: Build SwiftPM binary artifact - run: | - sdk/swift/scripts/build-xcframework.sh - mkdir -p dist - rm -f dist/MeshLLMFFI.xcframework.zip - ditto -c -k --sequesterRsrc --keepParent \ - sdk/swift/Generated/MeshLLMFFI.xcframework \ - dist/MeshLLMFFI.xcframework.zip - - - name: Prepare SwiftPM manifest for dispatched release - if: github.event_name == 'workflow_dispatch' - run: | - scripts/update-swift-package-manifest.sh \ - "${{ needs.metadata.outputs.tag }}" \ - dist/MeshLLMFFI.xcframework.zip - - - name: Verify SwiftPM binary artifact - run: | - scripts/verify-swift-release-artifact.sh dist/MeshLLMFFI.xcframework.zip - scripts/verify-swift-package-manifest.sh \ - "${{ needs.metadata.outputs.tag }}" \ - dist/MeshLLMFFI.xcframework.zip - - - name: Upload generated SwiftPM manifest - if: github.event_name == 'workflow_dispatch' - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: swift-package-manifest - path: Package.swift - if-no-files-found: error - - - name: Upload SwiftPM binary artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: release-swift-sdk - path: dist/MeshLLMFFI.xcframework.zip - if-no-files-found: error + permissions: + contents: read + uses: ./.github/workflows/swift-sdk-artifact.yml + with: + mode: full + artifact_name: release-swift-sdk + release_tag: ${{ needs.metadata.outputs.tag }} + prepare_release_version: ${{ github.event_name == 'workflow_dispatch' }} + update_release_manifest: ${{ github.event_name == 'workflow_dispatch' }} + require_tracked_console: ${{ github.event_name != 'workflow_dispatch' }} + retention_days: 90 + timeout_minutes: 90 build_linux_arm64: name: Build immutable host Linux ARM64 @@ -1729,6 +1599,13 @@ jobs: name: swift-package-manifest path: generated-swift-manifest + - name: Download generated Swift binding + if: github.event_name == 'workflow_dispatch' + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: generated-swift-binding-release-swift-sdk + path: generated-swift-binding + - name: Prepare dispatched release tag if: github.event_name == 'workflow_dispatch' env: @@ -1741,6 +1618,14 @@ jobs: fi scripts/release-version.sh "$RELEASE_TAG" cp generated-swift-manifest/Package.swift Package.swift + generated_binding="generated-swift-binding/mesh_ffi.swift" + tracked_binding="sdk/swift/Sources/MeshLLM/Generated/mesh_ffi.swift" + if [[ ! -f "$generated_binding" || -L "$generated_binding" ]]; then + echo "generated Swift binding artifact is missing or unsafe: $generated_binding" >&2 + exit 1 + fi + install -m 0644 "$generated_binding" "$tracked_binding" + cmp "$generated_binding" "$tracked_binding" scripts/package-sdk-console-assets.sh --sdk all scripts/verify-sdk-console-assets.sh --sdk all git config user.name "github-actions[bot]" diff --git a/.github/workflows/scripted-binary-smoke.yml b/.github/workflows/scripted-binary-smoke.yml index 316125129b..540b361ab4 100644 --- a/.github/workflows/scripted-binary-smoke.yml +++ b/.github/workflows/scripted-binary-smoke.yml @@ -36,10 +36,6 @@ on: smoke_script: required: true type: string - runs_on: - required: false - default: '"ubuntu-24.04"' - type: string timeout_minutes: required: false default: 20 @@ -52,16 +48,22 @@ env: MODEL_URL: ${{ inputs.model_url }} MODEL_FILE: ${{ inputs.model_file }} +permissions: + contents: read + jobs: scripted_binary_smoke: name: Scripted Binary Smoke - runs-on: ${{ fromJson(inputs.runs_on) }} + # Model credentials stay on an isolated GitHub-hosted runner. + runs-on: ubuntu-24.04 timeout-minutes: ${{ inputs.timeout_minutes }} env: HF_TOKEN: ${{ secrets.HF_TOKEN }} HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Install smoke dependencies run: sudo apt-get update && sudo apt-get install -y curl jq lsof diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index ebb1b3d6ef..8720f6311e 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -19,6 +19,38 @@ on: staged_binary_path: required: true type: string + swift_artifact_name: + required: false + default: '' + type: string + swift_artifact_path: + required: false + default: sdk-artifacts/swift + type: string + swift_artifact_mode: + required: false + default: '' + type: string + kotlin_artifact_name: + required: false + default: '' + type: string + kotlin_artifact_path: + required: false + default: sdk-artifacts/kotlin + type: string + kotlin_artifact_target: + required: false + default: '' + type: string + kotlin_artifact_backend: + required: false + default: '' + type: string + kotlin_artifact_profile: + required: false + default: '' + type: string model_url: required: false default: https://huggingface.co/unsloth/SmolLM2-135M-Instruct-GGUF/resolve/9e6855bc4be717fca1ef21360a1db4b29d5c559a/SmolLM2-135M-Instruct-Q8_0.gguf @@ -34,9 +66,6 @@ on: required: false default: '' type: string - runs_on: - required: true - type: string timeout_minutes: required: false default: 30 @@ -52,13 +81,106 @@ env: jobs: sdk_smoke: name: ${{ inputs.sdk_kind }} SDK Smoke - runs-on: ${{ fromJson(inputs.runs_on) }} + # Swift stays on the same pinned Apple toolchain boundary as its producer. + # Kotlin maps its validated artifact architecture to a bounded hosted + # runner; every other Linux SDK smoke stays on hosted x86 Ubuntu. + runs-on: ${{ inputs.sdk_kind == 'swift' && 'macos-15' || inputs.sdk_kind == 'kotlin' && inputs.kotlin_artifact_target == 'aarch64-unknown-linux-gnu' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} timeout-minutes: ${{ inputs.timeout_minutes }} + permissions: + contents: read env: HF_TOKEN: ${{ secrets.HF_TOKEN }} HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - name: Validate bounded SDK smoke inputs + shell: bash + env: + SDK_KIND: ${{ inputs.sdk_kind }} + run: | + set -euo pipefail + case "$SDK_KIND" in + rust|kotlin|swift) ;; + *) + echo "unsupported SDK smoke kind: $SDK_KIND" >&2 + exit 1 + ;; + esac + + - name: Validate Swift SDK producer contract + if: ${{ inputs.sdk_kind == 'swift' }} + env: + SWIFT_ARTIFACT_NAME: ${{ inputs.swift_artifact_name }} + SWIFT_ARTIFACT_MODE: ${{ inputs.swift_artifact_mode }} + run: | + set -euo pipefail + if [[ -z "$SWIFT_ARTIFACT_NAME" ]]; then + echo "swift_artifact_name is required for Swift SDK smoke" >&2 + exit 1 + fi + case "$SWIFT_ARTIFACT_MODE" in + host-only|full) ;; + *) + echo "unsupported Swift SDK artifact mode: $SWIFT_ARTIFACT_MODE" >&2 + exit 1 + ;; + esac + + - name: Download immutable Swift SDK input + if: ${{ inputs.sdk_kind == 'swift' }} + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ${{ inputs.swift_artifact_name }} + path: ${{ inputs.swift_artifact_path }} + + - name: Download immutable generated Swift binding + if: ${{ inputs.sdk_kind == 'swift' }} + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: generated-swift-binding-${{ inputs.swift_artifact_name }} + path: sdk-artifacts/swift-generated-binding + + - name: Validate Kotlin SDK producer contract + if: ${{ inputs.sdk_kind == 'kotlin' }} + env: + KOTLIN_ARTIFACT_NAME: ${{ inputs.kotlin_artifact_name }} + KOTLIN_ARTIFACT_TARGET: ${{ inputs.kotlin_artifact_target }} + KOTLIN_ARTIFACT_BACKEND: ${{ inputs.kotlin_artifact_backend }} + KOTLIN_ARTIFACT_PROFILE: ${{ inputs.kotlin_artifact_profile }} + run: | + set -euo pipefail + if [[ -z "$KOTLIN_ARTIFACT_NAME" ]]; then + echo "kotlin_artifact_name is required for Kotlin SDK smoke" >&2 + exit 1 + fi + case "$KOTLIN_ARTIFACT_TARGET" in + x86_64-unknown-linux-gnu|aarch64-unknown-linux-gnu) ;; + *) + echo "unsupported Kotlin native SDK target: $KOTLIN_ARTIFACT_TARGET" >&2 + exit 1 + ;; + esac + if [[ "$KOTLIN_ARTIFACT_BACKEND" != "cpu" ]]; then + echo "Kotlin smoke requires the CPU native SDK input" >&2 + exit 1 + fi + case "$KOTLIN_ARTIFACT_PROFILE" in + debug|release) ;; + *) + echo "unsupported Kotlin native SDK profile: $KOTLIN_ARTIFACT_PROFILE" >&2 + exit 1 + ;; + esac + + - name: Download immutable Kotlin native SDK input + if: ${{ inputs.sdk_kind == 'kotlin' }} + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ${{ inputs.kotlin_artifact_name }} + path: ${{ inputs.kotlin_artifact_path }} - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 if: ${{ inputs.sdk_kind == 'rust' }} @@ -82,26 +204,14 @@ jobs: cache-dependency-path: crates/mesh-llm-ui/pnpm-lock.yaml - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + if: ${{ inputs.sdk_kind == 'rust' }} - name: Install Linux SDK dependencies - if: ${{ runner.os == 'Linux' }} + if: ${{ runner.os == 'Linux' && inputs.sdk_kind == 'rust' }} run: sudo apt-get update && sudo apt-get install -y build-essential libdbus-1-dev curl jq lsof lld patchelf - - name: Install macOS SDK dependencies - if: ${{ runner.os == 'macOS' }} - # `lld` is required because the Swift smoke job's transitive - # cargo build (via `sdk/swift/scripts/generate-swift-bindings.sh` - # in a temp dir) ends up invoking `cc` with - # `-fuse-ld=/opt/homebrew/bin/ld64.lld`. Apple clang accepts - # that flag only when the binary actually exists on disk; - # without `lld` installed the link step fails with - # `clang: error: invalid linker name in argument`. The macOS - # host and unit-test jobs install `lld` for the same - # reason. Install it here too so the Swift smoke lane matches. - run: brew install jq lld - - name: Configure Linux Rust linker - if: ${{ runner.os == 'Linux' }} + if: ${{ runner.os == 'Linux' && inputs.sdk_kind == 'rust' }} run: | mkdir -p .cargo cat > .cargo/config.toml <<'EOF' @@ -109,6 +219,26 @@ jobs: rustflags = ["-C", "link-arg=-fuse-ld=lld"] EOF + - name: Verify Kotlin smoke runtime utilities + if: ${{ runner.os == 'Linux' && inputs.sdk_kind == 'kotlin' }} + run: | + set -euo pipefail + for utility in \ + bash \ + curl \ + ldd \ + pgrep \ + python3 \ + readelf \ + sort \ + tail \ + xargs; do + command -v "$utility" >/dev/null || { + echo "missing Kotlin smoke runtime utility: $utility" >&2 + exit 1 + } + done + - uses: ./.github/actions/restore-smoke-inputs with: artifact_name: ${{ inputs.artifact_name }} @@ -127,8 +257,28 @@ jobs: - name: Kotlin SDK smoke test if: ${{ inputs.sdk_kind == 'kotlin' }} - run: scripts/ci-kotlin-sdk-smoke.sh "${{ inputs.staged_binary_path }}" "${{ inputs.artifact_path }}" "$HOME/.models/${{ inputs.model_file }}" + run: | + scripts/ci-kotlin-sdk-smoke.sh \ + "${{ inputs.staged_binary_path }}" \ + "${{ inputs.artifact_path }}" \ + "$HOME/.models/${{ inputs.model_file }}" \ + "${{ inputs.kotlin_artifact_path }}" \ + "${{ inputs.kotlin_artifact_target }}" \ + "${{ inputs.kotlin_artifact_backend }}" \ + "${{ inputs.kotlin_artifact_profile }}" - name: Swift SDK smoke test if: ${{ inputs.sdk_kind == 'swift' }} - run: scripts/ci-swift-sdk-smoke.sh "${{ inputs.staged_binary_path }}" "$(dirname "${{ inputs.staged_binary_path }}")" "$HOME/.models/${{ inputs.model_file }}" + env: + STAGED_BINARY_PATH: ${{ inputs.staged_binary_path }} + SWIFT_INPUT_ARCHIVE: ${{ format('{0}/MeshLLMFFI.xcframework.zip', inputs.swift_artifact_path) }} + SWIFT_INPUT_BINDING: sdk-artifacts/swift-generated-binding/mesh_ffi.swift + SWIFT_INPUT_MODE: ${{ inputs.swift_artifact_mode }} + run: | + scripts/ci-swift-sdk-smoke.sh \ + "$STAGED_BINARY_PATH" \ + "$(dirname "$STAGED_BINARY_PATH")" \ + "$HOME/.models/${{ inputs.model_file }}" \ + "$SWIFT_INPUT_ARCHIVE" \ + "$SWIFT_INPUT_MODE" \ + "$SWIFT_INPUT_BINDING" diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 9bb027e1e7..c3e5c62ccf 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -16,10 +16,6 @@ on: required: false default: '' type: string - runs_on: - required: false - default: '"ubuntu-24.04"' - type: string secrets: HF_TOKEN: required: false @@ -29,15 +25,21 @@ env: MODEL_URL: https://huggingface.co/unsloth/SmolLM2-135M-Instruct-GGUF/resolve/9e6855bc4be717fca1ef21360a1db4b29d5c559a/SmolLM2-135M-Instruct-Q8_0.gguf MODEL_FILE: SmolLM2-135M-Instruct-Q8_0.gguf +permissions: + contents: read + jobs: smoke_tests: name: Skippy Inference Smoke Tests - runs-on: ${{ fromJson(inputs.runs_on) }} + # Model credentials stay on an isolated GitHub-hosted runner. + runs-on: ubuntu-24.04 env: HF_TOKEN: ${{ secrets.HF_TOKEN }} HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Prepare dispatched release version if: inputs.release_tag != '' diff --git a/.github/workflows/static-abi-artifact.yml b/.github/workflows/static-abi-artifact.yml new file mode 100644 index 0000000000..7733a558f3 --- /dev/null +++ b/.github/workflows/static-abi-artifact.yml @@ -0,0 +1,187 @@ +name: Reusable Static ABI Artifact + +on: + workflow_call: + inputs: + backend: + description: Static llama ABI backend. + required: true + type: string + target: + description: Native target triple represented by the ABI. + required: true + type: string + artifact_name: + description: Immutable GitHub artifact name. + required: true + type: string + runner_size: + description: Eligible Linux runner size (default, 4, 8, or 16). + required: false + default: '8' + type: string + retention_days: + description: Artifact retention in days. + required: false + default: 1 + type: number + timeout_minutes: + description: Producer timeout in minutes. + required: false + default: 45 + type: number + +env: + CACHE_NAMESPACE: mesh-llm + +jobs: + runner_policy: + name: Select protected static ABI runner + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: none + outputs: + runner: ${{ steps.policy.outputs.runner }} + allow_depot_remote_cache: ${{ steps.policy.outputs.allow_depot_remote_cache }} + steps: + - name: Derive protected runner policy + id: policy + shell: bash + env: + POLICY_REPOSITORY: ${{ github.repository }} + POLICY_REF: ${{ github.ref }} + POLICY_EVENT_NAME: ${{ github.event_name }} + POLICY_DEPOT_ENABLED: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + POLICY_MANUAL_USE_DEPOT: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.use_depot == 'true' }} + POLICY_TARGET: ${{ inputs.target }} + POLICY_RUNNER_SIZE: ${{ inputs.runner_size }} + run: | + set -euo pipefail + + case "$POLICY_RUNNER_SIZE" in + default|4|8|16) ;; + *) + echo "runner_size must be one of: default, 4, 8, 16" >&2 + exit 1 + ;; + esac + + case "$POLICY_TARGET" in + x86_64-unknown-linux-gnu) + hosted_runner=ubuntu-24.04 + depot_runner=depot-ubuntu-24.04 + ;; + aarch64-unknown-linux-gnu) + hosted_runner=ubuntu-24.04-arm + depot_runner=depot-ubuntu-24.04-arm + ;; + *) + echo "unsupported static ABI producer target: $POLICY_TARGET" >&2 + exit 1 + ;; + esac + + allow_depot_remote_cache=false + runner=$hosted_runner + if [[ "$POLICY_REPOSITORY" == "Mesh-LLM/mesh-llm" && + "$POLICY_REF" == "refs/heads/main" && + ( "$POLICY_EVENT_NAME" == "push" || + "$POLICY_EVENT_NAME" == "workflow_dispatch" ) && + ( "$POLICY_DEPOT_ENABLED" == "true" || + ( "$POLICY_EVENT_NAME" == "workflow_dispatch" && + "$POLICY_MANUAL_USE_DEPOT" == "true" ) ) ]]; then + suffix= + if [[ "$POLICY_RUNNER_SIZE" != "default" ]]; then + suffix="-$POLICY_RUNNER_SIZE" + fi + runner="${depot_runner}${suffix}" + allow_depot_remote_cache=true + fi + + { + echo "runner=$runner" + echo "allow_depot_remote_cache=$allow_depot_remote_cache" + } >> "$GITHUB_OUTPUT" + + static_abi_artifact: + needs: runner_policy + name: Static ABI input (${{ inputs.target }}, ${{ inputs.backend }}) + runs-on: ${{ needs.runner_policy.outputs.runner }} + timeout-minutes: ${{ inputs.timeout_minutes }} + permissions: + contents: read + packages: read + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ github.token }} + defaults: + run: + shell: bash + env: + LLAMA_STAGE_BACKEND: ${{ inputs.backend }} + LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static + MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + MESH_LLM_REQUIRE_SCCACHE: "1" + SCCACHE_GHA_ENABLED: "true" + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - name: Verify prebuilt static ABI environment + run: verify-runner-image public + + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} + + # Prepare before restoring the nested build cache. prepare-llama replaces + # an unowned .deps/llama.cpp directory, so doing this after cache restore + # would discard the restored build tree. + - name: Prepare patched llama.cpp checkout + run: scripts/prepare-llama.sh pinned + + - name: Cache portable static ABI input + id: static_abi_cache + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: static-abi-artifact-output + key: ${{ format('{0}-{1}-skippy-abi-{2}-{3}-{4}-{5}', env.CACHE_NAMESPACE, runner.os, inputs.backend, inputs.target, env.MESH_LLM_LLAMA_TOOLCHAIN_EPOCH, hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/restore-static-abi-input.sh', 'scripts/safe-extract-tar.py', 'scripts/verify-checksum-sidecar.py', 'scripts/verify-static-abi-build-stamp.py', '.github/actions/prepare-static-abi-input/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt')) }} + + - name: Restore and verify cached portable static ABI + if: ${{ steps.static_abi_cache.outputs.cache-hit == 'true' }} + env: + STATIC_ABI_TARGET: ${{ inputs.target }} + STATIC_ABI_BACKEND: ${{ inputs.backend }} + run: | + scripts/restore-static-abi-input.sh \ + static-abi-artifact-output \ + "$LLAMA_STAGE_BUILD_DIR" \ + "$STATIC_ABI_TARGET" \ + "$STATIC_ABI_BACKEND" + + - name: Build and archive immutable static ABI input + id: static-abi + if: ${{ steps.static_abi_cache.outputs.cache-hit != 'true' }} + uses: ./.github/actions/prepare-static-abi-input + with: + backend: ${{ inputs.backend }} + target: ${{ inputs.target }} + build: "true" + + - name: Upload immutable static ABI input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ inputs.artifact_name }} + path: static-abi-artifact-output/* + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} + + - name: Capture static ABI sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-static-abi-${{ inputs.target }}-${{ inputs.backend }}-${{ github.run_attempt }} diff --git a/.github/workflows/swift-sdk-artifact.yml b/.github/workflows/swift-sdk-artifact.yml new file mode 100644 index 0000000000..1b5502ae38 --- /dev/null +++ b/.github/workflows/swift-sdk-artifact.yml @@ -0,0 +1,227 @@ +name: Reusable Swift SDK Artifact + +on: + workflow_call: + inputs: + mode: + description: "XCFramework coverage: host-only for PR iteration or full for main/release." + required: true + type: string + artifact_name: + description: Immutable GitHub artifact name for the verified XCFramework ZIP. + required: true + type: string + release_tag: + description: Optional release tag used to verify/update Package.swift. + required: false + default: '' + type: string + prepare_release_version: + description: Prepare checked-out sources for release_tag before building. + required: false + default: false + type: boolean + update_release_manifest: + description: Update and upload Package.swift for a dispatched release. + required: false + default: false + type: boolean + require_tracked_console: + description: Require generated Swift console assets to be tracked. + required: false + default: false + type: boolean + retention_days: + description: Artifact retention in days. + required: false + default: 1 + type: number + timeout_minutes: + description: Producer timeout in minutes. + required: false + default: 45 + type: number + +jobs: + swift_sdk_artifact: + name: Swift SDK input (${{ inputs.mode }}) + runs-on: macos-15 + timeout-minutes: ${{ inputs.timeout_minutes }} + permissions: + contents: read + env: + CARGO_INCREMENTAL: "0" + CARGO_NET_RETRY: "10" + CARGO_HTTP_MULTIPLEXING: "false" + LLAMA_STAGE_BACKEND: metal + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + # Bump this epoch whenever the pinned macOS/Xcode native build boundary + # changes so an older C/C++ ABI cache cannot cross toolchains. + SWIFT_NATIVE_XCODE_CACHE_EPOCH: macos-15-arm64-xcode-default-v1 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - name: Validate typed producer inputs + shell: bash + env: + SWIFT_SDK_MODE: ${{ inputs.mode }} + RELEASE_TAG: ${{ inputs.release_tag }} + PREPARE_RELEASE_VERSION: ${{ inputs.prepare_release_version }} + UPDATE_RELEASE_MANIFEST: ${{ inputs.update_release_manifest }} + run: | + set -euo pipefail + case "$SWIFT_SDK_MODE" in + host-only|full) ;; + *) + echo "unsupported Swift SDK artifact mode: $SWIFT_SDK_MODE" >&2 + exit 1 + ;; + esac + if [[ ( "$PREPARE_RELEASE_VERSION" == "true" || + "$UPDATE_RELEASE_MANIFEST" == "true" ) && + -z "$RELEASE_TAG" ]]; then + echo "release_tag is required for release preparation" >&2 + exit 1 + fi + + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 + with: + version: 10 + + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: | + .github/cache-version.txt + crates/mesh-llm-ui/pnpm-lock.yaml + + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + + - name: Install macOS SDK build dependencies + run: brew install cmake ninja lld + + - name: Restore exact Swift native ABI cache + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: .deps/llama-build + # The full and host-only producers share target-specific ABI directories. + # GitHub scopes PR writes to the merge ref, so a trusted main full build + # can seed this exact mode-independent key without a PR shadowing it. + key: ${{ format('mesh-llm-swift-sdk-{0}-{1}-{2}-{3}', runner.os, runner.arch, env.SWIFT_NATIVE_XCODE_CACHE_EPOCH, hashFiles('Cargo.lock', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'sdk/swift/scripts/build-host-macos-xcframework.sh', 'sdk/swift/scripts/build-xcframework.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt')) }} + + - name: Prepare dispatched release version + if: ${{ inputs.prepare_release_version }} + env: + RELEASE_TAG: ${{ inputs.release_tag }} + run: scripts/release-version.sh "$RELEASE_TAG" + + - name: Prepare Swift console resources + run: | + scripts/package-sdk-console-assets.sh --sdk swift + scripts/verify-sdk-console-assets.sh --sdk swift + + - name: Verify tagged Swift console resources + if: ${{ inputs.require_tracked_console }} + run: git ls-files --error-unmatch sdk/swift/Sources/MeshLLM/Resources/Console/index.html + + - name: Build host-only Swift SDK input + if: ${{ inputs.mode == 'host-only' }} + run: sdk/swift/scripts/build-host-macos-xcframework.sh + + - name: Build full Swift SDK input + if: ${{ inputs.mode == 'full' }} + run: sdk/swift/scripts/build-xcframework.sh + + - name: Verify protected Swift binding source is current + if: ${{ !inputs.prepare_release_version && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) }} + run: | + set -euo pipefail + generated_binding="sdk/swift/Sources/MeshLLM/Generated/mesh_ffi.swift" + git ls-files --error-unmatch "$generated_binding" + if ! git diff --exit-code -- "$generated_binding"; then + echo "generated Swift UniFFI bindings changed during the protected build" >&2 + echo "regenerate and commit $generated_binding before publishing" >&2 + exit 1 + fi + + - name: Stage immutable generated Swift binding + run: | + set -euo pipefail + generated_binding="sdk/swift/Sources/MeshLLM/Generated/mesh_ffi.swift" + staged_binding="dist/swift-generated-binding/mesh_ffi.swift" + git ls-files --error-unmatch "$generated_binding" + mkdir -p "$(dirname "$staged_binding")" + install -m 0644 "$generated_binding" "$staged_binding" + cmp "$generated_binding" "$staged_binding" + + - name: Package Swift SDK input + run: | + set -euo pipefail + mkdir -p dist + rm -f dist/MeshLLMFFI.xcframework.zip + ditto -c -k --sequesterRsrc --keepParent \ + sdk/swift/Generated/MeshLLMFFI.xcframework \ + dist/MeshLLMFFI.xcframework.zip + + - name: Prepare SwiftPM manifest for dispatched release + if: ${{ inputs.update_release_manifest }} + env: + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + scripts/update-swift-package-manifest.sh \ + "$RELEASE_TAG" \ + dist/MeshLLMFFI.xcframework.zip + + - name: Verify immutable Swift SDK input + env: + SWIFT_SDK_MODE: ${{ inputs.mode }} + run: | + scripts/verify-swift-release-artifact.sh \ + dist/MeshLLMFFI.xcframework.zip \ + "$SWIFT_SDK_MODE" + + - name: Verify SwiftPM release manifest + if: ${{ inputs.release_tag != '' }} + env: + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + scripts/verify-swift-package-manifest.sh \ + "$RELEASE_TAG" \ + dist/MeshLLMFFI.xcframework.zip + + - name: Upload generated SwiftPM manifest + if: ${{ inputs.update_release_manifest }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: swift-package-manifest + path: Package.swift + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} + + - name: Upload immutable Swift SDK input + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ inputs.artifact_name }} + path: dist/MeshLLMFFI.xcframework.zip + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} + + - name: Upload immutable generated Swift binding + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: generated-swift-binding-${{ inputs.artifact_name }} + path: dist/swift-generated-binding + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} + + - name: Capture Swift SDK sccache statistics + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-swift-sdk-${{ inputs.mode }}-${{ github.run_attempt }} diff --git a/RELEASE.md b/RELEASE.md index e202acdbb2..5dd48ccdb8 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -145,17 +145,41 @@ git add Package.swift sdk/swift/Sources/MeshLLM/Generated/mesh_ffi.swift git commit -m "v0.X.Y: prepare Swift package artifact" ``` -The release workflow rebuilds `MeshLLMFFI.xcframework.zip`, verifies the macOS -framework layout, runs a zipped-artifact SwiftPM consumer smoke, and checks that -the tagged `Package.swift` already points at the exact release URL and checksum. -If `Package.swift` still contains placeholders on a tag push, or if the -checksum does not match the artifact built in release CI, the release fails -before publishing. +The release workflow invokes the shared typed Swift SDK producer in exhaustive +`full` mode to build `MeshLLMFFI.xcframework.zip`. That same producer is used +in `host-only` mode for PR iteration and `full` mode on main. It verifies the +exact platform and architecture slices plus the macOS framework layout, runs a +zipped-artifact SwiftPM consumer smoke, and checks that the tagged +`Package.swift` already points at the exact release URL and checksum. The +producer also uploads the generated `mesh_ffi.swift` as a separate immutable +companion artifact. Main and tag builds fail when that generated binding drifts +from the tracked source. If `Package.swift` still contains placeholders on a +tag push, if the generated binding is stale, or if the checksum does not match +the artifact built in release CI, the release fails before publishing. +Producer and smoke use the pinned `macos-15` image and an explicit native/Xcode +cache epoch. Downstream Swift smoke consumes both verified producer artifacts +and never compiles an XCFramework replacement. + +Native SDK release archives use the same typed `native-sdk-artifact.yml` +producer as PR and main Kotlin validation. Callers select an explicit target, +backend, Cargo profile, and bounded runner size; they cannot provide a runner +label or Depot-cache permission. Each Linux release invocation first nests the +shared `static-abi-artifact.yml` producer on the matching native architecture, +then restores its checksummed/stamped CPU ABI into the normal native-SDK +`--build` path so only the Rust FFI compilation remains. Both reusable +producers derive architecture-specific hosted/Depot placement and cache +authority from the protected workflow's repository/event/ref policy. Release +enables native runtime crate staging on the same verified archive path. The +producer keeps each +`release-native-sdk--` artifact flat with exactly one +archive, its checksum sidecar, and its target-specific `.crate`, preserving the +published asset names while Kotlin smoke remains a no-build consumer. For `workflow_dispatch` releases, the release workflow computes the SwiftPM -checksum from the XCFramework artifact it just built, patches `Package.swift` -in the workflow workspace, and creates the requested release tag at a -manifest-only commit before publishing. +checksum from the XCFramework artifact it just built, carries both the patched +`Package.swift` and the producer's exact generated `mesh_ffi.swift` into the +release workspace, and creates the requested release tag at that prepared +source commit before publishing. The current GitHub Actions release workflow publishes macOS aarch64, Linux x86_64 CPU, Linux ARM64 CPU, Linux ARM64 CUDA, Linux CUDA, Linux CUDA diff --git a/ci/DEPOT_MIGRATION.md b/ci/DEPOT_MIGRATION.md index 16834152d3..4678b99bf8 100644 --- a/ci/DEPOT_MIGRATION.md +++ b/ci/DEPOT_MIGRATION.md @@ -92,7 +92,7 @@ Artifact contracts: | Host input | executable, `.sha256`, `host-imports.json`; release adds attestation | immutable after checksum | | Runtime input | runtime directory, archive, archive checksum, `manifest.json` | immutable after verification | | Product input | host, host imports, `product-manifest.json`, one `native-runtimes/` | composer never compiles | -| Static ABI test input | one tarred CPU llama ABI build keyed by patch queue and build recipe | one producer per workflow; test rows only restore | +| Static ABI input | checksummed, target-described CPU llama link closure keyed by patch queue, pinned build-image epoch, and recipe | one producer per target; tests and native SDK rows verify and restore without fallback builds | PR artifacts are unstamped, retained for one day, and cannot be promoted into a release. Main and release exercise the same actions; release adds version @@ -112,9 +112,9 @@ PRs optimize for the earliest reliable signal: for manual, benchmark, and backend-affecting runs; - fan that one exact host artifact into the CPU and every selected backend runtime row; -- build or restore the static CPU llama ABI once, archive it, and fan those - exact bytes into every crate-test and grouped-test row instead of compiling - the same C++ graph concurrently; +- build or restore the static CPU llama ABI once per target, archive it, and fan + those exact bytes into every crate-test, grouped-test, and native-SDK row + instead of compiling the same C++ graph concurrently; - run runner-image contract checks only when their workflow, cache version, or cache integration changes; - make SDK smokes consume the staged product runtime and reject hidden rebuilds; @@ -157,7 +157,7 @@ Depot runners are selected with a single label such as | measured high-parallelism runtime builds | `depot-ubuntu-24.04-16` | compare wall time, peak disk, and cost before adopting | | hardware-qualified CUDA tests | dedicated GPU runner | requires a real device | -The current runner selector has one effective repository gate: +The current top-level runner selector has one effective repository gate: - `DEPOT_RUNNERS_ENABLED=true` enables eligible trusted `main` push and `main`-ref dispatch jobs. Tag pushes and every other ref remain hosted. @@ -171,6 +171,23 @@ never authority to run feature-branch code on Depot. The selector emits one typed cache permission from the same decision, so a caller cannot select a hosted runner while independently enabling Depot WebDAV. +Runner-owning reusable workflows require an additional boundary. A PR can call +the main-pinned `native-sdk-artifact.yml` or `static-abi-artifact.yml` while the +called workflow checks out PR contents, so neither workflow accepts +caller-provided `runs_on` or `allow_depot_remote_cache` inputs. A fixed +`ubuntu-24.04` policy job validates a bounded size enum (`default`, `4`, `8`, +`16`), maps the requested target architecture to checked-in labels, and emits +both the build runner and cache permission. Depot is selected only when the +caller context is the exact `Mesh-LLM/mesh-llm` repository, a `push` or +`workflow_dispatch` on `refs/heads/main`, and +`DEPOT_RUNNERS_ENABLED == 'true'`. Every PR event, tag, feature ref, external +repository, macOS target, or disabled gate without the authorized canary +receives a GitHub-hosted label and cache permission false. For the pre-variable +canary, the protected workflow may also read `use_depot == 'true'` from the +immutable `workflow_dispatch` event payload, but only the same exact +repository/main/dispatch guards can authorize it; no reusable-workflow input +can grant that authority. + This selector is defense in depth, not the primary security boundary. The current pull-request workflows and repository-local actions are evaluated from PR-controlled code, so a pull request can modify or bypass the selector itself. @@ -180,20 +197,28 @@ the current PR workflow safe for Depot. Activation prerequisites: 1. The Depot GitHub Apps remain connected to `Mesh-LLM`. -2. While public-repository access is still disabled, change GitHub's +2. Protect `main` with an enforceable review/ruleset gate for runner-owning + workflow changes. The current repository ruleset prevents deletion, + non-fast-forward updates, and non-linear history, but does not require a + pull request, review, or successful CI check. An exact-main workflow + allowlist is not a durable privilege boundary while an unreviewed direct + push can replace that workflow. +3. While public-repository access is still disabled, change GitHub's organization `Default` runner group to selected repository `Mesh-LLM/mesh-llm` and selected workflow `Mesh-LLM/mesh-llm/.github/workflows/depot-canary.yml@refs/heads/main`. -3. Only after both restrictions are saved, enable public repositories for the +4. Only after both restrictions are saved, enable public repositories for the `Default` group. Depot-managed ephemeral runners register in that group. -4. Dispatch `depot-canary.yml` from `refs/heads/main` twice. Verify all four +5. Dispatch `depot-canary.yml` from `refs/heads/main` twice. Verify all four Intel runner sizes, both ARM runner sizes, their reported architectures, and a cold-to-warm cache hit without printing credentials. -5. Dispatch the canary from a feature ref, prove that it cannot acquire a +6. Dispatch the canary from a feature ref, prove that it cannot acquire a Depot runner, and cancel that exact queued run. -6. Add exact default-branch workflow refs only as their phase starts. Reusable - workflows whose jobs run on Depot must be listed separately. -7. Set `DEPOT_RUNNERS_ENABLED=true` only after comparable trusted canaries meet +7. Add exact default-branch workflow refs only as their phase starts. Reusable + workflows whose jobs run on Depot must be listed separately, must derive + runner placement from immutable caller context inside the protected + workflow, and must never pass a caller-provided label to `runs-on`. +8. Set `DEPOT_RUNNERS_ENABLED=true` only after comparable trusted canaries meet the rollout targets. The initial main allowlist is: @@ -201,12 +226,25 @@ The initial main allowlist is: ```text Mesh-LLM/mesh-llm/.github/workflows/ci.yml@refs/heads/main Mesh-LLM/mesh-llm/.github/workflows/pr_quality.yml@refs/heads/main -Mesh-LLM/mesh-llm/.github/workflows/hf-download-smoke.yml@refs/heads/main -Mesh-LLM/mesh-llm/.github/workflows/smoke.yml@refs/heads/main -Mesh-LLM/mesh-llm/.github/workflows/scripted-binary-smoke.yml@refs/heads/main -Mesh-LLM/mesh-llm/.github/workflows/sdk-smoke.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/native-sdk-artifact.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/static-abi-artifact.yml@refs/heads/main ``` +`hf-download-smoke.yml`, `smoke.yml`, `scripted-binary-smoke.yml`, and +`sdk-smoke.yml` may receive model credentials from trusted callers and +intentionally allocate only bounded GitHub-hosted labels. They are not selected +for the Depot group. The Swift SDK producer is likewise fixed to +GitHub-hosted `macos-15` instead of accepting a runner input. This keeps a pull +request from invoking a +default-branch reusable workflow with a privileged Depot or dedicated-runner +label. + +Pull-request callers do not pass `HF_TOKEN` to these workflows at all; public +fixtures and merge-ref-scoped model caches provide the PR signal without +exposing a repository secret to checked-out PR code. Trusted main and release +callers may pass the optional token for rate-limit resilience, but those +credential-bearing invocations remain GitHub-hosted. + Add `pr_builds.yml@refs/heads/main` only for its trusted manual benchmark and `release.yml@refs/heads/main` only for the non-publishing release phase. Never select a feature ref, `refs/pull/*`, or “all workflows.” @@ -239,7 +277,10 @@ inspection found: The current `Default` state safely prevents Depot from serving this public repository, so a canary will queue until the ordered restriction changes above are made. The GPU group is separate from Depot and its all-workflows policy -must also be reviewed before treating those devices as a trusted-only pool. +must also be restricted to protected workflow entry points before treating +those devices as a trusted-only pool; a public repository with an +all-workflows runner group otherwise allows PR-controlled workflow definitions +to request those persistent runners directly. Depot redirects every GitHub Actions cache API consumer on its runners, including `actions/cache`, `actions/setup-node`, and third-party cache actions. @@ -247,7 +288,9 @@ Its namespace is repository-scoped and is not isolated by branch. Therefore: - current pull-request jobs never run on Depot and may use the normal `mesh-llm` key namespace in GitHub's native cache because GitHub scopes PR - writes to the merge ref and trusted main jobs do not restore from that ref; + writes to the merge ref and trusted main jobs do not restore from that ref. + The crate-test target shards deliberately restore the already-seeded + `main-rust-crate-tests-` keys with writes disabled; - a local sccache disk-only setting protects only that sccache child process; it does not remove the Depot token or prevent another cache API consumer from reading or poisoning the repository cache; @@ -275,7 +318,17 @@ Relevant Depot documentation: ### `Mesh-LLM/mesh-llm-runner-images` Runner images own stable tools and backend SDKs, not commit-specific products. -The next image revision should: +The first migration phase is implemented in draft +[`mesh-llm-runner-images#9`](https://github.com/Mesh-LLM/mesh-llm-runner-images/pull/9): +PRs route only affected image families plus the mandatory public CPU AMD64 +contract, never export BuildKit cache, and never stage or promote registry +content. Trusted main pushes stage candidates; weekly or explicit manual runs +promote a retained candidate cohort. The reusable family workflow independently +derives its runner/cache authority, verifies the source revision, identifies +candidate content by digest, and serially reconciles the complete `latest` +cohort. Deleted files participate in routing. + +The subsequent role-isolation revision should: 1. build the UI once in a Node-capable producer and upload it before any Node-free host role starts; `public-rust-host` consumes those prepared UI @@ -313,6 +366,17 @@ ROCm 7.2 AMD64 image in an 18m 03s `Build and push architecture image by digest` step. This demonstrates duplicate test/publish image construction; it does not measure image size, cold-pull time, or cache effectiveness. +The hardened build-once PR graph is measured by +[runner-images run 30504335079](https://github.com/Mesh-LLM/mesh-llm-runner-images/actions/runs/30504335079). +Because the PR changed the Dockerfile, the affected-family planner correctly +selected all 20 platform rows. All 22 allocated jobs stayed GitHub-hosted, +completed in 6m 22s wall and 1h 13m 07s aggregate, and emitted no real cache +export phase. Compared with the first build-once run's 22m 57s wall and +2h 52m 59s aggregate, that is a 72.3% wall reduction and 57.7% aggregate +reduction. The slowest self-hosted ROCm 7.2 row fell from 22m 20s to 5m 48s. +This validates read-only PR cache and change routing; trusted registry +stage/promotion remains to be canaried separately. + No retained audit evidence currently substantiates the previously cited 1.53 GB/1.92 GB compressed sizes or backend cold-initialization medians, so those values are not migration baselines. The following are provisional design @@ -330,21 +394,52 @@ token to PR code. ### `Mesh-LLM/mesh-packaging` -Packaging already consumes product-v2 and must not rebuild the CLI/runtime. -After the MeshLLM graph is stable: - -1. replace the global native-package matrix barrier with a reusable per-row - package → QA → image → QA pipeline; -2. publish a canonical release artifact index from MeshLLM containing source +Packaging already consumes product-v2 native products and must not rebuild the +CLI/runtime. Its current OCI publication path is not build-once: `runtime-image` +tests a `runtime-qa` cache-only build, while `publish-images` later rebuilds the +`runtime` target from mutable base tags. The published bytes are therefore not +guaranteed to be the bytes that passed QA. A filtered manual native run also +enables every npm lane and Homebrew regardless of the native filters. In +[run 30390907268](https://github.com/Mesh-LLM/mesh-packaging/actions/runs/30390907268), +the selected CPU package/image each took less than a minute, but the unintended +macOS and Windows addon builds extended the workflow to 22m 53s. The historical +11-row full run +[30327805587](https://github.com/Mesh-LLM/mesh-packaging/actions/runs/30327805587) +took 30m 36s and retained 3.09 GB of Actions artifacts: 1.46 GB of verified +products plus 1.63 GB of native packages. + +There is no successful current product-v2 full baseline yet. Run +[30460399813](https://github.com/Mesh-LLM/mesh-packaging/actions/runs/30460399813) +correctly failed closed because v0.74.0 predates that schema. After the MeshLLM +graph produces a complete product-v2 release: + +1. add typed `validate_npm` and `validate_homebrew` dispatch inputs so a + filtered native canary runs only the requested rows; publication still + requires the complete release validation set; +2. replace provider-name/source-build blacklists with positive runner-class + and artifact-consumer contracts; +3. publish a canonical release artifact index from MeshLLM containing source SHA, product/host/runtime/addon assets, schemas, and digests; -3. publish Node addon producer artifacts from MeshLLM so packaging assembles +4. publish Node addon producer artifacts from MeshLLM so packaging assembles npm instead of rebuilding five targets; -4. build each runtime image once at an immutable staging digest, test it, then - promote that digest; -5. route CPU-only packaging rows to Depot after the main repository canary. - -The first product-v2 release candidate is the compatibility baseline for that -work; older v0.74 assets predate the contract. +5. resolve every OCI base to a digest, build the final image once at an + immutable staging reference, test that exact digest, and promote versioned + and moving tags with registry manifest operations. An identical existing + version tag is a no-op; a mismatch fails closed; +6. replace the global native-package matrix barrier with a reusable per-row + product restore → package → package QA → image → image QA pipeline; +7. key BuildKit scope by base digest, product digest, architecture, distro, + Dockerfile inputs, and cache schema. PR canaries are restore-only; trusted + main/release may write. Re-verify every restored product; +8. establish one filtered product-v2 canary, one full non-publishing main run, + and one full release rehearsal before publication; +9. route eligible CPU-only Linux/ARM, Windows assembly, and compatible macOS + rows to Depot only after the main repository canary and runner-group trust + gates pass. + +The first successful product-v2 full run becomes the compatibility and timing +baseline. Record per-row queue, download, package, QA, image, cache, artifact, +and promotion timings plus the release-index and base-image digests. ## Measurement and rollout gates @@ -367,19 +462,25 @@ python3 scripts/collect-ci-metrics.py \ Rollout sequence: -1. restrict the `Default` runner group to the repository and only +1. require pull requests and review for changes to `main`, with runner-owning + workflow changes covered by the enforceable repository ruleset; +2. restrict the persistent `mesh-llm` GPU runner group to protected workflow + entry points before scheduling untrusted public-repository workflows on + those devices; +3. restrict the Depot-backed `Default` runner group to the repository and only `depot-canary.yml@refs/heads/main`, then enable public-repository access; -2. run the allowed-main and denied-feature-ref canaries; -3. compare `-4`, `-8`, and `-16` using Depot CPU/memory/disk utilization data; -4. allowlist main CI plus every directly invoked reusable workflow, then - canary routing, quality, and the Linux product graph from `main`; -5. collect five comparable green main canaries; -6. set `DEPOT_RUNNERS_ENABLED=true` for trusted main jobs after those canaries +4. run the allowed-main and denied-feature-ref canaries; +5. compare `-4`, `-8`, and `-16` using Depot CPU/memory/disk utilization data; +6. allowlist main CI plus only the hardened reusable producers that directly + allocate Depot runners, then canary routing, quality, and the Linux product + graph from `main`; keep credential-bearing smoke workflows GitHub-hosted; +7. collect five comparable green main canaries; +8. set `DEPOT_RUNNERS_ENABLED=true` for trusted main jobs after those canaries meet the targets; -7. allowlist `release.yml@refs/heads/main` and exercise the non-publishing, +9. allowlist `release.yml@refs/heads/main` and exercise the non-publishing, non-secret runtime/composition producers. Tag-push publishing remains hosted; -8. keep all PR-event code hosted while automatic Depot Cache is enabled; -9. migrate packaging only after product-v2 and addon contracts are stable. +10. keep all PR-event code hosted while automatic Depot Cache is enabled; +11. migrate packaging only after product-v2 and addon contracts are stable. Rollback for the currently implemented trusted lanes is one repository-variable change: diff --git a/ci/ci.md b/ci/ci.md index 262cbe91dc..c3961dcf8e 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -91,6 +91,8 @@ flowchart TD MacHost["macos_host_input\none immutable neutral host"] MacRuntime["macos_metal_runtime_input\none Metal runtime"] MacCPU["macos_cpu_artifact\ncompose host + Metal runtime\n→ ci-macos-inference-binaries"] + KotlinInput["kotlin_sdk_input\ndebug native SDK producer"] + SwiftInput["swift_sdk_input\nhost-only XCFramework producer"] MacTests["macos_unit_tests"] end @@ -98,7 +100,7 @@ flowchart TD Restore["restore-smoke-inputs action\ndownload artifact · stage binary · restore model"] Inference["smoke.yml\nLinux inference + OpenAI + split serving"] Scripted["scripted-binary-smoke.yml\ntwo-node client/serving"] - SDKSmoke["sdk-smoke.yml\nnative · Kotlin · Swift"] + SDKSmoke["sdk-smoke.yml\nRust · Kotlin · Swift"] end end @@ -109,12 +111,16 @@ flowchart TD LinuxCPU --> LinuxProduct Affected --> LinuxTests TestBins --> StaticABI + SDK --> StaticABI StaticABI --> RustCrateTests StaticABI --> LinuxTests + StaticABI --> KotlinInput InferenceArtifact --> MacHost InferenceArtifact --> MacRuntime MacHost --> MacCPU MacRuntime --> MacCPU + SDK --> KotlinInput + SDK --> SwiftInput Affected --> MacTests Backend --> LinuxCUDARuntime Backend --> LinuxROCmRuntime @@ -139,6 +145,8 @@ flowchart TD Restore --> Inference Restore --> Scripted Restore --> SDKSmoke + KotlinInput -- "artifact: pr-kotlin-native-sdk-input" --> SDKSmoke + SwiftInput -- "artifact: pr-swift-sdk-input" --> SDKSmoke SDK --> SDKSmoke subgraph Cleanup["pr_cleanup.yml · PR Cache Cleanup"] @@ -179,6 +187,9 @@ flowchart TD runtime. The Linux host and CPU runtime build independently, then a composition-only job uploads product-v2 for every downstream smoke. SDK smokes consume the staged runtime instead of compiling a private replacement. + The Swift XCFramework is also built by the same typed producer used by PR and + release: main requests exhaustive `full` mode and its smoke only verifies and + consumes that immutable artifact. - Main builds immutable Linux, macOS, and Windows release hosts independently from their CPU, Metal, CUDA, ROCm, and Vulkan runtimes. Composition-only jobs verify and combine those exact producer inputs. Each Linux GPU backend has @@ -190,12 +201,15 @@ flowchart TD qualification. - `.github/actions/prepare-host-input`, `.github/actions/prepare-windows-host-input`, - `.github/actions/prepare-native-runtime-input`, and + `.github/actions/prepare-native-runtime-input`, + `.github/actions/prepare-native-sdk-input`, + `.github/actions/prepare-static-abi-input`, and `.github/actions/compose-product-input` are the shared PR/main/release primitives. The composer never compiles either producer input. -- Linux crate-test and grouped-test matrices restore one tarred static CPU - llama ABI from `linux_static_abi_input`; individual rows never rebuild the - same patch queue concurrently. +- Linux crate-test and grouped-test matrices plus the Kotlin native-SDK producer + restore one checksummed, target-described static CPU llama ABI from + `linux_static_abi_input`; individual consumers never rebuild or raw-extract + the same patch queue concurrently. ### Current PR Builds contract @@ -235,7 +249,28 @@ flowchart TD immutable host without a matrix-wide fan-in barrier. Windows follows the same graph: one debug neutral host, independent CPU/CUDA/ROCm/Vulkan runtime inputs, and composition-only products. Unsupported macOS CUDA, ROCm, and - Vulkan rows are omitted. + Vulkan rows are omitted. The PR Swift `host-only` XCFramework producer starts + directly from change routing, in parallel with the macOS product and unit + tests; Swift smoke waits only for the macOS product and XCFramework inputs. + `sdk_smoke_required` makes the shared static CPU ABI producer eligible; the PR + Kotlin debug native-SDK producer restores that immutable ABI, validates its + complete link closure, pinned build-image epoch, and build stamp through the + verification-only `--require-prebuilt-llama` path, and compiles only the Rust + FFI while the Linux product proceeds independently. Both build-script and + Cargo auto-build fallbacks are disabled for that reuse path. Kotlin smoke + waits only for the product and native-SDK inputs and performs no native + compilation. +- **PR Builds Summary** is the stable, non-matrix branch-protection check for + this workflow. `changes` runs `scripts/plan-pr-build-jobs.py` once and exports + `required_jobs_json`; every conditional top-level job uses membership in + that plan as its route, while normal `needs` success semantics keep consumers + behind their producers. The summary directly depends on every other + top-level job and consumes the same plan. A skipped job is accepted only when + the planner did not require it, so a required producer or dependency chain + cannot disappear behind propagated skips. Any failure, cancellation, unknown + result, duplicate plan entry, or required ID outside the summary graph fails + the gate. The job uses `if: ${{ !cancelled() }}` so ordinary upstream + failures are still summarized without using `always()`. - Product readiness starts a local mDNS client and never depends on the mutable public mesh. The public `client --auto` admission probe is manual-only, so an external peer outage cannot block a pull request or release. @@ -330,10 +365,18 @@ role-isolated topology. The planned split has these prerequisites: contract even for Node-free images; - `self-hosted-*` runner/device overlays are added and verified last. -Runner-image publication will follow one immutable chain: +The first runner-image migration phase in draft +[`mesh-llm-runner-images#9`](https://github.com/Mesh-LLM/mesh-llm-runner-images/pull/9) +implements one immutable chain: `build once -> stage digest -> verify that exact digest -> promote digest`. -Manifest assembly and human-facing tags consume verified digests and must not -rebuild an architecture image. The latest measured compatibility-image +PRs build only affected families plus the mandatory public CPU AMD64 contract, +use BuildKit caches read-only, and cannot stage or promote. Main pushes stage +candidate digests; weekly and explicit manual runs promote a complete retained +cohort. The reusable stage workflow derives its own trusted runner/cache policy, +source revisions are verified, content-digest tags identify immutable +candidates, and one serial reconciliation updates the `latest` cohort. +Manifest assembly and human-facing tags consume verified digests and do not +rebuild an architecture image. The pre-migration compatibility-image [run 30248081255](https://github.com/Mesh-LLM/mesh-llm-runner-images/actions/runs/30248081255) took 39m 15s across 55 jobs; its slowest test build step was 14m 25s and a later second public ROCm 7.2 AMD64 publication build took 18m 03s. That run @@ -341,6 +384,16 @@ demonstrates duplicate construction, but it did not retain authoritative compressed-size or controlled cold-pull evidence. Role-size and pull-time thresholds remain proposed rollout gates until measured. +The replacement PR +[run 30504335079](https://github.com/Mesh-LLM/mesh-llm-runner-images/actions/runs/30504335079) +exercised all 20 platform rows because the Dockerfile changed. Its 22 allocated +jobs remained GitHub-hosted and completed in 6m 22s wall / 1h 13m 07s +aggregate, versus 22m 57s / 2h 52m 59s for the first build-once run. The +slowest self-hosted ROCm 7.2 row fell from 22m 20s to 5m 48s, and logs contained +no actual cache-export phase. This is PR-validation evidence only; registry +staging, cohort promotion, compressed-size, and controlled cold-pull gates +still require trusted runs. + Production workflows and Flux resources must pin the multi-architecture OCI digest, using `ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:`. Timestamp, source-revision, and `*-latest` tags are discovery or evaluation inputs only; @@ -418,7 +471,10 @@ Consequently, no untrusted PR code may run on Depot while automatic cache injection is enabled. GitHub-hosted jobs retain the existing best-effort `disk,gha` path or explicit disk-only mode. Cache read failures degrade to misses, cache write failures only warn, and a failed remote probe restarts -`sccache` with disk-only storage. +`sccache` with disk-only storage. PR crate-test shards restore the existing +`main-rust-crate-tests-` Cargo target caches read-only (`save-if: +false`), so trusted main owns the cache while PRs avoid recompiling the same +workspace graph. ## Depot rollout @@ -430,8 +486,23 @@ because PR workflow and local-action files are themselves PR-controlled. Trusted main/release jobs use `DEPOT_RUNNERS_ENABLED`, and a trusted main-ref manual dispatch can use `use_depot=true` for a bounded canary. The selector requires `refs/heads/main`; tag pushes and feature refs fall back to hosted -runners. It emits both Intel and ARM64 labels from the same trust decision, so -release CPU producers, composers, and smokes do not bypass the policy. +runners. It emits both Intel and ARM64 labels from the same trust decision for +eligible producers and composers. + +The selected `native-sdk-artifact.yml` and `static-abi-artifact.yml` reusable +workflows own a stricter policy because a caller can pin the protected workflow +while still asking it to check out caller-controlled contents. They accept only +a bounded `runner_size` (`default`, `4`, `8`, or `16`), then a fixed +GitHub-hosted policy job derives the build label and Depot-cache permission +from the exact repository, event, `refs/heads/main`, repository gate, and +target architecture. Callers cannot provide `runs-on` or independently enable +Depot WebDAV. PR and `pull_request_target` events, tags, feature refs, external +repositories, macOS targets, and a disabled gate without the authorized canary +always select the architecture-matching GitHub-hosted runner with cache +permission false. The event-owned `use_depot` canary may enable Depot only for +an exact `Mesh-LLM/mesh-llm` main-ref `workflow_dispatch`; it is read from the +immutable event payload and is not a reusable-workflow input. + Depot-managed runners register in the organization `Default` runner group. Before enabling public access, restrict that group to `Mesh-LLM/mesh-llm` and exact default-branch workflow refs, beginning with @@ -444,8 +515,12 @@ from that ref. Depot's cache is repository-scoped instead, so cache-key conventions or a trusted reusable caller are not sufficient protection from malicious checked-out PR code. PR events stay hosted while automatic Depot Cache is enabled. Runner placement does not alter build action inputs or -artifact contracts. Hardware-qualified GPU execution stays on dedicated -runners. See +artifact contracts. Credential-bearing Hugging Face, inference, scripted, and +SDK smoke reusable workflows accept no arbitrary runner label and stay on +GitHub-hosted runners. PR callers pass no `HF_TOKEN`; only trusted main/release +invocations receive the optional rate-limit credential. The Swift producer and +Swift smoke are fixed to the GitHub-hosted `macos-15` image. +Hardware-qualified GPU execution stays on dedicated runners. See [`DEPOT_MIGRATION.md`](DEPOT_MIGRATION.md) for activation prerequisites, baseline metrics, target service levels, and the cross-repository plan. @@ -478,13 +553,40 @@ baseline metrics, target service levels, and the cross-repository plan. directory via `MESH_HF_DOWNLOAD_TEST_CACHE_DIR`. - Shared model caches are restored in PRs and saved only from trusted `main` runs. -- Linux CPU artifacts feed inference, two-node, native SDK, and Kotlin SDK - smokes. macOS CPU artifacts feed Swift SDK smokes. +- Linux CPU artifacts feed inference, two-node, and SDK smokes. Kotlin also + restores the verified native SDK archive built by `native-sdk-artifact.yml` + from an explicit target/backend/profile contract. That producer starts + from the `sdk_smoke_required` static-ABI input and runs in parallel with the + Linux product; PR uses a debug package, while main and release use release + packages. Release creates one matching static ABI per Linux target through + `static-abi-artifact.yml` before invoking the same native-SDK action. + Both protected producers derive runner placement and Depot-cache authority + internally from a bounded runner-size and target-architecture contract; + callers cannot inject runner labels. The ABI v3 manifest covers the complete + static link closure and a pinned build-image/toolchain epoch. Its cached and + uploaded payload is a minimal path-normalized link bundle rather than a + producer-local CMake build tree, and the producer retains target/backend + sccache evidence on both cache hits and misses. Native SDK + packaging invokes `build-llama.sh --require-existing` with both build.rs + auto-build switches disabled, so a missing or stale restored archive fails + instead of silently compiling llama.cpp again. + Kotlin smoke verifies and extracts the immutable package without Cargo, + llama.cpp preparation/builds, or native-SDK packaging. macOS CPU artifacts + feed Swift SDK smokes. Swift additionally restores the verified XCFramework + and exact generated `mesh_ffi.swift` companion artifact built by + `swift-sdk-artifact.yml`; PR uses `host-only`, while main and release use + `full`. Producer and smoke are fixed to `macos-15`, and the shared native + cache includes an explicit macOS/Xcode epoch. Rust compilation is routed + through sccache, with per-mode/per-attempt statistics retained as CI evidence. + Main and tag producers reject tracked-binding drift; dispatched releases copy + the producer binding into the prepared tag commit. The Swift consumer cannot + invoke Cargo, llama.cpp compilation, native-SDK packaging, or an XCFramework + build. - Linux native-runtime packaging uses `patchelf` to make packaged shared libraries relocatable with `$ORIGIN`, then verifies them without - `LD_LIBRARY_PATH`. Release native-runtime jobs and Linux SDK smoke jobs need - `patchelf` because SDK smoke prepares native runtime packages through - `scripts/ci-prepare-native-runtime.sh`. + `LD_LIBRARY_PATH`. Release native-runtime jobs and Rust SDK smoke jobs need + `patchelf`; Kotlin and Swift smokes reuse the runtime adjacent to their + composed product without rebuilding it. - Artifact-consuming smokes are additionally gated on the matching CPU producer being eligible, so backend-only or cleanup-only PRs skip those jobs natively instead of attempting to download an artifact that was never uploaded. diff --git a/docs/AGENTS.md b/docs/AGENTS.md index ab32d8e1ab..aad9b04a52 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -254,8 +254,9 @@ without touching the endpoint or creating artifacts. Scheduled GitHub runs are opt-in via `MESH_NIGHTLY_STABILITY_ENABLED=1` plus a configured endpoint. The scheduled/manual wrapper delegates execution to the reusable `nightly-stability-run.yml` workflow, which owns the harness run, -artifact upload, and timing summary. Treat this as a trend/evidence harness, -not a required PR gate. +artifact upload, and timing summary. The reusable workflow uses GitHub-hosted +Ubuntu and does not accept a caller-selected runner label. Treat this as a +trend/evidence harness, not a required PR gate. ## curl or any OpenAI client diff --git a/docs/SDK.md b/docs/SDK.md index c6991d9936..ae85c83d15 100644 --- a/docs/SDK.md +++ b/docs/SDK.md @@ -376,6 +376,41 @@ node --test sdk/node/test/*.test.js Run serving smoke examples with a real model: ```bash -scripts/ci-swift-sdk-smoke.sh -scripts/ci-kotlin-sdk-smoke.sh +scripts/ci-swift-sdk-smoke.sh \ + \ + \ + \ + \ + \ + +scripts/ci-kotlin-sdk-smoke.sh \ + \ + \ + \ + \ + \ + \ + ``` + +The Swift smoke consumes a previously produced XCFramework ZIP and the exact +generated `mesh_ffi.swift` companion artifact; it does not compile +`mesh-llm-ffi` or llama.cpp. CI creates both immutable inputs through the +shared `swift-sdk-artifact.yml` producer (`host-only` on PRs and `full` on +main/release). The producer and smoke are pinned to `macos-15`; the native +llama.cpp cache carries an explicit macOS/Xcode epoch that must be bumped when +that toolchain boundary changes. Protected main and tag builds fail if the +generated binding differs from the tracked source. + +The Kotlin smoke similarly consumes a previously produced, checksummed native +SDK archive. CI creates that immutable input through the shared +`native-sdk-artifact.yml` producer with an explicit target, backend, and Cargo +profile. PR validation uses `debug`; main and release use `release`. Kotlin +native-SDK production restores the checksummed `static-abi-artifact.yml` input +for its target, then keeps `package-native-sdk.sh --build` as a stamp check and +Rust FFI build instead of recompiling llama.cpp. Both reusable producers accept +only a bounded runner size and derive the architecture-specific hosted/Depot +label and cache permission internally; callers cannot supply runner labels or +Depot-cache authority. Kotlin smoke verifies and +safely extracts the final package, and does not prepare or compile llama.cpp, +build `mesh-llm-ffi`, or package a replacement. diff --git a/docs/design/TESTING.md b/docs/design/TESTING.md index a11b91590b..7519d1a8fa 100644 --- a/docs/design/TESTING.md +++ b/docs/design/TESTING.md @@ -216,8 +216,11 @@ The scheduled/manual wrapper calls the reusable `nightly-stability-run.yml` workflow so maintainers can reuse the same harness execution from other workflows or lab jobs. The job summary includes the timing snapshot from `summary.md`, so day-over-day drift can be checked without opening JSONL -artifacts. It is intentionally evidence-producing and non-required: failed -nightlies should guide stabilization work, not block unrelated pull requests. +artifacts. The reusable workflow is fixed to GitHub-hosted Ubuntu; it never +accepts caller-provided runner labels because it checks out the caller's +repository content. It is intentionally evidence-producing and non-required: +failed nightlies should guide stabilization work, not block unrelated pull +requests. ### 0f. KV/tool-loop stability certification diff --git a/scripts/build-llama.sh b/scripts/build-llama.sh index 68141bcdfd..0a224fb19d 100755 --- a/scripts/build-llama.sh +++ b/scripts/build-llama.sh @@ -3,6 +3,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck disable=SC1091 source "$ROOT/scripts/lib/cuda-toolkit.sh" LLAMA_WORKDIR="${LLAMA_WORKDIR:-$ROOT/.deps/llama.cpp}" @@ -10,11 +11,23 @@ LLAMA_BUILD_ROOT="${MESH_LLM_LLAMA_BUILD_ROOT:-$ROOT/.deps/llama-build}" LLAMA_BACKEND="${LLAMA_STAGE_BACKEND:-${SKIPPY_LLAMA_BACKEND:-${LLAMA_BACKEND:-cpu}}}" LLAMA_LINK_MODE="${LLAMA_STAGE_LINK_MODE:-${SKIPPY_LLAMA_LINK_MODE:-static}}" PRINT_BUILD_DIR=0 +REQUIRE_EXISTING=0 -if [[ "${1:-}" == "--print-build-dir" ]]; then - PRINT_BUILD_DIR=1 - shift -fi +while [[ "$#" -gt 0 ]]; do + case "$1" in + --print-build-dir) + PRINT_BUILD_DIR=1 + shift + ;; + --require-existing) + REQUIRE_EXISTING=1 + shift + ;; + *) + break + ;; + esac +done case "$LLAMA_BACKEND" in cpu|cuda|rocm|hip|vulkan|metal) ;; @@ -68,18 +81,23 @@ detect_jobs() { fi } -required_archives() { +required_static_archives() { printf '%s\n' \ "$LLAMA_BUILD_DIR/src/libllama.a" \ "$LLAMA_BUILD_DIR/common/libllama-common.a" \ + "$LLAMA_BUILD_DIR/common/libllama-common-base.a" \ + "$LLAMA_BUILD_DIR/ggml/src/libggml.a" \ + "$LLAMA_BUILD_DIR/ggml/src/libggml-base.a" \ "$LLAMA_BUILD_DIR/tools/mtmd/libmtmd.a" } -required_archives_exist() { +required_static_archives_exist() { local archive while IFS= read -r archive; do [[ -f "$archive" ]] || return 1 - done < <(required_archives) + done < <(required_static_archives) + [[ -f "$LLAMA_BUILD_DIR/ggml/src/libggml-cpu.a" || + -f "$LLAMA_BUILD_DIR/ggml/src/ggml-cpu/libggml-cpu.a" ]] } dynamic_library_names() { @@ -97,9 +115,10 @@ dynamic_library_names() { } required_dynamic_libraries_exist() { - local name + local name found while IFS= read -r name; do - find "$LLAMA_BUILD_DIR" -type f -name "$name" -print -quit | grep -q . + found="$(find "$LLAMA_BUILD_DIR" -name "$name" -print -quit)" + [[ -n "$found" && -e "$found" ]] || return 1 done < <(dynamic_library_names) } @@ -107,7 +126,7 @@ required_outputs_exist() { if [[ "$LLAMA_LINK_MODE" == "dynamic" ]]; then required_dynamic_libraries_exist else - required_archives_exist + required_static_archives_exist fi } @@ -174,6 +193,17 @@ CMAKE_ARGS=( -DMTMD_VIDEO=OFF ) +# Static ABI inputs cross job and runner boundaries. Normalize compiler- +# embedded source/build paths so the archived link closure does not retain a +# producer-local workspace path. +if [[ "$LLAMA_LINK_MODE" == "static" ]]; then + PREFIX_MAP_FLAGS="-ffile-prefix-map=$ROOT=/mesh-llm -fdebug-prefix-map=$ROOT=/mesh-llm -fmacro-prefix-map=$ROOT=/mesh-llm" + CMAKE_ARGS+=( + "-DCMAKE_C_FLAGS=$PREFIX_MAP_FLAGS" + "-DCMAKE_CXX_FLAGS=$PREFIX_MAP_FLAGS" + ) +fi + if command -v ninja >/dev/null 2>&1; then CMAKE_ARGS=(-G Ninja "${CMAKE_ARGS[@]}") echo "using CMake generator: Ninja" @@ -215,6 +245,10 @@ esac USE_SCCACHE="${LLAMA_STAGE_USE_SCCACHE:-${SKIPPY_USE_SCCACHE:-1}}" if [[ "$USE_SCCACHE" != "0" && -n "$SCCACHE_BIN" ]] && ! "$SCCACHE_BIN" --start-server >/dev/null 2>&1; then + if [[ "${MESH_LLM_REQUIRE_SCCACHE:-0}" == "1" ]]; then + echo "sccache failed to start and MESH_LLM_REQUIRE_SCCACHE=1" >&2 + exit 1 + fi echo "sccache failed to start; llama.cpp build will run without compiler caching" >&2 USE_SCCACHE=0 fi @@ -234,6 +268,10 @@ if [[ "$USE_SCCACHE" != "0" && -n "$SCCACHE_BIN" ]]; then esac echo "using sccache for llama.cpp C/C++ compilation: $SCCACHE_BIN" elif [[ "$USE_SCCACHE" != "0" ]]; then + if [[ "${MESH_LLM_REQUIRE_SCCACHE:-0}" == "1" ]]; then + echo "sccache is required but was not found" >&2 + exit 1 + fi echo "sccache not found; llama.cpp build will run without compiler caching" >&2 else CMAKE_ARGS+=(-DGGML_CCACHE=OFF) @@ -245,11 +283,24 @@ fi PATCHED_SHA="$(tr -d '[:space:]' < "$LLAMA_WORKDIR/.mesh-llm-patched-sha" 2>/dev/null || git -C "$LLAMA_WORKDIR" rev-parse HEAD)" BUILD_STAMP="$LLAMA_BUILD_DIR/.mesh-llm-build-stamp" + +normalize_build_stamp_arg() { + local value="$1" + value="${value//"$LLAMA_BUILD_DIR"/@LLAMA_BUILD_DIR@}" + value="${value//"$LLAMA_WORKDIR"/@LLAMA_WORKDIR@}" + value="${value//"$ROOT"/@MESH_LLM_ROOT@}" + if [[ -n "$SCCACHE_BIN" ]]; then + value="${value//"$SCCACHE_BIN"/@SCCACHE@}" + fi + printf '%s\n' "$value" +} + CURRENT_BUILD_STAMP="$( - printf 'stamp-version=1\n' + printf 'stamp-version=3\n' printf 'patched-sha=%s\n' "$PATCHED_SHA" printf 'backend=%s\n' "$LLAMA_BACKEND" printf 'link-mode=%s\n' "$LLAMA_LINK_MODE" + printf 'toolchain-epoch=%s\n' "${MESH_LLM_LLAMA_TOOLCHAIN_EPOCH:-local}" printf 'build-type=%s\n' "${CMAKE_BUILD_TYPE:-Release}" printf 'ggml-native=%s\n' "${LLAMA_STAGE_GGML_NATIVE:-${SKIPPY_GGML_NATIVE:-OFF}}" printf 'cuda-architectures=%s\n' "${LLAMA_STAGE_CUDA_ARCHITECTURES:-${SKIPPY_CUDA_ARCHITECTURES:-}}" @@ -257,7 +308,7 @@ CURRENT_BUILD_STAMP="$( printf 'cuda-no-vmm=%s\n' "${GGML_CUDA_NO_VMM:-}" printf 'use-sccache=%s\n' "$USE_SCCACHE" for arg in "${CMAKE_ARGS[@]}"; do - printf 'cmake-arg=%s\n' "$arg" + printf 'cmake-arg=%s\n' "$(normalize_build_stamp_arg "$arg")" done )" @@ -272,11 +323,25 @@ if [[ "${LLAMA_STAGE_FORCE_BUILD:-${SKIPPY_FORCE_LLAMA_BUILD:-0}}" != "1" && exit 0 fi +if [[ "$REQUIRE_EXISTING" == "1" ]]; then + echo "prebuilt patched llama.cpp ABI did not match the requested build contract" >&2 + echo " backend: $LLAMA_BACKEND" >&2 + echo " link mode: $LLAMA_LINK_MODE" >&2 + echo " toolchain epoch: ${MESH_LLM_LLAMA_TOOLCHAIN_EPOCH:-local}" >&2 + echo " build dir: $LLAMA_BUILD_DIR" >&2 + echo "refusing to rebuild because --require-existing was set" >&2 + exit 1 +fi + cmake "${CMAKE_ARGS[@]}" cmake --build "$LLAMA_BUILD_DIR" --config "${CMAKE_BUILD_TYPE:-Release}" --parallel "$(detect_jobs)" --target llama llama-common mtmd printf '%s\n' "$CURRENT_BUILD_STAMP" > "$BUILD_STAMP" +if ! required_outputs_exist; then + echo "patched llama.cpp build completed without the full link closure" >&2 + exit 1 +fi echo "built patched llama.cpp" echo " backend: $LLAMA_BACKEND" diff --git a/scripts/ci-kotlin-sdk-smoke.sh b/scripts/ci-kotlin-sdk-smoke.sh index 55dfff34a0..1d495bcde6 100755 --- a/scripts/ci-kotlin-sdk-smoke.sh +++ b/scripts/ci-kotlin-sdk-smoke.sh @@ -1,61 +1,28 @@ #!/usr/bin/env bash set -euo pipefail -if [ "$#" -ne 3 ]; then - echo "Usage: $0 " >&2 +if [ "$#" -ne 7 ]; then + echo "Usage: $0 " >&2 exit 1 fi REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$REPO_ROOT" -export CARGO_HTTP_MULTIPLEXING="${CARGO_HTTP_MULTIPLEXING:-false}" -export CARGO_NET_RETRY="${CARGO_NET_RETRY:-10}" - -retry_transient() { - local attempt=1 - local max_attempts=3 - - while true; do - if "$@"; then - return 0 - fi - if [ "$attempt" -ge "$max_attempts" ]; then - return 1 - fi - echo "command failed; retrying ($attempt/$max_attempts): $*" >&2 - sleep $((attempt * 5)) - attempt=$((attempt + 1)) - done -} - scripts/check-sdk-contract.sh scripts/package-sdk-console-assets.sh --sdk kotlin scripts/verify-sdk-console-assets.sh --sdk kotlin -scripts/prepare-llama.sh "${MESH_LLM_LLAMA_PIN_SHA:-pinned}" -LLAMA_STAGE_BACKEND=cpu \ -LLAMA_STAGE_BUILD_DIR="$REPO_ROOT/.deps/llama-build/build-stage-abi-ci-kotlin-cpu" \ -LLAMA_BUILD_DIR="$REPO_ROOT/.deps/llama-build/build-stage-abi-ci-kotlin-cpu" \ - scripts/build-llama.sh - -LLAMA_STAGE_BACKEND=cpu \ -LLAMA_STAGE_BUILD_DIR="$REPO_ROOT/.deps/llama-build/build-stage-abi-ci-kotlin-cpu" \ - retry_transient cargo build -p mesh-llm-ffi --no-default-features --features host,embedded-runtime - -native_sdk_out="$REPO_ROOT/target/kotlin-native-sdk" -LLAMA_STAGE_BACKEND=cpu \ -LLAMA_STAGE_BUILD_DIR="$REPO_ROOT/.deps/llama-build/build-stage-abi-ci-kotlin-cpu" \ - retry_transient scripts/package-native-sdk.sh \ - --backend cpu \ - --profile debug \ - --out "$native_sdk_out" -scripts/verify-native-sdk-package.sh "$native_sdk_out"/meshllm-native-*.tar.gz -native_sdk_artifact_dir="$(find "$native_sdk_out" -mindepth 1 -maxdepth 1 -type d -name 'meshllm-native-*' -print -quit)" -if [[ -z "$native_sdk_artifact_dir" ]]; then - echo "native SDK artifact directory not found under $native_sdk_out" >&2 - exit 1 -fi +native_sdk_tmp="$(mktemp -d)" +trap 'rm -rf "$native_sdk_tmp"' EXIT +native_sdk_artifact_dir="$( + scripts/restore-native-sdk-input.sh \ + "$4" \ + "$native_sdk_tmp/extracted" \ + "$5" \ + "$6" \ + "$7" +)" native_sdk_uniffi_library="$( python3 - "$native_sdk_artifact_dir/manifest.json" <<'PY' import json diff --git a/scripts/ci-swift-sdk-smoke.sh b/scripts/ci-swift-sdk-smoke.sh index 0fb607b1d9..680a1ff9da 100755 --- a/scripts/ci-swift-sdk-smoke.sh +++ b/scripts/ci-swift-sdk-smoke.sh @@ -1,35 +1,48 @@ #!/usr/bin/env bash set -euo pipefail -if [ "$#" -ne 3 ]; then - echo "Usage: $0 " >&2 +if [ "$#" -ne 6 ]; then + echo "Usage: $0 " >&2 exit 1 fi REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$REPO_ROOT" +SWIFT_INPUT_ARCHIVE="$4" +SWIFT_INPUT_MODE="$5" +SWIFT_INPUT_BINDING="$6" +SWIFT_TRACKED_BINDING="sdk/swift/Sources/MeshLLM/Generated/mesh_ffi.swift" + +if [[ ! -f "$SWIFT_INPUT_BINDING" || -L "$SWIFT_INPUT_BINDING" ]]; then + echo "immutable generated Swift binding is missing or unsafe: $SWIFT_INPUT_BINDING" >&2 + exit 1 +fi +install -m 0644 "$SWIFT_INPUT_BINDING" "$SWIFT_TRACKED_BINDING" +cmp "$SWIFT_INPUT_BINDING" "$SWIFT_TRACKED_BINDING" scripts/check-sdk-contract.sh scripts/package-sdk-console-assets.sh --sdk swift scripts/verify-sdk-console-assets.sh --sdk swift -if [[ "${MESH_SWIFT_FULL_XCFRAMEWORK_SMOKE:-0}" == "1" ]]; then - ./sdk/swift/scripts/build-xcframework.sh -else - ./sdk/swift/scripts/build-host-macos-xcframework.sh +scripts/verify-swift-release-artifact.sh \ + "$SWIFT_INPUT_ARCHIVE" \ + "$SWIFT_INPUT_MODE" + +SWIFT_EXTRACT_DIR="$(mktemp -d)" +trap 'rm -rf "$SWIFT_EXTRACT_DIR"' EXIT +scripts/safe-extract-zip.py "$SWIFT_INPUT_ARCHIVE" "$SWIFT_EXTRACT_DIR" + +SWIFT_XCFRAMEWORK="sdk/swift/Generated/MeshLLMFFI.xcframework" +if [[ ! -d "$SWIFT_EXTRACT_DIR/MeshLLMFFI.xcframework" ]]; then + echo "verified Swift SDK input did not restore MeshLLMFFI.xcframework" >&2 + exit 1 fi +rm -rf "$SWIFT_XCFRAMEWORK" +mv "$SWIFT_EXTRACT_DIR/MeshLLMFFI.xcframework" "$SWIFT_XCFRAMEWORK" scripts/verify-swift-privacy-manifest.sh \ sdk/swift/PrivacyInfo.xcprivacy \ - sdk/swift/Generated/MeshLLMFFI.xcframework - -SWIFT_ARTIFACT_SMOKE_DIR="$(mktemp -d)" -trap 'rm -rf "$SWIFT_ARTIFACT_SMOKE_DIR"' EXIT -ditto -c -k --sequesterRsrc --keepParent \ - sdk/swift/Generated/MeshLLMFFI.xcframework \ - "$SWIFT_ARTIFACT_SMOKE_DIR/MeshLLMFFI.xcframework.zip" -scripts/verify-swift-release-artifact.sh \ - "$SWIFT_ARTIFACT_SMOKE_DIR/MeshLLMFFI.xcframework.zip" + "$SWIFT_XCFRAMEWORK" native_runtime_dir="$( scripts/ci-prepare-native-runtime.sh \ diff --git a/scripts/package-native-sdk.sh b/scripts/package-native-sdk.sh index 3c989ae9a0..155449b56a 100755 --- a/scripts/package-native-sdk.sh +++ b/scripts/package-native-sdk.sh @@ -5,12 +5,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" BUILD=0 +REQUIRE_PREBUILT_LLAMA=0 OUT_DIR="$REPO_ROOT/dist/native-sdk" BACKEND="${LLAMA_STAGE_BACKEND:-${SKIPPY_LLAMA_BACKEND:-cpu}}" TARGET_TRIPLE="${MESH_NATIVE_SDK_TARGET:-}" PROFILE="${MESH_NATIVE_SDK_PROFILE:-release}" LLAMA_WORKDIR="${LLAMA_WORKDIR:-$REPO_ROOT/.deps/llama.cpp}" -LLAMA_BUILD_ROOT="${MESH_LLM_LLAMA_BUILD_ROOT:-$REPO_ROOT/.deps/llama-build}" usage() { cat >&2 <<'EOF' @@ -20,6 +20,9 @@ Package a backend-flavoured MeshLLM native SDK runtime artifact. Options: --build Build patched llama.cpp and mesh-llm-ffi before packaging. + --require-prebuilt-llama + With --build, verify and reuse the existing llama.cpp ABI + instead of rebuilding it. --backend NAME cpu, metal, cuda, rocm, hip, or vulkan. --target TRIPLE Rust target triple. Defaults to the host target. --profile PROFILE Cargo profile to package: release or debug. Defaults to release. @@ -42,6 +45,10 @@ while [[ "$#" -gt 0 ]]; do BUILD=1 shift ;; + --require-prebuilt-llama) + REQUIRE_PREBUILT_LLAMA=1 + shift + ;; --backend) BACKEND="${2:?missing backend}" shift 2 @@ -86,6 +93,11 @@ case "$PROFILE" in ;; esac +if [[ "$REQUIRE_PREBUILT_LLAMA" == "1" && "$BUILD" != "1" ]]; then + echo "--require-prebuilt-llama requires --build" >&2 + exit 1 +fi + host_os() { case "$(uname -s)" in Darwin) printf 'darwin\n' ;; @@ -239,18 +251,27 @@ fi if [[ "$BUILD" == "1" ]]; then "$SCRIPT_DIR/prepare-llama.sh" "${MESH_LLM_LLAMA_PIN_SHA:-pinned}" + llama_build_dir="$LLAMA_STAGE_BUILD_DIR" + build_args=() + if [[ "$REQUIRE_PREBUILT_LLAMA" == "1" ]]; then + build_args+=(--require-existing) + fi LLAMA_STAGE_BACKEND="$(build_backend)" \ - LLAMA_BUILD_DIR="$LLAMA_STAGE_BUILD_DIR" \ - LLAMA_STAGE_BUILD_DIR="$LLAMA_STAGE_BUILD_DIR" \ - "$SCRIPT_DIR/build-llama.sh" + LLAMA_BUILD_DIR="$llama_build_dir" \ + LLAMA_STAGE_BUILD_DIR="$llama_build_dir" \ + "$SCRIPT_DIR/build-llama.sh" "${build_args[@]}" - cargo_args=(build -p mesh-llm-ffi --no-default-features --features host,embedded-runtime) + cargo_args=(build -p mesh-llm-ffi --no-default-features --features "host,embedded-runtime") if [[ "$PROFILE" == "release" ]]; then cargo_args+=(--release) fi if [[ "$TARGET_TRIPLE" != "$(default_target_triple)" ]]; then cargo_args+=(--target "$TARGET_TRIPLE") fi + if [[ "$REQUIRE_PREBUILT_LLAMA" == "1" ]]; then + export SKIPPY_LLAMA_AUTO_BUILD=0 + export MESH_LLM_AUTO_BUILD_LLAMA=0 + fi LLAMA_STAGE_BACKEND="$(build_backend)" \ LLAMA_STAGE_BUILD_DIR="$LLAMA_STAGE_BUILD_DIR" \ cargo "${cargo_args[@]}" diff --git a/scripts/plan-pr-build-jobs.py b/scripts/plan-pr-build-jobs.py new file mode 100644 index 0000000000..af1bbfb1ef --- /dev/null +++ b/scripts/plan-pr-build-jobs.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Plan the conditionally required top-level jobs in PR Builds.""" + +from __future__ import annotations + +from collections import Counter +import json +import sys +from typing import Any + + +BOOLEAN_FIELDS = frozenset( + { + "all_rust", + "backend_changed", + "benchmarks", + "docs_only", + "inference_artifact_required", + "linux_test_groups_nonempty", + "live_agent_enabled", + "runner_contract_required", + "sdk_smoke_required", + "test_batches_nonempty", + "windows_cpu_required", + "windows_gpu_required", + } +) + +# Keep this in workflow order. Every conditional top-level job must appear +# exactly once; the contract tests compare this table with pr_builds.yml. +JOB_ROUTES: tuple[tuple[str, str], ...] = ( + ("public_runner_image_contract", "runner_contract"), + ("linux_host_input", "linux_host"), + ("linux_cpu_runtime_input", "linux_cpu"), + ("linux_cpu_artifact", "linux_cpu"), + ("linux_cuda_runtime_input", "linux_backend"), + ("linux_cuda_product", "linux_backend"), + ("linux_rocm_runtime_input", "linux_backend"), + ("linux_rocm_product", "linux_backend"), + ("linux_vulkan_runtime_input", "linux_backend"), + ("linux_vulkan_product", "linux_backend"), + ("linux_static_abi_input", "static_abi"), + ("rust_crate_tests", "rust_crate_tests"), + ("linux_test_groups", "linux_test_groups"), + ("linux_public_mesh_admission", "public_mesh_admission"), + ("hf_download_smoke", "hf_download"), + ("inference_smoke_tests", "inference_smoke"), + ("agent_live_smokes", "agent_live"), + ("two_node_client_serving_smoke", "two_node_client"), + ("two_node_split_smoke", "two_node_split"), + ("rust_sdk_smoke", "linux_sdk_smoke"), + ("kotlin_sdk_input", "kotlin_sdk_input"), + ("kotlin_sdk_smoke", "linux_sdk_smoke"), + ("swift_sdk_input", "swift_sdk"), + ("macos_host_input", "macos_product"), + ("macos_metal_runtime_input", "macos_product"), + ("macos_cpu_artifact", "macos_product"), + ("swift_sdk_smoke", "swift_sdk"), + ("macos_unit_tests", "macos_unit_tests"), + ("windows_checks", "windows_checks"), + ("windows_host_input", "windows_host"), + ("windows_cpu_runtime_input", "windows_cpu"), + ("windows_gpu_runtime_inputs", "windows_gpu"), + ("windows_cpu_product", "windows_cpu"), + ("windows_gpu_products", "windows_gpu"), +) + + +class PlanError(ValueError): + """Raised when the planner input or static route table is invalid.""" + + +def _validate_payload(payload: object) -> dict[str, Any]: + if not isinstance(payload, dict): + raise PlanError("planner input must be a JSON object") + + expected_fields = BOOLEAN_FIELDS | {"affected_crates", "event_name"} + actual_fields = set(payload) + missing = sorted(expected_fields - actual_fields) + unknown = sorted(actual_fields - expected_fields) + if missing: + raise PlanError(f"planner input is missing fields: {', '.join(missing)}") + if unknown: + raise PlanError(f"planner input has unknown fields: {', '.join(unknown)}") + + event_name = payload["event_name"] + if event_name not in {"pull_request", "workflow_dispatch"}: + raise PlanError(f"unsupported PR Builds event: {event_name!r}") + + for field in sorted(BOOLEAN_FIELDS): + if type(payload[field]) is not bool: + raise PlanError(f"{field} must be a JSON boolean") + + affected_crates = payload["affected_crates"] + if not isinstance(affected_crates, list) or not all( + isinstance(crate, str) and crate for crate in affected_crates + ): + raise PlanError("affected_crates must be an array of non-empty strings") + + return payload + + +def _validate_job_routes() -> None: + job_ids = [job_id for job_id, _route in JOB_ROUTES] + duplicates = sorted( + job_id for job_id, count in Counter(job_ids).items() if count != 1 + ) + if duplicates: + raise PlanError(f"jobs mapped more than once: {', '.join(duplicates)}") + + +def route_requirements(raw_payload: object) -> dict[str, bool]: + payload = _validate_payload(raw_payload) + affected = set(payload["affected_crates"]) + dispatch = payload["event_name"] == "workflow_dispatch" + eligible = not payload["docs_only"] + linux_inference = dispatch or payload["inference_artifact_required"] + macos_inference = linux_inference or payload["benchmarks"] + linux_backend = ( + dispatch + or payload["backend_changed"] + or payload["benchmarks"] + ) + + client_runtime_changed = bool( + affected & {"mesh-llm", "mesh-llm-client", "openai-frontend"} + ) + inference_runtime_changed = bool( + affected + & { + "mesh-llm", + "model-artifact", + "openai-frontend", + "skippy-runtime", + "skippy-server", + } + ) + split_runtime_changed = bool( + affected + & { + "mesh-llm", + "model-artifact", + "skippy-runtime", + "skippy-server", + } + ) + + broad_runtime = dispatch or payload["all_rust"] + sdk_required = payload["sdk_smoke_required"] + return { + "runner_contract": dispatch or payload["runner_contract_required"], + "linux_host": eligible and (linux_inference or payload["benchmarks"]), + "linux_cpu": eligible and linux_inference, + "linux_backend": eligible and linux_backend, + "static_abi": eligible + and ( + payload["test_batches_nonempty"] + or payload["linux_test_groups_nonempty"] + or sdk_required + ), + "rust_crate_tests": eligible and payload["test_batches_nonempty"], + "linux_test_groups": eligible and payload["linux_test_groups_nonempty"], + # Preserve the existing operator-only public-mesh path. Its product + # dependency still has to succeed before GitHub starts the job. + "public_mesh_admission": eligible and dispatch, + "hf_download": eligible + and ( + broad_runtime + or bool(affected & {"mesh-llm", "model-artifact"}) + ), + "inference_smoke": eligible + and linux_inference + and (broad_runtime or inference_runtime_changed), + "agent_live": eligible + and linux_inference + and payload["live_agent_enabled"] + and (broad_runtime or client_runtime_changed), + "two_node_client": eligible + and linux_inference + and (broad_runtime or client_runtime_changed), + "two_node_split": eligible + and linux_inference + and (broad_runtime or split_runtime_changed), + "linux_sdk_smoke": eligible and linux_inference and sdk_required, + "kotlin_sdk_input": eligible and sdk_required, + "swift_sdk": eligible and macos_inference and sdk_required, + "macos_product": eligible and macos_inference, + "macos_unit_tests": eligible + and ( + payload["all_rust"] + or bool( + affected + & {"mesh-llm", "mesh-llm-host-runtime", "model-artifact"} + ) + ), + "windows_checks": eligible + and ( + dispatch + or payload["all_rust"] + or payload["windows_cpu_required"] + or payload["windows_gpu_required"] + ), + "windows_host": eligible + and ( + dispatch + or payload["windows_cpu_required"] + or payload["windows_gpu_required"] + ), + "windows_cpu": eligible + and (dispatch or payload["windows_cpu_required"]), + "windows_gpu": eligible + and (dispatch or payload["windows_gpu_required"]), + } + + +def required_jobs(raw_payload: object) -> list[str]: + _validate_job_routes() + requirements = route_requirements(raw_payload) + declared_routes = {route for _job_id, route in JOB_ROUTES} + unknown_routes = sorted(declared_routes - requirements.keys()) + unused_routes = sorted(requirements.keys() - declared_routes) + if unknown_routes: + raise PlanError(f"jobs use undefined routes: {', '.join(unknown_routes)}") + if unused_routes: + raise PlanError(f"routes have no jobs: {', '.join(unused_routes)}") + return [ + job_id + for job_id, route in JOB_ROUTES + if requirements[route] + ] + + +def main() -> int: + try: + payload = json.load(sys.stdin) + plan = required_jobs(payload) + except (json.JSONDecodeError, PlanError) as error: + print(f"ERROR: unable to plan PR Builds jobs: {error}", file=sys.stderr) + return 2 + + print(json.dumps(plan, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/restore-native-sdk-input.sh b/scripts/restore-native-sdk-input.sh new file mode 100755 index 0000000000..eb115660ee --- /dev/null +++ b/scripts/restore-native-sdk-input.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 5 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +download_dir="$1" +extract_dir="$2" +expected_target="$3" +expected_backend="$4" +expected_profile="$5" + +case "$expected_target" in + x86_64-*) expected_arch="x86_64" ;; + aarch64-*) expected_arch="aarch64" ;; + *) + echo "unsupported native SDK target architecture: $expected_target" >&2 + exit 1 + ;; +esac +case "$(uname -m)" in + x86_64|amd64) runner_arch="x86_64" ;; + aarch64|arm64) runner_arch="aarch64" ;; + *) runner_arch="$(uname -m)" ;; +esac +if [[ "$runner_arch" != "$expected_arch" ]]; then + echo "native SDK target/runner architecture mismatch: $expected_target on $runner_arch" >&2 + exit 1 +fi + +if [[ ! -d "$download_dir" || -L "$download_dir" ]]; then + echo "native SDK artifact download directory is invalid: $download_dir" >&2 + exit 1 +fi +if [[ -L "$extract_dir" ]]; then + echo "native SDK extraction directory must not be a symlink: $extract_dir" >&2 + exit 1 +fi +mkdir -p "$extract_dir" + +shopt -s nullglob dotglob +download_entries=("$download_dir"/*) +archives=("$download_dir"/*.tar.gz) +checksums=("$download_dir"/*.tar.gz.sha256) +extract_entries=("$extract_dir"/*) +if [[ "${#download_entries[@]}" -ne 2 || + "${#archives[@]}" -ne 1 || + "${#checksums[@]}" -ne 1 ]]; then + echo "native SDK input must contain exactly one archive and checksum" >&2 + exit 1 +fi +if [[ "${checksums[0]}" != "${archives[0]}.sha256" ]]; then + echo "native SDK checksum sidecar does not match archive" >&2 + exit 1 +fi +if [[ "${#extract_entries[@]}" -ne 0 ]]; then + echo "native SDK extraction directory must be empty: $extract_dir" >&2 + exit 1 +fi + +"$REPO_ROOT/scripts/verify-native-sdk-package.sh" "${archives[0]}" >&2 +"$REPO_ROOT/scripts/safe-extract-tar.py" "${archives[0]}" "$extract_dir" + +artifact_entries=("$extract_dir"/*) +if [[ "${#artifact_entries[@]}" -ne 1 || + ! -d "${artifact_entries[0]}" || + -L "${artifact_entries[0]}" ]]; then + echo "native SDK archive must extract one artifact directory" >&2 + exit 1 +fi +artifact_dir="${artifact_entries[0]}" +"$REPO_ROOT/scripts/verify-native-sdk-package.sh" "$artifact_dir" >&2 + +python3 - \ + "$artifact_dir/manifest.json" \ + "$expected_target" \ + "$expected_backend" \ + "$expected_profile" <<'PY' +import json +import sys + +manifest_path, target, backend, profile = sys.argv[1:] +with open(manifest_path, encoding="utf-8") as handle: + manifest = json.load(handle) +expected = { + "target_triple": target, + "backend": backend, + "cargo_profile": profile, +} +for field, value in expected.items(): + if manifest.get(field) != value: + raise SystemExit( + f"native SDK manifest {field} mismatch: " + f"expected {value!r}, got {manifest.get(field)!r}" + ) +PY + +printf '%s\n' "$artifact_dir" diff --git a/scripts/restore-static-abi-input.sh b/scripts/restore-static-abi-input.sh new file mode 100755 index 0000000000..d02d6dae9b --- /dev/null +++ b/scripts/restore-static-abi-input.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 4 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +download_dir="$1" +build_dir="$2" +expected_target="$3" +expected_backend="$4" +expected_basename="build-stage-abi-static" +expected_toolchain_epoch="${MESH_LLM_LLAMA_TOOLCHAIN_EPOCH:-}" + +if [[ -z "$expected_toolchain_epoch" || + ! "$expected_toolchain_epoch" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "MESH_LLM_LLAMA_TOOLCHAIN_EPOCH must identify the pinned build image" >&2 + exit 1 +fi + +case "$expected_target" in + x86_64-unknown-linux-gnu) expected_arch="x86_64" ;; + aarch64-unknown-linux-gnu) expected_arch="aarch64" ;; + *) + echo "unsupported static ABI target: $expected_target" >&2 + exit 1 + ;; +esac +case "$(uname -m)" in + x86_64|amd64) runner_arch="x86_64" ;; + aarch64|arm64) runner_arch="aarch64" ;; + *) runner_arch="$(uname -m)" ;; +esac +if [[ "$runner_arch" != "$expected_arch" ]]; then + echo "static ABI target/runner architecture mismatch: $expected_target on $runner_arch" >&2 + exit 1 +fi +if [[ ! -d "$download_dir" || -L "$download_dir" ]]; then + echo "static ABI artifact download directory is invalid: $download_dir" >&2 + exit 1 +fi +if [[ "$(basename "$build_dir")" != "$expected_basename" ]]; then + echo "static ABI destination must end in $expected_basename: $build_dir" >&2 + exit 1 +fi +if [[ -e "$build_dir" || -L "$build_dir" ]]; then + echo "static ABI destination must not already exist: $build_dir" >&2 + exit 1 +fi + +shopt -s nullglob dotglob +download_entries=("$download_dir"/*) +archive="$download_dir/mesh-llm-static-abi.tar.gz" +checksum="$archive.sha256" +if [[ "${#download_entries[@]}" -ne 2 || + ! -s "$archive" || + ! -s "$checksum" ]]; then + echo "static ABI input must contain exactly its archive and checksum" >&2 + exit 1 +fi +python3 "$REPO_ROOT/scripts/verify-checksum-sidecar.py" "$archive" + +extract_root="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mesh-static-abi.XXXXXX")" +trap 'rm -rf -- "$extract_root"' EXIT +python3 "$REPO_ROOT/scripts/safe-extract-tar.py" \ + "$archive" \ + "$extract_root" + +extract_entries=("$extract_root"/*) +if [[ "${#extract_entries[@]}" -ne 1 || + ! -d "${extract_entries[0]}" || + -L "${extract_entries[0]}" || + "$(basename "${extract_entries[0]}")" != "$expected_basename" ]]; then + echo "static ABI archive must contain exactly $expected_basename" >&2 + exit 1 +fi +restored_dir="${extract_entries[0]}" +manifest="$restored_dir/.mesh-llm-static-abi-input.json" +build_stamp="$restored_dir/.mesh-llm-build-stamp" +cmake_cache="$restored_dir/CMakeCache.txt" +required_archives=( + "$restored_dir/src/libllama.a" + "$restored_dir/common/libllama-common.a" + "$restored_dir/common/libllama-common-base.a" + "$restored_dir/ggml/src/libggml.a" + "$restored_dir/ggml/src/libggml-base.a" + "$restored_dir/tools/mtmd/libmtmd.a" +) +test -s "$manifest" +test -s "$build_stamp" +test -f "$cmake_cache" +for archive in "${required_archives[@]}"; do + if [[ ! -s "$archive" ]]; then + echo "static ABI is missing required archive: $archive" >&2 + exit 1 + fi +done +if [[ ! -s "$restored_dir/ggml/src/libggml-cpu.a" && + ! -s "$restored_dir/ggml/src/ggml-cpu/libggml-cpu.a" ]]; then + echo "static ABI is missing libggml-cpu.a" >&2 + exit 1 +fi + +python3 - \ + "$manifest" \ + "$build_stamp" \ + "$expected_target" \ + "$expected_backend" \ + "$expected_basename" \ + "$expected_toolchain_epoch" <<'PY' +import hashlib +import json +import sys + +( + manifest_path, + stamp_path, + target, + backend, + build_directory, + toolchain_epoch, +) = sys.argv[1:] +with open(manifest_path, encoding="utf-8") as handle: + manifest = json.load(handle) +expected = { + "schema_version": 3, + "contract": "mesh-llm-static-abi-v3", + "target_triple": target, + "backend": backend, + "build_directory": build_directory, + "toolchain_epoch": toolchain_epoch, +} +for field, value in expected.items(): + if manifest.get(field) != value: + raise SystemExit( + f"static ABI manifest {field} mismatch: " + f"expected {value!r}, got {manifest.get(field)!r}" + ) +with open(stamp_path, "rb") as handle: + stamp_bytes = handle.read() +stamp_sha256 = hashlib.sha256(stamp_bytes).hexdigest() +if manifest.get("build_stamp_sha256") != stamp_sha256: + raise SystemExit("static ABI build stamp checksum mismatch") +PY +python3 "$REPO_ROOT/scripts/verify-static-abi-build-stamp.py" \ + "$build_stamp" \ + --backend "$expected_backend" \ + --link-mode static \ + --stamp-version 3 \ + --toolchain-epoch "$expected_toolchain_epoch" + +mkdir -p "$(dirname "$build_dir")" +cp -a "$restored_dir" "$build_dir" +test -d "$build_dir" diff --git a/scripts/safe-extract-zip.py b/scripts/safe-extract-zip.py new file mode 100755 index 0000000000..1bfcc26c99 --- /dev/null +++ b/scripts/safe-extract-zip.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Extract a ZIP archive without allowing path or symlink escapes.""" + +from __future__ import annotations + +import re +import shutil +import stat +import sys +import zipfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import NoReturn + + +WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:") + + +@dataclass(frozen=True) +class Entry: + info: zipfile.ZipInfo + parts: tuple[str, ...] + kind: str + mode: int + link_target: str | None = None + + +def fail(message: str) -> NoReturn: + raise SystemExit(f"unsafe ZIP archive: {message}") + + +def portable_parts(name: str, *, label: str) -> tuple[str, ...]: + if ( + not name + or any(character in name for character in ("\0", "\r", "\n", "\t")) + or "\\" in name + or name.startswith("/") + or WINDOWS_DRIVE.match(name) + ): + fail(f"{label} is not a portable relative path: {name!r}") + + path = PurePosixPath(name) + if ( + path.is_absolute() + or not path.parts + or any(part in {"", ".", ".."} for part in path.parts) + ): + fail(f"{label} escapes the extraction root: {name!r}") + return tuple(path.parts) + + +def resolve_link(parts: tuple[str, ...], target: str) -> None: + if ( + not target + or any(character in target for character in ("\0", "\r", "\n", "\t")) + or "\\" in target + or target.startswith("/") + or WINDOWS_DRIVE.match(target) + ): + fail(f"symlink target is not portable: {target!r}") + + resolved = list(parts[:-1]) + for part in PurePosixPath(target).parts: + if part in {"", "."}: + continue + if part == "..": + if not resolved: + fail(f"symlink target escapes the extraction root: {target!r}") + resolved.pop() + continue + resolved.append(part) + if not resolved: + fail(f"symlink target resolves to the extraction root: {target!r}") + + +def classify( + archive: zipfile.ZipFile, + info: zipfile.ZipInfo, +) -> Entry: + name = info.filename.rstrip("/") if info.is_dir() else info.filename + parts = portable_parts(name, label="entry") + mode = info.external_attr >> 16 + file_type = stat.S_IFMT(mode) + + if info.is_dir() or file_type == stat.S_IFDIR: + return Entry(info, parts, "directory", mode) + if file_type == stat.S_IFLNK: + try: + target = archive.read(info).decode("utf-8") + except UnicodeDecodeError: + fail(f"symlink target is not UTF-8: {info.filename!r}") + resolve_link(parts, target) + return Entry(info, parts, "symlink", mode, target) + if file_type in {0, stat.S_IFREG}: + return Entry(info, parts, "file", mode) + fail(f"unsupported entry type for {info.filename!r}") + + +def inspect_archive(archive: zipfile.ZipFile) -> list[Entry]: + entries = [classify(archive, info) for info in archive.infolist()] + seen: set[tuple[str, ...]] = set() + symlinks = {entry.parts for entry in entries if entry.kind == "symlink"} + + for entry in entries: + if entry.parts in seen: + fail(f"duplicate entry path: {entry.info.filename!r}") + seen.add(entry.parts) + for index in range(1, len(entry.parts)): + if entry.parts[:index] in symlinks: + fail( + "entry is nested beneath an archive symlink: " + f"{entry.info.filename!r}" + ) + return entries + + +def extract(archive_path: Path, destination: Path) -> None: + if not archive_path.is_file(): + fail(f"archive does not exist: {archive_path}") + if destination.is_symlink(): + fail(f"destination cannot be a symlink: {destination}") + destination.mkdir(parents=True, exist_ok=True) + if any(destination.iterdir()): + fail(f"destination must be empty: {destination}") + + with zipfile.ZipFile(archive_path) as archive: + entries = inspect_archive(archive) + + for entry in entries: + if entry.kind == "directory": + destination.joinpath(*entry.parts).mkdir(parents=True, exist_ok=True) + + for entry in entries: + if entry.kind != "file": + continue + output = destination.joinpath(*entry.parts) + output.parent.mkdir(parents=True, exist_ok=True) + with archive.open(entry.info) as source, output.open("xb") as target: + shutil.copyfileobj(source, target) + permissions = entry.mode & 0o777 + if permissions: + output.chmod(permissions) + + for entry in entries: + if entry.kind != "symlink": + continue + output = destination.joinpath(*entry.parts) + output.parent.mkdir(parents=True, exist_ok=True) + output.symlink_to(entry.link_target) + + +def main() -> None: + if len(sys.argv) != 3: + raise SystemExit( + "usage: scripts/safe-extract-zip.py ARCHIVE.zip DESTINATION" + ) + extract(Path(sys.argv[1]), Path(sys.argv[2])) + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index 3fd78f8302..0c16fb8903 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -306,6 +306,66 @@ def run_runner_selector( for line in output.read_text(encoding="utf-8").splitlines() ) + def run_reusable_runner_policy( + self, + workflow_name: str, + *, + repository: str, + event_name: str, + ref: str, + depot_enabled: str, + target: str, + runner_size: str, + manual_use_depot: str = "false", + ) -> tuple[subprocess.CompletedProcess[str], dict[str, str]]: + workflow = ( + ROOT / ".github" / "workflows" / workflow_name + ).read_text(encoding="utf-8") + policy = workflow.split( + " - name: Derive protected runner policy\n", + maxsplit=1, + )[1] + run_block = policy.split(" run: |\n", maxsplit=1)[1] + script_lines: list[str] = [] + for line in run_block.splitlines(): + if line.startswith(" "): + script_lines.append(line[10:]) + elif not line: + script_lines.append("") + else: + break + script = "\n".join(script_lines) + + with tempfile.TemporaryDirectory() as temp_dir: + output = Path(temp_dir) / "github-output" + result = subprocess.run( + ["bash", "-c", script], + cwd=ROOT, + env={ + **os.environ, + "GITHUB_OUTPUT": str(output), + "POLICY_REPOSITORY": repository, + "POLICY_REF": ref, + "POLICY_EVENT_NAME": event_name, + "POLICY_DEPOT_ENABLED": depot_enabled, + "POLICY_MANUAL_USE_DEPOT": manual_use_depot, + "POLICY_TARGET": target, + "POLICY_RUNNER_SIZE": runner_size, + }, + check=False, + capture_output=True, + text=True, + ) + outputs = {} + if output.exists(): + outputs = dict( + line.split("=", maxsplit=1) + for line in output.read_text( + encoding="utf-8", + ).splitlines() + ) + return result, outputs + def test_host_action_uses_canonical_dynamic_host_builder(self) -> None: action = self.read_action("prepare-host-input") @@ -571,6 +631,590 @@ def test_sdk_routing_covers_every_direct_smoke_script(self) -> None: with self.subTest(script=script): self.assertRegex(script, direct_sdk_pattern) + def test_native_sdk_build_is_a_shared_immutable_producer(self) -> None: + producer = ( + ROOT / ".github" / "workflows" / "native-sdk-artifact.yml" + ).read_text(encoding="utf-8") + producer_action = self.read_action("prepare-native-sdk-input") + consumer_workflow = ( + ROOT / ".github" / "workflows" / "sdk-smoke.yml" + ).read_text(encoding="utf-8") + consumer_script = ( + ROOT / "scripts" / "ci-kotlin-sdk-smoke.sh" + ).read_text(encoding="utf-8") + restore_script = ( + ROOT / "scripts" / "restore-native-sdk-input.sh" + ).read_text(encoding="utf-8") + routing = self.read_action("compute-changes") + + self.assertIn( + "uses: ./.github/actions/prepare-native-sdk-input", + producer, + ) + self.assertIn( + "uses: ./.github/workflows/static-abi-artifact.yml", + producer, + ) + self.assertIn( + "scripts/restore-static-abi-input.sh", + producer, + ) + self.assertIn( + "LLAMA_STAGE_BUILD_DIR: " + ".deps/llama.cpp/build-stage-abi-static", + producer, + ) + self.assertIn("persist-credentials: false", producer) + self.assertIn("actions/upload-artifact@", producer) + self.assertIn("inputs.include_runtime_crate", producer) + self.assertIn("RUSTC_WRAPPER: sccache", producer) + self.assertEqual( + producer.count( + "uses: ./.github/actions/capture-sccache-stats", + ), + 2, + ) + self.assertIn( + "sccache-native-sdk-${{ inputs.target }}-" + "${{ inputs.backend }}-${{ inputs.profile }}-" + "${{ github.run_attempt }}", + producer, + ) + self.assertIn( + "require_prebuilt_static_abi: " + "${{ inputs.static_abi_artifact_name != '' }}", + producer, + ) + self.assertIn("scripts/package-native-sdk.sh", producer_action) + self.assertIn("--build", producer_action) + self.assertIn("--require-prebuilt-llama", producer_action) + self.assertIn( + "scripts/verify-native-sdk-package.sh", + producer_action, + ) + self.assertIn( + "scripts/package-native-sdk-crate.sh", + producer_action, + ) + self.assertIn( + "native SDK release asset basename collision", + producer_action, + ) + + self.assertIn( + "name: ${{ inputs.kotlin_artifact_name }}", + consumer_workflow, + ) + self.assertIn( + "actions/download-artifact@" + "37930b1c2abaa49bbe596cd826c3c89aef350131", + consumer_workflow, + ) + self.assertIn( + "scripts/restore-native-sdk-input.sh", + consumer_script, + ) + for forbidden in ( + "cargo ", + "prepare-llama.sh", + "build-llama.sh", + "package-native-sdk.sh", + ): + with self.subTest(forbidden=forbidden): + self.assertNotIn(forbidden, consumer_script) + self.assertIn("scripts/safe-extract-tar.py", restore_script) + self.assertIn("prepare-native-sdk-input", routing) + self.assertIn("native-sdk-artifact", routing) + self.assertIn("restore-native-sdk-input", routing) + + def test_static_abi_artifact_is_typed_and_safely_reused(self) -> None: + producer = ( + ROOT / ".github" / "workflows" / "static-abi-artifact.yml" + ).read_text(encoding="utf-8") + producer_action = self.read_action("prepare-static-abi-input") + restore_script = ( + ROOT / "scripts" / "restore-static-abi-input.sh" + ).read_text(encoding="utf-8") + native_sdk_producer = ( + ROOT / ".github" / "workflows" / "native-sdk-artifact.yml" + ).read_text(encoding="utf-8") + pr_workflow = ( + ROOT / ".github" / "workflows" / "pr_builds.yml" + ).read_text(encoding="utf-8") + main_workflow = ( + ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + routing = self.read_action("compute-changes") + + self.assertIn("CACHE_NAMESPACE: mesh-llm", producer) + self.assertIn( + "inputs.backend, inputs.target, " + "env.MESH_LLM_LLAMA_TOOLCHAIN_EPOCH, hashFiles(", + producer, + ) + self.assertIn("path: static-abi-artifact-output", producer) + self.assertNotIn( + "path: .deps/llama.cpp/build-stage-abi-static", + producer, + ) + self.assertIn( + "mesh-llm-cuda-runner-sha256-" + "8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d", + producer, + ) + epoch = ( + "MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: " + "mesh-llm-cuda-runner-sha256-" + "8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d" + ) + for consumer in (native_sdk_producer, pr_workflow, main_workflow): + self.assertIn(epoch, consumer) + self.assertIn( + "uses: ./.github/actions/prepare-static-abi-input", + producer, + ) + prepare_index = producer.index( + "name: Prepare patched llama.cpp checkout", + ) + cache_index = producer.index( + "name: Cache portable static ABI input", + ) + self.assertLess(prepare_index, cache_index) + for cache_input in ( + "scripts/prepare-llama.sh", + "scripts/restore-static-abi-input.sh", + "scripts/safe-extract-tar.py", + "scripts/verify-checksum-sidecar.py", + "scripts/verify-static-abi-build-stamp.py", + ".github/actions/prepare-static-abi-input/action.yml", + ): + self.assertIn(cache_input, producer) + self.assertIn("name: ${{ inputs.artifact_name }}", producer) + self.assertIn( + "scripts/restore-static-abi-input.sh", + producer, + ) + self.assertIn( + "artifact_name: sccache-static-abi-" + "${{ inputs.target }}-${{ inputs.backend }}-" + "${{ github.run_attempt }}", + producer, + ) + self.assertIn("target/runner architecture mismatch", producer_action) + self.assertIn("verify-static-abi-build-stamp.py", producer_action) + self.assertIn("--patched-sha", producer_action) + self.assertIn("Portable MeshLLM static ABI link metadata", producer_action) + self.assertIn("retained producer-local path", producer_action) + self.assertNotIn( + 'tar -C "$(dirname "$LLAMA_STAGE_BUILD_DIR")"', + producer_action, + ) + for archive in ( + "libllama-common-base.a", + "libggml.a", + "libggml-base.a", + "libggml-cpu.a", + ): + self.assertIn(archive, producer_action) + self.assertIn( + ".mesh-llm-static-abi-input.json", + producer_action, + ) + self.assertIn("verify-checksum-sidecar.py", producer_action) + + self.assertIn("scripts/safe-extract-tar.py", restore_script) + self.assertIn("mesh-llm-static-abi-v3", restore_script) + self.assertIn("toolchain_epoch", restore_script) + self.assertIn("verify-checksum-sidecar.py", restore_script) + self.assertIn("verify-static-abi-build-stamp.py", restore_script) + self.assertIn("target/runner architecture mismatch", restore_script) + self.assertNotIn("tar -x", restore_script) + self.assertIn("prepare-static-abi-input", routing) + self.assertIn("restore-static-abi-input", routing) + self.assertIn("static-abi-artifact", routing) + + def test_protected_reusable_producers_own_runner_and_cache_policy( + self, + ) -> None: + workflow_names = ( + "native-sdk-artifact.yml", + "static-abi-artifact.yml", + ) + for workflow_name in workflow_names: + workflow = ( + ROOT / ".github" / "workflows" / workflow_name + ).read_text(encoding="utf-8") + inputs = workflow[: workflow.index("\njobs:\n")] + with self.subTest(workflow=workflow_name): + self.assertIn("runner_size:", inputs) + self.assertIn("default: '8'", inputs) + self.assertNotIn("runs_on:", inputs) + self.assertNotIn("allow_depot_remote_cache:", inputs) + self.assertNotIn("inputs.runs_on", workflow) + self.assertNotIn( + "inputs.allow_depot_remote_cache", + workflow, + ) + self.assertIn( + "runs-on: ${{ needs.runner_policy.outputs.runner }}", + workflow, + ) + self.assertIn( + "allow_depot_remote_cache: " + "${{ needs.runner_policy.outputs." + "allow_depot_remote_cache }}", + workflow, + ) + self.assertIn( + 'POLICY_REPOSITORY" == "Mesh-LLM/mesh-llm"', + workflow, + ) + self.assertIn( + 'POLICY_REF" == "refs/heads/main"', + workflow, + ) + self.assertIn( + 'POLICY_EVENT_NAME" == "push"', + workflow, + ) + self.assertIn( + 'POLICY_EVENT_NAME" == "workflow_dispatch"', + workflow, + ) + self.assertIn( + "POLICY_DEPOT_ENABLED: " + "${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }}", + workflow, + ) + self.assertIn( + "POLICY_MANUAL_USE_DEPOT: " + "${{ github.event_name == 'workflow_dispatch' " + "&& github.event.inputs.use_depot == 'true' }}", + workflow, + ) + self.assertIn("default|4|8|16", workflow) + self.assertIn("depot-ubuntu-24.04-arm", workflow) + + def test_protected_reusable_runner_policy_is_fail_closed(self) -> None: + hosted_cases = ( + ( + "pull_request", + "refs/pull/12/merge", + "Mesh-LLM/mesh-llm", + ), + ( + "pull_request_target", + "refs/heads/main", + "Mesh-LLM/mesh-llm", + ), + ( + "push", + "refs/tags/v1.2.3", + "Mesh-LLM/mesh-llm", + ), + ( + "workflow_dispatch", + "refs/heads/feature", + "Mesh-LLM/mesh-llm", + ), + ( + "push", + "refs/heads/main", + "attacker/mesh-llm", + ), + ) + for workflow_name in ( + "native-sdk-artifact.yml", + "static-abi-artifact.yml", + ): + for event_name, ref, repository in hosted_cases: + with self.subTest( + workflow=workflow_name, + event_name=event_name, + ref=ref, + repository=repository, + ): + result, outputs = self.run_reusable_runner_policy( + workflow_name, + repository=repository, + event_name=event_name, + ref=ref, + depot_enabled="true", + target="x86_64-unknown-linux-gnu", + runner_size="16", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(outputs["runner"], "ubuntu-24.04") + self.assertEqual( + outputs["allow_depot_remote_cache"], + "false", + ) + + trusted_cases = ( + ( + "x86_64-unknown-linux-gnu", + "8", + "depot-ubuntu-24.04-8", + ), + ( + "aarch64-unknown-linux-gnu", + "4", + "depot-ubuntu-24.04-arm-4", + ), + ( + "x86_64-unknown-linux-gnu", + "default", + "depot-ubuntu-24.04", + ), + ) + for target, size, expected_runner in trusted_cases: + with self.subTest( + workflow=workflow_name, + target=target, + size=size, + ): + result, outputs = self.run_reusable_runner_policy( + workflow_name, + repository="Mesh-LLM/mesh-llm", + event_name="push", + ref="refs/heads/main", + depot_enabled="true", + target=target, + runner_size=size, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(outputs["runner"], expected_runner) + self.assertEqual( + outputs["allow_depot_remote_cache"], + "true", + ) + + result, outputs = self.run_reusable_runner_policy( + workflow_name, + repository="Mesh-LLM/mesh-llm", + event_name="push", + ref="refs/heads/main", + depot_enabled="false", + target="aarch64-unknown-linux-gnu", + runner_size="8", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(outputs["runner"], "ubuntu-24.04-arm") + self.assertEqual( + outputs["allow_depot_remote_cache"], + "false", + ) + + result, outputs = self.run_reusable_runner_policy( + workflow_name, + repository="Mesh-LLM/mesh-llm", + event_name="push", + ref="refs/heads/main", + depot_enabled="true", + target="x86_64-unknown-linux-gnu", + runner_size="unbounded", + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "runner_size must be one of", + result.stderr, + ) + self.assertEqual(outputs, {}) + + result, outputs = self.run_reusable_runner_policy( + workflow_name, + repository="Mesh-LLM/mesh-llm", + event_name="workflow_dispatch", + ref="refs/heads/main", + depot_enabled="false", + manual_use_depot="true", + target="x86_64-unknown-linux-gnu", + runner_size="8", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + outputs["runner"], + "depot-ubuntu-24.04-8", + ) + self.assertEqual( + outputs["allow_depot_remote_cache"], + "true", + ) + + for event_name, ref, manual_use_depot in ( + ("workflow_dispatch", "refs/heads/main", "false"), + ("workflow_dispatch", "refs/heads/feature", "true"), + ("pull_request", "refs/pull/12/merge", "true"), + ("push", "refs/heads/main", "true"), + ): + with self.subTest( + workflow=workflow_name, + event_name=event_name, + ref=ref, + manual_use_depot=manual_use_depot, + ): + result, outputs = self.run_reusable_runner_policy( + workflow_name, + repository="Mesh-LLM/mesh-llm", + event_name=event_name, + ref=ref, + depot_enabled="false", + manual_use_depot=manual_use_depot, + target="x86_64-unknown-linux-gnu", + runner_size="8", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(outputs["runner"], "ubuntu-24.04") + self.assertEqual( + outputs["allow_depot_remote_cache"], + "false", + ) + + result, outputs = self.run_reusable_runner_policy( + "native-sdk-artifact.yml", + repository="Mesh-LLM/mesh-llm", + event_name="push", + ref="refs/heads/main", + depot_enabled="true", + target="aarch64-apple-darwin", + runner_size="8", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(outputs["runner"], "macos-15") + self.assertEqual( + outputs["allow_depot_remote_cache"], + "false", + ) + + def test_swift_sdk_build_is_a_shared_immutable_producer(self) -> None: + producer = ( + ROOT / ".github" / "workflows" / "swift-sdk-artifact.yml" + ).read_text(encoding="utf-8") + consumer_workflow = ( + ROOT / ".github" / "workflows" / "sdk-smoke.yml" + ).read_text(encoding="utf-8") + consumer_script = ( + ROOT / "scripts" / "ci-swift-sdk-smoke.sh" + ).read_text(encoding="utf-8") + routing = self.read_action("compute-changes") + + self.assertIn("type: string", producer) + self.assertIn("host-only|full", producer) + self.assertIn( + "sdk/swift/scripts/build-host-macos-xcframework.sh", + producer, + ) + self.assertIn("sdk/swift/scripts/build-xcframework.sh", producer) + self.assertIn( + "scripts/verify-swift-release-artifact.sh", + producer, + ) + self.assertIn( + "scripts/verify-swift-xcframework.py", + ( + ROOT / "scripts" / "verify-swift-release-artifact.sh" + ).read_text(encoding="utf-8"), + ) + self.assertIn("persist-credentials: false", producer) + self.assertIn("actions/upload-artifact@", producer) + self.assertIn("runs-on: macos-15", producer) + self.assertIn("RUSTC_WRAPPER: sccache", producer) + self.assertNotIn("macos_runner:", producer) + self.assertIn( + "name: generated-swift-binding-${{ inputs.artifact_name }}", + producer, + ) + self.assertIn( + "git diff --exit-code -- \"$generated_binding\"", + producer, + ) + self.assertIn( + "uses: ./.github/actions/capture-sccache-stats", + producer, + ) + self.assertIn("if: ${{ !cancelled() }}", producer) + self.assertIn( + "artifact_name: sccache-swift-sdk-" + "${{ inputs.mode }}-${{ github.run_attempt }}", + producer, + ) + + self.assertIn( + "name: ${{ inputs.swift_artifact_name }}", + consumer_workflow, + ) + self.assertIn( + "name: generated-swift-binding-" + "${{ inputs.swift_artifact_name }}", + consumer_workflow, + ) + self.assertIn( + "actions/download-artifact@" + "37930b1c2abaa49bbe596cd826c3c89aef350131", + consumer_workflow, + ) + self.assertIn("persist-credentials: false", consumer_workflow) + self.assertIn( + "if: ${{ inputs.sdk_kind == 'rust' }}", + consumer_workflow, + ) + + for forbidden in ( + "cargo ", + "build-llama.sh", + "package-native-sdk.sh", + "build-xcframework.sh", + "build-host-macos-xcframework.sh", + ): + with self.subTest(forbidden=forbidden): + self.assertNotIn(forbidden, consumer_script) + self.assertIn( + 'scripts/safe-extract-zip.py "$SWIFT_INPUT_ARCHIVE"', + consumer_script, + ) + self.assertIn( + 'install -m 0644 "$SWIFT_INPUT_BINDING" ' + '"$SWIFT_TRACKED_BINDING"', + consumer_script, + ) + self.assertIn("safe-extract-(tar|zip)", routing) + self.assertIn("verify-swift-xcframework", routing) + self.assertIn( + "(native-sdk-artifact|sdk-smoke|static-abi-artifact|" + "swift-sdk-artifact)", + routing, + ) + + def test_swift_sdk_cache_is_mode_independent_and_target_specific( + self, + ) -> None: + producer = ( + ROOT / ".github" / "workflows" / "swift-sdk-artifact.yml" + ).read_text(encoding="utf-8") + host_builder = ( + ROOT / "sdk" / "swift" / "scripts" + / "build-host-macos-xcframework.sh" + ).read_text(encoding="utf-8") + + self.assertIn( + "format('mesh-llm-swift-sdk-{0}-{1}-{2}-{3}', " + "runner.os, runner.arch, " + "env.SWIFT_NATIVE_XCODE_CACHE_EPOCH, hashFiles(", + producer, + ) + self.assertNotIn("runner.arch, inputs.mode, hashFiles(", producer) + self.assertIn( + "SWIFT_NATIVE_XCODE_CACHE_EPOCH: " + "macos-15-arm64-xcode-default-v1", + producer, + ) + self.assertIn("trusted main full build", producer) + self.assertNotIn("build-stage-abi-host-metal", producer) + self.assertIn( + ".deps/llama-build/build-stage-abi-$RUST_TARGET-metal", + host_builder, + ) + def test_runtime_action_never_builds_the_host(self) -> None: action = self.read_action("prepare-native-runtime-input") @@ -962,6 +1606,14 @@ def test_pr_caches_rely_on_github_ref_scoping_while_depot_is_blocked( builds = ( ROOT / ".github" / "workflows" / "pr_builds.yml" ).read_text(encoding="utf-8") + self.assertIn( + "shared-key: main-rust-crate-tests-${{ matrix.batch.idx }}", + builds, + ) + self.assertIn( + "shared-key: main-rust-crate-tests-${{ matrix.batch.idx }}", + main, + ) self.assertIn( "allow_depot_remote_cache: " "${{ needs.changes.outputs.allow_depot_remote_cache }}", diff --git a/scripts/tests/test_ci_workflow_artifacts.py b/scripts/tests/test_ci_workflow_artifacts.py index 526af14111..c2b8b3a740 100644 --- a/scripts/tests/test_ci_workflow_artifacts.py +++ b/scripts/tests/test_ci_workflow_artifacts.py @@ -199,23 +199,32 @@ def test_linux_tests_share_one_static_abi_producer(self) -> None: crate_tests = job_section(self.workflow, "rust_crate_tests") grouped_tests = job_section(self.workflow, "linux_test_groups") - self.assertIn("run: scripts/build-llama.sh", producer) - self.assertIn("name: ci-linux-static-abi-input", producer) - self.assertIn("mesh-llm-static-abi.tar.gz", producer) + self.assertIn( + "uses: ./.github/workflows/static-abi-artifact.yml", + producer, + ) + self.assertIn("artifact_name: ci-linux-static-abi-input", producer) + self.assertIn("runner_size: '8'", producer) + self.assertNotIn("runs_on:", producer) + self.assertNotIn("allow_depot_remote_cache:", producer) + self.assertIn( + "needs.changes.outputs.sdk_smoke_required == 'true'", + producer, + ) for consumer in (crate_tests, grouped_tests): with self.subTest(consumer=consumer.splitlines()[0].strip()): self.assertIn("linux_static_abi_input", consumer) self.assertIn("name: ci-linux-static-abi-input", consumer) self.assertIn("Restore immutable static ABI input", consumer) + self.assertIn("scripts/restore-static-abi-input.sh", consumer) + self.assertNotIn("tar -xzf", consumer) self.assertNotIn("run: scripts/build-llama.sh", consumer) self.assertNotIn("Cache patched llama.cpp ABI build", consumer) def test_macos_host_and_runtime_are_independent_producers(self) -> None: route = ( - "if: ${{ (github.event_name == 'workflow_dispatch' || " - "needs.changes.outputs.rust == 'true' || " - "needs.changes.outputs.ui == 'true' || " - "needs.changes.outputs.benchmarks == 'true') && " + "if: ${{ needs.changes.outputs." + "macos_inference_artifact_required == 'true' && " "needs.changes.outputs.docs_only != 'true' }}" ) host = job_section(self.workflow, "macos_host_input") @@ -411,19 +420,85 @@ def test_windows_node_checks_remain_separate_from_product_builds(self) -> None: self.assertNotIn("prepare-native-runtime-input", checks) self.assertNotIn("compose-product-input", checks) + def test_kotlin_smoke_reuses_parallel_release_native_sdk_input(self) -> None: + producer = job_section(self.workflow, "kotlin_sdk_input") + consumer = job_section(self.workflow, "kotlin_sdk_smoke") + + self.assertIn( + "needs: [changes, linux_static_abi_input]", + producer, + ) + self.assertNotIn("linux_cpu_artifact", producer) + self.assertIn( + "needs.linux_static_abi_input.result == 'success'", + producer, + ) + self.assertIn( + "needs.changes.outputs.sdk_smoke_required == 'true'", + producer, + ) + self.assertIn( + "uses: ./.github/workflows/native-sdk-artifact.yml", + producer, + ) + self.assertIn("profile: release", producer) + self.assertIn( + "artifact_name: ci-kotlin-native-sdk-input", + producer, + ) + self.assertIn( + "static_abi_artifact_name: ci-linux-static-abi-input", + producer, + ) + self.assertIn("runner_size: '8'", producer) + self.assertNotIn("runs_on:", producer) + self.assertNotIn("allow_depot_remote_cache:", producer) + + self.assertIn( + "needs: [changes, linux_cpu_artifact, kotlin_sdk_input]", + consumer, + ) + self.assertIn( + "needs.kotlin_sdk_input.result == 'success'", + consumer, + ) + self.assertIn( + "kotlin_artifact_name: ci-kotlin-native-sdk-input", + consumer, + ) + self.assertIn("kotlin_artifact_profile: release", consumer) + self.assertIn( + "uses: ./.github/workflows/sdk-smoke.yml", + consumer, + ) + def test_swift_smoke_uses_composed_macos_product(self) -> None: + producer = job_section(self.workflow, "swift_sdk_input") swift = job_section(self.workflow, "swift_sdk_smoke") + self.assertIn("needs: changes", producer) + self.assertIn( + "uses: ./.github/workflows/swift-sdk-artifact.yml", + producer, + ) + self.assertIn("mode: full", producer) + self.assertIn("artifact_name: ci-swift-sdk-input", producer) + self.assertNotIn("macos_runner:", producer) + self.assertNotIn("macos_cpu_artifact", producer) + self.assertNotIn("macos_unit_tests", producer) + self.assertIn( - "needs: [changes, macos_cpu_artifact, macos_unit_tests]", + "needs: [changes, macos_cpu_artifact, swift_sdk_input]", swift, ) - self.assertIn("!cancelled()", swift) self.assertNotIn("always()", swift) self.assertIn("needs.macos_cpu_artifact.result == 'success'", swift) - self.assertIn("needs.macos_unit_tests.result == 'success'", swift) - self.assertIn("needs.macos_unit_tests.result == 'skipped'", swift) + self.assertIn("needs.swift_sdk_input.result == 'success'", swift) + self.assertNotIn("macos_unit_tests", swift) self.assertIn("artifact_name: ci-macos-inference-binaries", swift) + self.assertIn("swift_artifact_name: ci-swift-sdk-input", swift) + self.assertIn("swift_artifact_mode: full", swift) + self.assertNotIn("macos_runner:", swift) self.assertIn("staged_binary_path: target/release/mesh-llm", swift) def test_main_runner_policy_is_selected_once(self) -> None: diff --git a/scripts/tests/test_native_artifact_verifiers.py b/scripts/tests/test_native_artifact_verifiers.py index 69b39885e8..dd46490ec6 100644 --- a/scripts/tests/test_native_artifact_verifiers.py +++ b/scripts/tests/test_native_artifact_verifiers.py @@ -5,6 +5,7 @@ import json import os from pathlib import Path +import platform import shutil import subprocess import tarfile @@ -15,6 +16,7 @@ ROOT = Path(__file__).resolve().parents[2] RUNTIME_VERIFIER = ROOT / "scripts" / "verify-native-runtime-package.sh" SDK_VERIFIER = ROOT / "scripts" / "verify-native-sdk-package.sh" +SDK_RESTORE = ROOT / "scripts" / "restore-native-sdk-input.sh" def bash_executable() -> str: @@ -32,6 +34,19 @@ def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def native_architecture() -> str: + machine = platform.machine().lower() + if machine in {"amd64", "x86_64"}: + return "x86_64" + if machine in {"aarch64", "arm64"}: + return "aarch64" + raise unittest.SkipTest(f"unsupported native test architecture: {machine}") + + +def native_linux_target() -> str: + return f"{native_architecture()}-unknown-linux-gnu" + + class NativeArtifactVerifierTests(unittest.TestCase): def run_verifier( self, @@ -52,6 +67,31 @@ def run_verifier( text=True, ) + def run_sdk_restore( + self, + download_dir: Path, + extract_dir: Path, + *, + target: str | None = None, + backend: str = "cpu", + profile: str = "debug", + ) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + bash_executable(), + SDK_RESTORE.as_posix(), + download_dir.as_posix(), + extract_dir.as_posix(), + target or native_linux_target(), + backend, + profile, + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + def write_runtime_artifact(self, root: Path) -> tuple[Path, dict]: artifact = root / "meshllm-native-runtime-darwin-x86_64-cpu" library = artifact / "lib" / "llama.bin" @@ -85,7 +125,8 @@ def write_runtime_artifact(self, root: Path) -> tuple[Path, dict]: return artifact, manifest def write_sdk_artifact(self, root: Path) -> tuple[Path, dict]: - artifact = root / "meshllm-native-linux-x86_64-cpu" + architecture = native_architecture() + artifact = root / f"meshllm-native-linux-{architecture}-cpu" library = artifact / "lib" / "libmesh_llm_ffi.so" uniffi_library = artifact / "lib" / "libmesh_llm_uniffi.so" library.parent.mkdir(parents=True) @@ -97,12 +138,13 @@ def write_sdk_artifact(self, root: Path) -> tuple[Path, dict]: "native_runtime_id": artifact.name, "sdk_version": "0.75.0", "mesh_version": "0.75.0", - "target_triple": "x86_64-unknown-linux-gnu", - "platform": "linux-x86_64", + "target_triple": native_linux_target(), + "platform": f"linux-{architecture}", "os": "linux", - "arch": "x86_64", + "arch": architecture, "backend": "cpu", "flavor": "cpu", + "cargo_profile": "debug", "library": "lib/libmesh_llm_ffi.so", "library_paths": ["lib/libmesh_llm_ffi.so"], "uniffi_library": "lib/libmesh_llm_uniffi.so", @@ -389,7 +431,8 @@ def test_sdk_rejects_unknown_target_backend_and_flavor(self) -> None: mutate(manifest) if name == "flavor": renamed = artifact.with_name( - "meshllm-native-linux-x86_64-made-up", + "meshllm-native-linux-" + f"{native_architecture()}-made-up", ) artifact.rename(renamed) artifact = renamed @@ -405,6 +448,104 @@ def test_sdk_rejects_unknown_target_backend_and_flavor(self) -> None: result.stderr, ) + def test_native_sdk_restore_verifies_exact_typed_artifact(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source_dir = root / "source" + download_dir = root / "download" + source_dir.mkdir() + download_dir.mkdir() + artifact, _ = self.write_sdk_artifact(source_dir) + archive = download_dir / f"{artifact.name}.tar.gz" + self.archive_artifact(artifact, archive) + + result = self.run_sdk_restore( + download_dir, + root / "extracted", + ) + + self.assertEqual( + result.returncode, + 0, + result.stdout + result.stderr, + ) + restored = Path(result.stdout.strip()) + self.assertEqual(restored.name, artifact.name) + self.assertTrue((restored / "manifest.json").is_file()) + + def test_native_sdk_restore_rejects_runner_architecture_mismatch( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source_dir = root / "source" + download_dir = root / "download" + source_dir.mkdir() + download_dir.mkdir() + artifact, _ = self.write_sdk_artifact(source_dir) + archive = download_dir / f"{artifact.name}.tar.gz" + self.archive_artifact(artifact, archive) + other_arch = ( + "aarch64" + if native_architecture() == "x86_64" + else "x86_64" + ) + + result = self.run_sdk_restore( + download_dir, + root / "extracted", + target=f"{other_arch}-unknown-linux-gnu", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("target/runner architecture mismatch", result.stderr) + + def test_native_sdk_restore_rejects_manifest_contract_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source_dir = root / "source" + download_dir = root / "download" + source_dir.mkdir() + download_dir.mkdir() + artifact, _ = self.write_sdk_artifact(source_dir) + archive = download_dir / f"{artifact.name}.tar.gz" + self.archive_artifact(artifact, archive) + + result = self.run_sdk_restore( + download_dir, + root / "extracted", + profile="release", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("cargo_profile mismatch", result.stderr) + + def test_native_sdk_restore_rejects_extra_upload_entries(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source_dir = root / "source" + download_dir = root / "download" + source_dir.mkdir() + download_dir.mkdir() + artifact, _ = self.write_sdk_artifact(source_dir) + archive = download_dir / f"{artifact.name}.tar.gz" + self.archive_artifact(artifact, archive) + (download_dir / "unexpected.txt").write_text( + "unexpected", + encoding="utf-8", + ) + + result = self.run_sdk_restore( + download_dir, + root / "extracted", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "exactly one archive and checksum", + result.stderr, + ) + def test_sdk_manifest_omits_runner_local_build_directory(self) -> None: packager = ( ROOT / "scripts" / "package-native-sdk.sh" diff --git a/scripts/tests/test_plan_pr_build_jobs.py b/scripts/tests/test_plan_pr_build_jobs.py new file mode 100644 index 0000000000..44071ad073 --- /dev/null +++ b/scripts/tests/test_plan_pr_build_jobs.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +PLANNER_PATH = ROOT / "scripts" / "plan-pr-build-jobs.py" +PLANNER_SPEC = importlib.util.spec_from_file_location( + "plan_pr_build_jobs_under_test", + PLANNER_PATH, +) +if PLANNER_SPEC is None or PLANNER_SPEC.loader is None: + raise RuntimeError(f"unable to import {PLANNER_PATH}") +PLANNER = importlib.util.module_from_spec(PLANNER_SPEC) +PLANNER_SPEC.loader.exec_module(PLANNER) + + +def base_payload(**overrides: object) -> dict[str, object]: + payload: dict[str, object] = { + "event_name": "pull_request", + "all_rust": False, + "backend_changed": False, + "benchmarks": False, + "docs_only": False, + "inference_artifact_required": False, + "linux_test_groups_nonempty": False, + "live_agent_enabled": False, + "runner_contract_required": False, + "sdk_smoke_required": False, + "test_batches_nonempty": False, + "windows_cpu_required": False, + "windows_gpu_required": False, + "affected_crates": [], + } + payload.update(overrides) + return payload + + +class PrBuildJobPlanTests(unittest.TestCase): + def test_docs_only_pull_request_has_no_heavy_jobs(self) -> None: + self.assertEqual( + PLANNER.required_jobs(base_payload(docs_only=True)), + [], + ) + + def test_runner_contract_is_independent_of_docs_routing(self) -> None: + self.assertEqual( + PLANNER.required_jobs( + base_payload( + docs_only=True, + runner_contract_required=True, + ) + ), + ["public_runner_image_contract"], + ) + + def test_docs_only_dispatch_does_not_require_public_mesh_without_product(self) -> None: + jobs = set( + PLANNER.required_jobs( + base_payload( + docs_only=True, + event_name="workflow_dispatch", + ) + ) + ) + self.assertIn("public_runner_image_contract", jobs) + self.assertNotIn("linux_cpu_artifact", jobs) + self.assertNotIn("linux_public_mesh_admission", jobs) + + def test_inference_routes_complete_producer_consumer_chains(self) -> None: + jobs = set( + PLANNER.required_jobs( + base_payload( + affected_crates=["mesh-llm"], + inference_artifact_required=True, + linux_test_groups_nonempty=True, + test_batches_nonempty=True, + ) + ) + ) + expected = { + "linux_host_input", + "linux_cpu_runtime_input", + "linux_cpu_artifact", + "linux_static_abi_input", + "rust_crate_tests", + "linux_test_groups", + "hf_download_smoke", + "inference_smoke_tests", + "two_node_client_serving_smoke", + "two_node_split_smoke", + "macos_host_input", + "macos_metal_runtime_input", + "macos_cpu_artifact", + "macos_unit_tests", + } + self.assertEqual(jobs, expected) + + def test_benchmark_routes_native_backends_without_cpu_product(self) -> None: + jobs = set(PLANNER.required_jobs(base_payload(benchmarks=True))) + self.assertIn("linux_host_input", jobs) + self.assertNotIn("linux_cpu_runtime_input", jobs) + self.assertNotIn("linux_cpu_artifact", jobs) + for backend in ("cuda", "rocm", "vulkan"): + self.assertIn(f"linux_{backend}_runtime_input", jobs) + self.assertIn(f"linux_{backend}_product", jobs) + self.assertTrue( + { + "macos_host_input", + "macos_metal_runtime_input", + "macos_cpu_artifact", + }.issubset(jobs) + ) + + def test_sdk_route_reuses_static_abi_and_platform_products(self) -> None: + jobs = set( + PLANNER.required_jobs( + base_payload( + inference_artifact_required=True, + sdk_smoke_required=True, + ) + ) + ) + self.assertTrue( + { + "linux_host_input", + "linux_cpu_runtime_input", + "linux_cpu_artifact", + "linux_static_abi_input", + "rust_sdk_smoke", + "kotlin_sdk_input", + "kotlin_sdk_smoke", + "swift_sdk_input", + "macos_host_input", + "macos_metal_runtime_input", + "macos_cpu_artifact", + "swift_sdk_smoke", + }.issubset(jobs) + ) + + def test_live_agent_route_requires_both_endpoint_and_relevant_code(self) -> None: + without_endpoint = set( + PLANNER.required_jobs( + base_payload( + affected_crates=["mesh-llm-client"], + inference_artifact_required=True, + ) + ) + ) + with_endpoint = set( + PLANNER.required_jobs( + base_payload( + affected_crates=["mesh-llm-client"], + inference_artifact_required=True, + live_agent_enabled=True, + ) + ) + ) + self.assertNotIn("agent_live_smokes", without_endpoint) + self.assertIn("agent_live_smokes", with_endpoint) + self.assertIn("two_node_client_serving_smoke", without_endpoint) + + def test_manual_dispatch_preserves_public_mesh_and_platform_canaries(self) -> None: + jobs = set( + PLANNER.required_jobs( + base_payload( + event_name="workflow_dispatch", + sdk_smoke_required=True, + ) + ) + ) + self.assertIn("linux_public_mesh_admission", jobs) + self.assertIn("public_runner_image_contract", jobs) + self.assertIn("linux_cpu_artifact", jobs) + self.assertIn("linux_cuda_product", jobs) + self.assertIn("macos_cpu_artifact", jobs) + self.assertIn("windows_cpu_product", jobs) + self.assertIn("windows_gpu_products", jobs) + self.assertNotIn("agent_live_smokes", jobs) + + def test_invalid_or_incomplete_input_fails_closed(self) -> None: + missing = base_payload() + del missing["docs_only"] + with self.assertRaises(PLANNER.PlanError): + PLANNER.required_jobs(missing) + with self.assertRaises(PLANNER.PlanError): + PLANNER.required_jobs(base_payload(docs_only="false")) + with self.assertRaises(PLANNER.PlanError): + PLANNER.required_jobs(base_payload(event_name="push")) + + def test_cli_emits_compact_json_and_rejects_invalid_input(self) -> None: + payload = base_payload(runner_contract_required=True) + result = subprocess.run( + [sys.executable, str(PLANNER_PATH)], + cwd=ROOT, + input=json.dumps(payload), + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + result.stdout, + '["public_runner_image_contract"]\n', + ) + + invalid = subprocess.run( + [sys.executable, str(PLANNER_PATH)], + cwd=ROOT, + input="{}", + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(invalid.returncode, 2) + self.assertIn("ERROR: unable to plan PR Builds jobs", invalid.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_pr_builds_summary.py b/scripts/tests/test_pr_builds_summary.py new file mode 100644 index 0000000000..63f6d2d01a --- /dev/null +++ b/scripts/tests/test_pr_builds_summary.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +from collections import Counter +import importlib.util +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +PR_BUILDS = ROOT / ".github" / "workflows" / "pr_builds.yml" +PLANNER_PATH = ROOT / "scripts" / "plan-pr-build-jobs.py" +JOB_HEADER = re.compile(r"(?m)^ ([A-Za-z0-9_-]+):\n") + +PLANNER_SPEC = importlib.util.spec_from_file_location( + "plan_pr_build_jobs", + PLANNER_PATH, +) +if PLANNER_SPEC is None or PLANNER_SPEC.loader is None: + raise RuntimeError(f"unable to import {PLANNER_PATH}") +PLANNER = importlib.util.module_from_spec(PLANNER_SPEC) +PLANNER_SPEC.loader.exec_module(PLANNER) + + +def job_sections(workflow: str) -> dict[str, str]: + jobs_start = workflow.index("jobs:\n") + len("jobs:\n") + jobs_body = workflow[jobs_start:] + matches = list(JOB_HEADER.finditer(jobs_body)) + sections: dict[str, str] = {} + for index, match in enumerate(matches): + end = matches[index + 1].start() if index + 1 < len(matches) else len(jobs_body) + sections[match.group(1)] = jobs_body[match.start() : end] + return sections + + +def summary_needs(summary: str) -> list[str]: + needs_match = re.search( + r"(?ms)^ needs:\n(?P(?: - [A-Za-z0-9_-]+\n)+)", + summary, + ) + if needs_match is None: + raise AssertionError("PR Builds summary must use an explicit needs list") + return re.findall(r"(?m)^ - ([A-Za-z0-9_-]+)$", needs_match.group("items")) + + +class PrBuildsSummaryTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.workflow = PR_BUILDS.read_text(encoding="utf-8") + cls.jobs = job_sections(cls.workflow) + cls.summary = cls.jobs["summary"] + + def test_summary_has_a_stable_branch_protection_name(self) -> None: + self.assertIn(" name: PR Builds Summary\n", self.summary) + self.assertNotIn("${{", self.summary.split(" name:", 1)[1].splitlines()[0]) + self.assertIn(" if: ${{ !cancelled() }}\n", self.summary) + self.assertNotIn("always()", self.summary) + + def test_summary_directly_needs_every_other_top_level_job(self) -> None: + expected = [job for job in self.jobs if job != "summary"] + self.assertEqual(summary_needs(self.summary), expected) + + def test_every_conditional_job_is_mapped_exactly_once(self) -> None: + mapped_jobs = [job_id for job_id, _route in PLANNER.JOB_ROUTES] + counts = Counter(mapped_jobs) + self.assertTrue(all(count == 1 for count in counts.values())) + self.assertEqual( + mapped_jobs, + [job for job in self.jobs if job not in {"changes", "summary"}], + ) + + def test_every_conditional_job_consumes_the_shared_plan(self) -> None: + for job_name, section in self.jobs.items(): + if job_name in {"changes", "summary"}: + continue + with self.subTest(job=job_name): + expected = ( + " if: ${{ contains(" + "fromJson(needs.changes.outputs.required_jobs_json), " + f"'{job_name}') }}}}\n" + ) + job_level_conditions = re.findall( + r"(?m)^ if: .+$", + section, + ) + self.assertEqual(job_level_conditions, [expected.rstrip()]) + + def test_changes_exports_the_checked_in_plan(self) -> None: + changes = self.jobs["changes"] + self.assertIn( + "required_jobs_json: ${{ steps.plan.outputs.required_jobs_json }}", + changes, + ) + self.assertIn("python3 scripts/plan-pr-build-jobs.py", changes) + self.assertIn("PR_BUILD_PLAN_INPUT:", changes) + + def test_summary_evaluates_the_complete_needs_result_object(self) -> None: + self.assertIn("NEEDS_RESULTS: ${{ toJson(needs) }}", self.summary) + self.assertIn( + "REQUIRED_JOBS: ${{ needs.changes.outputs.required_jobs_json }}", + self.summary, + ) + self.assertIn("set -euo pipefail", self.summary) + self.assertIn('$entry.value.result == "success"', self.summary) + self.assertIn('$entry.value.result == "skipped"', self.summary) + self.assertIn("($required | index($entry.key)) == null", self.summary) + self.assertIn("has($job)", self.summary) + self.assertIn("select(accepted | not)", self.summary) + self.assertIn("exit 1", self.summary) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_pr_workflow_artifacts.py b/scripts/tests/test_pr_workflow_artifacts.py index 817a469e65..0d7960f9fb 100644 --- a/scripts/tests/test_pr_workflow_artifacts.py +++ b/scripts/tests/test_pr_workflow_artifacts.py @@ -20,6 +20,14 @@ def job_section( return workflow[start:end] +def planned_condition(job_name: str) -> str: + return ( + "if: ${{ contains(" + "fromJson(needs.changes.outputs.required_jobs_json), " + f"'{job_name}') }}}}" + ) + + class PrWorkflowArtifactTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: @@ -78,6 +86,11 @@ def setUpClass(cls) -> None: "macos_host_input", "macos_metal_runtime_input", ) + cls.swift_input = job_section( + cls.workflow, + "swift_sdk_input", + "macos_host_input", + ) cls.macos_runtime = job_section( cls.workflow, "macos_metal_runtime_input", @@ -116,14 +129,11 @@ def setUpClass(cls) -> None: cls.windows_gpu_products = job_section( cls.workflow, "windows_gpu_products", + "summary", ) def test_host_profile_covers_every_backend_product_route(self) -> None: - self.assertIn( - "needs.changes.outputs.linux_inference_artifact_required == 'true' " - "|| needs.changes.outputs.benchmarks == 'true'", - self.host, - ) + self.assertIn(planned_condition("linux_host_input"), self.host) self.assertIn( "needs.changes.outputs.backend_changed == 'true' " "|| needs.changes.outputs.benchmarks == 'true'", @@ -131,20 +141,17 @@ def test_host_profile_covers_every_backend_product_route(self) -> None: ) self.assertIn("&& 'release' || 'debug'", self.host) - for runtime in self.backend_runtimes.values(): + for backend, runtime in self.backend_runtimes.items(): self.assertIn( - "github.event_name == 'workflow_dispatch' " - "|| needs.changes.outputs.backend_changed == 'true' " - "|| needs.changes.outputs.benchmarks == 'true'", + planned_condition(f"linux_{backend}_runtime_input"), runtime, ) def test_cpu_runtime_only_runs_for_cpu_product_consumers(self) -> None: - condition = ( - "if: ${{ needs.changes.outputs.linux_inference_artifact_required " - "== 'true' && needs.changes.outputs.docs_only != 'true' }}" + self.assertIn( + planned_condition("linux_cpu_runtime_input"), + self.cpu_runtime, ) - self.assertIn(condition, self.cpu_runtime) self.assertNotIn("benchmarks", self.cpu_runtime) def test_cpu_product_uses_matching_immutable_inputs(self) -> None: @@ -226,8 +233,7 @@ def test_backend_products_reuse_exact_immutable_inputs(self) -> None: product, ) self.assertIn( - "needs.linux_host_input.result == 'success' " - f"&& needs.linux_{backend}_runtime_input.result == 'success'", + planned_condition(f"linux_{backend}_product"), product, ) self.assertIn( @@ -275,7 +281,10 @@ def test_public_mesh_admission_is_manual_not_a_pr_gate(self) -> None: "hf_download_smoke", ) - self.assertIn("github.event_name == 'workflow_dispatch'", admission) + self.assertIn( + planned_condition("linux_public_mesh_admission"), + admission, + ) self.assertNotIn("linux_client_auto_boot:", self.workflow) self.assertIn("scripts/ci-client-auto-test.sh", admission) self.assertIn("uses: ./.github/actions/restore-smoke-inputs", admission) @@ -332,27 +341,39 @@ def test_linux_tests_share_one_static_abi_producer(self) -> None: "linux_public_mesh_admission", ) - self.assertIn("run: scripts/build-llama.sh", producer) - self.assertIn("name: pr-linux-static-abi-input", producer) - self.assertIn("mesh-llm-static-abi.tar.gz", producer) + self.assertIn( + "uses: ./.github/workflows/static-abi-artifact.yml", + producer, + ) + self.assertIn("artifact_name: pr-linux-static-abi-input", producer) + self.assertIn("runner_size: '8'", producer) + self.assertNotIn("runs_on:", producer) + self.assertNotIn("allow_depot_remote_cache:", producer) + self.assertIn( + planned_condition("linux_static_abi_input"), + producer, + ) for consumer in (crate_tests, grouped_tests): with self.subTest(consumer=consumer.splitlines()[0].strip()): self.assertIn("linux_static_abi_input", consumer) self.assertIn("name: pr-linux-static-abi-input", consumer) self.assertIn("Restore immutable static ABI input", consumer) + self.assertIn("scripts/restore-static-abi-input.sh", consumer) + self.assertNotIn("tar -xzf", consumer) self.assertNotIn("run: scripts/build-llama.sh", consumer) self.assertNotIn("Cache patched llama.cpp ABI build", consumer) def test_macos_producers_keep_the_existing_product_route(self) -> None: - route = ( - "if: ${{ needs.changes.outputs.macos_inference_artifact_required " - "== 'true' && needs.changes.outputs.docs_only != 'true' }}" - ) - self.assertIn("needs: changes", self.macos_host) - self.assertIn(route, self.macos_host) + self.assertIn( + planned_condition("macos_host_input"), + self.macos_host, + ) self.assertIn("needs: changes", self.macos_runtime) - self.assertIn(route, self.macos_runtime) + self.assertIn( + planned_condition("macos_metal_runtime_input"), + self.macos_runtime, + ) def test_macos_host_and_runtime_are_independent_producers(self) -> None: self.assertIn( @@ -389,8 +410,7 @@ def test_macos_product_only_composes_immutable_inputs(self) -> None: self.macos_product, ) self.assertIn( - "needs.macos_host_input.result == 'success' " - "&& needs.macos_metal_runtime_input.result == 'success'", + planned_condition("macos_cpu_artifact"), self.macos_product, ) self.assertIn("name: pr-macos-host-input", self.macos_product) @@ -422,6 +442,62 @@ def test_macos_product_only_composes_immutable_inputs(self) -> None: self.assertNotIn("brew install", self.macos_product) self.assertNotIn("Swatinem/rust-cache", self.macos_product) + def test_kotlin_smoke_reuses_parallel_debug_native_sdk_input(self) -> None: + producer = job_section( + self.workflow, + "kotlin_sdk_input", + "kotlin_sdk_smoke", + ) + consumer = job_section( + self.workflow, + "kotlin_sdk_smoke", + "swift_sdk_input", + ) + + self.assertIn( + "needs: [changes, linux_static_abi_input]", + producer, + ) + self.assertNotIn("linux_cpu_artifact", producer) + self.assertIn( + planned_condition("kotlin_sdk_input"), + producer, + ) + self.assertIn( + "uses: ./.github/workflows/native-sdk-artifact.yml", + producer, + ) + self.assertIn("profile: debug", producer) + self.assertIn( + "artifact_name: pr-kotlin-native-sdk-input", + producer, + ) + self.assertIn( + "static_abi_artifact_name: pr-linux-static-abi-input", + producer, + ) + self.assertIn("runner_size: '8'", producer) + self.assertNotIn("runs_on:", producer) + self.assertNotIn("allow_depot_remote_cache:", producer) + + self.assertIn( + "needs: [changes, linux_cpu_artifact, kotlin_sdk_input]", + consumer, + ) + self.assertIn( + planned_condition("kotlin_sdk_smoke"), + consumer, + ) + self.assertIn( + "kotlin_artifact_name: pr-kotlin-native-sdk-input", + consumer, + ) + self.assertIn("kotlin_artifact_profile: debug", consumer) + self.assertIn( + "uses: ./.github/workflows/sdk-smoke.yml", + consumer, + ) + def test_macos_swift_gate_and_supported_targets_are_preserved(self) -> None: swift = job_section( self.workflow, @@ -434,16 +510,32 @@ def test_macos_swift_gate_and_supported_targets_are_preserved(self) -> None: "windows_checks", ) + self.assertIn("needs: changes", self.swift_input) + self.assertIn( + "uses: ./.github/workflows/swift-sdk-artifact.yml", + self.swift_input, + ) + self.assertIn("mode: host-only", self.swift_input) + self.assertIn("artifact_name: pr-swift-sdk-input", self.swift_input) + self.assertNotIn("macos_runner:", self.swift_input) + self.assertIn( + planned_condition("swift_sdk_input"), + self.swift_input, + ) + self.assertNotIn("macos_cpu_artifact", self.swift_input) + self.assertNotIn("macos_unit_tests", self.swift_input) + self.assertIn( - "needs: [changes, macos_cpu_artifact, macos_unit_tests]", + "needs: [changes, macos_cpu_artifact, swift_sdk_input]", swift, ) - self.assertIn("!cancelled()", swift) self.assertNotIn("always()", swift) - self.assertIn("needs.macos_cpu_artifact.result == 'success'", swift) - self.assertIn("needs.macos_unit_tests.result == 'success'", swift) - self.assertIn("needs.macos_unit_tests.result == 'skipped'", swift) + self.assertIn(planned_condition("swift_sdk_smoke"), swift) + self.assertNotIn("macos_unit_tests", swift) self.assertIn("artifact_name: ci-macos-inference-binaries", swift) + self.assertIn("swift_artifact_name: pr-swift-sdk-input", swift) + self.assertIn("swift_artifact_mode: host-only", swift) + self.assertNotIn("macos_runner:", swift) self.assertIn("needs: changes", unit_tests) self.assertNotIn("macos_cpu_artifact", unit_tests) self.assertIn( @@ -475,7 +567,10 @@ def test_macos_swift_gate_and_supported_targets_are_preserved(self) -> None: self.assertNotIn("Skip unsupported macOS GPU backend", self.workflow) def test_windows_pr_keeps_broad_rust_signals_lightweight(self) -> None: - self.assertIn("needs.changes.outputs.all_rust == 'true'", self.windows_checks) + self.assertIn( + planned_condition("windows_checks"), + self.windows_checks, + ) self.assertIn("name: Windows lightweight checks", self.windows_checks) self.assertIn("cargo check --locked -p mesh-llm --bin mesh-llm", self.windows_checks) self.assertNotIn("prepare-windows-host-input", self.windows_checks) @@ -501,11 +596,7 @@ def test_windows_pr_builds_one_debug_host_and_independent_runtimes(self) -> None self.assertNotIn("compose-product-input", self.windows_host) self.assertIn( - "needs.changes.outputs.windows_cpu == 'true'", - self.windows_cpu_runtime, - ) - self.assertNotIn( - "needs.changes.outputs.windows_gpu == 'true'", + planned_condition("windows_cpu_runtime_input"), self.windows_cpu_runtime, ) self.assertIn( @@ -521,11 +612,7 @@ def test_windows_pr_builds_one_debug_host_and_independent_runtimes(self) -> None self.assertNotIn("compose-product-input", self.windows_cpu_runtime) self.assertIn( - "needs.changes.outputs.windows_gpu == 'true'", - self.windows_gpu_runtimes, - ) - self.assertNotIn( - "needs.changes.outputs.windows_cpu == 'true'", + planned_condition("windows_gpu_runtime_inputs"), self.windows_gpu_runtimes, ) for backend in ("cuda", "rocm", "vulkan"): diff --git a/scripts/tests/test_release_workflow_artifacts.py b/scripts/tests/test_release_workflow_artifacts.py index 62738112db..4316a3c303 100644 --- a/scripts/tests/test_release_workflow_artifacts.py +++ b/scripts/tests/test_release_workflow_artifacts.py @@ -118,7 +118,11 @@ def test_release_routes_only_initial_non_secret_linux_lanes( self.assertIn("runs-on: ${{ matrix.os }}", host) self.assertIn("RELEASE_ATTESTATION_SIGNING_KEY", host) - for producer in (sdk_runtime, native_runtime): + self.assertIn("runner_size: '8'", sdk_runtime) + self.assertNotIn("runs_on:", sdk_runtime) + self.assertNotIn("allow_depot_remote_cache:", sdk_runtime) + self.assertNotIn("needs.metadata.outputs.runner", sdk_runtime) + for producer in (native_runtime,): self.assertIn( "matrix.target == 'x86_64-unknown-linux-gnu'", producer, @@ -178,9 +182,59 @@ def test_inference_smoke_consumes_composed_product(self) -> None: "inference_smoke_tests", "build_native_sdk_runtime", ) + self.assertNotIn("runs_on:", inference) + smoke = ( + ROOT / ".github" / "workflows" / "smoke.yml" + ).read_text(encoding="utf-8") + self.assertIn("runs-on: ubuntu-24.04", smoke) + + def test_swift_release_reuses_full_typed_producer(self) -> None: + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + reusable = ( + ROOT / ".github" / "workflows" / "swift-sdk-artifact.yml" + ).read_text(encoding="utf-8") + producer = job_block( + workflow, + "build_swift_sdk_artifact", + "build_linux_arm64", + ) + + self.assertIn( + "uses: ./.github/workflows/swift-sdk-artifact.yml", + producer, + ) + self.assertIn("mode: full", producer) + self.assertIn("artifact_name: release-swift-sdk", producer) + self.assertNotIn("macos_runner:", producer) + self.assertIn( + "release_tag: ${{ needs.metadata.outputs.tag }}", + producer, + ) + self.assertIn( + "prepare_release_version: " + "${{ github.event_name == 'workflow_dispatch' }}", + producer, + ) + self.assertNotIn("build-xcframework.sh", producer) + self.assertNotIn("cargo ", producer) + self.assertIn("name: swift-package-manifest", reusable) self.assertIn( - "runs_on: ${{ toJson(needs.metadata.outputs.runner_8) }}", - inference, + "name: generated-swift-binding-${{ inputs.artifact_name }}", + reusable, + ) + self.assertIn("name: ${{ inputs.artifact_name }}", reusable) + publish = job_block( + workflow, + "publish", + "dispatch_packaging_release", + ) + self.assertIn( + "name: generated-swift-binding-release-swift-sdk", + publish, + ) + self.assertIn( + 'install -m 0644 "$generated_binding" "$tracked_binding"', + publish, ) def test_release_permissions_are_least_privilege(self) -> None: @@ -287,26 +341,76 @@ def test_arm64_smoke_requires_integrity_and_safe_extraction(self) -> None: def test_native_sdk_assets_are_staged_flat_for_publishing(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") - producer = job_block( + reusable = ( + ROOT / ".github" / "workflows" / "native-sdk-artifact.yml" + ).read_text(encoding="utf-8") + producer_action = ( + ROOT / ".github" / "actions" + / "prepare-native-sdk-input" / "action.yml" + ).read_text(encoding="utf-8") + caller = job_block( workflow, "build_native_sdk_runtime", "build_native_runtime", ) - upload = producer[producer.index("- name: Upload native SDK runtime") :] publish = job_block( workflow, "publish", "dispatch_packaging_release", ) - self.assertIn("- name: Stage flat native SDK release assets", producer) + self.assertIn( + "uses: ./.github/workflows/native-sdk-artifact.yml", + caller, + ) + self.assertIn("profile: release", caller) + self.assertIn("include_runtime_crate: true", caller) + self.assertIn( + "static_abi_artifact_name: " + "ci-release-native-sdk-static-abi-${{ matrix.artifact_suffix }}", + caller, + ) + self.assertIn( + "produce_static_abi: " + "${{ endsWith(matrix.target, '-unknown-linux-gnu') }}", + caller, + ) + self.assertIn( + "artifact_name: " + "release-native-sdk-${{ matrix.artifact_suffix }}", + caller, + ) + self.assertIn("runner_size: '8'", caller) + self.assertNotIn("runs_on:", caller) + self.assertNotIn("allow_depot_remote_cache:", caller) + self.assertNotIn("scripts/package-native-sdk.sh", caller) + self.assertIn( + "uses: ./.github/actions/prepare-native-sdk-input", + reusable, + ) + self.assertIn( + "uses: ./.github/workflows/static-abi-artifact.yml", + reusable, + ) + self.assertIn( + "scripts/restore-static-abi-input.sh", + reusable, + ) + self.assertIn("name: ${{ inputs.artifact_name }}", reusable) + self.assertIn( + "path: ${{ steps.native-sdk.outputs.upload_path }}", + reusable, + ) + self.assertIn( + "scripts/package-native-sdk-crate.sh", + producer_action, + ) self.assertIn( "native SDK release asset basename collision", - producer, + producer_action, ) - self.assertIn("path: release-native-sdk-assets/*", upload) - self.assertNotIn("dist/native-sdk/", upload) - self.assertNotIn("dist/native-sdk-crates/", upload) + self.assertIn('upload_sources=("$archive_path" "$checksum_path")', producer_action) + self.assertIn('upload_sources+=("${runtime_crates[0]}")', producer_action) self.assertIn("files: release-artifacts/*", publish) def test_windows_host_publishes_prebuilt_attestation_verifier(self) -> None: diff --git a/scripts/tests/test_reusable_workflow_runner_trust.py b/scripts/tests/test_reusable_workflow_runner_trust.py new file mode 100644 index 0000000000..79ce4e8383 --- /dev/null +++ b/scripts/tests/test_reusable_workflow_runner_trust.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS = ROOT / ".github" / "workflows" + + +class ReusableWorkflowRunnerTrustTests(unittest.TestCase): + def workflow(self, name: str) -> str: + return (WORKFLOWS / name).read_text(encoding="utf-8") + + def test_no_reusable_workflow_allocates_a_raw_caller_label(self) -> None: + offenders = [] + for path in sorted(WORKFLOWS.glob("*.yml")): + workflow = path.read_text(encoding="utf-8") + if "workflow_call:" not in workflow: + continue + if "fromJson(inputs.runs_on)" in workflow: + offenders.append(path.name) + + self.assertEqual([], offenders) + + def test_credential_bearing_smokes_are_github_hosted(self) -> None: + for name in ( + "hf-download-smoke.yml", + "smoke.yml", + "scripted-binary-smoke.yml", + ): + with self.subTest(workflow=name): + workflow = self.workflow(name) + self.assertIn("runs-on: ubuntu-24.04", workflow) + self.assertNotIn("runs_on:", workflow) + self.assertNotIn("depot-ubuntu", workflow) + self.assertIn("persist-credentials: false", workflow) + + def test_sdk_and_swift_runners_are_fixed_hosted_labels(self) -> None: + sdk_smoke = self.workflow("sdk-smoke.yml") + swift_producer = self.workflow("swift-sdk-artifact.yml") + + for workflow in (sdk_smoke, swift_producer): + self.assertIn("macos-15", workflow) + self.assertNotIn("macos_runner:", workflow) + self.assertNotIn("macos-latest", workflow) + self.assertNotIn("runs_on:", workflow) + self.assertNotIn("fromJson(inputs.", workflow) + self.assertNotIn("depot-ubuntu", workflow) + self.assertNotIn("self-hosted", workflow) + + self.assertIn("'ubuntu-24.04'", sdk_smoke) + self.assertIn("'ubuntu-24.04-arm'", sdk_smoke) + self.assertIn( + "inputs.kotlin_artifact_target == " + "'aarch64-unknown-linux-gnu'", + sdk_smoke, + ) + self.assertIn( + "x86_64-unknown-linux-gnu|aarch64-unknown-linux-gnu", + sdk_smoke, + ) + + def test_main_macos_product_graph_uses_the_swift_toolchain_boundary( + self, + ) -> None: + main = self.workflow("ci.yml") + + self.assertNotIn("macos-latest", main) + for job_id in ( + "macos_host_input", + "macos_metal_runtime_input", + "macos_cpu_artifact", + "macos_unit_tests", + ): + with self.subTest(job=job_id): + match = re.search( + rf"(?ms)^ {re.escape(job_id)}:\n" + r"(?P.*?)(?=^ [A-Za-z0-9_]+:\n|\Z)", + main, + ) + self.assertIsNotNone(match) + self.assertIn( + "runs-on: macos-15", + match.group("body") if match else "", + ) + + def test_nightly_reusable_workflow_cannot_select_a_runner(self) -> None: + wrapper = self.workflow("nightly-stability.yml") + reusable = self.workflow("nightly-stability-run.yml") + + self.assertNotIn("runs_on:", wrapper) + self.assertNotIn("MESH_NIGHTLY_STABILITY_RUNS_ON", wrapper) + self.assertNotIn("runs_on:", reusable) + self.assertIn("runs-on: ubuntu-24.04", reusable) + self.assertIn("persist-credentials: false", reusable) + + def test_depot_allowlist_excludes_credential_smokes(self) -> None: + migration = (ROOT / "ci" / "DEPOT_MIGRATION.md").read_text( + encoding="utf-8", + ) + allowlist_start = migration.index("The initial main allowlist is:") + allowlist_end = migration.index("```", allowlist_start) + allowlist_end = migration.index("```", allowlist_end + 3) + allowlist = migration[allowlist_start:allowlist_end] + + self.assertIn("native-sdk-artifact.yml@refs/heads/main", allowlist) + self.assertIn("static-abi-artifact.yml@refs/heads/main", allowlist) + for name in ( + "hf-download-smoke.yml", + "smoke.yml", + "scripted-binary-smoke.yml", + "sdk-smoke.yml", + "swift-sdk-artifact.yml", + ): + with self.subTest(workflow=name): + self.assertNotIn(name, allowlist) + + def test_pull_request_builds_do_not_receive_hugging_face_secret(self) -> None: + pr_builds = self.workflow("pr_builds.yml") + + self.assertNotIn("secrets.HF_TOKEN", pr_builds) + self.assertNotIn("HUGGING_FACE_HUB_TOKEN:", pr_builds) + + def test_pr_facing_checkouts_do_not_persist_job_credentials(self) -> None: + for name in ( + "docker-precheck.yml", + "pr_quality.yml", + "pr_website.yml", + ): + with self.subTest(workflow=name): + workflow = self.workflow(name) + checkout_count = workflow.count("uses: actions/checkout@") + self.assertGreater(checkout_count, 0) + self.assertEqual( + checkout_count, + workflow.count("persist-credentials: false"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_safe_extract_zip.py b/scripts/tests/test_safe_extract_zip.py new file mode 100644 index 0000000000..3d55f059e1 --- /dev/null +++ b/scripts/tests/test_safe_extract_zip.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import stat +import subprocess +import sys +import tempfile +import unittest +import warnings +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "safe-extract-zip.py" + + +def symlink_info(name: str) -> zipfile.ZipInfo: + info = zipfile.ZipInfo(name) + info.create_system = 3 + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + return info + + +class SafeExtractZipTests(unittest.TestCase): + def run_extractor( + self, + archive: Path, + destination: Path, + ) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), str(archive), str(destination)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + def test_extracts_files_and_safe_framework_symlinks(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive = root / "input.zip" + destination = root / "output" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr( + "MeshLLMFFI.xcframework/Info.plist", + "plist", + ) + bundle.writestr( + "MeshLLMFFI.xcframework/" + "macos/MeshLLMFFI.framework/Versions/A/MeshLLMFFI", + "library", + ) + bundle.writestr( + symlink_info( + "MeshLLMFFI.xcframework/" + "macos/MeshLLMFFI.framework/Versions/Current" + ), + "A", + ) + bundle.writestr( + symlink_info( + "MeshLLMFFI.xcframework/" + "macos/MeshLLMFFI.framework/MeshLLMFFI" + ), + "Versions/Current/MeshLLMFFI", + ) + + result = self.run_extractor(archive, destination) + + self.assertEqual(result.returncode, 0, result.stderr) + framework = ( + destination + / "MeshLLMFFI.xcframework" + / "macos" + / "MeshLLMFFI.framework" + ) + self.assertTrue((framework / "Versions/Current").is_symlink()) + self.assertEqual( + (framework / "Versions/Current").readlink(), + Path("A"), + ) + self.assertEqual( + (framework / "MeshLLMFFI").read_text(encoding="utf-8"), + "library", + ) + + def test_rejects_entry_path_escapes(self) -> None: + for name in ("../escape", "/absolute", r"C:\escape"): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive = root / "input.zip" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr(name, "bad") + + result = self.run_extractor(archive, root / "output") + + self.assertNotEqual(result.returncode, 0) + self.assertIn("unsafe ZIP archive", result.stderr) + self.assertFalse((root / "escape").exists()) + + def test_rejects_escaping_or_ancestor_symlinks(self) -> None: + fixtures = ( + (("link", "../../escape"), None), + (("alias", "real"), ("alias/file", "bad")), + ) + for link, nested in fixtures: + with self.subTest(link=link), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive = root / "input.zip" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr(symlink_info(link[0]), link[1]) + if nested is not None: + bundle.writestr(*nested) + + result = self.run_extractor(archive, root / "output") + + self.assertNotEqual(result.returncode, 0) + self.assertIn("unsafe ZIP archive", result.stderr) + + def test_rejects_duplicate_entries_and_nonempty_destination(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive = root / "input.zip" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("duplicate", "one") + bundle.writestr("duplicate", "two") + + duplicate = self.run_extractor(archive, root / "duplicates") + self.assertNotEqual(duplicate.returncode, 0) + self.assertIn("duplicate entry path", duplicate.stderr) + + clean_archive = root / "clean.zip" + with zipfile.ZipFile(clean_archive, "w") as bundle: + bundle.writestr("file", "content") + destination = root / "nonempty" + destination.mkdir() + (destination / "sentinel").write_text("keep", encoding="utf-8") + + nonempty = self.run_extractor(clean_archive, destination) + + self.assertNotEqual(nonempty.returncode, 0) + self.assertIn("destination must be empty", nonempty.stderr) + self.assertEqual( + (destination / "sentinel").read_text(encoding="utf-8"), + "keep", + ) + + actual = root / "actual" + actual.mkdir() + symlink_destination = root / "symlink-destination" + symlink_destination.symlink_to(actual, target_is_directory=True) + symlinked = self.run_extractor(clean_archive, symlink_destination) + self.assertNotEqual(symlinked.returncode, 0) + self.assertIn("destination cannot be a symlink", symlinked.stderr) + self.assertEqual(list(actual.iterdir()), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_sccache_evidence.py b/scripts/tests/test_sccache_evidence.py index 417f089a7c..c72eaf014d 100644 --- a/scripts/tests/test_sccache_evidence.py +++ b/scripts/tests/test_sccache_evidence.py @@ -21,6 +21,7 @@ "pr-builds": ROOT / ".github" / "workflows" / "pr_builds.yml", "pr-quality": ROOT / ".github" / "workflows" / "pr_quality.yml", "main": ROOT / ".github" / "workflows" / "ci.yml", + "swift-sdk": ROOT / ".github" / "workflows" / "swift-sdk-artifact.yml", } @@ -56,6 +57,7 @@ def run_capture( payload: dict[str, object], *, artifact_name: str = "sccache-test-1", + sccache_error: str = "", ) -> tuple[subprocess.CompletedProcess[str], Path, Path]: temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup) @@ -64,9 +66,14 @@ def run_capture( fake_sccache.write_text( "#!/bin/sh\n" "if [ \"$#\" -eq 1 ] && [ \"$1\" = \"--show-stats\" ]; then\n" - " printf '%s\\n' 'Compile requests 12'\n" + " printf '%s\\n' " + "'Cache location: https://human-stats-secret.example/private'\n" "elif [ \"$#\" -eq 3 ] && [ \"$1\" = \"--show-stats\" ] " "&& [ \"$2\" = \"--stats-format\" ] && [ \"$3\" = \"json\" ]; then\n" + " if [ -n \"$FAKE_SCCACHE_ERROR\" ]; then\n" + " printf '%s\\n' \"$FAKE_SCCACHE_ERROR\" >&2\n" + " exit 23\n" + " fi\n" " printf '%s\\n' \"$FAKE_SCCACHE_JSON\"\n" "else\n" " exit 2\n" @@ -93,6 +100,7 @@ def run_capture( **os.environ, "PATH": f"{root}{os.pathsep}{os.environ['PATH']}", "FAKE_SCCACHE_JSON": json.dumps(payload), + "FAKE_SCCACHE_ERROR": sccache_error, }, check=False, capture_output=True, @@ -100,20 +108,104 @@ def run_capture( ) return result, stats_file, github_output - def test_capture_emits_human_stats_and_machine_readable_counters(self) -> None: + def test_capture_writes_only_sanitized_machine_readable_counters(self) -> None: payload = valid_payload() result, stats_file, github_output = self.run_capture(payload) self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("Human-readable sccache statistics", result.stdout) - self.assertIn("Compile requests 12", result.stdout) - self.assertEqual(json.loads(stats_file.read_text()), payload) + self.assertNotIn("Human-readable sccache statistics", result.stdout) + self.assertNotIn("human-stats-secret", result.stdout) + self.assertEqual( + json.loads(stats_file.read_text()), + { + "schema": "mesh-llm.sccache-stats", + "schema_version": 1, + "stats": { + "compile_requests": 12, + "requests_executed": 10, + "compilations": 4, + "cache_writes": 3, + "cache_read_errors": 0, + "cache_write_errors": 0, + "cache_hits": {"counts": {"total": 6}}, + "cache_misses": {"counts": {"total": 4}}, + "cache_errors": {"counts": {"total": 0}}, + }, + }, + ) outputs = github_output.read_text(encoding="utf-8") self.assertIn("compile_requests=12", outputs) self.assertIn("requests_executed=10", outputs) self.assertIn("cache_hits=6", outputs) self.assertIn("cache_misses=4", outputs) + def test_raw_secrets_urls_and_paths_cannot_reach_logs_or_evidence(self) -> None: + payload = valid_payload() + stats = payload["stats"] + self.assertIsInstance(stats, dict) + cache_hits = stats["cache_hits"] + self.assertIsInstance(cache_hits, dict) + cache_hits["counts"] = { + "/Users/private/cache/path?token=count-key-secret": 6, + } + stats["not_cached"] = { + "/home/runner/private-source": 1, + "https://stats-secret.example/cache": 2, + } + payload.update( + { + "cache_location": ( + "WebDAV: https://cache-user:location-secret@cache.example" + ), + "basedirs": ["/home/runner/work/private-repository"], + "version": "raw-version-secret", + "url": "https://payload-secret.example/cache", + "absolute_path": "/Users/private/sccache", + }, + ) + + result, stats_file, github_output = self.run_capture(payload) + + self.assertEqual(result.returncode, 0, result.stderr) + evidence = json.loads(stats_file.read_text(encoding="utf-8")) + self.assertEqual(evidence["stats"]["cache_hits"], {"counts": {"total": 6}}) + exposed_surface = "\n".join( + ( + result.stdout, + result.stderr, + stats_file.read_text(encoding="utf-8"), + github_output.read_text(encoding="utf-8"), + ), + ) + for forbidden in ( + "human-stats-secret", + "count-key-secret", + "private-source", + "stats-secret", + "cache_location", + "basedirs", + "location-secret", + "raw-version-secret", + "payload-secret", + "/Users/private", + "/home/runner/work/private-repository", + ): + with self.subTest(forbidden=forbidden): + self.assertNotIn(forbidden, exposed_surface) + + def test_sccache_error_output_cannot_leak_into_the_job_log(self) -> None: + secret = "https://cache-user:stderr-secret@cache.example/private" + result, stats_file, _ = self.run_capture( + valid_payload(), + sccache_error=secret, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(stats_file.exists()) + self.assertNotIn(secret, result.stdout) + self.assertNotIn(secret, result.stderr) + self.assertIn("exit code 23", result.stderr) + def test_zero_compile_requests_warns_but_remains_valid_evidence(self) -> None: result, stats_file, _ = self.run_capture( valid_payload(compile_requests=0), @@ -196,6 +288,9 @@ def test_instrumented_workflows_use_unique_evidence_artifacts(self) -> None: "sccache-main-rust-crate-tests-${{ matrix.batch.idx }}-${{ github.run_attempt }}", "sccache-main-linux-tests-${{ matrix.group }}-${{ github.run_attempt }}", ), + "swift-sdk": ( + "sccache-swift-sdk-${{ inputs.mode }}-${{ github.run_attempt }}", + ), } for workflow_name, path in WORKFLOWS.items(): diff --git a/scripts/tests/test_static_abi_artifacts.py b/scripts/tests/test_static_abi_artifacts.py new file mode 100644 index 0000000000..4a9ff517a3 --- /dev/null +++ b/scripts/tests/test_static_abi_artifacts.py @@ -0,0 +1,338 @@ +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import platform +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +RESTORE = ROOT / "scripts" / "restore-static-abi-input.sh" +STAMP_VERIFIER = ROOT / "scripts" / "verify-static-abi-build-stamp.py" +TOOLCHAIN_EPOCH = "test-runner-image-sha256-deadbeef" + + +def bash_executable() -> str: + if os.name != "nt": + return shutil.which("bash") or "bash" + git = shutil.which("git") + if git: + candidate = Path(git).parent.parent / "bin" / "bash.exe" + if candidate.is_file(): + return str(candidate) + raise RuntimeError("Git Bash is required for static ABI tests") + + +def native_linux_target() -> str: + machine = platform.machine().lower() + if machine in {"arm64", "aarch64"}: + return "aarch64-unknown-linux-gnu" + if machine in {"amd64", "x86_64"}: + return "x86_64-unknown-linux-gnu" + raise unittest.SkipTest(f"unsupported test architecture: {machine}") + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +class StaticAbiArtifactTests(unittest.TestCase): + def test_native_sdk_reuse_is_verification_only(self) -> None: + build_script = (ROOT / "scripts" / "build-llama.sh").read_text( + encoding="utf-8", + ) + package_script = ( + ROOT / "scripts" / "package-native-sdk.sh" + ).read_text(encoding="utf-8") + + self.assertIn("--require-existing", build_script) + self.assertIn("refusing to rebuild", build_script) + self.assertIn("toolchain-epoch=", build_script) + self.assertIn("stamp-version=3", build_script) + self.assertIn("@LLAMA_BUILD_DIR@", build_script) + self.assertIn("@LLAMA_WORKDIR@", build_script) + self.assertIn("-ffile-prefix-map=", build_script) + self.assertIn("MESH_LLM_REQUIRE_SCCACHE=1", build_script) + for archive in ( + "libllama-common-base.a", + "libggml.a", + "libggml-base.a", + "libggml-cpu.a", + ): + self.assertIn(archive, build_script) + self.assertIn("--require-prebuilt-llama", package_script) + self.assertIn("SKIPPY_LLAMA_AUTO_BUILD=0", package_script) + self.assertIn("MESH_LLM_AUTO_BUILD_LLAMA=0", package_script) + + def test_dynamic_output_probe_is_pipefail_safe(self) -> None: + build_script = (ROOT / "scripts" / "build-llama.sh").read_text( + encoding="utf-8", + ) + function = build_script.split( + "required_dynamic_libraries_exist() {", + maxsplit=1, + )[1].split("\n}", maxsplit=1)[0] + + self.assertIn('found="$(find ', function) + self.assertIn('[[ -n "$found" && -e "$found" ]] || return 1', function) + self.assertNotIn("| grep -q", function) + + def write_artifact( + self, + root: Path, + *, + manifest_target: str | None = None, + link_mode: str = "static", + sibling: bool = False, + toolchain_epoch: str = TOOLCHAIN_EPOCH, + omit_archive: str | None = None, + ) -> Path: + target = native_linux_target() + build_dir = root / "source" / "build-stage-abi-static" + for relative in ( + "src/libllama.a", + "common/libllama-common.a", + "common/libllama-common-base.a", + "ggml/src/libggml.a", + "ggml/src/libggml-base.a", + "ggml/src/ggml-cpu/libggml-cpu.a", + "tools/mtmd/libmtmd.a", + ): + if relative == omit_archive: + continue + path = build_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(relative.encode()) + stamp = build_dir / ".mesh-llm-build-stamp" + stamp.write_text( + "\n".join( + ( + "stamp-version=3", + "patched-sha=0123456789abcdef", + "backend=cpu", + f"link-mode={link_mode}", + f"toolchain-epoch={toolchain_epoch}", + "build-type=Release", + "cmake-arg=-DGGML_NATIVE=OFF", + "cmake-arg=-DGGML_OPENMP=OFF", + ) + ) + + "\n", + encoding="utf-8", + ) + (build_dir / "CMakeCache.txt").write_text( + "# Portable MeshLLM static ABI link metadata\n" + "GGML_OPENMP_ENABLED:BOOL=OFF\n", + encoding="utf-8", + ) + manifest = { + "schema_version": 3, + "contract": "mesh-llm-static-abi-v3", + "target_triple": manifest_target or target, + "backend": "cpu", + "build_directory": build_dir.name, + "toolchain_epoch": toolchain_epoch, + "build_stamp_sha256": sha256(stamp), + } + (build_dir / ".mesh-llm-static-abi-input.json").write_text( + json.dumps(manifest), + encoding="utf-8", + ) + + download = root / "download" + download.mkdir() + archive = download / "mesh-llm-static-abi.tar.gz" + with tarfile.open(archive, "w:gz") as bundle: + bundle.add(build_dir, arcname=build_dir.name) + if sibling: + sibling_file = root / "unexpected.txt" + sibling_file.write_text("unexpected", encoding="utf-8") + bundle.add(sibling_file, arcname=sibling_file.name) + archive.with_name(f"{archive.name}.sha256").write_text( + f"{sha256(archive)} {archive.name}\n", + encoding="utf-8", + ) + return download + + def restore( + self, + download: Path, + destination: Path, + ) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env["MESH_LLM_LLAMA_TOOLCHAIN_EPOCH"] = TOOLCHAIN_EPOCH + return subprocess.run( + [ + bash_executable(), + RESTORE.as_posix(), + download.as_posix(), + destination.as_posix(), + native_linux_target(), + "cpu", + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + env=env, + ) + + def test_restore_accepts_exact_typed_static_abi(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + download = self.write_artifact(root) + destination = root / "restored" / "build-stage-abi-static" + + result = self.restore(download, destination) + + self.assertEqual( + result.returncode, + 0, + result.stdout + result.stderr, + ) + self.assertTrue((destination / "src/libllama.a").is_file()) + + def test_stamp_verifier_allows_repeated_cmake_arguments(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + download = self.write_artifact(root) + with tarfile.open( + download / "mesh-llm-static-abi.tar.gz", + "r:gz", + ) as bundle: + stamp = bundle.extractfile( + "build-stage-abi-static/.mesh-llm-build-stamp", + ) + self.assertIsNotNone(stamp) + stamp_path = root / "build-stamp" + stamp_path.write_bytes(stamp.read()) + + result = subprocess.run( + [ + sys.executable, + str(STAMP_VERIFIER), + str(stamp_path), + "--backend", + "cpu", + "--link-mode", + "static", + "--stamp-version", + "3", + "--toolchain-epoch", + TOOLCHAIN_EPOCH, + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("cmake_arguments=2", result.stdout) + + def test_restore_rejects_manifest_target_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + download = self.write_artifact( + root, + manifest_target="mismatched-target", + ) + + result = self.restore( + download, + root / "restored" / "build-stage-abi-static", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("target_triple mismatch", result.stderr) + + def test_restore_rejects_non_static_build_stamp(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + download = self.write_artifact(root, link_mode="dynamic") + + result = self.restore( + download, + root / "restored" / "build-stage-abi-static", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("link-mode mismatch", result.stderr) + + def test_restore_rejects_incomplete_link_closure(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + download = self.write_artifact( + root, + omit_archive="common/libllama-common-base.a", + ) + + result = self.restore( + download, + root / "restored" / "build-stage-abi-static", + ) + + self.assertNotEqual(result.returncode, 0) + + def test_restore_rejects_toolchain_epoch_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + download = self.write_artifact( + root, + toolchain_epoch="different-runner-image", + ) + + result = self.restore( + download, + root / "restored" / "build-stage-abi-static", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("toolchain_epoch mismatch", result.stderr) + + def test_restore_rejects_archive_sibling(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + download = self.write_artifact(root, sibling=True) + + result = self.restore( + download, + root / "restored" / "build-stage-abi-static", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "must contain exactly build-stage-abi-static", + result.stderr, + ) + + def test_restore_rejects_extra_upload_entry(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + download = self.write_artifact(root) + (download / "unexpected.txt").write_text( + "unexpected", + encoding="utf-8", + ) + + result = self.restore( + download, + root / "restored" / "build-stage-abi-static", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "exactly its archive and checksum", + result.stderr, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_verify_swift_xcframework.py b/scripts/tests/test_verify_swift_xcframework.py new file mode 100644 index 0000000000..d04ecc1d00 --- /dev/null +++ b/scripts/tests/test_verify_swift_xcframework.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import os +from pathlib import Path +import plistlib +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +VERIFIER = ROOT / "scripts" / "verify-swift-xcframework.py" + + +FULL_SLICES = [ + ("ios-arm64", "ios", "", ["arm64"]), + ("ios-arm64_x86_64-simulator", "ios", "simulator", ["arm64", "x86_64"]), + ( + "ios-arm64_x86_64-maccatalyst", + "ios", + "maccatalyst", + ["arm64", "x86_64"], + ), + ("macos-arm64_x86_64", "macos", "", ["arm64", "x86_64"]), +] + + +class SwiftXCFrameworkVerifierTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.xcframework = self.root / "MeshLLMFFI.xcframework" + self.xcframework.mkdir() + self.lipo = self.root / "fake-lipo" + self.lipo.write_text( + "#!/usr/bin/env python3\n" + "from pathlib import Path\n" + "import sys\n" + "print(Path(sys.argv[-1]).read_text(encoding='utf-8').strip())\n", + encoding="utf-8", + ) + self.lipo.chmod(0o755) + + def tearDown(self) -> None: + self.temp.cleanup() + + def write_framework( + self, + identifier: str, + platform: str, + variant: str, + declared_architectures: list[str], + binary_architectures: list[str] | None = None, + ) -> dict[str, object]: + framework = ( + self.xcframework + / identifier + / "MeshLLMFFI.framework" + ) + framework.mkdir(parents=True) + binary_contents = " ".join(binary_architectures or declared_architectures) + + if platform == "macos" and not variant: + version = framework / "Versions" / "A" + (version / "Headers").mkdir(parents=True) + (version / "Modules").mkdir() + (version / "Resources").mkdir() + (version / "MeshLLMFFI").write_text( + binary_contents, + encoding="utf-8", + ) + (version / "Modules" / "module.modulemap").write_text( + "framework module MeshLLMFFI {}\n", + encoding="utf-8", + ) + (version / "Resources" / "Info.plist").write_bytes( + plistlib.dumps({"CFBundleName": "MeshLLMFFI"}), + ) + (version / "Resources" / "PrivacyInfo.xcprivacy").write_bytes( + plistlib.dumps({"NSPrivacyTracking": False}), + ) + (framework / "Versions" / "Current").symlink_to("A") + (framework / "MeshLLMFFI").symlink_to( + "Versions/Current/MeshLLMFFI", + ) + (framework / "Headers").symlink_to("Versions/Current/Headers") + (framework / "Modules").symlink_to("Versions/Current/Modules") + (framework / "Resources").symlink_to("Versions/Current/Resources") + else: + (framework / "MeshLLMFFI").write_text( + binary_contents, + encoding="utf-8", + ) + + entry: dict[str, object] = { + "LibraryIdentifier": identifier, + "LibraryPath": "MeshLLMFFI.framework", + "SupportedArchitectures": declared_architectures, + "SupportedPlatform": platform, + } + if variant: + entry["SupportedPlatformVariant"] = variant + return entry + + def write_info(self, slices: list[tuple[str, str, str, list[str]]]) -> None: + libraries = [ + self.write_framework(identifier, platform, variant, architectures) + for identifier, platform, variant, architectures in slices + ] + (self.xcframework / "Info.plist").write_bytes( + plistlib.dumps({"AvailableLibraries": libraries}), + ) + + def run_verifier(self, mode: str) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env["LIPO"] = str(self.lipo) + return subprocess.run( + [sys.executable, str(VERIFIER), str(self.xcframework), mode], + check=False, + capture_output=True, + text=True, + env=env, + ) + + def test_accepts_exact_full_architecture_matrix(self) -> None: + self.write_info(FULL_SLICES) + + result = self.run_verifier("full") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("verified 4 XCFramework slice(s)", result.stdout) + + def test_accepts_exact_host_only_arm64_slice(self) -> None: + self.write_info( + [("macos-arm64", "macos", "", ["arm64"])], + ) + + result = self.run_verifier("host-only") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("host-only mode", result.stdout) + + def test_rejects_full_slice_missing_declared_x86_64(self) -> None: + slices = list(FULL_SLICES) + slices[1] = ( + "ios-arm64-simulator", + "ios", + "simulator", + ["arm64"], + ) + self.write_info(slices) + + result = self.run_verifier("full") + + self.assertNotEqual(result.returncode, 0) + self.assertIn("unexpected architecture contract", result.stderr) + + def test_rejects_lipo_slices_that_disagree_with_plist(self) -> None: + libraries = [] + for identifier, platform, variant, architectures in FULL_SLICES: + binary_architectures = ( + ["arm64"] + if variant == "maccatalyst" + else architectures + ) + libraries.append( + self.write_framework( + identifier, + platform, + variant, + architectures, + binary_architectures, + ), + ) + (self.xcframework / "Info.plist").write_bytes( + plistlib.dumps({"AvailableLibraries": libraries}), + ) + + result = self.run_verifier("full") + + self.assertNotEqual(result.returncode, 0) + self.assertIn("lipo architectures", result.stderr) + + def test_mode_independent_verification_still_requires_macos(self) -> None: + self.write_info( + [("ios-arm64", "ios", "", ["arm64"])], + ) + env = os.environ.copy() + env["LIPO"] = str(self.lipo) + + result = subprocess.run( + [sys.executable, str(VERIFIER), str(self.xcframework)], + check=False, + capture_output=True, + text=True, + env=env, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("does not contain a macOS framework slice", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify-static-abi-build-stamp.py b/scripts/verify-static-abi-build-stamp.py new file mode 100644 index 0000000000..0b981380f9 --- /dev/null +++ b/scripts/verify-static-abi-build-stamp.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Validate the portable llama.cpp static-ABI build-stamp contract.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import re + + +FIELD_NAME = re.compile(r"^[a-z][a-z0-9-]*$") +REQUIRED_FIELDS = ( + "stamp-version", + "patched-sha", + "backend", + "link-mode", + "toolchain-epoch", +) + + +class StampError(RuntimeError): + """Raised when a build stamp does not satisfy the portable ABI contract.""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("stamp", type=Path) + parser.add_argument("--backend", required=True) + parser.add_argument("--link-mode", required=True) + parser.add_argument("--stamp-version", required=True) + parser.add_argument("--toolchain-epoch", required=True) + parser.add_argument("--patched-sha") + return parser.parse_args() + + +def parse_stamp(path: Path) -> tuple[dict[str, str], list[str]]: + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeError) as error: + raise StampError(f"unable to read static ABI build stamp: {error}") from error + + fields: dict[str, str] = {} + cmake_arguments: list[str] = [] + for line_number, line in enumerate(lines, start=1): + key, separator, value = line.partition("=") + if not separator or not FIELD_NAME.fullmatch(key): + raise StampError( + f"static ABI build stamp line {line_number} is malformed", + ) + if key == "cmake-arg": + cmake_arguments.append(value) + continue + if key in fields: + raise StampError( + f"static ABI build stamp repeats singleton field {key!r}", + ) + fields[key] = value + + missing = [name for name in REQUIRED_FIELDS if not fields.get(name)] + if missing: + raise StampError( + "static ABI build stamp is missing required fields: " + + ", ".join(missing), + ) + if not cmake_arguments: + raise StampError("static ABI build stamp must contain at least one cmake-arg") + return fields, cmake_arguments + + +def require_equal(fields: dict[str, str], name: str, expected: str) -> None: + actual = fields.get(name) + if actual != expected: + raise StampError( + f"static ABI build stamp {name} mismatch: " + f"expected {expected!r}, got {actual!r}", + ) + + +def main() -> int: + arguments = parse_args() + try: + fields, cmake_arguments = parse_stamp(arguments.stamp) + require_equal(fields, "backend", arguments.backend) + require_equal(fields, "link-mode", arguments.link_mode) + require_equal(fields, "stamp-version", arguments.stamp_version) + require_equal(fields, "toolchain-epoch", arguments.toolchain_epoch) + if arguments.patched_sha is not None: + require_equal(fields, "patched-sha", arguments.patched_sha) + except StampError as error: + raise SystemExit(str(error)) from error + + print( + "verified static ABI build stamp: " + f"backend={fields['backend']} " + f"cmake_arguments={len(cmake_arguments)}", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify-swift-release-artifact.sh b/scripts/verify-swift-release-artifact.sh index 30137be84f..b6bdbd724f 100755 --- a/scripts/verify-swift-release-artifact.sh +++ b/scripts/verify-swift-release-artifact.sh @@ -3,15 +3,16 @@ set -euo pipefail usage() { cat >&2 <<'EOF' -Usage: scripts/verify-swift-release-artifact.sh +Usage: scripts/verify-swift-release-artifact.sh [host-only|full] Verifies the SwiftPM release artifact shape by checking the zipped XCFramework, its embedded privacy manifests, and a temporary Swift package consumer that -depends on the zipped binary target. +depends on the zipped binary target. When a mode is supplied, also verifies the +expected platform/slice contract. EOF } -if [[ "$#" -ne 1 ]]; then +if [[ "$#" -lt 1 || "$#" -gt 2 ]]; then usage exit 1 fi @@ -23,6 +24,15 @@ fi REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" ARTIFACT_ZIP="$1" +EXPECTED_MODE="${2:-}" + +case "$EXPECTED_MODE" in + ""|host-only|full) ;; + *) + echo "unsupported Swift SDK artifact mode: $EXPECTED_MODE" >&2 + exit 1 + ;; +esac if [[ ! -f "$ARTIFACT_ZIP" ]]; then echo "Swift release artifact does not exist: $ARTIFACT_ZIP" >&2 @@ -49,7 +59,9 @@ trap 'rm -rf "$TMP_ROOT"' EXIT EXTRACT_DIR="$TMP_ROOT/extract" mkdir -p "$EXTRACT_DIR" -ditto -x -k "$ARTIFACT_ZIP" "$EXTRACT_DIR" +python3 "$REPO_ROOT/scripts/safe-extract-zip.py" \ + "$ARTIFACT_ZIP" \ + "$EXTRACT_DIR" XCFRAMEWORK_COUNT="$(find "$EXTRACT_DIR" -mindepth 1 -maxdepth 1 -name '*.xcframework' -type d | wc -l | tr -d ' ')" if [[ "$XCFRAMEWORK_COUNT" != "1" ]]; then @@ -68,58 +80,12 @@ plutil -lint "$XCFRAMEWORK_PATH/Info.plist" >/dev/null "$REPO_ROOT/sdk/swift/PrivacyInfo.xcprivacy" \ "$XCFRAMEWORK_PATH" -python3 - "$XCFRAMEWORK_PATH" <<'PY' -import os -import plistlib -import sys - -xcframework = sys.argv[1] -with open(os.path.join(xcframework, "Info.plist"), "rb") as fh: - info = plistlib.load(fh) - -macos_frameworks = [] -for library in info.get("AvailableLibraries", []): - if library.get("SupportedPlatform") != "macos": - continue - library_path = library.get("LibraryPath") - identifier = library.get("LibraryIdentifier") - if not library_path or not identifier: - raise SystemExit(f"invalid macOS library entry: {library!r}") - macos_frameworks.append(os.path.join(xcframework, identifier, library_path)) - -if not macos_frameworks: - raise SystemExit("XCFramework does not contain a macOS framework slice") - -for framework in macos_frameworks: - name = os.path.splitext(os.path.basename(framework))[0] - expected = { - "Versions/Current": "A", - name: f"Versions/Current/{name}", - "Headers": "Versions/Current/Headers", - "Modules": "Versions/Current/Modules", - "Resources": "Versions/Current/Resources", - } - for relative, target in expected.items(): - path = os.path.join(framework, relative) - if not os.path.islink(path): - raise SystemExit(f"macOS framework is not versioned; missing symlink: {path}") - actual = os.readlink(path) - if actual != target: - raise SystemExit(f"unexpected symlink target for {path}: {actual!r} != {target!r}") - - required_paths = [ - os.path.join(framework, "Versions", "A", name), - os.path.join(framework, "Versions", "A", "Headers"), - os.path.join(framework, "Versions", "A", "Modules", "module.modulemap"), - os.path.join(framework, "Versions", "A", "Resources", "Info.plist"), - os.path.join(framework, "Versions", "A", "Resources", "PrivacyInfo.xcprivacy"), - ] - for path in required_paths: - if not os.path.exists(path): - raise SystemExit(f"macOS framework versioned layout is incomplete: {path}") - -print(f"verified {len(macos_frameworks)} versioned macOS framework slice(s)") -PY +xcframework_args=("$XCFRAMEWORK_PATH") +if [[ -n "$EXPECTED_MODE" ]]; then + xcframework_args+=("$EXPECTED_MODE") +fi +python3 "$REPO_ROOT/scripts/verify-swift-xcframework.py" \ + "${xcframework_args[@]}" CONSUMER_DIR="$TMP_ROOT/consumer" mkdir -p "$CONSUMER_DIR/Sources" "$CONSUMER_DIR/Sources/Consumer" diff --git a/scripts/verify-swift-xcframework.py b/scripts/verify-swift-xcframework.py new file mode 100755 index 0000000000..679cac6bf5 --- /dev/null +++ b/scripts/verify-swift-xcframework.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Verify the immutable Swift XCFramework slice and architecture contract.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path, PurePosixPath +import plistlib +import subprocess +import sys +from typing import Any + + +PlatformKey = tuple[str, str] + +EXPECTED_ARCHITECTURES: dict[str, dict[PlatformKey, frozenset[str]]] = { + "host-only": { + ("macos", ""): frozenset({"arm64"}), + }, + "full": { + ("ios", ""): frozenset({"arm64"}), + ("ios", "maccatalyst"): frozenset({"arm64", "x86_64"}), + ("ios", "simulator"): frozenset({"arm64", "x86_64"}), + ("macos", ""): frozenset({"arm64", "x86_64"}), + }, +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Verify an XCFramework's declared architectures, binary slices, " + "and versioned macOS framework layout." + ), + ) + parser.add_argument("xcframework", type=Path) + parser.add_argument("mode", nargs="?", choices=sorted(EXPECTED_ARCHITECTURES)) + return parser.parse_args() + + +def fail(message: str) -> None: + raise ValueError(message) + + +def require_safe_component(value: Any, field: str) -> str: + if not isinstance(value, str) or not value: + fail(f"XCFramework {field} must be a non-empty string") + path = PurePosixPath(value) + if path.is_absolute() or len(path.parts) != 1 or path.parts[0] in {".", ".."}: + fail(f"XCFramework {field} must be one safe path component: {value!r}") + return value + + +def platform_key(library: dict[str, Any]) -> PlatformKey: + platform = library.get("SupportedPlatform") + variant = library.get("SupportedPlatformVariant", "") + if not isinstance(platform, str) or not platform: + fail(f"invalid SupportedPlatform in XCFramework entry: {library!r}") + if not isinstance(variant, str): + fail(f"invalid SupportedPlatformVariant in XCFramework entry: {library!r}") + return platform, variant + + +def declared_architectures( + library: dict[str, Any], + key: PlatformKey, +) -> frozenset[str]: + architectures = library.get("SupportedArchitectures") + if not isinstance(architectures, list) or not architectures: + fail(f"XCFramework slice {key!r} must declare SupportedArchitectures") + if not all( + isinstance(architecture, str) and architecture + for architecture in architectures + ): + fail(f"XCFramework slice {key!r} has invalid SupportedArchitectures") + declared = frozenset(architectures) + if len(declared) != len(architectures): + fail(f"XCFramework slice {key!r} declares duplicate architectures") + return declared + + +def framework_path( + xcframework: Path, + library: dict[str, Any], +) -> Path: + identifier = require_safe_component( + library.get("LibraryIdentifier"), + "LibraryIdentifier", + ) + library_path = require_safe_component(library.get("LibraryPath"), "LibraryPath") + if not library_path.endswith(".framework"): + fail(f"XCFramework LibraryPath must name a framework: {library_path!r}") + framework = xcframework / identifier / library_path + resolved_root = xcframework.resolve() + resolved_framework = framework.resolve() + if resolved_root not in resolved_framework.parents: + fail(f"XCFramework library escapes its root: {framework}") + if not framework.is_dir(): + fail(f"XCFramework library is missing: {framework}") + return framework + + +def framework_binary(framework: Path) -> Path: + name = framework.stem + binary = framework / name + if not binary.exists() or not binary.is_file(): + fail(f"XCFramework binary is missing: {binary}") + return binary + + +def verify_macos_layout(framework: Path) -> None: + name = framework.stem + expected_symlinks = { + "Versions/Current": "A", + name: f"Versions/Current/{name}", + "Headers": "Versions/Current/Headers", + "Modules": "Versions/Current/Modules", + "Resources": "Versions/Current/Resources", + } + for relative, target in expected_symlinks.items(): + path = framework / relative + if not path.is_symlink(): + fail(f"macOS framework is not versioned; missing symlink: {path}") + actual = os.readlink(path) + if actual != target: + fail(f"unexpected symlink target for {path}: {actual!r} != {target!r}") + + required_paths = [ + framework / "Versions" / "A" / name, + framework / "Versions" / "A" / "Headers", + framework / "Versions" / "A" / "Modules" / "module.modulemap", + framework / "Versions" / "A" / "Resources" / "Info.plist", + framework / "Versions" / "A" / "Resources" / "PrivacyInfo.xcprivacy", + ] + for path in required_paths: + if not path.exists(): + fail(f"macOS framework versioned layout is incomplete: {path}") + + +def lipo_architectures(binary: Path) -> frozenset[str]: + lipo = os.environ.get("LIPO", "lipo") + try: + result = subprocess.run( + [lipo, "-archs", str(binary)], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError) as error: + fail(f"failed to inspect XCFramework binary with lipo: {binary}: {error}") + architectures = result.stdout.strip().split() + if not architectures: + fail(f"lipo reported no architectures for XCFramework binary: {binary}") + return frozenset(architectures) + + +def verify_xcframework(xcframework: Path, mode: str | None) -> None: + info_path = xcframework / "Info.plist" + if not xcframework.is_dir() or not info_path.is_file(): + fail(f"XCFramework or Info.plist is missing: {xcframework}") + with info_path.open("rb") as handle: + info = plistlib.load(handle) + + libraries = info.get("AvailableLibraries") + if not isinstance(libraries, list) or not libraries: + fail("XCFramework AvailableLibraries must be a non-empty array") + + entries: dict[PlatformKey, dict[str, Any]] = {} + for library in libraries: + if not isinstance(library, dict): + fail(f"invalid XCFramework library entry: {library!r}") + key = platform_key(library) + if key in entries: + fail(f"XCFramework contains a duplicate platform slice: {key!r}") + entries[key] = library + + if ("macos", "") not in entries: + fail("XCFramework does not contain a macOS framework slice") + + expected = EXPECTED_ARCHITECTURES.get(mode) if mode else None + if expected is not None and entries.keys() != expected.keys(): + fail( + f"{mode} Swift SDK input has an unexpected platform matrix: " + f"{sorted(entries)!r}; expected {sorted(expected)!r}" + ) + + for key, library in entries.items(): + declared = declared_architectures(library, key) + if expected is not None and declared != expected[key]: + fail( + f"{mode} Swift SDK slice {key!r} has an unexpected architecture " + f"contract: {sorted(declared)!r}; expected {sorted(expected[key])!r}" + ) + framework = framework_path(xcframework, library) + if key == ("macos", ""): + verify_macos_layout(framework) + binary_architectures = lipo_architectures(framework_binary(framework)) + if binary_architectures != declared: + fail( + f"XCFramework slice {key!r} lipo architectures " + f"{sorted(binary_architectures)!r} do not match " + f"SupportedArchitectures {sorted(declared)!r}" + ) + + print( + f"verified {len(entries)} XCFramework slice(s)" + + (f" for {mode} mode" if mode else "") + ) + + +def main() -> int: + args = parse_args() + try: + verify_xcframework(args.xcframework, args.mode) + except (OSError, ValueError, plistlib.InvalidFileException) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sdk/swift/scripts/build-host-macos-xcframework.sh b/sdk/swift/scripts/build-host-macos-xcframework.sh index 3a57fbfc77..09d23706a9 100755 --- a/sdk/swift/scripts/build-host-macos-xcframework.sh +++ b/sdk/swift/scripts/build-host-macos-xcframework.sh @@ -37,7 +37,7 @@ rustup target add "$RUST_TARGET" 2>/dev/null || true export MACOSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-13.0}" export LLAMA_STAGE_BACKEND="${LLAMA_STAGE_BACKEND:-metal}" -export LLAMA_STAGE_BUILD_DIR="${LLAMA_STAGE_BUILD_DIR:-$REPO_ROOT/.deps/llama.cpp/build-stage-abi-metal}" +export LLAMA_STAGE_BUILD_DIR="${LLAMA_STAGE_BUILD_DIR:-$REPO_ROOT/.deps/llama-build/build-stage-abi-$RUST_TARGET-metal}" echo "Preparing embedded llama.cpp ABI libraries..." "$REPO_ROOT/scripts/prepare-llama.sh" "${MESH_LLM_LLAMA_PIN_SHA:-pinned}" diff --git a/tools/xtask/src/workflow_checks.rs b/tools/xtask/src/workflow_checks.rs index cdc014b16a..e8fb9efcfd 100644 --- a/tools/xtask/src/workflow_checks.rs +++ b/tools/xtask/src/workflow_checks.rs @@ -13,6 +13,12 @@ pub(crate) fn check_docs_and_workflow_invariants(repo_root: &Path) -> DynResult< let release = fs::read_to_string(repo_root.join("RELEASE.md"))?; let justfile = fs::read_to_string(repo_root.join("Justfile"))?; let release_workflow = fs::read_to_string(repo_root.join(".github/workflows/release.yml"))?; + let native_sdk_artifact_workflow = + fs::read_to_string(repo_root.join(".github/workflows/native-sdk-artifact.yml"))?; + let static_abi_artifact_workflow = + fs::read_to_string(repo_root.join(".github/workflows/static-abi-artifact.yml"))?; + let swift_sdk_artifact_workflow = + fs::read_to_string(repo_root.join(".github/workflows/swift-sdk-artifact.yml"))?; let ci_workflow = fs::read_to_string(repo_root.join(".github/workflows/ci.yml"))?; let pr_builds_workflow = fs::read_to_string(repo_root.join(".github/workflows/pr_builds.yml"))?; let pr_quality_workflow = @@ -396,18 +402,159 @@ pub(crate) fn check_docs_and_workflow_invariants(repo_root: &Path) -> DynResult< &prepare_native_runtime_action, &compose_product_action, )?; - check_release_dispatch_version_preparation(&release_workflow)?; + for (workflow, context) in [ + (&ci_workflow, "main shared static ABI producer"), + (&pr_builds_workflow, "PR shared static ABI producer"), + ] { + ensure_contains( + workflow, + "uses: ./.github/workflows/static-abi-artifact.yml", + context, + )?; + ensure_contains( + workflow, + "scripts/restore-static-abi-input.sh", + &format!("{context} consumer restore"), + )?; + let static_abi_caller = workflow_job_section(workflow, "linux_static_abi_input") + .ok_or_else(|| format!("{context}: missing `linux_static_abi_input` job"))?; + ensure_contains( + static_abi_caller, + "runner_size: '8'", + &format!("{context} bounded runner size"), + )?; + ensure_not_contains( + static_abi_caller, + "runs_on:", + &format!("{context} must not supply a runner label"), + )?; + ensure_not_contains( + static_abi_caller, + "allow_depot_remote_cache:", + &format!("{context} must not supply Depot cache authority"), + )?; + + let native_sdk_caller = workflow_job_section(workflow, "kotlin_sdk_input") + .ok_or_else(|| format!("{context}: missing `kotlin_sdk_input` job"))?; + ensure_contains( + native_sdk_caller, + "runner_size: '8'", + &format!("{context} native SDK bounded runner size"), + )?; + ensure_not_contains( + native_sdk_caller, + "runs_on:", + &format!("{context} native SDK must not supply a runner label"), + )?; + ensure_not_contains( + native_sdk_caller, + "allow_depot_remote_cache:", + &format!("{context} native SDK must not supply Depot cache authority"), + )?; + } + check_protected_reusable_runner_policy( + &native_sdk_artifact_workflow, + "native SDK reusable workflow", + )?; + check_protected_reusable_runner_policy( + &static_abi_artifact_workflow, + "static ABI reusable workflow", + )?; + ensure_contains( + &native_sdk_artifact_workflow, + "uses: ./.github/workflows/static-abi-artifact.yml", + "native SDK nested release static ABI producer", + )?; + ensure_contains( + &native_sdk_artifact_workflow, + "scripts/restore-static-abi-input.sh", + "native SDK static ABI consumer restore", + )?; + ensure_contains( + &static_abi_artifact_workflow, + "CACHE_NAMESPACE: mesh-llm", + "static ABI reusable cache namespace", + )?; + check_release_dispatch_version_preparation( + &release_workflow, + &native_sdk_artifact_workflow, + &swift_sdk_artifact_workflow, + )?; check_release_container_contracts(&release_workflow, &configure_sccache_action)?; check_ci_crate_test_coverage(&ci_workflow, &pr_builds_workflow, &compute_changes_action)?; Ok(()) } -fn check_release_dispatch_version_preparation(release_workflow: &str) -> DynResult<()> { +fn check_protected_reusable_runner_policy(workflow: &str, context: &str) -> DynResult<()> { + for (required, contract) in [ + ("runner_size:", "bounded runner-size input"), + ("default: '8'", "bounded runner-size default"), + ("runner_policy:", "protected runner policy job"), + ("runs-on: ubuntu-24.04", "fixed hosted policy runner"), + ( + "POLICY_REPOSITORY: ${{ github.repository }}", + "immutable repository context", + ), + ("POLICY_REF: ${{ github.ref }}", "immutable ref context"), + ( + "POLICY_EVENT_NAME: ${{ github.event_name }}", + "immutable event context", + ), + ( + "POLICY_DEPOT_ENABLED: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }}", + "repository Depot gate", + ), + ( + "POLICY_MANUAL_USE_DEPOT: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.use_depot == 'true' }}", + "immutable main-dispatch canary flag", + ), + ( + r#"POLICY_REPOSITORY" == "Mesh-LLM/mesh-llm""#, + "exact repository guard", + ), + ( + r#"POLICY_REF" == "refs/heads/main""#, + "exact main-ref guard", + ), + ( + r#"POLICY_MANUAL_USE_DEPOT" == "true""#, + "main-dispatch canary decision", + ), + ("default|4|8|16", "bounded runner-size validation"), + ("depot-ubuntu-24.04", "allowlisted Depot AMD64 label"), + ("depot-ubuntu-24.04-arm", "allowlisted Depot ARM64 label"), + ( + "runs-on: ${{ needs.runner_policy.outputs.runner }}", + "derived producer runner", + ), + ( + "allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }}", + "derived Depot cache authority", + ), + ] { + ensure_contains(workflow, required, &format!("{context} {contract}"))?; + } + for (forbidden, contract) in [ + ("inputs.runs_on", "caller-controlled runner label"), + ( + "inputs.allow_depot_remote_cache", + "caller-controlled Depot cache authority", + ), + ("fromJson(inputs.runs_on)", "caller-controlled runner JSON"), + ] { + ensure_not_contains(workflow, forbidden, &format!("{context} {contract}"))?; + } + Ok(()) +} + +fn check_release_dispatch_version_preparation( + release_workflow: &str, + native_sdk_artifact_workflow: &str, + swift_sdk_artifact_workflow: &str, +) -> DynResult<()> { const DISPATCH_RELEASE_JOBS: &[&str] = &[ "build", - "build_native_sdk_runtime", - "build_swift_sdk_artifact", "build_linux_arm64", "compose_linux_aarch64_cuda", "compose_linux_cuda", @@ -439,6 +586,103 @@ fn check_release_dispatch_version_preparation(release_workflow: &str) -> DynResu )?; } + let native_sdk_caller = workflow_job_section(release_workflow, "build_native_sdk_runtime") + .ok_or("release workflow: missing `build_native_sdk_runtime` job")?; + for (required, context) in [ + ( + "uses: ./.github/workflows/native-sdk-artifact.yml", + "release native SDK shared producer call", + ), + ("profile: release", "release native SDK producer profile"), + ( + "artifact_name: release-native-sdk-${{ matrix.artifact_suffix }}", + "release native SDK artifact name", + ), + ( + "include_runtime_crate: true", + "release native SDK runtime crate staging", + ), + ( + "static_abi_artifact_name: ci-release-native-sdk-static-abi-${{ matrix.artifact_suffix }}", + "release native SDK static ABI artifact", + ), + ( + "produce_static_abi: ${{ endsWith(matrix.target, '-unknown-linux-gnu') }}", + "release native SDK per-target static ABI producer", + ), + ("runner_size: '8'", "release native SDK bounded runner size"), + ( + "release_tag: ${{ needs.metadata.outputs.tag }}", + "release native SDK producer tag input", + ), + ( + "prepare_release_version: ${{ github.event_name == 'workflow_dispatch' }}", + "release native SDK dispatch version input", + ), + ] { + ensure_contains(native_sdk_caller, required, context)?; + } + ensure_not_contains( + native_sdk_caller, + "runs_on:", + "release native SDK must not supply a runner label", + )?; + ensure_not_contains( + native_sdk_caller, + "allow_depot_remote_cache:", + "release native SDK must not supply Depot cache authority", + )?; + ensure_contains( + native_sdk_artifact_workflow, + REQUIRED_STEP, + "shared native SDK producer dispatch version step", + )?; + ensure_contains( + native_sdk_artifact_workflow, + "if: ${{ inputs.prepare_release_version }}", + "shared native SDK producer dispatch version condition", + )?; + ensure_contains( + native_sdk_artifact_workflow, + REQUIRED_COMMAND, + "shared native SDK producer dispatch version command", + )?; + + let swift_caller = workflow_job_section(release_workflow, "build_swift_sdk_artifact") + .ok_or("release workflow: missing `build_swift_sdk_artifact` job")?; + for (required, context) in [ + ( + "uses: ./.github/workflows/swift-sdk-artifact.yml", + "release Swift shared producer call", + ), + ("mode: full", "release Swift exhaustive producer mode"), + ( + "release_tag: ${{ needs.metadata.outputs.tag }}", + "release Swift producer tag input", + ), + ( + "prepare_release_version: ${{ github.event_name == 'workflow_dispatch' }}", + "release Swift dispatch version input", + ), + ] { + ensure_contains(swift_caller, required, context)?; + } + ensure_contains( + swift_sdk_artifact_workflow, + REQUIRED_STEP, + "shared Swift producer dispatch version step", + )?; + ensure_contains( + swift_sdk_artifact_workflow, + "if: ${{ inputs.prepare_release_version }}", + "shared Swift producer dispatch version condition", + )?; + ensure_contains( + swift_sdk_artifact_workflow, + REQUIRED_COMMAND, + "shared Swift producer dispatch version command", + )?; + Ok(()) } From 80f82e03a665450f76856105b28aa9e82ffdc775 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 22:02:46 -0400 Subject: [PATCH 17/23] fix(ci): reuse configured sccache server --- scripts/build-llama.sh | 15 +++++++++------ scripts/tests/test_static_abi_artifacts.py | 5 +++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/scripts/build-llama.sh b/scripts/build-llama.sh index 0a224fb19d..0dbc563723 100755 --- a/scripts/build-llama.sh +++ b/scripts/build-llama.sh @@ -244,13 +244,16 @@ esac USE_SCCACHE="${LLAMA_STAGE_USE_SCCACHE:-${SKIPPY_USE_SCCACHE:-1}}" if [[ "$USE_SCCACHE" != "0" && -n "$SCCACHE_BIN" ]] && - ! "$SCCACHE_BIN" --start-server >/dev/null 2>&1; then - if [[ "${MESH_LLM_REQUIRE_SCCACHE:-0}" == "1" ]]; then - echo "sccache failed to start and MESH_LLM_REQUIRE_SCCACHE=1" >&2 - exit 1 + ! "$SCCACHE_BIN" --show-stats >/dev/null 2>&1; then + if ! "$SCCACHE_BIN" --start-server >/dev/null 2>&1 || + ! "$SCCACHE_BIN" --show-stats >/dev/null 2>&1; then + if [[ "${MESH_LLM_REQUIRE_SCCACHE:-0}" == "1" ]]; then + echo "sccache is unavailable and MESH_LLM_REQUIRE_SCCACHE=1" >&2 + exit 1 + fi + echo "sccache is unavailable; llama.cpp build will run without compiler caching" >&2 + USE_SCCACHE=0 fi - echo "sccache failed to start; llama.cpp build will run without compiler caching" >&2 - USE_SCCACHE=0 fi if [[ "$USE_SCCACHE" != "0" && -n "$SCCACHE_BIN" ]]; then diff --git a/scripts/tests/test_static_abi_artifacts.py b/scripts/tests/test_static_abi_artifacts.py index 4a9ff517a3..2d5c272a0a 100644 --- a/scripts/tests/test_static_abi_artifacts.py +++ b/scripts/tests/test_static_abi_artifacts.py @@ -60,6 +60,11 @@ def test_native_sdk_reuse_is_verification_only(self) -> None: self.assertIn("@LLAMA_WORKDIR@", build_script) self.assertIn("-ffile-prefix-map=", build_script) self.assertIn("MESH_LLM_REQUIRE_SCCACHE=1", build_script) + self.assertIn('"$SCCACHE_BIN" --show-stats', build_script) + self.assertLess( + build_script.index('"$SCCACHE_BIN" --show-stats'), + build_script.index('"$SCCACHE_BIN" --start-server'), + ) for archive in ( "libllama-common-base.a", "libggml.a", From 26156d0d32b3de9f66f90e25d26c8441f9556f7b Mon Sep 17 00:00:00 2001 From: Nick DiZazzo <728690+ndizazzo@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:13:54 -0400 Subject: [PATCH 18/23] fix(ci): harden exact native cache reuse --- .../manage-ci/references/current-inventory.md | 15 +- .github/actions/compute-changes/action.yml | 10 +- .../resolve-native-toolchain-epoch/action.yml | 104 ++++++++++++++ .../restore-windows-abi-cache/action.yml | 20 ++- .github/workflows/ci.yml | 48 ++++++- .github/workflows/depot-canary.yml | 95 ++++++++++++- .github/workflows/native-sdk-artifact.yml | 11 +- .github/workflows/pr_builds.yml | 54 +++++++- .github/workflows/release.yml | 41 +++++- .github/workflows/static-abi-artifact.yml | 9 +- .github/workflows/swift-sdk-artifact.yml | 11 +- .github/workflows/windows-warm-caches.yml | 73 ++++++++-- ci/DEPOT_MIGRATION.md | 11 +- ci/ci.md | 8 ++ crates/mesh-llm/tests/qa_nightly_stability.rs | 11 +- scripts/build-llama.sh | 24 +++- scripts/build-windows.ps1 | 35 ++++- scripts/ci-swift-sdk-smoke.sh | 9 +- scripts/prepare-llama.sh | 34 ++++- scripts/tests/test_build_windows.py | 37 +++++ scripts/tests/test_ci_artifact_actions.py | 128 +++++++++++++++++- scripts/tests/test_depot_canary_workflow.py | 17 +++ scripts/tests/test_prepare_llama.py | 77 ++++++++++- scripts/tests/test_static_abi_artifacts.py | 3 + 24 files changed, 818 insertions(+), 67 deletions(-) create mode 100644 .github/actions/resolve-native-toolchain-epoch/action.yml diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 575ca522c3..65199efef5 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -129,14 +129,20 @@ Local actions: archive containing only the path-normalized static link closure and portable OpenMP metadata. The reusable workflow caches that archive, not the local CMake build graph; crate tests and native SDK producers consume it. +- `.github/actions/resolve-native-toolchain-epoch` exports one cache-safe + identity to both native build stamps and cache keys. Digest-pinned Linux + containers use their immutable image digest; hosted macOS and Windows jobs + use the exact runner image revision, with compiler/CMake/Ninja versions added + where hosted or Depot Linux/macOS toolchains are not otherwise pinned. - `.github/actions/compose-product-input` verifies producer inputs, creates one product-v2 tree without compiling, and runs CLI/client readiness. - `.github/actions/restore-smoke-inputs` owns producer artifact staging and model restoration for smoke consumers. - `.github/actions/restore-windows-abi-cache` owns the exact Windows CPU, CUDA, ROCm, and Vulkan ABI cache identity shared by the trusted warmer and - PR/main/release runtime producers. Architecture sets and toolchain versions - are compatibility boundaries; the action never uses restore prefixes. + PR/main/release runtime producers. The hosted-image epoch, architecture sets, + and toolchain versions are compatibility boundaries; the action requires the + key epoch to equal the build-stamp epoch and never uses restore prefixes. - `.github/actions/setup-windows-rocm-sdk` owns reusable Windows ROCm setup. Routing and test-planning scripts: @@ -313,6 +319,11 @@ GitHub-hosted jobs retain `disk,gha` or explicit disk-only mode. Persistent Cargo target and ABI reuse remains owned by `Swatinem/rust-cache` and `actions/cache`. Current PR jobs use the normal `mesh-llm` key namespace and GitHub's merge-ref scoping; trusted main does not restore PR-written entries. +Raw native ABI caches also include the exact native toolchain epoch used by the +build stamp. Linux container jobs use the pinned OCI digest, macOS keys include +the hosted image revision and native-tool fingerprint, and Windows warmer, +PR, main, and release jobs share the same hosted image revision. A reported +cache hit is verified against the current build contract before reuse. A future Depot PR entrypoint must instead use keys that trusted main/release jobs never restore because Depot cache entries are repository-scoped. Key separation alone is not a security boundary while the job receives Depot cache diff --git a/.github/actions/compute-changes/action.yml b/.github/actions/compute-changes/action.yml index 413dcf71c5..40e9e1d849 100644 --- a/.github/actions/compute-changes/action.yml +++ b/.github/actions/compute-changes/action.yml @@ -164,7 +164,7 @@ runs: if [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]; then RUNNER_CONTRACT_REQUIRED="true" elif [[ -n "$CHANGED_FILES" ]]; then - RUNNER_CONTRACT_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^\.github/cache-version\.txt$|^\.github/actionlint\.yaml$|^\.github/actions/(capture-sccache-stats|configure-sccache-gha|select-ci-runners)/|^\.github/workflows/(ci|depot-canary|native-sdk-artifact|pr_builds|pr_quality|release|sdk-smoke|static-abi-artifact|swift-sdk-artifact)\.yml$)' || true) + RUNNER_CONTRACT_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^\.github/cache-version\.txt$|^\.github/actionlint\.yaml$|^\.github/actions/(capture-sccache-stats|configure-sccache-gha|resolve-native-toolchain-epoch|select-ci-runners)/|^\.github/workflows/(ci|depot-canary|native-sdk-artifact|pr_builds|pr_quality|release|sdk-smoke|static-abi-artifact|swift-sdk-artifact)\.yml$)' || true) if [[ -n "$RUNNER_CONTRACT_INPUTS" ]]; then RUNNER_CONTRACT_REQUIRED="true" fi @@ -315,7 +315,7 @@ runs: if [[ "$ALL_RUST" == "true" ]]; then BACKEND_CHANGED="true" elif [[ -n "$CHANGED_FILES" ]]; then - BACKEND_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^third_party/llama\.cpp/|^crates/skippy-ffi/|^scripts/(build-llama|prepare-llama|build-linux|build-linux-rocm|build-mac|build-windows|install-windows-sdk|build-host|build-release|package-release|package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^\.github/actions/(prepare-host-input|prepare-windows-host-input|prepare-native-runtime-input|compose-product-input|restore-smoke-inputs|restore-windows-abi-cache|setup-windows-rocm-sdk)/|^\.github/workflows/(ci|pr_builds|release|sdk-smoke|smoke)\.yml$|^\.github/cache-version\.txt$)' || true) + BACKEND_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^third_party/llama\.cpp/|^crates/skippy-ffi/|^scripts/(build-llama|prepare-llama|build-linux|build-linux-rocm|build-mac|build-windows|install-windows-sdk|build-host|build-release|package-release|package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^\.github/actions/(prepare-host-input|prepare-windows-host-input|prepare-native-runtime-input|compose-product-input|resolve-native-toolchain-epoch|restore-smoke-inputs|restore-windows-abi-cache|setup-windows-rocm-sdk)/|^\.github/workflows/(ci|pr_builds|release|sdk-smoke|smoke)\.yml$|^\.github/cache-version\.txt$)' || true) if [[ -n "$BACKEND_INPUTS" ]] || [[ "$BACKEND_RECIPE_CHANGED" == "true" ]]; then BACKEND_CHANGED="true" fi @@ -327,8 +327,8 @@ runs: WINDOWS_CPU_BUILD_REQUIRED="true" WINDOWS_GPU_BUILD_REQUIRED="true" elif [[ -n "$CHANGED_FILES" ]]; then - WINDOWS_CPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/mesh-llm-release-footer/|^crates/mesh-llm-nodejs/|^crates/skippy-ffi/|^scripts/(build-windows|package-release)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^third_party/llama\.cpp/|^Cargo\.toml$|^Cargo\.lock$|^\.github/cache-version\.txt$|^\.github/workflows/(ci|pr_builds|release|windows-warm-caches)\.yml$|^\.github/actions/(compute-changes/action\.yml$|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|restore-windows-abi-cache/))' || true) - WINDOWS_GPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/skippy-ffi/|^scripts/(build-windows|install-windows-sdk|package-release)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^scripts/windows-native-runtime-deps\.py$|^scripts/tests/test_windows_native_runtime_deps\.py$|^third_party/llama\.cpp/|^\.github/cache-version\.txt$|^\.github/workflows/(ci|pr_builds|release|windows-warm-caches)\.yml$|^\.github/actions/(compute-changes/action\.yml$|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|restore-windows-abi-cache/|setup-windows-rocm-sdk/))' || true) + WINDOWS_CPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/mesh-llm-release-footer/|^crates/mesh-llm-nodejs/|^crates/skippy-ffi/|^scripts/(build-windows|package-release)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^third_party/llama\.cpp/|^Cargo\.toml$|^Cargo\.lock$|^\.github/cache-version\.txt$|^\.github/workflows/(ci|pr_builds|release|windows-warm-caches)\.yml$|^\.github/actions/(compute-changes/action\.yml$|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|resolve-native-toolchain-epoch/|restore-windows-abi-cache/))' || true) + WINDOWS_GPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/skippy-ffi/|^scripts/(build-windows|install-windows-sdk|package-release)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^scripts/windows-native-runtime-deps\.py$|^scripts/tests/test_windows_native_runtime_deps\.py$|^third_party/llama\.cpp/|^\.github/cache-version\.txt$|^\.github/workflows/(ci|pr_builds|release|windows-warm-caches)\.yml$|^\.github/actions/(compute-changes/action\.yml$|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|resolve-native-toolchain-epoch/|restore-windows-abi-cache/|setup-windows-rocm-sdk/))' || true) if [[ -n "$WINDOWS_CPU_INPUTS" ]] || [[ "$BACKEND_RECIPE_CHANGED" == "true" ]]; then WINDOWS_CPU_BUILD_REQUIRED="true" fi @@ -343,7 +343,7 @@ runs: if [[ "${{ inputs.event_name }}" == "workflow_dispatch" ]]; then SDK_SMOKE_REQUIRED="true" elif [[ -n "$CHANGED_FILES" ]]; then - DIRECT_SDK_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^sdk/|^Package\.swift$|^scripts/ci-(rust|kotlin|swift)-sdk-smoke\.sh$|^scripts/ci-prepare-native-runtime\.sh$|^scripts/ci-sdk-fixture\.sh$|^scripts/(check-sdk-contract|package-sdk-console-assets|restore-native-sdk-input|restore-static-abi-input|verify-sdk-console-assets|verify-swift-privacy-manifest|verify-swift-release-artifact|prepare-llama|build-llama)\.sh$|^scripts/(package-native-sdk|package-native-sdk-crate|verify-native-sdk-package|verify-checksum-sidecar|verify-static-abi-build-stamp|safe-extract-(tar|zip)|verify-swift-xcframework)\.(sh|py)$|^\.github/actions/(prepare-native-sdk-input|prepare-static-abi-input|restore-smoke-inputs)/|^\.github/workflows/(native-sdk-artifact|sdk-smoke|static-abi-artifact|swift-sdk-artifact)\.yml$)' || true) + DIRECT_SDK_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^sdk/|^Package\.swift$|^scripts/ci-(rust|kotlin|swift)-sdk-smoke\.sh$|^scripts/ci-prepare-native-runtime\.sh$|^scripts/ci-sdk-fixture\.sh$|^scripts/(check-sdk-contract|package-sdk-console-assets|restore-native-sdk-input|restore-static-abi-input|verify-sdk-console-assets|verify-swift-privacy-manifest|verify-swift-release-artifact|prepare-llama|build-llama)\.sh$|^scripts/(package-native-sdk|package-native-sdk-crate|verify-native-sdk-package|verify-checksum-sidecar|verify-static-abi-build-stamp|safe-extract-(tar|zip)|verify-swift-xcframework)\.(sh|py)$|^\.github/actions/(prepare-native-sdk-input|prepare-static-abi-input|resolve-native-toolchain-epoch|restore-smoke-inputs)/|^\.github/workflows/(native-sdk-artifact|sdk-smoke|static-abi-artifact|swift-sdk-artifact)\.yml$)' || true) if [[ -n "$DIRECT_SDK_INPUTS" ]]; then SDK_SMOKE_REQUIRED="true" elif echo "$AFFECTED_CRATES" | jq -e 'index("mesh-llm-client") or index("mesh-llm-api-client") or index("mesh-llm-api-server") or index("mesh-llm-config") or index("mesh-llm-console-server") or index("mesh-llm-ffi") or index("mesh-llm-native-runtime") or index("mesh-llm-protocol") or index("mesh-llm-routing") or index("mesh-llm-types")' >/dev/null; then diff --git a/.github/actions/resolve-native-toolchain-epoch/action.yml b/.github/actions/resolve-native-toolchain-epoch/action.yml new file mode 100644 index 0000000000..b0890d2ff5 --- /dev/null +++ b/.github/actions/resolve-native-toolchain-epoch/action.yml @@ -0,0 +1,104 @@ +name: Resolve native toolchain epoch +description: Derive one exact, cache-safe native build identity and export it to later steps. + +inputs: + pinned_epoch: + description: Exact prevalidated identity for an immutable toolchain image. + required: false + default: "" + include_tool_versions: + description: Include the selected native compiler, CMake, and Ninja versions. + required: false + default: "false" + +outputs: + epoch: + description: Validated native toolchain identity used by build stamps and cache keys. + value: ${{ steps.resolve.outputs.epoch }} + +runs: + using: composite + steps: + - name: Resolve exact native toolchain identity + id: resolve + shell: bash + env: + INPUT_PINNED_EPOCH: ${{ inputs.pinned_epoch }} + INPUT_INCLUDE_TOOL_VERSIONS: ${{ inputs.include_tool_versions }} + RUNNER_OS_VALUE: ${{ runner.os }} + RUNNER_ARCH_VALUE: ${{ runner.arch }} + run: | + set -euo pipefail + + case "$INPUT_INCLUDE_TOOL_VERSIONS" in + true|false) ;; + *) + echo "include_tool_versions must be true or false" >&2 + exit 1 + ;; + esac + + if [[ -n "$INPUT_PINNED_EPOCH" ]]; then + epoch="$INPUT_PINNED_EPOCH" + if [[ "$INPUT_INCLUDE_TOOL_VERSIONS" == "true" ]]; then + echo "include_tool_versions cannot be combined with pinned_epoch" >&2 + exit 1 + fi + else + image_os="${ImageOS:-}" + image_version="${ImageVersion:-}" + if [[ -z "$image_os" || -z "$image_version" ]]; then + echo "ImageOS and ImageVersion are required for a hosted native toolchain epoch" >&2 + exit 1 + fi + epoch="github-${image_os}-${image_version}-${RUNNER_ARCH_VALUE}" + + if [[ "$INPUT_INCLUDE_TOOL_VERSIONS" == "true" ]]; then + case "$RUNNER_OS_VALUE" in + macOS) + version_commands=(xcodebuild clang cmake ninja) + ;; + Linux) + version_commands=(cc c++ cmake ninja) + ;; + *) + echo "native tool version fingerprinting is unsupported on $RUNNER_OS_VALUE" >&2 + exit 1 + ;; + esac + for command_name in "${version_commands[@]}" shasum; do + if ! command -v "$command_name" >/dev/null 2>&1; then + echo "$command_name is required to resolve the native toolchain epoch" >&2 + exit 1 + fi + done + if [[ "$RUNNER_OS_VALUE" == "macOS" ]]; then + tool_digest="$( + { + xcodebuild -version + clang --version + cmake --version + ninja --version + } | shasum -a 256 | awk '{print $1}' + )" + else + tool_digest="$( + { + for command_name in "${version_commands[@]}"; do + "$command_name" --version + done + } | shasum -a 256 | awk '{print $1}' + )" + fi + epoch="${epoch}-native-${tool_digest}" + fi + fi + + if [[ ! "$epoch" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "resolved native toolchain epoch contains unsafe characters" >&2 + exit 1 + fi + + echo "epoch=$epoch" >> "$GITHUB_OUTPUT" + echo "MESH_LLM_LLAMA_TOOLCHAIN_EPOCH=$epoch" >> "$GITHUB_ENV" + echo "Resolved native toolchain epoch: $epoch" diff --git a/.github/actions/restore-windows-abi-cache/action.yml b/.github/actions/restore-windows-abi-cache/action.yml index 82e81e88f1..839a64ed10 100644 --- a/.github/actions/restore-windows-abi-cache/action.yml +++ b/.github/actions/restore-windows-abi-cache/action.yml @@ -8,6 +8,9 @@ inputs: build_dir: description: Repository-relative Windows native ABI build directory. required: true + toolchain_epoch: + description: Exact hosted Windows image/toolchain identity shared with the build stamp. + required: true architecture_set: description: Canonical backend architecture set; CUDA and ROCm require a non-empty value. required: false @@ -42,11 +45,12 @@ runs: env: INPUT_BACKEND: ${{ inputs.backend }} INPUT_BUILD_DIR: ${{ inputs.build_dir }} + INPUT_TOOLCHAIN_EPOCH: ${{ inputs.toolchain_epoch }} INPUT_ARCHITECTURE_SET: ${{ inputs.architecture_set }} INPUT_CUDA_TOOLCHAIN_VERSION: ${{ inputs.cuda_toolchain_version }} INPUT_VULKAN_TOOLCHAIN_VERSION: ${{ inputs.vulkan_toolchain_version }} INPUT_ROCM_TOOLCHAIN_VERSION: ${{ inputs.rocm_toolchain_version }} - CACHE_INPUT_HASH: ${{ hashFiles('.github/actions/restore-windows-abi-cache/action.yml', '.github/actions/prepare-native-runtime-input/action.yml', '.github/actions/setup-windows-rocm-sdk/action.yml', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} + CACHE_INPUT_HASH: ${{ hashFiles('.github/actions/restore-windows-abi-cache/action.yml', '.github/actions/resolve-native-toolchain-epoch/action.yml', '.github/actions/prepare-native-runtime-input/action.yml', '.github/actions/setup-windows-rocm-sdk/action.yml', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} run: | $ErrorActionPreference = "Stop" @@ -78,6 +82,18 @@ runs: if ($backend -notin @("cpu", "cuda", "rocm", "vulkan")) { throw "unsupported Windows ABI cache backend: $backend" } + $toolchainEpoch = Normalize-KeySegment ` + -Value "$env:INPUT_TOOLCHAIN_EPOCH" ` + -Name "toolchain_epoch" + $buildStampEpoch = Normalize-KeySegment ` + -Value "$env:MESH_LLM_LLAMA_TOOLCHAIN_EPOCH" ` + -Name "MESH_LLM_LLAMA_TOOLCHAIN_EPOCH" + if ($toolchainEpoch -ne $buildStampEpoch) { + throw ( + "toolchain_epoch must match MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: " + + "$toolchainEpoch != $buildStampEpoch" + ) + } $buildDir = "$env:INPUT_BUILD_DIR".Trim() if (-not $buildDir) { @@ -176,7 +192,7 @@ runs: } New-Item -ItemType Directory -Force -Path $resolvedBuildDir | Out-Null - $cacheKey = "mesh-llm-windows-2022-skippy-abi-$backend-$architectureSet-$toolchain-$inputHash" + $cacheKey = "mesh-llm-windows-2022-skippy-abi-$backend-$architectureSet-$toolchain-$toolchainEpoch-$inputHash" @( "build-dir=$resolvedBuildDir" "cache-key=$cacheKey" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b8bc005ca..93cfcb699e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,6 @@ env: CARGO_HTTP_MULTIPLEXING: "false" SCCACHE_GHA_ENABLED: "true" LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static - MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d permissions: contents: read @@ -291,6 +290,10 @@ jobs: - name: Verify prebuilt CI environment run: verify-runner-image public + - name: Resolve CPU native toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -416,6 +419,10 @@ jobs: persist-credentials: false - name: Verify prebuilt CI environment run: verify-runner-image public + - name: Resolve static ABI toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -490,6 +497,10 @@ jobs: - name: Verify prebuilt CI environment run: verify-runner-image public + - name: Resolve static ABI toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -843,6 +854,10 @@ jobs: persist-credentials: false - name: Install native runtime dependencies run: brew install cmake ninja jq + - name: Resolve macOS native toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + include_tool_versions: "true" - name: Prepare Metal native runtime uses: ./.github/actions/prepare-native-runtime-input with: @@ -924,6 +939,11 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install unit-test dependencies run: brew install cmake ninja jq lld + - name: Resolve macOS native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + include_tool_versions: "true" - name: Configure macOS Rust linker run: | mkdir -p .cargo @@ -942,10 +962,12 @@ jobs: uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ env.LLAMA_STAGE_BUILD_DIR }} - key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-static-metal-${{ hashFiles('scripts/build-mac.sh', 'scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} + key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-${{ runner.arch }}-skippy-abi-static-metal-${{ steps.native_toolchain.outputs.epoch }}-${{ hashFiles('scripts/build-mac.sh', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - name: Prepare patched llama.cpp ABI checkout - if: ${{ steps.llama_cache.outputs.cache-hit != 'true' }} run: scripts/prepare-llama.sh pinned + - name: Verify cached patched llama.cpp ABI libraries + if: ${{ steps.llama_cache.outputs.cache-hit == 'true' }} + run: scripts/build-llama.sh --require-existing - name: Build patched llama.cpp ABI libraries if: ${{ steps.llama_cache.outputs.cache-hit != 'true' }} run: scripts/build-llama.sh @@ -994,6 +1016,10 @@ jobs: persist-credentials: false - name: Verify prebuilt backend environment run: verify-runner-image public cuda + - name: Resolve CUDA native toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -1089,6 +1115,10 @@ jobs: persist-credentials: false - name: Verify prebuilt backend environment run: verify-runner-image public rocm + - name: Resolve ROCm native toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-0e13e5d2d2c121df265ff6c69be81e468989e09f81d6b7ff049b110cc0bb0d2b - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -1182,6 +1212,10 @@ jobs: persist-credentials: false - name: Verify prebuilt backend environment run: verify-runner-image public vulkan + - name: Resolve Vulkan native toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-ce55fed5c680cd3184b5d4770d9a77c43a702687690906e5753efd2cea27ed80 - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -1329,11 +1363,15 @@ jobs: prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} shared-key: windows-runtime-${{ matrix.backend }} save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Resolve Windows native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch - name: Restore exact Windows GPU ABI cache uses: ./.github/actions/restore-windows-abi-cache with: backend: ${{ matrix.backend }} build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + toolchain_epoch: ${{ steps.native_toolchain.outputs.epoch }} architecture_set: ${{ matrix.cuda_architectures || matrix.rocm_architectures || '' }} cuda_toolchain_version: ${{ env.WINDOWS_CUDA_VERSION }} vulkan_toolchain_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} @@ -1471,11 +1509,15 @@ jobs: prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} shared-key: windows-runtime-cpu save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Resolve Windows native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch - name: Restore exact Windows CPU ABI cache uses: ./.github/actions/restore-windows-abi-cache with: backend: cpu build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + toolchain_epoch: ${{ steps.native_toolchain.outputs.epoch }} - name: Initialize MSVC for CPU runtime uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: diff --git a/.github/workflows/depot-canary.yml b/.github/workflows/depot-canary.yml index 9eef1e82d3..e96084cc0b 100644 --- a/.github/workflows/depot-canary.yml +++ b/.github/workflows/depot-canary.yml @@ -2,6 +2,12 @@ name: Depot Runner Canary on: workflow_dispatch: + inputs: + expect_cache_hit: + description: Fail unless this is a warm Depot Cache run. + required: false + default: false + type: boolean permissions: {} @@ -41,6 +47,19 @@ jobs: echo "runner architecture mismatch: expected $expected_arch, got $actual_arch" >&2 exit 1 fi + if [[ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]]; then + echo "Depot Cache WebDAV endpoint was not injected" >&2 + exit 1 + fi + if [[ -z "${ImageOS:-}" || -z "${ImageVersion:-}" ]]; then + echo "Depot runner image identity was not injected" >&2 + exit 1 + fi + if [[ -z "${DEPOT_CACHE_TOKEN:-}" && + -z "${SCCACHE_WEBDAV_TOKEN:-}" ]]; then + echo "Depot Cache authentication was not injected" >&2 + exit 1 + fi { echo "### ${{ matrix.runner }}" @@ -51,10 +70,77 @@ jobs: printf '| logical CPUs | %s |\n' "$(nproc)" printf '| memory KiB | %s |\n' "$(awk '/MemTotal/ { print $2 }' /proc/meminfo)" printf '| workspace free KiB | %s |\n' "$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR == 2 { print $4 }')" - printf '| Depot cache token injected | %s |\n' \ - "$([[ -n "${DEPOT_CACHE_TOKEN:-}" ]] && echo yes || echo no)" + printf '| runner image | %s %s |\n' "$ImageOS" "$ImageVersion" + echo "| Depot cache endpoint injected | yes |" + echo "| Depot cache authentication injected | yes |" } >> "$GITHUB_STEP_SUMMARY" + - name: Install sccache + uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + + - name: Exercise authenticated Depot sccache + shell: bash + env: + EXPECT_CACHE_HIT: ${{ inputs.expect_cache_hit }} + run: | + set -euo pipefail + + if [[ -z "${SCCACHE_WEBDAV_TOKEN:-}" ]]; then + export SCCACHE_WEBDAV_TOKEN="$DEPOT_CACHE_TOKEN" + fi + if sccache --stop-server >/dev/null 2>&1; then + : + fi + sccache --zero-stats + mkdir -p .depot-sccache-canary + printf 'int depot_sccache_probe(void) { return 42; }\n' \ + > .depot-sccache-canary/probe.c + sccache cc \ + -c .depot-sccache-canary/probe.c \ + -o .depot-sccache-canary/probe.o + sccache --show-stats --stats-format json \ + > "$RUNNER_TEMP/depot-sccache-stats.json" + + python3 - \ + "$RUNNER_TEMP/depot-sccache-stats.json" \ + "$EXPECT_CACHE_HIT" \ + "$GITHUB_STEP_SUMMARY" <<'PY' + import json + import sys + + stats_path, expect_cache_hit, summary_path = sys.argv[1:] + with open(stats_path, encoding="utf-8") as handle: + payload = json.load(handle) + + location = str(payload.get("cache_location", "")).lower() + if "webdav" not in location and "depot" not in location: + raise SystemExit("sccache did not select the Depot WebDAV backend") + + stats = payload["stats"] + hits = sum(stats["cache_hits"]["counts"].values()) + misses = sum(stats["cache_misses"]["counts"].values()) + cache_errors = sum(stats["cache_errors"]["counts"].values()) + cache_errors += stats["cache_read_errors"] + cache_errors += stats["cache_write_errors"] + writes = stats["cache_writes"] + + if cache_errors: + raise SystemExit( + f"Depot sccache reported {cache_errors} cache errors" + ) + if not hits and not writes: + raise SystemExit( + "Depot sccache produced neither a cache hit nor a cache write" + ) + if expect_cache_hit == "true" and not hits: + raise SystemExit("expected a warm Depot sccache hit") + + with open(summary_path, "a", encoding="utf-8") as summary: + summary.write(f"| sccache hits | {hits} |\n") + summary.write(f"| sccache misses | {misses} |\n") + summary.write(f"| sccache writes | {writes} |\n") + PY + - name: Restore Depot cache probe id: cache uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 @@ -66,9 +152,14 @@ jobs: shell: bash env: CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} + EXPECT_CACHE_HIT: ${{ inputs.expect_cache_hit }} run: | set -euo pipefail + if [[ "$EXPECT_CACHE_HIT" == "true" && "$CACHE_HIT" != "true" ]]; then + echo "expected a warm Depot Cache hit for ${{ matrix.runner }}" >&2 + exit 1 + fi mkdir -p .depot-canary-cache printf 'Depot cache canary\n' > .depot-canary-cache/probe.txt printf '| cache hit | %s |\n' "${CACHE_HIT:-false}" \ diff --git a/.github/workflows/native-sdk-artifact.yml b/.github/workflows/native-sdk-artifact.yml index 7613cc666f..03057f70a4 100644 --- a/.github/workflows/native-sdk-artifact.yml +++ b/.github/workflows/native-sdk-artifact.yml @@ -182,7 +182,6 @@ jobs: shell: bash env: LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static - MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -209,6 +208,11 @@ jobs: - name: Verify prebuilt native SDK environment run: verify-runner-image public + - name: Resolve static ABI toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} @@ -302,6 +306,11 @@ jobs: - name: Install macOS native SDK dependencies run: brew install cmake ninja lld + - name: Resolve macOS native toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + include_tool_versions: "true" + - name: Prepare dispatched release version if: ${{ inputs.prepare_release_version }} env: diff --git a/.github/workflows/pr_builds.yml b/.github/workflows/pr_builds.yml index c1367d2727..36659a8f12 100644 --- a/.github/workflows/pr_builds.yml +++ b/.github/workflows/pr_builds.yml @@ -32,7 +32,6 @@ env: CARGO_HTTP_MULTIPLEXING: "false" SCCACHE_GHA_ENABLED: "true" LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static - MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d jobs: changes: @@ -308,6 +307,10 @@ jobs: persist-credentials: false - name: Verify prebuilt CI environment run: verify-runner-image public + - name: Resolve CPU native toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -409,6 +412,10 @@ jobs: - name: Verify prebuilt backend environment run: verify-runner-image public cuda + - name: Resolve CUDA native toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -504,6 +511,10 @@ jobs: - name: Verify prebuilt backend environment run: verify-runner-image public rocm + - name: Resolve ROCm native toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-6b88ca9371ada2c507d6e36b71f0e0538fee378c6a5e2b39c17249b4b7e5088a - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -597,6 +608,11 @@ jobs: - name: Verify prebuilt backend environment run: verify-runner-image public vulkan + - name: Resolve Vulkan native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-ce55fed5c680cd3184b5d4770d9a77c43a702687690906e5753efd2cea27ed80 - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -607,7 +623,7 @@ jobs: uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ env.LLAMA_STAGE_BUILD_DIR }} - key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-skippy-abi-vulkan-${{ hashFiles('scripts/build-linux.sh', 'scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} + key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-${{ runner.arch }}-skippy-abi-vulkan-${{ steps.native_toolchain.outputs.epoch }}-${{ hashFiles('scripts/build-linux.sh', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', '.github/actions/prepare-native-runtime-input/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - name: Prepare Vulkan native runtime uses: ./.github/actions/prepare-native-runtime-input @@ -720,6 +736,10 @@ jobs: persist-credentials: false - name: Verify prebuilt CI environment run: verify-runner-image public + - name: Resolve static ABI toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -792,6 +812,10 @@ jobs: - name: Verify prebuilt CI environment run: verify-runner-image public + - name: Resolve static ABI toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.changes.outputs.allow_depot_remote_cache }} @@ -1111,13 +1135,18 @@ jobs: persist-credentials: false - name: Install native-runtime build dependencies run: brew install cmake ninja + - name: Resolve macOS native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + include_tool_versions: "true" - name: Ensure Metal ABI cache directory run: mkdir -p "$LLAMA_STAGE_BUILD_DIR" - name: Cache Metal ABI build uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: .deps/llama-build/build-stage-abi-dynamic-metal - key: ${{ env.CACHE_NAMESPACE }}-github-macos-15-skippy-abi-dynamic-metal-${{ hashFiles('scripts/build-mac.sh', 'scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} + key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-${{ runner.arch }}-skippy-abi-dynamic-metal-${{ steps.native_toolchain.outputs.epoch }}-${{ hashFiles('scripts/build-mac.sh', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', '.github/actions/prepare-native-runtime-input/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - name: Prepare Metal native runtime uses: ./.github/actions/prepare-native-runtime-input with: @@ -1209,6 +1238,11 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install build dependencies run: brew install cmake ninja jq lld + - name: Resolve macOS native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + include_tool_versions: "true" - name: Configure macOS Rust linker run: | mkdir -p .cargo @@ -1227,10 +1261,12 @@ jobs: uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ env.LLAMA_STAGE_BUILD_DIR }} - key: ${{ env.CACHE_NAMESPACE }}-github-macos-15-skippy-abi-static-metal-${{ hashFiles('scripts/build-mac.sh', 'scripts/build-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} + key: ${{ env.CACHE_NAMESPACE }}-${{ runner.os }}-${{ runner.arch }}-skippy-abi-static-metal-${{ steps.native_toolchain.outputs.epoch }}-${{ hashFiles('scripts/build-mac.sh', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt') }} - name: Prepare patched llama.cpp ABI checkout - if: ${{ steps.llama_cache.outputs.cache-hit != 'true' }} run: scripts/prepare-llama.sh pinned + - name: Verify cached patched llama.cpp ABI libraries + if: ${{ steps.llama_cache.outputs.cache-hit == 'true' }} + run: scripts/build-llama.sh --require-existing - name: Build patched llama.cpp ABI libraries if: ${{ steps.llama_cache.outputs.cache-hit != 'true' }} run: scripts/build-llama.sh @@ -1352,11 +1388,15 @@ jobs: prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} shared-key: windows-runtime-cpu save-if: "false" + - name: Resolve Windows native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch - name: Restore exact Windows CPU ABI cache uses: ./.github/actions/restore-windows-abi-cache with: backend: cpu build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + toolchain_epoch: ${{ steps.native_toolchain.outputs.epoch }} - name: Initialize MSVC for CPU runtime uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: @@ -1422,11 +1462,15 @@ jobs: prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} shared-key: windows-runtime-${{ matrix.backend }} save-if: "false" + - name: Resolve Windows native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch - name: Restore exact Windows GPU ABI cache uses: ./.github/actions/restore-windows-abi-cache with: backend: ${{ matrix.backend }} build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + toolchain_epoch: ${{ steps.native_toolchain.outputs.epoch }} architecture_set: ${{ matrix.cuda_architectures || matrix.rocm_architectures || '' }} cuda_toolchain_version: ${{ env.WINDOWS_CUDA_VERSION }} vulkan_toolchain_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c9da391cab..f55700a8bb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -407,6 +407,10 @@ jobs: - name: Install macOS dependencies if: runner.os == 'macOS' run: brew install cmake ninja lld + - name: Resolve hosted native toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + include_tool_versions: "true" - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' @@ -442,9 +446,11 @@ jobs: - cuda_version: '12.9.2' cuda_major: '12' runner_image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd + toolchain_epoch: mesh-llm-cuda-runner-sha256-c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd - cuda_version: '13.1.2' cuda_major: '13' runner_image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:6b87598605f5d8deeafecfb1a55027e0ca9e47f4fc6f230d030487c450c31aa6 + toolchain_epoch: mesh-llm-cuda-runner-sha256-6b87598605f5d8deeafecfb1a55027e0ca9e47f4fc6f230d030487c450c31aa6 container: image: ${{ matrix.runner_image }} credentials: @@ -463,6 +469,10 @@ jobs: run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Verify prebuilt backend environment run: verify-runner-image public cuda + - name: Resolve CUDA native toolchain epoch + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: ${{ matrix.toolchain_epoch }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} @@ -498,11 +508,13 @@ jobs: - cuda_version: '12.9.2' cuda_major: '12' runner_image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd + toolchain_epoch: mesh-llm-cuda-runner-sha256-c5b85ef527230f77cf9933ef40bcb44316f9bbcb8fd2ce0651b58acda5143dfd cuda_architectures: '75;80;86;87;89;90' cuda_architectures_cache: '75_80_86_87_89_90' - cuda_version: '13.1.2' cuda_major: '13' runner_image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:6b87598605f5d8deeafecfb1a55027e0ca9e47f4fc6f230d030487c450c31aa6 + toolchain_epoch: mesh-llm-cuda-runner-sha256-6b87598605f5d8deeafecfb1a55027e0ca9e47f4fc6f230d030487c450c31aa6 cuda_architectures: '75;80;86;87;89;90;100;103;120;121' cuda_architectures_cache: '75_80_86_87_89_90_100_103_120_121' container: @@ -525,6 +537,11 @@ jobs: run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Verify prebuilt backend environment run: verify-runner-image public cuda + - name: Resolve CUDA native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: ${{ matrix.toolchain_epoch }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} @@ -532,7 +549,7 @@ jobs: uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ env.LLAMA_STAGE_BUILD_DIR }} - key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-cuda${{ matrix.cuda_major }}-${{ matrix.cuda_version }}-dynamic-${{ matrix.cuda_architectures_cache }}-${{ hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} + key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-cuda${{ matrix.cuda_major }}-${{ matrix.cuda_version }}-dynamic-${{ matrix.cuda_architectures_cache }}-${{ steps.native_toolchain.outputs.epoch }}-${{ hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', '.github/actions/prepare-native-runtime-input/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' env: @@ -577,6 +594,11 @@ jobs: run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Verify prebuilt backend environment run: verify-runner-image public rocm + - name: Resolve ROCm native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-0e13e5d2d2c121df265ff6c69be81e468989e09f81d6b7ff049b110cc0bb0d2b - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} @@ -584,7 +606,7 @@ jobs: uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ env.LLAMA_STAGE_BUILD_DIR }} - key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-rocm7.0-dynamic-gfx90a_gfx942_gfx1100_gfx1101_gfx1102_gfx1103_gfx1151_gfx1200_gfx1201-${{ hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} + key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-rocm7.0-dynamic-gfx90a_gfx942_gfx1100_gfx1101_gfx1102_gfx1103_gfx1151_gfx1200_gfx1201-${{ steps.native_toolchain.outputs.epoch }}-${{ hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', '.github/actions/prepare-native-runtime-input/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' env: @@ -628,6 +650,11 @@ jobs: run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Verify prebuilt backend environment run: verify-runner-image public vulkan + - name: Resolve Vulkan native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-ce55fed5c680cd3184b5d4770d9a77c43a702687690906e5753efd2cea27ed80 - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.metadata.outputs.allow_depot_remote_cache }} @@ -635,7 +662,7 @@ jobs: uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ env.LLAMA_STAGE_BUILD_DIR }} - key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-vulkan-dynamic-${{ hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} + key: ${{ env.CACHE_NAMESPACE }}-release-llama-linux-x86_64-vulkan-dynamic-${{ steps.native_toolchain.outputs.epoch }}-${{ hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/package-native-runtime.sh', '.github/actions/prepare-native-runtime-input/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt') }} - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' env: @@ -1327,11 +1354,15 @@ jobs: python-version: '3.x' - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - name: Resolve Windows native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch - name: Restore exact Windows CPU ABI cache uses: ./.github/actions/restore-windows-abi-cache with: backend: cpu build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + toolchain_epoch: ${{ steps.native_toolchain.outputs.epoch }} - name: Initialize MSVC for native runtime uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: @@ -1398,11 +1429,15 @@ jobs: python-version: '3.x' - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - name: Resolve Windows native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch - name: Restore exact Windows GPU ABI cache uses: ./.github/actions/restore-windows-abi-cache with: backend: ${{ matrix.backend }} build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + toolchain_epoch: ${{ steps.native_toolchain.outputs.epoch }} architecture_set: ${{ matrix.cuda_architectures || matrix.rocm_architectures || '' }} cuda_toolchain_version: ${{ env.WINDOWS_CUDA_VERSION }} vulkan_toolchain_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} diff --git a/.github/workflows/static-abi-artifact.yml b/.github/workflows/static-abi-artifact.yml index 7733a558f3..775399530e 100644 --- a/.github/workflows/static-abi-artifact.yml +++ b/.github/workflows/static-abi-artifact.yml @@ -123,7 +123,6 @@ jobs: env: LLAMA_STAGE_BACKEND: ${{ inputs.backend }} LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static - MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d MESH_LLM_REQUIRE_SCCACHE: "1" SCCACHE_GHA_ENABLED: "true" steps: @@ -134,6 +133,12 @@ jobs: - name: Verify prebuilt static ABI environment run: verify-runner-image public + - name: Resolve static ABI toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + pinned_epoch: mesh-llm-cuda-runner-sha256-8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} @@ -149,7 +154,7 @@ jobs: uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: static-abi-artifact-output - key: ${{ format('{0}-{1}-skippy-abi-{2}-{3}-{4}-{5}', env.CACHE_NAMESPACE, runner.os, inputs.backend, inputs.target, env.MESH_LLM_LLAMA_TOOLCHAIN_EPOCH, hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/restore-static-abi-input.sh', 'scripts/safe-extract-tar.py', 'scripts/verify-checksum-sidecar.py', 'scripts/verify-static-abi-build-stamp.py', '.github/actions/prepare-static-abi-input/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt')) }} + key: ${{ format('{0}-{1}-skippy-abi-{2}-{3}-{4}-{5}', env.CACHE_NAMESPACE, runner.os, inputs.backend, inputs.target, steps.native_toolchain.outputs.epoch, hashFiles('scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'scripts/restore-static-abi-input.sh', 'scripts/safe-extract-tar.py', 'scripts/verify-checksum-sidecar.py', 'scripts/verify-static-abi-build-stamp.py', '.github/actions/prepare-static-abi-input/action.yml', '.github/actions/resolve-native-toolchain-epoch/action.yml', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', 'Justfile', '.github/cache-version.txt')) }} - name: Restore and verify cached portable static ABI if: ${{ steps.static_abi_cache.outputs.cache-hit == 'true' }} diff --git a/.github/workflows/swift-sdk-artifact.yml b/.github/workflows/swift-sdk-artifact.yml index 1b5502ae38..92a09a8ea7 100644 --- a/.github/workflows/swift-sdk-artifact.yml +++ b/.github/workflows/swift-sdk-artifact.yml @@ -56,9 +56,6 @@ jobs: LLAMA_STAGE_BACKEND: metal RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "true" - # Bump this epoch whenever the pinned macOS/Xcode native build boundary - # changes so an older C/C++ ABI cache cannot cross toolchains. - SWIFT_NATIVE_XCODE_CACHE_EPOCH: macos-15-arm64-xcode-default-v1 steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -106,6 +103,12 @@ jobs: - name: Install macOS SDK build dependencies run: brew install cmake ninja lld + - name: Resolve macOS native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch + with: + include_tool_versions: "true" + - name: Restore exact Swift native ABI cache uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: @@ -113,7 +116,7 @@ jobs: # The full and host-only producers share target-specific ABI directories. # GitHub scopes PR writes to the merge ref, so a trusted main full build # can seed this exact mode-independent key without a PR shadowing it. - key: ${{ format('mesh-llm-swift-sdk-{0}-{1}-{2}-{3}', runner.os, runner.arch, env.SWIFT_NATIVE_XCODE_CACHE_EPOCH, hashFiles('Cargo.lock', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'sdk/swift/scripts/build-host-macos-xcframework.sh', 'sdk/swift/scripts/build-xcframework.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/cache-version.txt')) }} + key: ${{ format('mesh-llm-swift-sdk-{0}-{1}-{2}-{3}', runner.os, runner.arch, steps.native_toolchain.outputs.epoch, hashFiles('Cargo.lock', 'scripts/build-llama.sh', 'scripts/prepare-llama.sh', 'sdk/swift/scripts/build-host-macos-xcframework.sh', 'sdk/swift/scripts/build-xcframework.sh', 'third_party/llama.cpp/upstream.txt', 'third_party/llama.cpp/patches/**', '.github/actions/resolve-native-toolchain-epoch/action.yml', '.github/cache-version.txt')) }} - name: Prepare dispatched release version if: ${{ inputs.prepare_release_version }} diff --git a/.github/workflows/windows-warm-caches.yml b/.github/workflows/windows-warm-caches.yml index 0cab14e9c8..95cd332dc6 100644 --- a/.github/workflows/windows-warm-caches.yml +++ b/.github/workflows/windows-warm-caches.yml @@ -15,6 +15,7 @@ on: - 'scripts/windows-native-runtime-deps.py' - '.github/cache-version.txt' - '.github/actions/prepare-native-runtime-input/action.yml' + - '.github/actions/resolve-native-toolchain-epoch/action.yml' - '.github/actions/setup-windows-rocm-sdk/action.yml' - '.github/actions/restore-windows-abi-cache/action.yml' - '.github/workflows/ci.yml' @@ -58,12 +59,16 @@ jobs: with: python-version: '3.x' - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - name: Resolve Windows native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch - name: Restore Windows CPU ABI cache id: llama_cache uses: ./.github/actions/restore-windows-abi-cache with: backend: cpu build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + toolchain_epoch: ${{ steps.native_toolchain.outputs.epoch }} - name: Initialize MSVC for CPU runtime if: steps.llama_cache.outputs.cache-hit != 'true' uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 @@ -80,14 +85,33 @@ jobs: shell: pwsh run: | $stamp = Join-Path $env:LLAMA_STAGE_BUILD_DIR ".mesh-llm-build-stamp" - if (-not (Test-Path $stamp -PathType Leaf)) { - throw "Canonical build stamp was not found: $stamp" + python scripts/verify-static-abi-build-stamp.py ` + $stamp ` + --backend cpu ` + --link-mode dynamic ` + --stamp-version 3 ` + --toolchain-epoch $env:MESH_LLM_LLAMA_TOOLCHAIN_EPOCH + $requiredLibraries = @{ + llama = @("llama.dll", "libllama.dll") + common = @("llama-common.dll", "libllama-common.dll") + mtmd = @("mtmd.dll", "libmtmd.dll") } - $libraries = Get-ChildItem -Path $env:LLAMA_STAGE_BUILD_DIR -Recurse -File -Filter *.dll -ErrorAction SilentlyContinue | Select-Object -First 20 - if (-not $libraries) { - throw "No dynamic runtime libraries were found under $env:LLAMA_STAGE_BUILD_DIR." + foreach ($entry in $requiredLibraries.GetEnumerator()) { + $library = Get-ChildItem ` + -Path $env:LLAMA_STAGE_BUILD_DIR ` + -Recurse ` + -File ` + -ErrorAction SilentlyContinue | + Where-Object { $entry.Value -contains $_.Name } | + Select-Object -First 1 + if (-not $library) { + throw ( + "Windows CPU ABI cache is missing $($entry.Key): " + + "$($entry.Value -join ', ')" + ) + } + Write-Host $library.FullName } - $libraries | ForEach-Object { Write-Host $_.FullName } - name: Save Windows CPU ABI cache if: steps.llama_cache.outputs.cache-hit != 'true' uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 @@ -122,12 +146,16 @@ jobs: with: python-version: '3.x' - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - name: Resolve Windows native toolchain epoch + id: native_toolchain + uses: ./.github/actions/resolve-native-toolchain-epoch - name: Restore Windows GPU ABI cache id: llama_cache uses: ./.github/actions/restore-windows-abi-cache with: backend: ${{ matrix.backend }} build_dir: ${{ env.LLAMA_STAGE_BUILD_DIR }} + toolchain_epoch: ${{ steps.native_toolchain.outputs.epoch }} architecture_set: ${{ matrix.build_args }} cuda_toolchain_version: ${{ env.WINDOWS_CUDA_VERSION }} vulkan_toolchain_version: ${{ env.WINDOWS_VULKAN_SDK_VERSION }} @@ -201,16 +229,37 @@ jobs: output_dir: cache-runtime-input/${{ matrix.backend }} - name: Verify Windows GPU ABI cache shell: pwsh + env: + CACHE_BACKEND: ${{ matrix.backend }} run: | $stamp = Join-Path $env:LLAMA_STAGE_BUILD_DIR ".mesh-llm-build-stamp" - if (-not (Test-Path $stamp -PathType Leaf)) { - throw "Canonical build stamp was not found: $stamp" + python scripts/verify-static-abi-build-stamp.py ` + $stamp ` + --backend $env:CACHE_BACKEND ` + --link-mode dynamic ` + --stamp-version 3 ` + --toolchain-epoch $env:MESH_LLM_LLAMA_TOOLCHAIN_EPOCH + $requiredLibraries = @{ + llama = @("llama.dll", "libllama.dll") + common = @("llama-common.dll", "libllama-common.dll") + mtmd = @("mtmd.dll", "libmtmd.dll") } - $libraries = Get-ChildItem -Path $env:LLAMA_STAGE_BUILD_DIR -Recurse -File -Filter *.dll -ErrorAction SilentlyContinue | Select-Object -First 20 - if (-not $libraries) { - throw "No dynamic runtime libraries were found under $env:LLAMA_STAGE_BUILD_DIR." + foreach ($entry in $requiredLibraries.GetEnumerator()) { + $library = Get-ChildItem ` + -Path $env:LLAMA_STAGE_BUILD_DIR ` + -Recurse ` + -File ` + -ErrorAction SilentlyContinue | + Where-Object { $entry.Value -contains $_.Name } | + Select-Object -First 1 + if (-not $library) { + throw ( + "Windows $env:CACHE_BACKEND ABI cache is missing " + + "$($entry.Key): $($entry.Value -join ', ')" + ) + } + Write-Host $library.FullName } - $libraries | ForEach-Object { Write-Host $_.FullName } - name: Save Windows GPU ABI cache if: steps.llama_cache.outputs.cache-hit != 'true' uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 diff --git a/ci/DEPOT_MIGRATION.md b/ci/DEPOT_MIGRATION.md index 4678b99bf8..eb5f8326ef 100644 --- a/ci/DEPOT_MIGRATION.md +++ b/ci/DEPOT_MIGRATION.md @@ -209,9 +209,14 @@ Activation prerequisites: `Mesh-LLM/mesh-llm/.github/workflows/depot-canary.yml@refs/heads/main`. 4. Only after both restrictions are saved, enable public repositories for the `Default` group. Depot-managed ephemeral runners register in that group. -5. Dispatch `depot-canary.yml` from `refs/heads/main` twice. Verify all four - Intel runner sizes, both ARM runner sizes, their reported architectures, - and a cold-to-warm cache hit without printing credentials. +5. Dispatch `depot-canary.yml` from `refs/heads/main` once with + `expect_cache_hit=false`, then again with `expect_cache_hit=true`. The + workflow fails closed unless Depot injects its WebDAV endpoint, + authentication, and runner-image identity. It compiles a deterministic + no-checkout probe through sccache and rejects WebDAV read/write errors; the + warm pass requires both sccache and Actions cache hits. Verify all four Intel + runner sizes, both ARM runner sizes, and their reported architectures + without printing credentials. 6. Dispatch the canary from a feature ref, prove that it cannot acquire a Depot runner, and cancel that exact queued run. 7. Add exact default-branch workflow refs only as their phase starts. Reusable diff --git a/ci/ci.md b/ci/ci.md index c3961dcf8e..b346d9fe07 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -476,6 +476,14 @@ misses, cache write failures only warn, and a failed remote probe restarts false`), so trusted main owns the cache while PRs avoid recompiling the same workspace graph. +Native ABI cache keys and llama build stamps share one resolved toolchain epoch. +Digest-pinned Linux jobs use the immutable runner-image digest. Hosted macOS +keys include the exact image revision plus Xcode/Clang/CMake/Ninja fingerprint; +Windows warmer, PR, main, and release jobs use the exact hosted image revision +alongside backend SDK and architecture inputs. Cache-hit paths validate the +stored build stamp and required link closure instead of trusting the cache API +result alone. + ## Depot rollout Every current `pull_request` job selects GitHub-hosted runners, regardless of diff --git a/crates/mesh-llm/tests/qa_nightly_stability.rs b/crates/mesh-llm/tests/qa_nightly_stability.rs index c6ea18ecd9..d8cdeb47ff 100644 --- a/crates/mesh-llm/tests/qa_nightly_stability.rs +++ b/crates/mesh-llm/tests/qa_nightly_stability.rs @@ -66,7 +66,6 @@ fn nightly_stability_wrapper_calls_reusable_workflow() { "agent_smokes:", "skip_streaming:", "timeout:", - "runs_on:", "output_dir:", ] { assert!( @@ -74,6 +73,10 @@ fn nightly_stability_wrapper_calls_reusable_workflow() { "wrapper should expose {expected:?}; workflow was:\n{wrapper}" ); } + assert!( + !wrapper.contains("runs_on:"), + "wrapper must not allow caller-selected runner labels" + ); } #[test] @@ -82,7 +85,7 @@ fn nightly_stability_reusable_workflow_owns_execution() { .expect("reusable nightly stability workflow should be readable"); for expected in [ "workflow_call:", - "runs-on: ${{ fromJson(inputs.runs_on) }}", + "runs-on: ubuntu-24.04", "scripts/qa-nightly-stability.py", "Publish run summary", "$GITHUB_STEP_SUMMARY", @@ -93,6 +96,10 @@ fn nightly_stability_reusable_workflow_owns_execution() { "reusable workflow should contain {expected:?}; workflow was:\n{reusable}" ); } + assert!( + !reusable.contains("inputs.runs_on"), + "reusable workflow must own its GitHub-hosted runner selection" + ); } #[test] diff --git a/scripts/build-llama.sh b/scripts/build-llama.sh index 0dbc563723..b05ea0a297 100755 --- a/scripts/build-llama.sh +++ b/scripts/build-llama.sh @@ -100,13 +100,19 @@ required_static_archives_exist() { -f "$LLAMA_BUILD_DIR/ggml/src/ggml-cpu/libggml-cpu.a" ]] } -dynamic_library_names() { +dynamic_library_name_groups() { case "$(uname -s)" in Darwin) printf '%s\n' libllama.dylib libllama-common.dylib libmtmd.dylib ;; MINGW*|MSYS*|CYGWIN*) - printf '%s\n' llama.dll llama-common.dll mtmd.dll + # CMake's MinGW generator normally prefixes these DLLs with "lib"; + # retain the unprefixed MSVC-compatible spelling as an accepted + # alternative because both are valid runtime package inputs. + printf '%s\n' \ + 'libllama.dll|llama.dll' \ + 'libllama-common.dll|llama-common.dll' \ + 'libmtmd.dll|mtmd.dll' ;; *) printf '%s\n' libllama.so libllama-common.so libmtmd.so @@ -115,11 +121,17 @@ dynamic_library_names() { } required_dynamic_libraries_exist() { - local name found - while IFS= read -r name; do - found="$(find "$LLAMA_BUILD_DIR" -name "$name" -print -quit)" + local candidates name found + local -a names + while IFS= read -r candidates; do + IFS='|' read -r -a names <<< "$candidates" + found="" + for name in "${names[@]}"; do + found="$(find "$LLAMA_BUILD_DIR" -name "$name" -print -quit)" + [[ -z "$found" ]] || break + done [[ -n "$found" && -e "$found" ]] || return 1 - done < <(dynamic_library_names) + done < <(dynamic_library_name_groups) } required_outputs_exist() { diff --git a/scripts/build-windows.ps1 b/scripts/build-windows.ps1 index f1f5f0e645..1938993576 100755 --- a/scripts/build-windows.ps1 +++ b/scripts/build-windows.ps1 @@ -60,8 +60,39 @@ function Prepare-Llama { Invoke-NativeCommand "git" @("clean", "-fdx", "-e", "build/") $patches = Get-ChildItem -Path $patchDir -Filter "*.patch" | Sort-Object Name - foreach ($patch in $patches) { - Invoke-NativeCommand "git" @("am", "--3way", $patch.FullName) + $gitIdentityVariables = @( + "GIT_AUTHOR_DATE", + "GIT_AUTHOR_EMAIL", + "GIT_AUTHOR_NAME", + "GIT_COMMITTER_DATE", + "GIT_COMMITTER_EMAIL", + "GIT_COMMITTER_NAME" + ) + $savedGitIdentity = @{} + foreach ($variable in $gitIdentityVariables) { + if (Test-Path "Env:$variable") { + $savedGitIdentity[$variable] = (Get-Item "Env:$variable").Value + } + Remove-Item "Env:$variable" -ErrorAction SilentlyContinue + } + try { + foreach ($patch in $patches) { + Invoke-NativeCommand "git" @( + "am", + "--3way", + "--committer-date-is-author-date", + "--no-gpg-sign", + "--no-verify", + $patch.FullName + ) + } + } finally { + foreach ($variable in $gitIdentityVariables) { + Remove-Item "Env:$variable" -ErrorAction SilentlyContinue + } + foreach ($entry in $savedGitIdentity.GetEnumerator()) { + Set-Item "Env:$($entry.Key)" $entry.Value + } } $patchedSha = (& git rev-parse HEAD).Trim() diff --git a/scripts/ci-swift-sdk-smoke.sh b/scripts/ci-swift-sdk-smoke.sh index 680a1ff9da..10b6733a90 100755 --- a/scripts/ci-swift-sdk-smoke.sh +++ b/scripts/ci-swift-sdk-smoke.sh @@ -32,7 +32,14 @@ SWIFT_EXTRACT_DIR="$(mktemp -d)" trap 'rm -rf "$SWIFT_EXTRACT_DIR"' EXIT scripts/safe-extract-zip.py "$SWIFT_INPUT_ARCHIVE" "$SWIFT_EXTRACT_DIR" -SWIFT_XCFRAMEWORK="sdk/swift/Generated/MeshLLMFFI.xcframework" +SWIFT_GENERATED_DIR="sdk/swift/Generated" +if [[ -L "$SWIFT_GENERATED_DIR" ]] \ + || [[ -e "$SWIFT_GENERATED_DIR" && ! -d "$SWIFT_GENERATED_DIR" ]]; then + echo "Swift generated artifact directory is unsafe: $SWIFT_GENERATED_DIR" >&2 + exit 1 +fi +mkdir -p "$SWIFT_GENERATED_DIR" +SWIFT_XCFRAMEWORK="$SWIFT_GENERATED_DIR/MeshLLMFFI.xcframework" if [[ ! -d "$SWIFT_EXTRACT_DIR/MeshLLMFFI.xcframework" ]]; then echo "verified Swift SDK input did not restore MeshLLMFFI.xcframework" >&2 exit 1 diff --git a/scripts/prepare-llama.sh b/scripts/prepare-llama.sh index 75f1fb4835..016134c1aa 100755 --- a/scripts/prepare-llama.sh +++ b/scripts/prepare-llama.sh @@ -8,6 +8,7 @@ LLAMA_UPSTREAM_URL="${LLAMA_UPSTREAM_URL:-https://github.com/ggml-org/llama.cpp. LLAMA_WORKDIR="${LLAMA_WORKDIR:-$ROOT/.deps/llama.cpp}" PIN_FILE="${LLAMA_PIN_FILE:-$ROOT/third_party/llama.cpp/upstream.txt}" PATCH_DIR="${LLAMA_PATCH_DIR:-$ROOT/third_party/llama.cpp/patches}" +PREPARE_SCHEMA=2 if [[ ! -f "$PIN_FILE" ]]; then echo "missing llama upstream pin: $PIN_FILE" >&2 @@ -192,13 +193,16 @@ PATCH_DIGEST="$(compute_patch_digest)" if [[ -f "$LLAMA_WORKDIR/.mesh-llm-upstream-sha" && -f "$LLAMA_WORKDIR/.mesh-llm-patched-sha" && - -f "$LLAMA_WORKDIR/.mesh-llm-patch-digest" ]]; then + -f "$LLAMA_WORKDIR/.mesh-llm-patch-digest" && + -f "$LLAMA_WORKDIR/.mesh-llm-prepare-schema" ]]; then PREPARED_UPSTREAM="$(tr -d '[:space:]' < "$LLAMA_WORKDIR/.mesh-llm-upstream-sha")" PREPARED_PATCHED="$(tr -d '[:space:]' < "$LLAMA_WORKDIR/.mesh-llm-patched-sha")" PREPARED_DIGEST="$(tr -d '[:space:]' < "$LLAMA_WORKDIR/.mesh-llm-patch-digest")" + PREPARED_SCHEMA="$(tr -d '[:space:]' < "$LLAMA_WORKDIR/.mesh-llm-prepare-schema")" CURRENT_HEAD="$(git -C "$LLAMA_WORKDIR" rev-parse HEAD 2>/dev/null || true)" - if [[ "$PREPARED_UPSTREAM" == "$TARGET_SHA" && + if [[ "$PREPARED_SCHEMA" == "$PREPARE_SCHEMA" && + "$PREPARED_UPSTREAM" == "$TARGET_SHA" && "$PREPARED_PATCHED" == "$CURRENT_HEAD" && "$PREPARED_DIGEST" == "$PATCH_DIGEST" && ! -d "$LLAMA_WORKDIR/.git/rebase-apply" ]] && @@ -216,8 +220,11 @@ git -C "$LLAMA_WORKDIR" remote set-url origin "$LLAMA_UPSTREAM_URL" if [[ "$MODE" != "latest" ]]; then git_retry git -C "$LLAMA_WORKDIR" fetch origin master --tags fi -git -C "$LLAMA_WORKDIR" config user.name "${GIT_AUTHOR_NAME:-Mesh-LLM CI}" -git -C "$LLAMA_WORKDIR" config user.email "${GIT_AUTHOR_EMAIL:-ci@mesh-llm.local}" +# The patched checkout is an artifact identity shared across CI jobs. Keep the +# synthetic committer and timestamp deterministic so applying the same ordered +# patch queue to the same upstream pin always produces the same HEAD. +git -C "$LLAMA_WORKDIR" config user.name "Mesh-LLM CI" +git -C "$LLAMA_WORKDIR" config user.email "ci@mesh-llm.local" # The llama.cpp checkout is a generated dependency worktree. Local edits there # should live in third_party/llama.cpp/patches, so reset before switching pins. @@ -230,11 +237,28 @@ git -C "$LLAMA_WORKDIR" clean -fdx printf '%s\n' "$TARGET_SHA" > "$LLAMA_WORKDIR/.mesh-llm-upstream-sha" if (( ${#PATCHES[@]} > 0 )); then - git -C "$LLAMA_WORKDIR" am --3way "${PATCHES[@]}" + ( + # Do not let ambient Git identity/date overrides make the generated + # patched commit graph job-specific. + unset \ + GIT_AUTHOR_DATE \ + GIT_AUTHOR_EMAIL \ + GIT_AUTHOR_NAME \ + GIT_COMMITTER_DATE \ + GIT_COMMITTER_EMAIL \ + GIT_COMMITTER_NAME + git -C "$LLAMA_WORKDIR" am \ + --3way \ + --committer-date-is-author-date \ + --no-gpg-sign \ + --no-verify \ + "${PATCHES[@]}" + ) fi git -C "$LLAMA_WORKDIR" rev-parse HEAD > "$LLAMA_WORKDIR/.mesh-llm-patched-sha" printf '%s\n' "$PATCH_DIGEST" > "$LLAMA_WORKDIR/.mesh-llm-patch-digest" +printf '%s\n' "$PREPARE_SCHEMA" > "$LLAMA_WORKDIR/.mesh-llm-prepare-schema" echo "prepared llama.cpp" echo " upstream: $TARGET_SHA" diff --git a/scripts/tests/test_build_windows.py b/scripts/tests/test_build_windows.py index 2e17fa49b1..ecf8e8e394 100644 --- a/scripts/tests/test_build_windows.py +++ b/scripts/tests/test_build_windows.py @@ -80,6 +80,43 @@ def test_every_windows_runtime_graph_consumes_the_shared_abi_cache( ), 2, ) + self.assertEqual( + workflow.count( + "name: Resolve Windows native toolchain epoch", + ), + 2, + ) + self.assertEqual( + workflow.count( + "toolchain_epoch: " + "${{ steps.native_toolchain.outputs.epoch }}", + ), + 2, + ) + + warmer = WINDOWS_WARM_CACHES.read_text(encoding="utf-8") + self.assertIn( + "'.github/actions/resolve-native-toolchain-epoch/action.yml'", + warmer, + ) + self.assertEqual( + warmer.count("scripts/verify-static-abi-build-stamp.py"), + 2, + ) + self.assertEqual( + warmer.count( + "--toolchain-epoch " + "$env:MESH_LLM_LLAMA_TOOLCHAIN_EPOCH", + ), + 2, + ) + for library_group in ( + '@("llama.dll", "libllama.dll")', + '@("llama-common.dll", "libllama-common.dll")', + '@("mtmd.dll", "libmtmd.dll")', + ): + with self.subTest(library_group=library_group): + self.assertEqual(warmer.count(library_group), 2) def test_windows_abi_caches_live_outside_the_llama_worktree(self) -> None: workflows = { diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index 0c16fb8903..9756325644 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -467,6 +467,7 @@ def test_windows_abi_cache_action_keys_every_compatibility_boundary( for action_input in ( "backend:", "build_dir:", + "toolchain_epoch:", "architecture_set:", "cuda_toolchain_version:", "vulkan_toolchain_version:", @@ -506,6 +507,7 @@ def test_windows_abi_cache_action_keys_every_compatibility_boundary( expected_hash = ( "${{ hashFiles(" "'.github/actions/restore-windows-abi-cache/action.yml', " + "'.github/actions/resolve-native-toolchain-epoch/action.yml', " "'.github/actions/prepare-native-runtime-input/action.yml', " "'.github/actions/setup-windows-rocm-sdk/action.yml', " "'scripts/build-llama.sh', 'scripts/prepare-llama.sh', " @@ -517,7 +519,11 @@ def test_windows_abi_cache_action_keys_every_compatibility_boundary( self.assertIn(expected_hash, action) self.assertIn( '"mesh-llm-windows-2022-skippy-abi-' - '$backend-$architectureSet-$toolchain-$inputHash"', + '$backend-$architectureSet-$toolchain-$toolchainEpoch-$inputHash"', + action, + ) + self.assertIn( + "toolchain_epoch must match MESH_LLM_LLAMA_TOOLCHAIN_EPOCH", action, ) self.assertIn( @@ -535,6 +541,62 @@ def test_windows_abi_cache_action_keys_every_compatibility_boundary( action, ) + def test_native_toolchain_epoch_is_exact_and_shared_with_build_stamp( + self, + ) -> None: + resolver = self.read_action("resolve-native-toolchain-epoch") + pr_workflow = ( + ROOT / ".github" / "workflows" / "pr_builds.yml" + ).read_text(encoding="utf-8") + main_workflow = ( + ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + release_workflow = ( + ROOT / ".github" / "workflows" / "release.yml" + ).read_text(encoding="utf-8") + warmer = ( + ROOT / ".github" / "workflows" / "windows-warm-caches.yml" + ).read_text(encoding="utf-8") + + for contract in ( + 'image_os="${ImageOS:-}"', + 'image_version="${ImageVersion:-}"', + 'INPUT_PINNED_EPOCH: ${{ inputs.pinned_epoch }}', + 'echo "epoch=$epoch" >> "$GITHUB_OUTPUT"', + 'echo "MESH_LLM_LLAMA_TOOLCHAIN_EPOCH=$epoch" >> "$GITHUB_ENV"', + "xcodebuild -version", + "cmake --version", + "ninja --version", + ): + with self.subTest(contract=contract): + self.assertIn(contract, resolver) + + for workflow in ( + pr_workflow, + main_workflow, + release_workflow, + warmer, + ): + self.assertIn( + "uses: ./.github/actions/resolve-native-toolchain-epoch", + workflow, + ) + for workflow in (pr_workflow, main_workflow): + global_env = workflow.split("\npermissions:", maxsplit=1)[0] + self.assertNotIn("MESH_LLM_LLAMA_TOOLCHAIN_EPOCH:", global_env) + + for workflow in (pr_workflow, main_workflow, release_workflow): + for cache_block in re.findall( + r"uses: actions/cache@[^\n]+\n" + r"(?:[ \t]+[^\n]*\n){1,8}", + workflow, + ): + if "LLAMA_STAGE_BUILD_DIR" in cache_block: + self.assertIn( + "native_toolchain.outputs.epoch", + cache_block, + ) + def test_push_routing_diffs_the_complete_event_range(self) -> None: action = self.read_action("compute-changes") push_start = action.index( @@ -574,11 +636,27 @@ def test_runner_contract_routing_covers_cache_evidence_actions( for local_action in ( "capture-sccache-stats", "configure-sccache-gha", + "resolve-native-toolchain-epoch", "select-ci-runners", ): with self.subTest(local_action=local_action): self.assertIn(local_action, routing) + epoch_resolver = "resolve-native-toolchain-epoch" + for route_start, route_end in ( + ("BACKEND_INPUTS=", "WINDOWS_CPU_BUILD_REQUIRED="), + ("WINDOWS_CPU_INPUTS=", "# SDK smokes are consumer tests"), + ("DIRECT_SDK_INPUTS=", "# Inference artifacts are needed"), + ): + with self.subTest(route=route_start): + route = action[ + action.index(route_start) : action.index( + route_end, + action.index(route_start), + ) + ] + self.assertIn(epoch_resolver, route) + def test_justfile_release_primitives_route_backend_builds(self) -> None: action = self.read_action("compute-changes") match = re.search( @@ -749,9 +827,21 @@ def test_static_abi_artifact_is_typed_and_safely_reused(self) -> None: self.assertIn("CACHE_NAMESPACE: mesh-llm", producer) self.assertIn( "inputs.backend, inputs.target, " - "env.MESH_LLM_LLAMA_TOOLCHAIN_EPOCH, hashFiles(", + "steps.native_toolchain.outputs.epoch, hashFiles(", producer, ) + self.assertIn( + "uses: ./.github/actions/resolve-native-toolchain-epoch", + producer, + ) + self.assertIn( + "uses: ./.github/actions/resolve-native-toolchain-epoch", + native_sdk_producer, + ) + self.assertIn( + 'include_tool_versions: "true"', + native_sdk_producer, + ) self.assertIn("path: static-abi-artifact-output", producer) self.assertNotIn( "path: .deps/llama.cpp/build-stage-abi-static", @@ -763,12 +853,20 @@ def test_static_abi_artifact_is_typed_and_safely_reused(self) -> None: producer, ) epoch = ( - "MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: " "mesh-llm-cuda-runner-sha256-" "8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d" ) for consumer in (native_sdk_producer, pr_workflow, main_workflow): self.assertIn(epoch, consumer) + for workflow in (pr_workflow, main_workflow): + self.assertNotIn( + f"env:\n MESH_LLM_LLAMA_TOOLCHAIN_EPOCH: {epoch}", + workflow, + ) + self.assertIn( + "uses: ./.github/actions/resolve-native-toolchain-epoch", + workflow, + ) self.assertIn( "uses: ./.github/actions/prepare-static-abi-input", producer, @@ -1177,6 +1275,23 @@ def test_swift_sdk_build_is_a_shared_immutable_producer(self) -> None: '"$SWIFT_TRACKED_BINDING"', consumer_script, ) + self.assertIn( + '[[ -L "$SWIFT_GENERATED_DIR" ]]', + consumer_script, + ) + self.assertIn( + '[[ -e "$SWIFT_GENERATED_DIR" ' + '&& ! -d "$SWIFT_GENERATED_DIR" ]]', + consumer_script, + ) + mkdir_index = consumer_script.index( + 'mkdir -p "$SWIFT_GENERATED_DIR"', + ) + move_index = consumer_script.index( + 'mv "$SWIFT_EXTRACT_DIR/MeshLLMFFI.xcframework" ' + '"$SWIFT_XCFRAMEWORK"', + ) + self.assertLess(mkdir_index, move_index) self.assertIn("safe-extract-(tar|zip)", routing) self.assertIn("verify-swift-xcframework", routing) self.assertIn( @@ -1199,15 +1314,16 @@ def test_swift_sdk_cache_is_mode_independent_and_target_specific( self.assertIn( "format('mesh-llm-swift-sdk-{0}-{1}-{2}-{3}', " "runner.os, runner.arch, " - "env.SWIFT_NATIVE_XCODE_CACHE_EPOCH, hashFiles(", + "steps.native_toolchain.outputs.epoch, hashFiles(", producer, ) self.assertNotIn("runner.arch, inputs.mode, hashFiles(", producer) self.assertIn( - "SWIFT_NATIVE_XCODE_CACHE_EPOCH: " - "macos-15-arm64-xcode-default-v1", + "uses: ./.github/actions/resolve-native-toolchain-epoch", producer, ) + self.assertIn('include_tool_versions: "true"', producer) + self.assertNotIn("SWIFT_NATIVE_XCODE_CACHE_EPOCH", producer) self.assertIn("trusted main full build", producer) self.assertNotIn("build-stage-abi-host-metal", producer) self.assertIn( diff --git a/scripts/tests/test_depot_canary_workflow.py b/scripts/tests/test_depot_canary_workflow.py index 6f74287c00..58349bbb29 100644 --- a/scripts/tests/test_depot_canary_workflow.py +++ b/scripts/tests/test_depot_canary_workflow.py @@ -40,6 +40,23 @@ def test_canary_uses_a_pinned_cache_action_without_printing_tokens( self.workflow, ) self.assertIn("${DEPOT_CACHE_TOKEN:-}", self.workflow) + self.assertIn("${SCCACHE_WEBDAV_TOKEN:-}", self.workflow) + self.assertIn("${SCCACHE_WEBDAV_ENDPOINT:-}", self.workflow) + self.assertIn("Depot Cache authentication was not injected", self.workflow) + self.assertIn("Depot runner image identity was not injected", self.workflow) + self.assertIn('"$ImageOS" "$ImageVersion"', self.workflow) + self.assertIn("mozilla-actions/sccache-action@", self.workflow) + self.assertIn("sccache cc", self.workflow) + self.assertIn( + "sccache did not select the Depot WebDAV backend", + self.workflow, + ) + self.assertIn("expected a warm Depot sccache hit", self.workflow) + self.assertIn("expect_cache_hit:", self.workflow) + self.assertIn( + '[[ "$EXPECT_CACHE_HIT" == "true" && "$CACHE_HIT" != "true" ]]', + self.workflow, + ) self.assertNotIn("echo \"$DEPOT_CACHE_TOKEN\"", self.workflow) self.assertNotIn("printenv", self.workflow) diff --git a/scripts/tests/test_prepare_llama.py b/scripts/tests/test_prepare_llama.py index a8ef42da3e..f95d4e730d 100644 --- a/scripts/tests/test_prepare_llama.py +++ b/scripts/tests/test_prepare_llama.py @@ -19,7 +19,7 @@ def run_git( self, cwd: Path, *args: str, capture_output: bool = False ) -> subprocess.CompletedProcess[str]: return subprocess.run( - ["git", *args], + ["git", "-c", "commit.gpgsign=false", *args], cwd=cwd, check=True, text=True, @@ -119,6 +119,81 @@ def test_blobless_checkout_is_replaced_before_three_way_patch_application( ) self.assertNotEqual(partial_clone.returncode, 0) + def test_patch_application_has_deterministic_commit_identity(self) -> None: + """Equivalent clean preparations produce one reusable patched SHA.""" + with tempfile.TemporaryDirectory(prefix="mesh-llm-prepare-identity-") as temp_dir: + root = Path(temp_dir) + upstream = root / "upstream" + author = root / "author" + patch_dir = root / "patches" + pin_file = root / "upstream.txt" + + self.run_git(root, "init", "--initial-branch=master", str(upstream)) + self.run_git(upstream, "config", "user.name", "Patch Author") + self.run_git(upstream, "config", "user.email", "author@example.com") + (upstream / "sample.txt").write_text("base\n", encoding="utf-8") + self.run_git(upstream, "add", "sample.txt") + self.run_git(upstream, "commit", "-m", "base") + pin_file.write_text( + f"{self.run_git(upstream, 'rev-parse', 'HEAD', capture_output=True).stdout.strip()}\n", + encoding="utf-8", + ) + + self.run_git(root, "clone", str(upstream), str(author)) + self.run_git(author, "config", "user.name", "Patch Author") + self.run_git(author, "config", "user.email", "author@example.com") + (author / "sample.txt").write_text("patched\n", encoding="utf-8") + self.run_git(author, "commit", "-am", "local patch") + patch_dir.mkdir() + patch = self.run_git( + author, "format-patch", "-1", "--stdout", capture_output=True + ).stdout + (patch_dir / "0001-local.patch").write_text(patch, encoding="utf-8") + + patched_shas = [] + for index, (committer_date, timezone) in enumerate( + ( + ("2001-01-01T00:00:00Z", "UTC"), + ("2031-01-01T00:00:00Z", "America/Toronto"), + ) + ): + workdir = root / f"workdir-{index}" + env = os.environ | { + "GIT_AUTHOR_EMAIL": f"author-{index}@example.com", + "GIT_AUTHOR_NAME": f"Ambient Author {index}", + "GIT_COMMITTER_DATE": committer_date, + "GIT_COMMITTER_EMAIL": f"committer-{index}@example.com", + "GIT_COMMITTER_NAME": f"Ambient Committer {index}", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "commit.gpgsign", + "GIT_CONFIG_VALUE_0": "true", + "TZ": timezone, + "LLAMA_UPSTREAM_URL": f"file://{upstream}", + "LLAMA_WORKDIR": str(workdir), + "LLAMA_PIN_FILE": str(pin_file), + "LLAMA_PATCH_DIR": str(patch_dir), + "LLAMA_GIT_MAX_ATTEMPTS": "1", + } + subprocess.run( + [str(PREPARE_LLAMA), "pinned"], + cwd=ROOT, + check=True, + env=env, + ) + patched_shas.append( + (workdir / ".mesh-llm-patched-sha") + .read_text(encoding="utf-8") + .strip() + ) + self.assertEqual( + (workdir / ".mesh-llm-prepare-schema") + .read_text(encoding="utf-8") + .strip(), + "2", + ) + + self.assertEqual(patched_shas[0], patched_shas[1]) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_static_abi_artifacts.py b/scripts/tests/test_static_abi_artifacts.py index 2d5c272a0a..f691e19c33 100644 --- a/scripts/tests/test_static_abi_artifacts.py +++ b/scripts/tests/test_static_abi_artifacts.py @@ -87,6 +87,9 @@ def test_dynamic_output_probe_is_pipefail_safe(self) -> None: self.assertIn('found="$(find ', function) self.assertIn('[[ -n "$found" && -e "$found" ]] || return 1', function) + self.assertIn("libllama.dll|llama.dll", build_script) + self.assertIn("libllama-common.dll|llama-common.dll", build_script) + self.assertIn("libmtmd.dll|mtmd.dll", build_script) self.assertNotIn("| grep -q", function) def write_artifact( From c5eb3494c7b3c1e51dde38b08994c5759d70f96a Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Thu, 30 Jul 2026 00:01:58 -0400 Subject: [PATCH 19/23] fix(ci): make PR compiler cache read-only --- .agents/skills/manage-ci/SKILL.md | 8 ++++ .../manage-ci/references/current-inventory.md | 18 +++++--- .../actions/configure-sccache-gha/action.yml | 9 +++- .github/workflows/hf-download-smoke.yml | 1 + .github/workflows/pr_builds.yml | 9 ++-- .github/workflows/swift-sdk-artifact.yml | 15 +++++++ ci/DEPOT_MIGRATION.md | 14 ++++--- ci/ci.md | 28 ++++++++----- scripts/tests/test_ci_artifact_actions.py | 24 +++++++++-- scripts/tests/test_sccache_evidence.py | 42 +++++++++++++++++++ 10 files changed, 139 insertions(+), 29 deletions(-) diff --git a/.agents/skills/manage-ci/SKILL.md b/.agents/skills/manage-ci/SKILL.md index fdfc634357..6d3bc3e10b 100644 --- a/.agents/skills/manage-ci/SKILL.md +++ b/.agents/skills/manage-ci/SKILL.md @@ -314,6 +314,14 @@ update the skill resources in the same change. - GitHub-hosted PR jobs may share the normal key namespace with main because GitHub scopes PR writes to the merge ref and trusted main does not restore them. Do not assume that isolation applies to another cache provider. +- Keep the GitHub Actions sccache remote tier read-only for `pull_request` and + `pull_request_target` events. PR jobs may read trusted default-branch entries + and write their job-local disk tier, but only trusted main, release, + scheduled warmer, or explicitly authorized dispatch paths may publish shared + sccache entries. Apply the same event-derived mode to direct + `mozilla-actions/sccache-action` users and to + `.github/actions/configure-sccache-gha`; do not let a reusable workflow + silently restore read-write PR publication. - Depot's GitHub cache namespace is repository-scoped and has no branch isolation. With automatic Depot Cache enabled, its authority is injected into the whole runner job and cannot be contained by sccache disk-only mode or diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 65199efef5..0861651525 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -107,8 +107,10 @@ Local actions: permission is derived from the same typed trust decision. - `.github/actions/configure-sccache-gha` exports ephemeral Actions cache credentials to the baked `sccache`, permits Depot WebDAV only for an explicit - trusted call, uses disk-only storage if a future pull-request trust context is - ever evaluated on Depot, and resets counters after configuring the server. + trusted call, keeps the GitHub Actions remote tier read-only for PR events + while retaining a writable job-local disk tier, uses disk-only storage if a + future pull-request trust context is ever evaluated on Depot, and resets + counters after configuring the server. - `.github/actions/capture-sccache-stats` validates and uploads one machine-readable sccache evidence artifact per instrumented job or matrix row. Evidence is retained for 14 days so cold/warm samples span the configured @@ -315,10 +317,14 @@ Public-image Rust jobs use the baked `sccache` binary. Trusted calls to `SCCACHE_WEBDAV_TOKEN`/`DEPOT_CACHE_TOKEN` in a fail-open `disk,webdav` chain. When that permission is false and Depot is detected, the action gives the sccache server a credential-free environment and uses job-local disk only. -GitHub-hosted jobs retain `disk,gha` or explicit disk-only mode. Persistent -Cargo target and ABI reuse remains owned by `Swatinem/rust-cache` and -`actions/cache`. Current PR jobs use the normal `mesh-llm` key namespace and -GitHub's merge-ref scoping; trusted main does not restore PR-written entries. +GitHub-hosted jobs retain `disk,gha` or explicit disk-only mode. The GHA tier is +read-only for PR events, including direct sccache-action users, while trusted +main, release, warmer, and dispatch paths may seed it. Swift restores a +mode-independent Rust dependency cache that only trusted main pushes save. +Persistent Cargo target and ABI reuse remains owned by +`Swatinem/rust-cache` and `actions/cache`. Current PR jobs use the normal +`mesh-llm` key namespace; native `actions/cache` writes remain merge-ref scoped, +and trusted main does not restore PR-written entries. Raw native ABI caches also include the exact native toolchain epoch used by the build stamp. Linux container jobs use the pinned OCI digest, macOS keys include the hosted image revision and native-tool fingerprint, and Windows warmer, diff --git a/.github/actions/configure-sccache-gha/action.yml b/.github/actions/configure-sccache-gha/action.yml index 20350b5eeb..19242069b4 100644 --- a/.github/actions/configure-sccache-gha/action.yml +++ b/.github/actions/configure-sccache-gha/action.yml @@ -28,6 +28,13 @@ runs: return; } + const eventName = process.env.GITHUB_EVENT_NAME || ''; + const ghaRemoteMode = + eventName === 'pull_request' || eventName === 'pull_request_target' + ? 'READ_ONLY' + : 'READ_WRITE'; + core.exportVariable('SCCACHE_GHA_RW_MODE', ghaRemoteMode); + const webdavEndpoint = process.env.SCCACHE_WEBDAV_ENDPOINT || ''; const webdavToken = process.env.SCCACHE_WEBDAV_TOKEN || @@ -240,7 +247,7 @@ runs: return; } core.info( - 'Baked sccache is using job-local disk with a best-effort GitHub Actions cache.', + `Baked sccache is using job-local disk with a ${ghaRemoteMode.toLowerCase()} GitHub Actions cache.`, ); return; } diff --git a/.github/workflows/hf-download-smoke.yml b/.github/workflows/hf-download-smoke.yml index 2d157f509b..20959069ad 100644 --- a/.github/workflows/hf-download-smoke.yml +++ b/.github/workflows/hf-download-smoke.yml @@ -13,6 +13,7 @@ on: env: CACHE_NAMESPACE: mesh-llm + SCCACHE_GHA_RW_MODE: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && 'READ_ONLY' || 'READ_WRITE' }} permissions: contents: read diff --git a/.github/workflows/pr_builds.yml b/.github/workflows/pr_builds.yml index 36659a8f12..b0ccc90960 100644 --- a/.github/workflows/pr_builds.yml +++ b/.github/workflows/pr_builds.yml @@ -21,9 +21,11 @@ concurrency: env: # GitHub scopes pull-request cache writes to refs/pull//merge while still - # allowing read-only reuse of the base branch. Depot cache access remains - # disabled for pull_request events because Depot's repository cache is not - # branch-isolated. + # allowing read-only reuse of the base branch. Keep sccache's per-object + # GitHub tier read-only on PRs so concurrent compiler jobs cannot exhaust the + # repository upload-rate budget; trusted dispatches remain cache seeders. + # Depot cache access remains disabled for pull_request events because Depot's + # repository cache is not branch-isolated. CACHE_NAMESPACE: mesh-llm MODEL_URL: https://huggingface.co/unsloth/SmolLM2-135M-Instruct-GGUF/resolve/9e6855bc4be717fca1ef21360a1db4b29d5c559a/SmolLM2-135M-Instruct-Q8_0.gguf MODEL_FILE: SmolLM2-135M-Instruct-Q8_0.gguf @@ -31,6 +33,7 @@ env: CARGO_NET_RETRY: "10" CARGO_HTTP_MULTIPLEXING: "false" SCCACHE_GHA_ENABLED: "true" + SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'pull_request' && 'READ_ONLY' || 'READ_WRITE' }} LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static jobs: diff --git a/.github/workflows/swift-sdk-artifact.yml b/.github/workflows/swift-sdk-artifact.yml index 92a09a8ea7..e5496b67c3 100644 --- a/.github/workflows/swift-sdk-artifact.yml +++ b/.github/workflows/swift-sdk-artifact.yml @@ -50,12 +50,14 @@ jobs: permissions: contents: read env: + CACHE_NAMESPACE: mesh-llm CARGO_INCREMENTAL: "0" CARGO_NET_RETRY: "10" CARGO_HTTP_MULTIPLEXING: "false" LLAMA_STAGE_BACKEND: metal RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "true" + SCCACHE_GHA_RW_MODE: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && 'READ_ONLY' || 'READ_WRITE' }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: @@ -100,6 +102,8 @@ jobs: - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: ./.github/actions/configure-sccache-gha + - name: Install macOS SDK build dependencies run: brew install cmake ninja lld @@ -109,6 +113,17 @@ jobs: with: include_tool_versions: "true" + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 + continue-on-error: true + with: + workspaces: . -> target + cache-bin: "false" + prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ runner.os }}-${{ hashFiles('.github/cache-version.txt') }} + shared-key: swift-sdk + key: ${{ steps.native_toolchain.outputs.epoch }} + add-job-id-key: "false" + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + - name: Restore exact Swift native ABI cache uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: diff --git a/ci/DEPOT_MIGRATION.md b/ci/DEPOT_MIGRATION.md index eb5f8326ef..e8ea35c0dc 100644 --- a/ci/DEPOT_MIGRATION.md +++ b/ci/DEPOT_MIGRATION.md @@ -292,9 +292,12 @@ including `actions/cache`, `actions/setup-node`, and third-party cache actions. Its namespace is repository-scoped and is not isolated by branch. Therefore: - current pull-request jobs never run on Depot and may use the normal - `mesh-llm` key namespace in GitHub's native cache because GitHub scopes PR - writes to the merge ref and trusted main jobs do not restore from that ref. - The crate-test target shards deliberately restore the already-seeded + `mesh-llm` key namespace in GitHub's native `actions/cache` because GitHub + scopes PR writes to the merge ref and trusted main jobs do not restore from + that ref. Their sccache GHA remote tier is read-only: PR jobs may consume + trusted default-branch entries and write job-local disk, while trusted + main/release/warmers publish shared compiler-cache entries. The crate-test + target shards deliberately restore the already-seeded `main-rust-crate-tests-` keys with writes disabled; - a local sccache disk-only setting protects only that sccache child process; it does not remove the Depot token or prevent another cache API consumer @@ -304,8 +307,9 @@ Its namespace is repository-scoped and is not isolated by branch. Therefore: - trusted main/release jobs may explicitly enable the `disk,webdav` chain and fall back to job-local disk. -GitHub-hosted jobs retain the existing disk/GitHub Actions cache path. Never -print a cache token. +GitHub-hosted jobs retain the existing disk/GitHub Actions cache path. PR +sccache reads are remote and writes are job-local; native `actions/cache` +continues to use GitHub's merge-ref isolation. Never print a cache token. Relevant Depot documentation: diff --git a/ci/ci.md b/ci/ci.md index b346d9fe07..735d5b09a3 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -468,13 +468,16 @@ the sccache child with a credential-free, job-local disk backend. That isolates only sccache: Depot's automatically injected job token and transparent GitHub-cache API redirection remain available to other code on a Depot runner. Consequently, no untrusted PR code may run on Depot while automatic cache -injection is enabled. GitHub-hosted jobs retain the existing best-effort -`disk,gha` path or explicit disk-only mode. Cache read failures degrade to -misses, cache write failures only warn, and a failed remote probe restarts -`sccache` with disk-only storage. PR crate-test shards restore the existing -`main-rust-crate-tests-` Cargo target caches read-only (`save-if: -false`), so trusted main owns the cache while PRs avoid recompiling the same -workspace graph. +injection is enabled. GitHub-hosted jobs retain the existing `disk,gha` path or +explicit disk-only mode. Pull-request jobs keep the GHA sccache tier read-only +and the job-local disk tier writable; trusted main, release, warmer, and +dispatch paths own remote publication. This avoids repository-wide per-object +upload throttling while preserving trusted default-branch hits. Cache read +failures degrade to misses, cache write failures only warn, and a failed remote +probe restarts `sccache` with disk-only storage. PR crate-test shards restore +the existing `main-rust-crate-tests-` Cargo target caches read-only +(`save-if: false`), so trusted main owns the cache while PRs avoid recompiling +the same workspace graph. Native ABI cache keys and llama build stamps share one resolved toolchain epoch. Digest-pinned Linux jobs use the immutable runner-image digest. Hosted macOS @@ -517,9 +520,11 @@ Before enabling public access, restrict that group to `depot-canary.yml@refs/heads/main`. The existing `mesh-llm` runner group owns the dedicated GPU scale sets and is not the Depot group. -Current GitHub-hosted PR jobs may share the `mesh-llm` key namespace because -GitHub scopes PR cache writes to the merge ref; trusted main does not restore -from that ref. Depot's cache is repository-scoped instead, so cache-key +Current GitHub-hosted PR jobs may share the `mesh-llm` native-cache key +namespace because GitHub scopes `actions/cache` PR writes to the merge ref; +trusted main does not restore from that ref. Their sccache GHA remote tier is +read-only, so trusted main/release/warmers own shared compiler-cache +publication. Depot's cache is repository-scoped instead, so cache-key conventions or a trusted reusable caller are not sufficient protection from malicious checked-out PR code. PR events stay hosted while automatic Depot Cache is enabled. Runner placement does not alter build action inputs or @@ -527,7 +532,8 @@ artifact contracts. Credential-bearing Hugging Face, inference, scripted, and SDK smoke reusable workflows accept no arbitrary runner label and stay on GitHub-hosted runners. PR callers pass no `HF_TOKEN`; only trusted main/release invocations receive the optional rate-limit credential. The Swift producer and -Swift smoke are fixed to the GitHub-hosted `macos-15` image. +Swift smoke are fixed to the GitHub-hosted `macos-15` image. Swift restores one +mode-independent Rust dependency cache and only trusted main pushes save it. Hardware-qualified GPU execution stays on dedicated runners. See [`DEPOT_MIGRATION.md`](DEPOT_MIGRATION.md) for activation prerequisites, baseline metrics, target service levels, and the cross-repository plan. diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index 9756325644..a2aaabe65f 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -1217,6 +1217,17 @@ def test_swift_sdk_build_is_a_shared_immutable_producer(self) -> None: self.assertIn("actions/upload-artifact@", producer) self.assertIn("runs-on: macos-15", producer) self.assertIn("RUSTC_WRAPPER: sccache", producer) + self.assertIn("SCCACHE_GHA_RW_MODE:", producer) + self.assertIn( + "uses: ./.github/actions/configure-sccache-gha", + producer, + ) + self.assertIn("shared-key: swift-sdk", producer) + self.assertIn( + "save-if: ${{ github.event_name == 'push' " + "&& github.ref == 'refs/heads/main' }}", + producer, + ) self.assertNotIn("macos_runner:", producer) self.assertIn( "name: generated-swift-binding-${{ inputs.artifact_name }}", @@ -1711,6 +1722,16 @@ def test_pr_caches_rely_on_github_ref_scoping_while_depot_is_blocked( workflow, ) + builds = ( + ROOT / ".github" / "workflows" / "pr_builds.yml" + ).read_text(encoding="utf-8") + self.assertIn( + "SCCACHE_GHA_RW_MODE: " + "${{ github.event_name == 'pull_request' " + "&& 'READ_ONLY' || 'READ_WRITE' }}", + builds, + ) + main = ( ROOT / ".github" / "workflows" / "ci.yml" ).read_text(encoding="utf-8") @@ -1719,9 +1740,6 @@ def test_pr_caches_rely_on_github_ref_scoping_while_depot_is_blocked( quality = ( ROOT / ".github" / "workflows" / "pr_quality.yml" ).read_text(encoding="utf-8") - builds = ( - ROOT / ".github" / "workflows" / "pr_builds.yml" - ).read_text(encoding="utf-8") self.assertIn( "shared-key: main-rust-crate-tests-${{ matrix.batch.idx }}", builds, diff --git a/scripts/tests/test_sccache_evidence.py b/scripts/tests/test_sccache_evidence.py index c72eaf014d..e968f7c087 100644 --- a/scripts/tests/test_sccache_evidence.py +++ b/scripts/tests/test_sccache_evidence.py @@ -23,6 +23,7 @@ "main": ROOT / ".github" / "workflows" / "ci.yml", "swift-sdk": ROOT / ".github" / "workflows" / "swift-sdk-artifact.yml", } +HF_WORKFLOW = ROOT / ".github" / "workflows" / "hf-download-smoke.yml" def valid_payload(*, compile_requests: int = 12) -> dict[str, object]: @@ -271,6 +272,47 @@ def test_remote_multilevel_writes_finish_before_ephemeral_job_exit(self) -> None configure, ) + def test_pull_request_remote_sccache_is_read_only(self) -> None: + configure = CONFIGURE_ACTION.read_text(encoding="utf-8") + builds = WORKFLOWS["pr-builds"].read_text(encoding="utf-8") + swift = WORKFLOWS["swift-sdk"].read_text(encoding="utf-8") + hf_download = HF_WORKFLOW.read_text(encoding="utf-8") + + self.assertIn("eventName === 'pull_request'", configure) + self.assertIn("eventName === 'pull_request_target'", configure) + self.assertIn( + "core.exportVariable('SCCACHE_GHA_RW_MODE', ghaRemoteMode)", + configure, + ) + for workflow in (builds, swift, hf_download): + with self.subTest(workflow=workflow.splitlines()[0]): + self.assertIn("SCCACHE_GHA_RW_MODE:", workflow) + self.assertIn("'READ_ONLY'", workflow) + self.assertIn("'READ_WRITE'", workflow) + + self.assertIn( + "uses: ./.github/actions/configure-sccache-gha", + swift, + ) + self.assertNotIn("SCCACHE_WEBDAV_RW_MODE", configure) + + def test_swift_uses_trusted_main_seeded_dependency_cache(self) -> None: + swift = WORKFLOWS["swift-sdk"].read_text(encoding="utf-8") + + self.assertIn( + "uses: Swatinem/rust-cache@" + "e18b497796c12c097a38f9edb9d0641fb99eee32", + swift, + ) + self.assertIn("shared-key: swift-sdk", swift) + self.assertIn("key: ${{ steps.native_toolchain.outputs.epoch }}", swift) + self.assertIn('add-job-id-key: "false"', swift) + self.assertIn( + "save-if: ${{ github.event_name == 'push' " + "&& github.ref == 'refs/heads/main' }}", + swift, + ) + def test_instrumented_workflows_use_unique_evidence_artifacts(self) -> None: expected_names = { "pr-builds": ( From 323e66330cf2ec39cc6e886196dbe93072b66154 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Thu, 30 Jul 2026 00:14:17 -0400 Subject: [PATCH 20/23] fix(ci): isolate pull request compiler writes --- .agents/skills/manage-ci/SKILL.md | 14 +++--- .../manage-ci/references/current-inventory.md | 16 +++--- .../actions/configure-sccache-gha/action.yml | 38 ++++++++++++++ .github/workflows/hf-download-smoke.yml | 1 + .github/workflows/pr_builds.yml | 4 ++ ci/DEPOT_MIGRATION.md | 16 +++--- ci/ci.md | 17 ++++--- scripts/tests/test_sccache_evidence.py | 49 ++++++++++++++++++- 8 files changed, 125 insertions(+), 30 deletions(-) diff --git a/.agents/skills/manage-ci/SKILL.md b/.agents/skills/manage-ci/SKILL.md index 6d3bc3e10b..057175ddc5 100644 --- a/.agents/skills/manage-ci/SKILL.md +++ b/.agents/skills/manage-ci/SKILL.md @@ -314,12 +314,14 @@ update the skill resources in the same change. - GitHub-hosted PR jobs may share the normal key namespace with main because GitHub scopes PR writes to the merge ref and trusted main does not restore them. Do not assume that isolation applies to another cache provider. -- Keep the GitHub Actions sccache remote tier read-only for `pull_request` and - `pull_request_target` events. PR jobs may read trusted default-branch entries - and write their job-local disk tier, but only trusted main, release, - scheduled warmer, or explicitly authorized dispatch paths may publish shared - sccache entries. Apply the same event-derived mode to direct - `mozilla-actions/sccache-action` users and to +- Keep sccache disk-only for `pull_request` and `pull_request_target` events. + The pinned sccache treats a mixed `disk,gha` chain as wholly read-only when + the GHA tier is read-only, so every miss records a rejected cache write and + cannot populate L0. PR jobs therefore use a writable job-local disk tier plus + bulk Rust and exact native `actions/cache` restores. Only trusted main, + release, scheduled warmer, or explicitly authorized dispatch paths may + publish shared sccache entries. Apply the same event-derived policy to direct + `mozilla-actions/sccache-action` users through `.github/actions/configure-sccache-gha`; do not let a reusable workflow silently restore read-write PR publication. - Depot's GitHub cache namespace is repository-scoped and has no branch diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 0861651525..982a2fc0a1 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -107,10 +107,11 @@ Local actions: permission is derived from the same typed trust decision. - `.github/actions/configure-sccache-gha` exports ephemeral Actions cache credentials to the baked `sccache`, permits Depot WebDAV only for an explicit - trusted call, keeps the GitHub Actions remote tier read-only for PR events - while retaining a writable job-local disk tier, uses disk-only storage if a - future pull-request trust context is ever evaluated on Depot, and resets - counters after configuring the server. + trusted call, uses writable job-local disk only for PR events because the + pinned sccache makes a mixed chain wholly read-only and records rejected + writes after misses, uses disk-only + storage if a future pull-request trust context is ever evaluated on Depot, + and resets counters after configuring the server. - `.github/actions/capture-sccache-stats` validates and uploads one machine-readable sccache evidence artifact per instrumented job or matrix row. Evidence is retained for 14 days so cold/warm samples span the configured @@ -317,9 +318,10 @@ Public-image Rust jobs use the baked `sccache` binary. Trusted calls to `SCCACHE_WEBDAV_TOKEN`/`DEPOT_CACHE_TOKEN` in a fail-open `disk,webdav` chain. When that permission is false and Depot is detected, the action gives the sccache server a credential-free environment and uses job-local disk only. -GitHub-hosted jobs retain `disk,gha` or explicit disk-only mode. The GHA tier is -read-only for PR events, including direct sccache-action users, while trusted -main, release, warmer, and dispatch paths may seed it. Swift restores a +GitHub-hosted trusted jobs retain `disk,gha` or explicit disk-only mode. PR +events use job-local disk only, including direct sccache-action users routed +through the configure action, while trusted main, release, warmer, and dispatch +paths may seed the GHA tier. Swift restores a mode-independent Rust dependency cache that only trusted main pushes save. Persistent Cargo target and ABI reuse remains owned by `Swatinem/rust-cache` and `actions/cache`. Current PR jobs use the normal diff --git a/.github/actions/configure-sccache-gha/action.yml b/.github/actions/configure-sccache-gha/action.yml index 19242069b4..b4efa29e7f 100644 --- a/.github/actions/configure-sccache-gha/action.yml +++ b/.github/actions/configure-sccache-gha/action.yml @@ -215,6 +215,44 @@ runs: return; } + // The runner image's pinned sccache treats a mixed disk,gha chain as + // wholly read-only when the GHA tier is read-only. Each miss then + // records a rejected cache write and cannot populate L0. Pull + // requests therefore use the writable job-local disk tier only. + // Cross-run PR reuse remains covered by the bulk Rust and exact + // native actions/cache entries, while trusted main, release, warmer, + // and authorized dispatch jobs populate the remote compiler cache + // without exposing a repository-scoped write path to PR code. + if (ghaRemoteMode === 'READ_ONLY') { + core.info( + 'Pull-request trust context detected; using baked sccache with job-local disk only.', + ); + core.exportVariable('SCCACHE_GHA_ENABLED', 'false'); + core.exportVariable('SCCACHE_MULTILEVEL_CHAIN', 'disk'); + core.exportVariable('SCCACHE_IGNORE_SERVER_IO_ERROR', '1'); + await exec.exec('sccache', ['--stop-server'], { + ignoreReturnCode: true, + }); + const pullRequestDiskExitCode = await exec.exec( + 'sccache', + ['--start-server'], + { + env: diskOnlyEnvironment(), + ignoreReturnCode: true, + }, + ); + if (pullRequestDiskExitCode !== 0) { + core.setFailed( + 'Unable to start baked sccache with its pull-request-local disk cache.', + ); + return; + } + if (!(await resetStatistics(diskOnlyEnvironment()))) { + return; + } + return; + } + const cacheUrl = process.env.ACTIONS_CACHE_URL || ''; const resultsUrl = process.env.ACTIONS_RESULTS_URL || ''; const runtimeToken = process.env.ACTIONS_RUNTIME_TOKEN || ''; diff --git a/.github/workflows/hf-download-smoke.yml b/.github/workflows/hf-download-smoke.yml index 20959069ad..73973d7bcc 100644 --- a/.github/workflows/hf-download-smoke.yml +++ b/.github/workflows/hf-download-smoke.yml @@ -46,6 +46,7 @@ jobs: rustflags = ["-C", "link-arg=-fuse-ld=lld"] EOF - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: ./.github/actions/configure-sccache-gha - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 with: workspaces: . -> target diff --git a/.github/workflows/pr_builds.yml b/.github/workflows/pr_builds.yml index b0ccc90960..97b0340d7c 100644 --- a/.github/workflows/pr_builds.yml +++ b/.github/workflows/pr_builds.yml @@ -1303,6 +1303,7 @@ jobs: persist-credentials: false - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: ./.github/actions/configure-sccache-gha - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: @@ -1340,6 +1341,7 @@ jobs: python-version: '3.x' - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: ./.github/actions/configure-sccache-gha - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: @@ -1383,6 +1385,7 @@ jobs: python-version: '3.x' - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: ./.github/actions/configure-sccache-gha - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: @@ -1457,6 +1460,7 @@ jobs: run: python -m unittest scripts.tests.test_windows_native_runtime_deps -v - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: ./.github/actions/configure-sccache-gha - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 snapshot 2026-03-12 continue-on-error: true with: diff --git a/ci/DEPOT_MIGRATION.md b/ci/DEPOT_MIGRATION.md index e8ea35c0dc..fe24e8b9ec 100644 --- a/ci/DEPOT_MIGRATION.md +++ b/ci/DEPOT_MIGRATION.md @@ -294,10 +294,11 @@ Its namespace is repository-scoped and is not isolated by branch. Therefore: - current pull-request jobs never run on Depot and may use the normal `mesh-llm` key namespace in GitHub's native `actions/cache` because GitHub scopes PR writes to the merge ref and trusted main jobs do not restore from - that ref. Their sccache GHA remote tier is read-only: PR jobs may consume - trusted default-branch entries and write job-local disk, while trusted - main/release/warmers publish shared compiler-cache entries. The crate-test - target shards deliberately restore the already-seeded + that ref. Their sccache backend is writable job-local disk only because the + pinned sccache makes a mixed chain wholly read-only and records each miss as + a rejected write; + trusted main/release/warmers publish shared compiler-cache entries. The + crate-test target shards deliberately restore the already-seeded `main-rust-crate-tests-` keys with writes disabled; - a local sccache disk-only setting protects only that sccache child process; it does not remove the Depot token or prevent another cache API consumer @@ -307,9 +308,10 @@ Its namespace is repository-scoped and is not isolated by branch. Therefore: - trusted main/release jobs may explicitly enable the `disk,webdav` chain and fall back to job-local disk. -GitHub-hosted jobs retain the existing disk/GitHub Actions cache path. PR -sccache reads are remote and writes are job-local; native `actions/cache` -continues to use GitHub's merge-ref isolation. Never print a cache token. +GitHub-hosted trusted jobs retain the existing disk/GitHub Actions cache path. +PR sccache is job-local, while bulk Rust and exact native `actions/cache` +restores provide safe cross-run reuse under GitHub's merge-ref isolation. +Never print a cache token. Relevant Depot documentation: diff --git a/ci/ci.md b/ci/ci.md index 735d5b09a3..5124f6edb2 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -468,13 +468,14 @@ the sccache child with a credential-free, job-local disk backend. That isolates only sccache: Depot's automatically injected job token and transparent GitHub-cache API redirection remain available to other code on a Depot runner. Consequently, no untrusted PR code may run on Depot while automatic cache -injection is enabled. GitHub-hosted jobs retain the existing `disk,gha` path or -explicit disk-only mode. Pull-request jobs keep the GHA sccache tier read-only -and the job-local disk tier writable; trusted main, release, warmer, and +injection is enabled. GitHub-hosted trusted jobs retain the existing `disk,gha` +path or explicit disk-only mode. Pull-request jobs use writable job-local disk +only; the pinned sccache otherwise makes the mixed chain wholly read-only and +records every miss as a rejected write. Trusted main, release, warmer, and dispatch paths own remote publication. This avoids repository-wide per-object -upload throttling while preserving trusted default-branch hits. Cache read -failures degrade to misses, cache write failures only warn, and a failed remote -probe restarts `sccache` with disk-only storage. PR crate-test shards restore +upload throttling and misleading PR write errors. Cache read failures degrade +to misses, cache write failures only warn, and a failed remote probe restarts +`sccache` with disk-only storage. PR crate-test shards restore the existing `main-rust-crate-tests-` Cargo target caches read-only (`save-if: false`), so trusted main owns the cache while PRs avoid recompiling the same workspace graph. @@ -522,8 +523,8 @@ the dedicated GPU scale sets and is not the Depot group. Current GitHub-hosted PR jobs may share the `mesh-llm` native-cache key namespace because GitHub scopes `actions/cache` PR writes to the merge ref; -trusted main does not restore from that ref. Their sccache GHA remote tier is -read-only, so trusted main/release/warmers own shared compiler-cache +trusted main does not restore from that ref. Their sccache backend is job-local +disk only, so trusted main/release/warmers own shared compiler-cache publication. Depot's cache is repository-scoped instead, so cache-key conventions or a trusted reusable caller are not sufficient protection from malicious checked-out PR code. PR events stay hosted while automatic Depot diff --git a/scripts/tests/test_sccache_evidence.py b/scripts/tests/test_sccache_evidence.py index e968f7c087..3cdf657311 100644 --- a/scripts/tests/test_sccache_evidence.py +++ b/scripts/tests/test_sccache_evidence.py @@ -24,6 +24,9 @@ "swift-sdk": ROOT / ".github" / "workflows" / "swift-sdk-artifact.yml", } HF_WORKFLOW = ROOT / ".github" / "workflows" / "hf-download-smoke.yml" +NATIVE_SDK_WORKFLOW = ( + ROOT / ".github" / "workflows" / "native-sdk-artifact.yml" +) def valid_payload(*, compile_requests: int = 12) -> dict[str, object]: @@ -254,7 +257,7 @@ def test_configure_action_resets_each_successful_server_route(self) -> None: configure = CONFIGURE_ACTION.read_text(encoding="utf-8") self.assertIn("['--zero-stats']", configure) - self.assertEqual(configure.count("await resetStatistics("), 6) + self.assertEqual(configure.count("await resetStatistics("), 7) def test_remote_multilevel_writes_finish_before_ephemeral_job_exit(self) -> None: configure = CONFIGURE_ACTION.read_text(encoding="utf-8") @@ -272,7 +275,7 @@ def test_remote_multilevel_writes_finish_before_ephemeral_job_exit(self) -> None configure, ) - def test_pull_request_remote_sccache_is_read_only(self) -> None: + def test_pull_request_sccache_is_disk_only(self) -> None: configure = CONFIGURE_ACTION.read_text(encoding="utf-8") builds = WORKFLOWS["pr-builds"].read_text(encoding="utf-8") swift = WORKFLOWS["swift-sdk"].read_text(encoding="utf-8") @@ -284,6 +287,12 @@ def test_pull_request_remote_sccache_is_read_only(self) -> None: "core.exportVariable('SCCACHE_GHA_RW_MODE', ghaRemoteMode)", configure, ) + self.assertIn("if (ghaRemoteMode === 'READ_ONLY')", configure) + self.assertIn( + "Pull-request trust context detected; using baked sccache " + "with job-local disk only.", + configure, + ) for workflow in (builds, swift, hf_download): with self.subTest(workflow=workflow.splitlines()[0]): self.assertIn("SCCACHE_GHA_RW_MODE:", workflow) @@ -294,8 +303,44 @@ def test_pull_request_remote_sccache_is_read_only(self) -> None: "uses: ./.github/actions/configure-sccache-gha", swift, ) + self.assertGreaterEqual( + builds.count("uses: ./.github/actions/configure-sccache-gha"), + 11, + ) + self.assertIn( + "uses: ./.github/actions/configure-sccache-gha", + hf_download, + ) self.assertNotIn("SCCACHE_WEBDAV_RW_MODE", configure) + def test_pull_request_direct_sccache_users_are_reconfigured(self) -> None: + workflows = ( + WORKFLOWS["pr-builds"], + WORKFLOWS["swift-sdk"], + HF_WORKFLOW, + NATIVE_SDK_WORKFLOW, + ) + + for path in workflows: + lines = path.read_text(encoding="utf-8").splitlines() + direct_users = [ + index + for index, line in enumerate(lines) + if "uses: mozilla-actions/sccache-action@" in line + ] + self.assertTrue(direct_users, path) + for index in direct_users: + with self.subTest(workflow=path.name, line=index + 1): + next_step = next( + line.strip() + for line in lines[index + 1 :] + if line.strip() + ) + self.assertEqual( + next_step, + "- uses: ./.github/actions/configure-sccache-gha", + ) + def test_swift_uses_trusted_main_seeded_dependency_cache(self) -> None: swift = WORKFLOWS["swift-sdk"].read_text(encoding="utf-8") From 3c4a2942f35a3870fbb06f72f836f3efd1026727 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Thu, 30 Jul 2026 01:59:07 -0400 Subject: [PATCH 21/23] feat(ci): produce immutable Node addon artifacts --- .../manage-ci/references/current-inventory.md | 9 + .github/workflows/node-sdk-addon-artifact.yml | 370 ++++++++++++++++++ .github/workflows/release.yml | 32 +- ci/ci.md | 6 + scripts/tests/test_ci_workflow_artifacts.py | 80 ++++ 5 files changed, 496 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/node-sdk-addon-artifact.yml diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 982a2fc0a1..4fd5e564f1 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -22,6 +22,7 @@ the commands at the end before operational changes. | `native-sdk-artifact.yml` | Reusable call | Typed target/backend/profile native SDK producer with protected runner/cache policy | | `static-abi-artifact.yml` | Reusable call | Typed target/backend static llama ABI producer with protected runner/cache policy | | `swift-sdk-artifact.yml` | Reusable call | Typed host-only/full Swift XCFramework producer | +| `node-sdk-addon-artifact.yml` | Reusable call | Five-target Node native-addon producer with fresh-install smoke, manifest, and checksum | | `hf-download-smoke.yml` | Reusable call | Hugging Face download smoke | | `nightly-stability.yml` | Schedule, dispatch | Nightly operator entry point | | `nightly-stability-run.yml` | Reusable call | Stability probes and evidence | @@ -53,6 +54,14 @@ place runtimes at `mesh-bundle/native-runtimes/`; Debian/Arch packages use `/usr/local/lib/mesh-llm//native-runtimes`; Homebrew uses formula-owned `libexec/native-runtimes`. +Release also fans out `node-sdk-addon-artifact.yml` across Darwin ARM64/x64, +Linux ARM64/x64, and Windows x64. Each producer compiles once on its matching +GitHub-hosted platform, packs and fresh-installs the SDK, verifies +`currentMeshVersion()`, and emits a versioned addon archive with a strict +manifest and SHA-256 sidecar. The release publisher requires all five producers +and attaches those exact artifacts; downstream packaging verifies and assembles +them without recompiling native source. + The Windows host input also carries the checksum-protected `xtask` executable that performed producer-side attestation. Windows product composers invoke that prebuilt verifier for the immutable host instead of compiling workspace code. diff --git a/.github/workflows/node-sdk-addon-artifact.yml b/.github/workflows/node-sdk-addon-artifact.yml new file mode 100644 index 0000000000..84e3fb3f35 --- /dev/null +++ b/.github/workflows/node-sdk-addon-artifact.yml @@ -0,0 +1,370 @@ +name: Reusable Node SDK Addon Artifact + +on: + workflow_call: + inputs: + target: + description: Node platform and architecture target. + required: true + type: string + artifact_name: + description: Immutable GitHub artifact name. + required: true + type: string + release_tag: + description: Release tag whose version the addon must report. + required: true + type: string + prepare_release_version: + description: Prepare checked-out sources for release_tag before building. + required: false + default: false + type: boolean + retention_days: + description: Artifact retention in days. + required: false + default: 90 + type: number + timeout_minutes: + description: Producer timeout in minutes. + required: false + default: 90 + type: number + +env: + CARGO_INCREMENTAL: "0" + CARGO_NET_RETRY: "10" + CARGO_HTTP_MULTIPLEXING: "false" + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + +jobs: + validate_inputs: + name: Validate Node SDK addon producer inputs + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: none + steps: + - name: Validate typed producer contract + shell: bash + env: + ARTIFACT_NAME: ${{ inputs.artifact_name }} + NODE_SDK_TARGET: ${{ inputs.target }} + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + set -euo pipefail + case "$NODE_SDK_TARGET" in + darwin-arm64|darwin-x64|linux-arm64|linux-x64|win32-x64) ;; + *) echo "unsupported Node SDK target: $NODE_SDK_TARGET" >&2; exit 1 ;; + esac + [[ "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || { + echo "invalid release tag: $RELEASE_TAG" >&2 + exit 1 + } + [[ "$ARTIFACT_NAME" =~ ^[A-Za-z0-9._-]+$ ]] || { + echo "invalid artifact name: $ARTIFACT_NAME" >&2 + exit 1 + } + + linux_addon: + name: Node SDK addon (${{ inputs.target }}) + needs: validate_inputs + if: ${{ startsWith(inputs.target, 'linux-') }} + runs-on: ${{ inputs.target == 'linux-arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + timeout-minutes: ${{ inputs.timeout_minutes }} + permissions: + contents: read + packages: read + container: + image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d + credentials: + username: ${{ github.actor }} + password: ${{ github.token }} + defaults: + run: + shell: bash + env: + LLAMA_STAGE_BACKEND: cpu + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - name: Trust checkout directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Verify prebuilt addon environment + run: verify-runner-image public cpu + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: "false" + - name: Build, smoke, and stage immutable addon + env: + NODE_SDK_TARGET: ${{ inputs.target }} + RELEASE_TAG: ${{ inputs.release_tag }} + PREPARE_RELEASE_VERSION: ${{ inputs.prepare_release_version }} + run: | + set -euo pipefail + case "$NODE_SDK_TARGET" in + linux-arm64|linux-x64) ;; + *) echo "unsupported Linux Node SDK target: $NODE_SDK_TARGET" >&2; exit 1 ;; + esac + [[ "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || { + echo "invalid release tag: $RELEASE_TAG" >&2 + exit 1 + } + if [[ "$PREPARE_RELEASE_VERSION" == "true" ]]; then + scripts/release-version.sh "$RELEASE_TAG" + fi + expected_version="${RELEASE_TAG#v}" + actual_version="$(node -p "require('./sdk/node/package.json').version")" + [[ "$actual_version" == "$expected_version" ]] || { + echo "Node SDK version mismatch: expected $expected_version, got $actual_version" >&2 + exit 1 + } + npm test --prefix sdk/node + npm run build:native --prefix sdk/node + source="sdk/node/native/$NODE_SDK_TARGET/mesh_llm_nodejs.node" + test -s "$source" + smoke_root="$(mktemp -d)" + trap 'rm -rf "$smoke_root"' EXIT + mkdir -p "$smoke_root/tarball" "$smoke_root/consumer" + (cd sdk/node && npm pack --pack-destination "$smoke_root/tarball") + tarball="$(find "$smoke_root/tarball" -maxdepth 1 -type f -name '*.tgz' -print -quit)" + test -n "$tarball" + (cd "$smoke_root/consumer" && npm init --yes >/dev/null && npm install "$tarball") + node - "$smoke_root/consumer" "$expected_version" <<'NODE' + const [root, expected] = process.argv.slice(2) + const sdk = require(root) + const actual = sdk.currentMeshVersion() + if (actual !== expected) { + throw new Error(`native addon version mismatch: expected ${expected}, got ${actual}`) + } + NODE + stage_root="dist/node-sdk-addon-stage" + output_root="dist/node-sdk-addon-release" + mkdir -p "$stage_root/$NODE_SDK_TARGET" "$output_root" + cp "$source" "$stage_root/$NODE_SDK_TARGET/mesh_llm_nodejs.node" + addon_digest="$(node -e "const fs=require('fs'),c=require('crypto'); console.log(c.createHash('sha256').update(fs.readFileSync(process.argv[1])).digest('hex'))" "$source")" + printf '{\n "schema": "mesh-llm-node-sdk-addon-v1",\n "version": "%s",\n "target": "%s",\n "file": "mesh_llm_nodejs.node",\n "sha256": "%s"\n}\n' \ + "$expected_version" "$NODE_SDK_TARGET" "$addon_digest" \ + > "$stage_root/$NODE_SDK_TARGET/manifest.json" + archive="mesh-llm-node-sdk-addon-$expected_version-$NODE_SDK_TARGET.tar.gz" + tar -C "$stage_root" -czf "$output_root/$archive" "$NODE_SDK_TARGET" + archive_digest="$(node -e "const fs=require('fs'),c=require('crypto'); console.log(c.createHash('sha256').update(fs.readFileSync(process.argv[1])).digest('hex'))" "$output_root/$archive")" + printf '%s %s\n' "$archive_digest" "$archive" > "$output_root/$archive.sha256" + - name: Capture Node SDK addon sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-node-sdk-addon-${{ inputs.target }}-${{ github.run_attempt }} + - name: Upload immutable Node SDK addon + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ inputs.artifact_name }} + path: dist/node-sdk-addon-release/* + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} + + macos_addon: + name: Node SDK addon (${{ inputs.target }}) + needs: validate_inputs + if: ${{ startsWith(inputs.target, 'darwin-') }} + runs-on: ${{ inputs.target == 'darwin-x64' && 'macos-15-intel' || 'macos-15' }} + timeout-minutes: ${{ inputs.timeout_minutes }} + permissions: + contents: read + defaults: + run: + shell: bash + env: + LLAMA_STAGE_BACKEND: metal + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: 24 + package-manager-cache: false + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: "false" + - name: Install macOS native dependencies + run: brew install cmake ninja lld + - name: Configure and validate macOS x64 linker + if: ${{ inputs.target == 'darwin-x64' }} + run: | + set -euo pipefail + lld_bin="$(brew --prefix lld)/bin" + test -x "$lld_bin/ld64.lld" + echo "$lld_bin" >> "$GITHUB_PATH" + echo 'CARGO_ENCODED_RUSTFLAGS=-Clink-arg=-fuse-ld=lld' >> "$GITHUB_ENV" + probe="$RUNNER_TEMP/mesh-cargo-linker-check" + mkdir -p "$probe/src" + printf '[package]\nname = "mesh-cargo-linker-check"\nversion = "0.0.0"\nedition = "2024"\n' > "$probe/Cargo.toml" + printf 'fn main() {}\n' > "$probe/src/main.rs" + PATH="$lld_bin:$PATH" CARGO_ENCODED_RUSTFLAGS='-Clink-arg=-fuse-ld=lld' \ + cargo rustc --quiet --manifest-path "$probe/Cargo.toml" --target x86_64-apple-darwin + - name: Build, smoke, and stage immutable addon + env: + NODE_SDK_TARGET: ${{ inputs.target }} + RELEASE_TAG: ${{ inputs.release_tag }} + PREPARE_RELEASE_VERSION: ${{ inputs.prepare_release_version }} + run: | + set -euo pipefail + case "$NODE_SDK_TARGET" in + darwin-arm64|darwin-x64) ;; + *) echo "unsupported macOS Node SDK target: $NODE_SDK_TARGET" >&2; exit 1 ;; + esac + [[ "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || { + echo "invalid release tag: $RELEASE_TAG" >&2 + exit 1 + } + if [[ "$PREPARE_RELEASE_VERSION" == "true" ]]; then + scripts/release-version.sh "$RELEASE_TAG" + fi + expected_version="${RELEASE_TAG#v}" + actual_version="$(node -p "require('./sdk/node/package.json').version")" + [[ "$actual_version" == "$expected_version" ]] || { + echo "Node SDK version mismatch: expected $expected_version, got $actual_version" >&2 + exit 1 + } + npm test --prefix sdk/node + npm run build:native --prefix sdk/node + source="sdk/node/native/$NODE_SDK_TARGET/mesh_llm_nodejs.node" + test -s "$source" + smoke_root="$(mktemp -d)" + trap 'rm -rf "$smoke_root"' EXIT + mkdir -p "$smoke_root/tarball" "$smoke_root/consumer" + (cd sdk/node && npm pack --pack-destination "$smoke_root/tarball") + tarball="$(find "$smoke_root/tarball" -maxdepth 1 -type f -name '*.tgz' -print -quit)" + test -n "$tarball" + (cd "$smoke_root/consumer" && npm init --yes >/dev/null && npm install "$tarball") + node - "$smoke_root/consumer" "$expected_version" <<'NODE' + const [root, expected] = process.argv.slice(2) + const sdk = require(root) + const actual = sdk.currentMeshVersion() + if (actual !== expected) { + throw new Error(`native addon version mismatch: expected ${expected}, got ${actual}`) + } + NODE + stage_root="dist/node-sdk-addon-stage" + output_root="dist/node-sdk-addon-release" + mkdir -p "$stage_root/$NODE_SDK_TARGET" "$output_root" + cp "$source" "$stage_root/$NODE_SDK_TARGET/mesh_llm_nodejs.node" + addon_digest="$(node -e "const fs=require('fs'),c=require('crypto'); console.log(c.createHash('sha256').update(fs.readFileSync(process.argv[1])).digest('hex'))" "$source")" + printf '{\n "schema": "mesh-llm-node-sdk-addon-v1",\n "version": "%s",\n "target": "%s",\n "file": "mesh_llm_nodejs.node",\n "sha256": "%s"\n}\n' \ + "$expected_version" "$NODE_SDK_TARGET" "$addon_digest" \ + > "$stage_root/$NODE_SDK_TARGET/manifest.json" + archive="mesh-llm-node-sdk-addon-$expected_version-$NODE_SDK_TARGET.tar.gz" + tar -C "$stage_root" -czf "$output_root/$archive" "$NODE_SDK_TARGET" + archive_digest="$(node -e "const fs=require('fs'),c=require('crypto'); console.log(c.createHash('sha256').update(fs.readFileSync(process.argv[1])).digest('hex'))" "$output_root/$archive")" + printf '%s %s\n' "$archive_digest" "$archive" > "$output_root/$archive.sha256" + - name: Capture Node SDK addon sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-node-sdk-addon-${{ inputs.target }}-${{ github.run_attempt }} + - name: Upload immutable Node SDK addon + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ inputs.artifact_name }} + path: dist/node-sdk-addon-release/* + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} + + windows_addon: + name: Node SDK addon (${{ inputs.target }}) + needs: validate_inputs + if: ${{ inputs.target == 'win32-x64' }} + runs-on: windows-2022 + timeout-minutes: ${{ inputs.timeout_minutes }} + permissions: + contents: read + defaults: + run: + shell: bash + env: + LLAMA_STAGE_BACKEND: cpu + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: 24 + package-manager-cache: false + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable 2026-07-16 + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: "false" + - name: Build, smoke, and stage immutable addon + env: + NODE_SDK_TARGET: ${{ inputs.target }} + RELEASE_TAG: ${{ inputs.release_tag }} + PREPARE_RELEASE_VERSION: ${{ inputs.prepare_release_version }} + run: | + set -euo pipefail + [[ "$NODE_SDK_TARGET" == "win32-x64" ]] || { + echo "unsupported Windows Node SDK target: $NODE_SDK_TARGET" >&2 + exit 1 + } + [[ "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || { + echo "invalid release tag: $RELEASE_TAG" >&2 + exit 1 + } + if [[ "$PREPARE_RELEASE_VERSION" == "true" ]]; then + scripts/release-version.sh "$RELEASE_TAG" + fi + expected_version="${RELEASE_TAG#v}" + actual_version="$(node -p "require('./sdk/node/package.json').version")" + [[ "$actual_version" == "$expected_version" ]] || { + echo "Node SDK version mismatch: expected $expected_version, got $actual_version" >&2 + exit 1 + } + npm test --prefix sdk/node + npm run build:native --prefix sdk/node + source="sdk/node/native/$NODE_SDK_TARGET/mesh_llm_nodejs.node" + test -s "$source" + smoke_root="$(mktemp -d)" + trap 'rm -rf "$smoke_root"' EXIT + mkdir -p "$smoke_root/tarball" "$smoke_root/consumer" + (cd sdk/node && npm pack --pack-destination "$smoke_root/tarball") + tarball="$(find "$smoke_root/tarball" -maxdepth 1 -type f -name '*.tgz' -print -quit)" + test -n "$tarball" + (cd "$smoke_root/consumer" && npm init --yes >/dev/null && npm install "$tarball") + node - "$smoke_root/consumer" "$expected_version" <<'NODE' + const [root, expected] = process.argv.slice(2) + const sdk = require(root) + const actual = sdk.currentMeshVersion() + if (actual !== expected) { + throw new Error(`native addon version mismatch: expected ${expected}, got ${actual}`) + } + NODE + stage_root="dist/node-sdk-addon-stage" + output_root="dist/node-sdk-addon-release" + mkdir -p "$stage_root/$NODE_SDK_TARGET" "$output_root" + cp "$source" "$stage_root/$NODE_SDK_TARGET/mesh_llm_nodejs.node" + addon_digest="$(node -e "const fs=require('fs'),c=require('crypto'); console.log(c.createHash('sha256').update(fs.readFileSync(process.argv[1])).digest('hex'))" "$source")" + printf '{\n "schema": "mesh-llm-node-sdk-addon-v1",\n "version": "%s",\n "target": "%s",\n "file": "mesh_llm_nodejs.node",\n "sha256": "%s"\n}\n' \ + "$expected_version" "$NODE_SDK_TARGET" "$addon_digest" \ + > "$stage_root/$NODE_SDK_TARGET/manifest.json" + archive="mesh-llm-node-sdk-addon-$expected_version-$NODE_SDK_TARGET.tar.gz" + tar -C "$stage_root" -czf "$output_root/$archive" "$NODE_SDK_TARGET" + archive_digest="$(node -e "const fs=require('fs'),c=require('crypto'); console.log(c.createHash('sha256').update(fs.readFileSync(process.argv[1])).digest('hex'))" "$output_root/$archive")" + printf '%s %s\n' "$archive_digest" "$archive" > "$output_root/$archive.sha256" + - name: Capture Node SDK addon sccache evidence + if: ${{ !cancelled() }} + uses: ./.github/actions/capture-sccache-stats + with: + artifact_name: sccache-node-sdk-addon-${{ inputs.target }}-${{ github.run_attempt }} + - name: Upload immutable Node SDK addon + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ inputs.artifact_name }} + path: dist/node-sdk-addon-release/* + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f55700a8bb..c284ed0030 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -362,6 +362,35 @@ jobs: retention_days: 90 timeout_minutes: 90 + build_node_sdk_addon: + name: Build Node SDK addon ${{ matrix.name }} + needs: metadata + permissions: + contents: read + packages: read + strategy: + fail-fast: false + matrix: + include: + - name: macOS ARM64 + target: darwin-arm64 + - name: macOS x64 + target: darwin-x64 + - name: Linux ARM64 + target: linux-arm64 + - name: Linux x64 + target: linux-x64 + - name: Windows x64 + target: win32-x64 + uses: ./.github/workflows/node-sdk-addon-artifact.yml + with: + target: ${{ matrix.target }} + artifact_name: release-node-sdk-addon-${{ matrix.target }} + release_tag: ${{ needs.metadata.outputs.tag }} + prepare_release_version: ${{ github.event_name == 'workflow_dispatch' }} + retention_days: 90 + timeout_minutes: 90 + build_native_runtime: name: Build native runtime ${{ matrix.name }} needs: metadata @@ -1532,6 +1561,7 @@ jobs: - compose_cpu_products - inference_smoke_tests - build_native_sdk_runtime + - build_node_sdk_addon - build_native_runtime - build_native_runtime_linux_aarch64_cuda - build_native_runtime_linux_x86_64_cuda @@ -1550,7 +1580,7 @@ jobs: - compose_windows_gpu - build_native_runtime_windows_cpu - build_native_runtime_windows_gpu - if: ${{ !cancelled() && needs.metadata.result == 'success' && needs.metadata.outputs.canary != 'true' && needs.compose_cpu_products.result == 'success' && needs.inference_smoke_tests.result == 'success' && needs.build_native_sdk_runtime.result == 'success' && needs.build_native_runtime.result == 'success' && (needs.build_native_runtime_linux_aarch64_cuda.result == 'success' || needs.build_native_runtime_linux_aarch64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_cuda.result == 'success' || needs.build_native_runtime_linux_x86_64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_rocm.result == 'success' || needs.build_native_runtime_linux_x86_64_rocm.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_vulkan.result == 'success' || needs.build_native_runtime_linux_x86_64_vulkan.result == 'skipped') && needs.build_swift_sdk_artifact.result == 'success' && needs.compose_linux_arm64_cpu.result == 'success' && needs.smoke_linux_arm64_artifact.result == 'success' && (needs.compose_linux_aarch64_cuda.result == 'success' || needs.compose_linux_aarch64_cuda.result == 'skipped') && (needs.compose_linux_cuda.result == 'success' || needs.compose_linux_cuda.result == 'skipped') && (needs.compose_linux_rocm.result == 'success' || needs.compose_linux_rocm.result == 'skipped') && (needs.compose_linux_vulkan.result == 'success' || needs.compose_linux_vulkan.result == 'skipped') && needs.compose_windows_cpu.result == 'success' && (needs.compose_windows_gpu.result == 'success' || needs.compose_windows_gpu.result == 'skipped') && needs.build_native_runtime_windows_cpu.result == 'success' && (needs.build_native_runtime_windows_gpu.result == 'success' || needs.build_native_runtime_windows_gpu.result == 'skipped') }} + if: ${{ !cancelled() && needs.metadata.result == 'success' && needs.metadata.outputs.canary != 'true' && needs.compose_cpu_products.result == 'success' && needs.inference_smoke_tests.result == 'success' && needs.build_native_sdk_runtime.result == 'success' && needs.build_node_sdk_addon.result == 'success' && needs.build_native_runtime.result == 'success' && (needs.build_native_runtime_linux_aarch64_cuda.result == 'success' || needs.build_native_runtime_linux_aarch64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_cuda.result == 'success' || needs.build_native_runtime_linux_x86_64_cuda.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_rocm.result == 'success' || needs.build_native_runtime_linux_x86_64_rocm.result == 'skipped') && (needs.build_native_runtime_linux_x86_64_vulkan.result == 'success' || needs.build_native_runtime_linux_x86_64_vulkan.result == 'skipped') && needs.build_swift_sdk_artifact.result == 'success' && needs.compose_linux_arm64_cpu.result == 'success' && needs.smoke_linux_arm64_artifact.result == 'success' && (needs.compose_linux_aarch64_cuda.result == 'success' || needs.compose_linux_aarch64_cuda.result == 'skipped') && (needs.compose_linux_cuda.result == 'success' || needs.compose_linux_cuda.result == 'skipped') && (needs.compose_linux_rocm.result == 'success' || needs.compose_linux_rocm.result == 'skipped') && (needs.compose_linux_vulkan.result == 'success' || needs.compose_linux_vulkan.result == 'skipped') && needs.compose_windows_cpu.result == 'success' && (needs.compose_windows_gpu.result == 'success' || needs.compose_windows_gpu.result == 'skipped') && needs.build_native_runtime_windows_cpu.result == 'success' && (needs.build_native_runtime_windows_gpu.result == 'success' || needs.build_native_runtime_windows_gpu.result == 'skipped') }} runs-on: ubuntu-24.04 permissions: contents: write diff --git a/ci/ci.md b/ci/ci.md index 5124f6edb2..48e0850799 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -199,6 +199,12 @@ flowchart TD host. Each product requires `--version`, `runtime list`, and client readiness without a driver stub. GPU availability remains separate hardware qualification. +- Release builds Node native addons through one typed reusable producer fanned + out across Darwin ARM64/x64, Linux ARM64/x64, and Windows x64. Every lane + performs a fresh npm install and native version smoke before emitting a + manifest-bound, checksummed archive. Release publication requires all five + artifacts, and `mesh-packaging` consumes those exact release assets instead + of compiling addon source again. - `.github/actions/prepare-host-input`, `.github/actions/prepare-windows-host-input`, `.github/actions/prepare-native-runtime-input`, diff --git a/scripts/tests/test_ci_workflow_artifacts.py b/scripts/tests/test_ci_workflow_artifacts.py index c2b8b3a740..b04ede2799 100644 --- a/scripts/tests/test_ci_workflow_artifacts.py +++ b/scripts/tests/test_ci_workflow_artifacts.py @@ -7,6 +7,10 @@ ROOT = Path(__file__).resolve().parents[2] CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" +RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "release.yml" +NODE_ADDON_WORKFLOW = ( + ROOT / ".github" / "workflows" / "node-sdk-addon-artifact.yml" +) def job_section(workflow: str, job_name: str) -> str: @@ -529,5 +533,81 @@ def test_linux_product_consumers_stage_the_release_profile(self) -> None: self.assertIn("target/release/mesh-llm", linux_consumers) +class ReleaseNodeAddonArtifactTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.release = RELEASE_WORKFLOW.read_text(encoding="utf-8") + cls.producer = NODE_ADDON_WORKFLOW.read_text(encoding="utf-8") + + def test_release_uses_one_reusable_producer_for_all_node_targets( + self, + ) -> None: + producer = job_section(self.release, "build_node_sdk_addon") + + self.assertIn( + "uses: ./.github/workflows/node-sdk-addon-artifact.yml", + producer, + ) + self.assertIn( + "artifact_name: release-node-sdk-addon-${{ matrix.target }}", + producer, + ) + for target in ( + "darwin-arm64", + "darwin-x64", + "linux-arm64", + "linux-x64", + "win32-x64", + ): + with self.subTest(target=target): + self.assertEqual(producer.count(f"target: {target}"), 1) + self.assertNotIn("npm run build:native", producer) + self.assertNotIn("actions/upload-artifact", producer) + + def test_node_addon_producer_is_non_publishing_and_checksummed( + self, + ) -> None: + self.assertIn("on:\n workflow_call:", self.producer) + self.assertIn( + "mesh-llm-node-sdk-addon-$expected_version-$NODE_SDK_TARGET.tar.gz", + self.producer, + ) + self.assertIn( + '"schema": "mesh-llm-node-sdk-addon-v1"', + self.producer, + ) + self.assertIn("currentMeshVersion()", self.producer) + self.assertIn("npm pack", self.producer) + self.assertIn("$output_root/$archive.sha256", self.producer) + self.assertIn( + "ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:", + self.producer, + ) + self.assertIn( + "darwin-arm64|darwin-x64|linux-arm64|linux-x64|win32-x64", + job_section(self.producer, "validate_inputs"), + ) + for job_name in ("linux_addon", "macos_addon", "windows_addon"): + with self.subTest(job=job_name): + self.assertIn( + "needs: validate_inputs", + job_section(self.producer, job_name), + ) + self.assertNotIn("softprops/action-gh-release", self.producer) + self.assertNotIn("npm publish", self.producer) + self.assertNotIn("contents: write", self.producer) + + def test_release_publish_requires_and_uploads_node_addons(self) -> None: + publish = job_section(self.release, "publish") + + self.assertIn("- build_node_sdk_addon", publish) + self.assertIn( + "needs.build_node_sdk_addon.result == 'success'", + publish, + ) + self.assertIn("pattern: release-*", publish) + self.assertIn("files: release-artifacts/*", publish) + + if __name__ == "__main__": unittest.main() From c94045d0bc0d998b42b2e0fd0bf23c3db9f99095 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Thu, 30 Jul 2026 02:44:29 -0400 Subject: [PATCH 22/23] fix(ci): restrict Depot canary to main --- .github/workflows/depot-canary.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/depot-canary.yml b/.github/workflows/depot-canary.yml index e96084cc0b..3b81598385 100644 --- a/.github/workflows/depot-canary.yml +++ b/.github/workflows/depot-canary.yml @@ -18,6 +18,10 @@ concurrency: jobs: runner: name: ${{ matrix.runner }} + if: >- + github.repository == 'Mesh-LLM/mesh-llm' && + github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/main' strategy: fail-fast: false matrix: From 2e6dfa7368048a78b929563b840330d3b190f2c6 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Thu, 30 Jul 2026 03:11:05 -0400 Subject: [PATCH 23/23] fix(ci): isolate Depot canary cache keys --- .github/workflows/depot-canary.yml | 4 +++- scripts/tests/test_depot_canary_workflow.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/depot-canary.yml b/.github/workflows/depot-canary.yml index 3b81598385..5285f3b22d 100644 --- a/.github/workflows/depot-canary.yml +++ b/.github/workflows/depot-canary.yml @@ -150,7 +150,9 @@ jobs: uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: .depot-canary-cache - key: depot-runner-canary-v1-${{ matrix.runner }} + # Keep the complete label between fixed delimiters so a shorter + # runner label cannot prefix-match a sibling size. + key: depot-runner-canary-v2-exact-${{ matrix.runner }}-probe - name: Record cache probe shell: bash diff --git a/scripts/tests/test_depot_canary_workflow.py b/scripts/tests/test_depot_canary_workflow.py index 58349bbb29..c75ee954f1 100644 --- a/scripts/tests/test_depot_canary_workflow.py +++ b/scripts/tests/test_depot_canary_workflow.py @@ -57,6 +57,11 @@ def test_canary_uses_a_pinned_cache_action_without_printing_tokens( '[[ "$EXPECT_CACHE_HIT" == "true" && "$CACHE_HIT" != "true" ]]', self.workflow, ) + self.assertIn( + "key: depot-runner-canary-v2-exact-${{ matrix.runner }}-probe", + self.workflow, + ) + self.assertNotIn("key: depot-runner-canary-v1-", self.workflow) self.assertNotIn("echo \"$DEPOT_CACHE_TOKEN\"", self.workflow) self.assertNotIn("printenv", self.workflow)