diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 029e3a8ca2..8d307a4a8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,20 +94,34 @@ jobs: - uses: mozilla-actions/sccache-action@v0.0.9 + # Workspace `. -> target` because the cargo workspace is at repo root + # (`Cargo.toml` members = ["mesh-llm", ...]) and cargo writes to + # `./target/`, NOT `mesh-llm/target/`. Using `workspaces: mesh-llm` + # silently caches an empty dir and gives zero PR cache hits — we + # measured a 4-minute regression from this exact misconfiguration. - uses: Swatinem/rust-cache@v2 with: - workspaces: mesh-llm + workspaces: . -> target prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ hashFiles('.github/cache-version.txt') }} - - name: Unit tests - run: cargo test --release - - - name: Build mesh-llm - run: cargo build --release -p mesh-llm --bin mesh-llm - + # Early-fail gate: runs before expensive compile/test work. - name: Check formatting working-directory: mesh-llm - run: cargo fmt -- --check || true + run: cargo fmt -- --check + + # Build in dev/debug profile, NOT release. Two reasons: + # 1. Build + Unit tests steps share one target/ subdir and one + # incremental cache — zero duplicated codegen. + # 2. debug has `incremental = true`; release has `incremental = false`. + # Warm rebuild after a lib edit: release ~43s, debug ~4s (M4 Pro). + # mesh-llm is a thin orchestrator around llama-server; the hot loop is + # inside llama-server C++, so debug vs release binary perf is + # negligible for smoke tests. Integration steps below use target/debug. + - name: Build mesh-llm binary (debug) + run: cargo build -p mesh-llm --bin mesh-llm + + - name: Unit tests + run: cargo test --lib --tests - name: Clippy working-directory: mesh-llm @@ -154,21 +168,21 @@ jobs: - name: Smoke test (real inference) run: | scripts/ci-smoke-test.sh \ - target/release/mesh-llm \ + target/debug/mesh-llm \ llama.cpp/build/bin \ ~/.models/$MODEL_FILE - name: OpenAI Python compat smoke run: | scripts/ci-compat-smoke.sh \ - target/release/mesh-llm \ + target/debug/mesh-llm \ llama.cpp/build/bin \ ~/.models/$MODEL_FILE - name: Split-mode test (host/worker routing) run: | scripts/ci-split-test.sh \ - target/release/mesh-llm \ + target/debug/mesh-llm \ llama.cpp/build/bin \ ~/.models/$MODEL_FILE @@ -196,17 +210,17 @@ jobs: - name: MoE mesh test (expert sharding end-to-end) run: | scripts/ci-moe-mesh-test.sh \ - target/release/mesh-llm \ + target/debug/mesh-llm \ llama.cpp/build/bin \ ~/.models/$MOE_MODEL_FILE - name: CLI smoke test run: | - target/release/mesh-llm --version - target/release/mesh-llm --help | head -5 + target/debug/mesh-llm --version + target/debug/mesh-llm --help | head -5 - name: Client-auto boot test - run: scripts/ci-client-auto-test.sh target/release/mesh-llm + run: scripts/ci-client-auto-test.sh target/debug/mesh-llm - name: Show sccache stats if: always() @@ -237,16 +251,23 @@ jobs: - uses: mozilla-actions/sccache-action@v0.0.9 + # See linux job for explanation of `. -> target`. - uses: Swatinem/rust-cache@v2 with: - workspaces: mesh-llm + workspaces: . -> target prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ hashFiles('.github/cache-version.txt') }} - - name: Unit tests - run: cargo test --release + # Early-fail gate: runs before expensive compile/test work. + - name: Check formatting + working-directory: mesh-llm + run: cargo fmt -- --check - - name: Build mesh-llm - run: cargo build --release -p mesh-llm --bin mesh-llm + # See linux job for rationale on building in dev/debug, not release. + - name: Build mesh-llm binary (debug) + run: cargo build -p mesh-llm --bin mesh-llm + + - name: Unit tests + run: cargo test -p mesh-llm --lib --tests # ── llama.cpp (Metal + RPC) ── - name: Clone llama.cpp fork @@ -288,14 +309,14 @@ jobs: - name: Smoke test (real inference) run: | scripts/ci-smoke-test.sh \ - target/release/mesh-llm \ + target/debug/mesh-llm \ llama.cpp/build/bin \ ~/.models/$MODEL_FILE - name: Split-mode test (host/worker routing) run: | scripts/ci-split-test.sh \ - target/release/mesh-llm \ + target/debug/mesh-llm \ llama.cpp/build/bin \ ~/.models/$MODEL_FILE @@ -323,17 +344,17 @@ jobs: - name: MoE mesh test (expert sharding end-to-end) run: | scripts/ci-moe-mesh-test.sh \ - target/release/mesh-llm \ + target/debug/mesh-llm \ llama.cpp/build/bin \ ~/.models/$MOE_MODEL_FILE - name: CLI smoke test run: | - target/release/mesh-llm --version - target/release/mesh-llm --help | head -5 + target/debug/mesh-llm --version + target/debug/mesh-llm --help | head -5 - name: Client-auto boot test - run: scripts/ci-client-auto-test.sh target/release/mesh-llm + run: scripts/ci-client-auto-test.sh target/debug/mesh-llm - name: Show sccache stats if: always() @@ -393,16 +414,126 @@ jobs: . -> target prefix-key: ${{ env.CACHE_NAMESPACE }}-rust-${{ hashFiles('.github/cache-version.txt') }} - - name: Build Linux CUDA backend + # ── Cross-PR llama.cpp / CUDA artifact cache (read-only consumer) ── + # + # Goal: every PR gets a "warm" llama.cpp CUDA build for free. Without + # this cache the CUDA job pays ~194s warm (sccache hit) or ~26m cold + # (sccache eviction) on every PR. With it, the llama.cpp build is + # skipped entirely on cache hit and the only remaining work is the + # mesh-llm cargo build. + # + # This job RESTORES ONLY. It never writes the cache. All writes + # happen in .github/workflows/warm-caches.yml, which runs on push + # to main (paths-filtered) and on workflow_dispatch. Splitting the + # writer into its own workflow prevents PR runs from polluting + # PR-scoped cache storage with a ~500 MB `llama.cpp/build/bin` + # entry on every push, and lets the warmup workflow prune old + # versions (see warm-caches.yml for the retention policy). + # + # The cache key MUST stay byte-identical to the one in + # warm-caches.yml or PRs will never hit the main-warmed entry. + # Any change to the hashFiles() list or literal suffix must be + # mirrored in both files in the same commit. + # + # Cross-branch read: pushes to `main` write the cache under + # `refs/heads/main`. PR runs on `refs/pull//merge` automatically + # fall back to the base branch's cache scope, so PRs hit the + # main-warmed cache without needing any cache to exist on the PR + # ref itself. + # + # The cache key embeds EVERY input that affects the produced binaries: + # - llama.cpp upstream commit SHA, resolved at workflow time below + # via `git ls-remote`. + # - hash of the build inputs in this repo: scripts/build-linux.sh, + # Justfile, ci.yml, warm-caches.yml, cache-version.txt. + # - the CUDA arch literal (`arch89`), the GGML_CUDA_FA_ALL_QUANTS + # state (`fa-off`), and the CUDA toolkit version pulled from the + # container image. + - name: Resolve llama.cpp upstream SHA + id: llama_sha + shell: bash + run: | + set -euo pipefail + SHA=$(git ls-remote https://github.com/michaelneale/llama.cpp.git refs/heads/upstream-latest | cut -f1) + if [[ -z "$SHA" ]]; then + echo "Failed to resolve llama.cpp upstream-latest SHA" >&2 + exit 1 + fi + echo "Resolved llama.cpp upstream-latest SHA: $SHA" + echo "sha=$SHA" >> "$GITHUB_OUTPUT" + + # `actions/cache/restore@v5` (NOT `actions/cache@v5`): restore-only, + # never saves. PR runs on `refs/pull//merge` never write anything + # to PR-scoped cache storage. The only writer is warm-caches.yml. + # On cache miss the full llama.cpp build still runs to prove the + # change works, but the result is discarded at job end. + - name: Restore llama.cpp CUDA cache + id: llama_cuda_cache + uses: actions/cache/restore@v5 + with: + path: llama.cpp/build/bin + key: ${{ env.CACHE_NAMESPACE }}-llama-cuda-${{ steps.llama_sha.outputs.sha }}-${{ hashFiles('scripts/build-linux.sh', 'Justfile', '.github/workflows/ci.yml', '.github/workflows/warm-caches.yml', '.github/cache-version.txt') }}-arch89-fa-off-cuda12.8 + + # CI is a validation build only: we only need to prove CUDA compiles + # and the Rust binary runs. User-facing release artifacts are produced + # by release.yml which keeps the full default cuda_arch list, release + # profile, and GGML_CUDA_FA_ALL_QUANTS=ON. Here we pass four CI-only + # opt-outs, all documented in scripts/build-linux.sh: + # 1. single cuda_arch 89 (Ada Lovelace) — ~3.7x faster llama.cpp build + # 2. MESH_LLM_BUILD_PROFILE=dev — debug profile for mesh-llm itself + # 3. MESH_LLM_CUDA_FA_ALL_QUANTS=off — skip the full FlashAttention + # kernel matrix. Safe ONLY because the CUDA smoke test is + # `mesh-llm --version` — the asymmetric K/V cache path that would + # crash rpc-server with BEST_FATTN_KERNEL_NONE is never exercised. + # NEVER set this in release.yml. + # 4. MESH_LLM_LLAMA_PIN_SHA — pin the llama.cpp checkout to the SHA + # embedded in the artifact cache key (above). NEVER set this in + # release.yml; release artifacts must build the current + # upstream-latest tip. + # + # Split into two mutually exclusive steps keyed off the artifact cache: + # + # - Cache miss → do the fast PR-only llama.cpp + mesh-llm build via the + # build script. This validates the same arch89/fa-off shape that + # main warms, but PR runs never write the shared cache. + # - Cache hit → skip llama.cpp entirely (we already proved it builds + # at this SHA + flag combo) and run only the mesh-llm cargo build. + # This is the warm path every PR hits when main has already run. + - name: Build Linux CUDA backend (cache miss — full llama.cpp build) + if: steps.llama_cuda_cache.outputs.cache-hit != 'true' + shell: bash + env: + MESH_LLM_BUILD_PROFILE: dev + MESH_LLM_CUDA_FA_ALL_QUANTS: off + MESH_LLM_LLAMA_PIN_SHA: ${{ steps.llama_sha.outputs.sha }} + run: | + just --shell bash --shell-arg -lc release-build-cuda 89 + + - name: Build mesh-llm binary only (cache hit — llama.cpp skipped) + if: steps.llama_cuda_cache.outputs.cache-hit == 'true' shell: bash + env: + MESH_LLM_BUILD_PROFILE: dev run: | - just --shell bash --shell-arg -lc release-build-cuda + set -euo pipefail + echo "✓ llama.cpp CUDA build restored from cache (SHA ${{ steps.llama_sha.outputs.sha }})" + if [[ ! -d llama.cpp/build/bin ]]; then + echo "ERROR: cache hit but llama.cpp/build/bin is missing" >&2 + exit 1 + fi + if ! find llama.cpp/build/bin -maxdepth 1 -type f | grep -q .; then + echo "ERROR: cache hit but llama.cpp/build/bin contains no cached binaries" >&2 + exit 1 + fi + ls -lh llama.cpp/build/bin/ | head -20 + cargo build -p mesh-llm --bin mesh-llm + ls -lh target/debug/mesh-llm - name: CLI smoke test shell: bash run: | - target/release/mesh-llm --version - target/release/mesh-llm --help | head -5 + target/debug/mesh-llm --version + target/debug/mesh-llm --help | head -5 - name: Show sccache stats if: always() diff --git a/.github/workflows/warm-caches.yml b/.github/workflows/warm-caches.yml index 9f1b01e405..dac1ce791a 100644 --- a/.github/workflows/warm-caches.yml +++ b/.github/workflows/warm-caches.yml @@ -147,7 +147,7 @@ jobs: uses: actions/cache@v5 with: path: llama.cpp/build/bin - key: ${{ env.CACHE_NAMESPACE }}-llama-cuda-${{ steps.llama_sha.outputs.sha }}-${{ hashFiles('scripts/build-linux.sh', 'Justfile', '.github/workflows/ci.yml', '.github/workflows/warm-caches.yml', '.github/cache-version.txt') }}-arch89-fa-on-cuda12.8 + key: ${{ env.CACHE_NAMESPACE }}-llama-cuda-${{ steps.llama_sha.outputs.sha }}-${{ hashFiles('scripts/build-linux.sh', 'Justfile', '.github/workflows/ci.yml', '.github/workflows/warm-caches.yml', '.github/cache-version.txt') }}-arch89-fa-off-cuda12.8 - name: Short-circuit if cache already warm if: steps.llama_cuda_cache.outputs.cache-hit == 'true' @@ -161,6 +161,7 @@ jobs: if: steps.llama_cuda_cache.outputs.cache-hit != 'true' shell: bash env: + MESH_LLM_CUDA_FA_ALL_QUANTS: off MESH_LLM_LLAMA_PIN_SHA: ${{ steps.llama_sha.outputs.sha }} run: | just --shell bash --shell-arg -lc release-build-cuda 89 diff --git a/scripts/build-linux.sh b/scripts/build-linux.sh index 96abd880e5..326255eddb 100755 --- a/scripts/build-linux.sh +++ b/scripts/build-linux.sh @@ -236,11 +236,26 @@ case "$BACKEND" in ;; esac +# MESH_LLM_LLAMA_PIN_SHA pins the llama.cpp checkout to a specific commit and +# disables the `git pull` that would otherwise move the working tree forward. +# This is required by the cross-PR llama.cpp / CUDA artifact cache in +# .github/workflows/ci.yml: the cache key embeds the resolved upstream SHA, so +# the actual checkout MUST match that SHA byte-for-byte or the restored +# `llama.cpp/build/` directory will be inconsistent with the source tree and +# cmake will silently rebuild things. +# +# When unset (the default for local `just build`), behaviour is unchanged: +# clone-or-pull `upstream-latest` HEAD as before. LLAMA_PIN_SHA="${MESH_LLM_LLAMA_PIN_SHA:-}" if [[ ! -d "$LLAMA_DIR" ]]; then if [[ -n "$LLAMA_PIN_SHA" ]]; then echo "Cloning michaelneale/llama.cpp pinned to $LLAMA_PIN_SHA..." + # Shallow clone of upstream-latest first (the common case is that + # $LLAMA_PIN_SHA == upstream-latest HEAD because ci.yml resolves it + # via `git ls-remote ... refs/heads/upstream-latest`). If the branch + # has moved between resolve and clone, fall back to fetching the + # specific commit. git clone -b upstream-latest --depth 1 \ https://github.com/michaelneale/llama.cpp.git "$LLAMA_DIR" if ! (cd "$LLAMA_DIR" && git cat-file -e "${LLAMA_PIN_SHA}^{commit}" 2>/dev/null); then @@ -256,6 +271,10 @@ if [[ ! -d "$LLAMA_DIR" ]]; then else cd "$LLAMA_DIR" if [[ -n "$LLAMA_PIN_SHA" ]]; then + # Pinned mode: do NOT pull. Fetch the requested SHA if missing and + # check it out in detached HEAD. Skipping `git pull` is the whole + # point — it keeps the working tree byte-identical to what the cache + # key promises. if ! git cat-file -e "${LLAMA_PIN_SHA}^{commit}" 2>/dev/null; then echo "Fetching pinned llama.cpp SHA $LLAMA_PIN_SHA..." git fetch --depth 1 origin "$LLAMA_PIN_SHA" @@ -305,10 +324,16 @@ elif [[ "$BACKEND" == "cuda" ]]; then # GGML_CUDA_FA_ALL_QUANTS compiles the full matrix of FlashAttention # kernels so mismatched K/V cache quantization types (e.g. K=q8_0, V=q4_0) # don't hit BEST_FATTN_KERNEL_NONE and crash the rpc-server. - # Increases compile time but is required for any asymmetric KV cache. - # Tracking: https://github.com/ggml-org/llama.cpp/issues/20866 - # Once that upstream issue is resolved and our fork is rebased past the - # fix, this flag can be dropped. + # Required for any asymmetric KV cache; the default (ON) is what user- + # facing release artifacts must ship. Tracking: + # https://github.com/ggml-org/llama.cpp/issues/20866 + # + # CI may opt out via MESH_LLM_CUDA_FA_ALL_QUANTS=off because ci.yml does + # only a --version smoke test on the CUDA binary and never exercises the + # asymmetric KV cache path. Dropping the flag shrinks the FlashAttention + # kernel matrix drastically (~177 fattn .cu instantiations \u2192 a fraction) + # and cuts llama.cpp CUDA compile time significantly. NEVER use this + # opt-out for release builds. CUDA_FA_ALL_QUANTS_FLAG="-DGGML_CUDA_FA_ALL_QUANTS=ON" if [[ "${MESH_LLM_CUDA_FA_ALL_QUANTS:-on}" == "off" ]]; then CUDA_FA_ALL_QUANTS_FLAG="-DGGML_CUDA_FA_ALL_QUANTS=OFF" @@ -348,9 +373,9 @@ cmake_flags+=("${compiler_launcher_flags[@]}") cmake "${cmake_flags[@]}" -# Post-configure assertion: guarantee GGML_CUDA_FA_ALL_QUANTS actually landed -# in the CMake cache for CUDA builds. If someone deletes the flag above by -# mistake, this trips before we burn 90s rebuilding llama.cpp. Tracking: +# Post-configure assertion: guarantee the CUDA cmake cache reflects the +# intended GGML_CUDA_FA_ALL_QUANTS state. The default path must ship ON; the +# CI opt-out must explicitly pass MESH_LLM_CUDA_FA_ALL_QUANTS=off. Tracking: # https://github.com/ggml-org/llama.cpp/issues/20866 if [[ "$BACKEND" == "cuda" ]]; then EXPECTED_FA_ALL_QUANTS="ON" @@ -359,6 +384,7 @@ if [[ "$BACKEND" == "cuda" ]]; then fi if ! grep -q "^GGML_CUDA_FA_ALL_QUANTS:BOOL=${EXPECTED_FA_ALL_QUANTS}" "$BUILD_DIR/CMakeCache.txt"; then echo "ERROR: GGML_CUDA_FA_ALL_QUANTS is not ${EXPECTED_FA_ALL_QUANTS} in $BUILD_DIR/CMakeCache.txt" >&2 + echo " Expected state derived from MESH_LLM_CUDA_FA_ALL_QUANTS=${MESH_LLM_CUDA_FA_ALL_QUANTS:-on}." >&2 echo " Release builds MUST ship ON (asymmetric K/V cache crash risk)." >&2 echo " See scripts/build-linux.sh and ggml-org/llama.cpp#20866." >&2 exit 1 @@ -372,7 +398,17 @@ if [[ -d "$MESH_DIR" ]]; then if [[ -d "$UI_DIR" ]]; then "$SCRIPT_DIR/build-ui.sh" "$UI_DIR" fi - echo "Building mesh-llm..." - (cd "$MESH_DIR" && cargo build --release) - echo "Mesh binary: target/release/mesh-llm" + + # MESH_LLM_BUILD_PROFILE=dev|debug lets CI opt into dev profile (single + # target subdir, only the bin target — same shape as linux+macos jobs). + # Default stays release so local `just build` is unchanged. + if [[ "${MESH_LLM_BUILD_PROFILE:-release}" == "dev" || "${MESH_LLM_BUILD_PROFILE:-release}" == "debug" ]]; then + echo "Building mesh-llm (profile: dev, bin only)..." + (cd "$REPO_ROOT" && cargo build -p mesh-llm --bin mesh-llm) + echo "Mesh binary: target/debug/mesh-llm" + else + echo "Building mesh-llm (profile: release)..." + (cd "$MESH_DIR" && cargo build --release) + echo "Mesh binary: target/release/mesh-llm" + fi fi