From 54f435d15feaa0e6fe437bea89db2cf5906f89d1 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Tue, 28 Jul 2026 20:47:45 -0400 Subject: [PATCH 1/5] feat(packaging): consume product-v2 bundles --- .github/workflows/images-precheck.yml | 5 +- .github/workflows/images-release.yml | 30 +++- .skills/distribution-certification/SKILL.md | 44 ++++-- .../references/format-checks.md | 46 ++++-- .../references/report-contract.md | 3 + AGENTS.md | 11 +- README.md | 8 +- TODO.md | 25 ++-- docker/Dockerfile.mesh-llm | 14 +- docker/qa-runtime-image.sh | 20 ++- docs/gpu-runbooks.md | 11 +- docs/matrix.md | 12 +- docs/native-packages.md | 35 +++-- docs/package-signing.md | 12 +- docs/packaging-readiness-gaps.md | 7 +- docs/packaging-readiness-scorecard.md | 10 +- docs/publishing.md | 11 +- docs/release-checklist.md | 13 +- docs/runner-capacity.md | 16 ++- docs/tagging.md | 6 +- .../homebrew/Formula/mesh-llm.rb.template | 6 +- packaging/homebrew/README.md | 11 +- packaging/native/README.md | 11 +- packaging/native/build-package.sh | 20 ++- schemas/product-v2.schema.json | 86 ++++++++++++ scripts/native-package-qa.sh | 21 +-- scripts/upstream-archive.ts | 132 +++++++++++++++++- tests/upstream-archive.test.ts | 80 ++++++++++- 28 files changed, 573 insertions(+), 133 deletions(-) create mode 100644 schemas/product-v2.schema.json diff --git a/.github/workflows/images-precheck.yml b/.github/workflows/images-precheck.yml index 1758be4..386c2d4 100644 --- a/.github/workflows/images-precheck.yml +++ b/.github/workflows/images-precheck.yml @@ -57,8 +57,11 @@ jobs: - uses: docker/setup-buildx-action@v4 - name: Check package-first Dockerfile targets run: | - mkdir -p artifacts/upstream artifacts/native-package + mkdir -p artifacts/upstream/native-runtimes/test-runtime/lib artifacts/native-package touch artifacts/upstream/mesh-llm + touch artifacts/upstream/product-manifest.json artifacts/upstream/host-imports.json + touch artifacts/upstream/native-runtimes/test-runtime/manifest.json + touch artifacts/upstream/native-runtimes/test-runtime/lib/libllama.so touch artifacts/native-package/mesh-llm-0.73.1-ubuntu-amd64-cpu.deb chmod +x artifacts/upstream/mesh-llm docker buildx build --check --platform linux/amd64 --target native-package-artifact --file docker/Dockerfile.mesh-llm \ diff --git a/.github/workflows/images-release.yml b/.github/workflows/images-release.yml index b8678b7..923e830 100644 --- a/.github/workflows/images-release.yml +++ b/.github/workflows/images-release.yml @@ -233,7 +233,7 @@ jobs: name: ${{ matrix.upstream_artifact_id }} path: artifacts/upstream - uses: docker/setup-buildx-action@v4 - - name: Build native package from verified upstream binary + - name: Build native package from verified upstream product bundle uses: docker/build-push-action@v7 with: context: . @@ -291,6 +291,17 @@ jobs: name: ${{ matrix.native_package_artifact_name }} path: artifacts/native-package - uses: docker/setup-buildx-action@v4 + - name: Read immutable product inputs + id: product + shell: bash + run: | + set -euo pipefail + provenance="$(find artifacts/native-package -name '*.upstream-provenance.json' -type f -print -quit)" + { + echo "host_sha=$(jq -r .host_sha256 "$provenance")" + echo "runtime_id=$(jq -r .runtime_id "$provenance")" + echo "runtime_sha=$(jq -r .runtime_sha256 "$provenance")" + } >> "$GITHUB_OUTPUT" - name: Build and QA package-installed runtime image uses: docker/build-push-action@v7 with: @@ -307,6 +318,9 @@ jobs: MESH_LLM_VERSION=${{ matrix.mesh_version }} MESH_LLM_REF=${{ needs.plan.outputs.mesh_ref }} MESH_LLM_SOURCE_SHA=${{ needs.plan.outputs.mesh_sha }} + MESH_LLM_HOST_SHA=${{ steps.product.outputs.host_sha }} + MESH_LLM_RUNTIME_ID=${{ steps.product.outputs.runtime_id }} + MESH_LLM_RUNTIME_SHA=${{ steps.product.outputs.runtime_sha }} homebrew: name: Verify Homebrew formula from upstream macOS archive @@ -568,6 +582,17 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ github.token }} + - name: Read immutable product inputs + id: product + shell: bash + run: | + set -euo pipefail + provenance="$(find artifacts/native-package -name '*.upstream-provenance.json' -type f -print -quit)" + { + echo "host_sha=$(jq -r .host_sha256 "$provenance")" + echo "runtime_id=$(jq -r .runtime_id "$provenance")" + echo "runtime_sha=$(jq -r .runtime_sha256 "$provenance")" + } >> "$GITHUB_OUTPUT" - id: push uses: docker/build-push-action@v7 with: @@ -587,6 +612,9 @@ jobs: MESH_LLM_VERSION=${{ matrix.mesh_version }} MESH_LLM_REF=${{ needs.plan.outputs.mesh_ref }} MESH_LLM_SOURCE_SHA=${{ needs.plan.outputs.mesh_sha }} + MESH_LLM_HOST_SHA=${{ steps.product.outputs.host_sha }} + MESH_LLM_RUNTIME_ID=${{ steps.product.outputs.runtime_id }} + MESH_LLM_RUNTIME_SHA=${{ steps.product.outputs.runtime_sha }} - uses: actions/attest-build-provenance@v4 with: subject-name: ${{ env.IMAGE_NAME }} diff --git a/.skills/distribution-certification/SKILL.md b/.skills/distribution-certification/SKILL.md index fa8d7e2..8cb92e0 100644 --- a/.skills/distribution-certification/SKILL.md +++ b/.skills/distribution-certification/SKILL.md @@ -130,24 +130,34 @@ Snapshot the packaging release before other tests: 4. Download the aggregate checksum, aggregate provenance, formula, every native package, every package sidecar, and every expected row SBOM/upstream provenance file. -5. Verify every downloaded asset byte-for-byte against the server-reported +5. Download each upstream MeshLLM product archive and require the product-v2 + layout: one backend-neutral `mesh-llm` host, `product-manifest.json`, + `host-imports.json`, and exactly one selected runtime below + `native-runtimes//manifest.json`. +6. Verify every downloaded asset byte-for-byte against the server-reported digest where the release surface provides one, then verify every native package against both its sidecar and aggregate checksum. -6. Run the aggregate checksum across every file it names, not just native +7. Run the aggregate checksum across every file it names, not just native packages. A stale formula or metadata checksum is a release failure. -7. Compare asset timestamps/digests with the release snapshot before and after +8. Compare asset timestamps/digests with the release snapshot before and after checksum verification. Reject moved tags, replaced assets, stale aggregate checksums, and any checksum generated before the asset bytes it claims. -8. Download the immutable packaging tag source and derive the expected matrix +9. Download the immutable packaging tag source and derive the expected matrix using its checked-in matrix and tag-generation code. Do not build product code. -9. Confirm one SBOM and one upstream-provenance file for every enabled native +10. Confirm one SBOM and one upstream-provenance file for every enabled native row. Cryptographically verify SBOM and provenance subjects against the exact artifact digests they claim, and reject unrelated or stale subjects. Explicitly record disabled/unsupported rows. -10. Compare tagged channel identity—especially npm package scope—with the +11. Compare tagged channel identity—especially npm package scope—with the artifact actually published. +For each product archive, validate every digest recorded by +`product-manifest.json`, verify that the host dependency report rejects no +backend runtime imports, and confirm that the selected runtime ID, platform, +backend, and MeshLLM version match the release row. Reject absolute build paths +or additional top-level payload files. + Preserve both redacted machine-readable inventory and a concise human-readable validation log. @@ -160,14 +170,18 @@ Common runtime rule: 1. Resolve the package-owned or channel-owned executable directly. 2. Run `--version` and require exact MeshLLM semantic version equality. 3. Run `runtime list` where the channel exposes the CLI. -4. Isolate `HOME`, XDG paths, cache directory, and runtime directory. -5. Reserve both an API port and a console port. Pass both explicitly even when +4. Require `runtime list` to discover the channel-owned adjacent runtime while + the user cache is empty; it must not copy that runtime into the cache. +5. Run `--version`, `--help`, and `runtime list` without GPU/device passthrough. + A backend driver loader failure is a product failure. +6. Isolate `HOME`, XDG paths, cache directory, and runtime directory. +7. Reserve both an API port and a console port. Pass both explicitly even when using `--no-console`; some versions may still initialize the web server. -6. Start client mode with `--log-format json --no-console --auto`. -7. Require the process/container to remain alive and emit a real ready event +8. Start client mode with `--log-format json --no-console --auto`. +9. Require the process/container to remain alive and emit a real ready event such as `Client ready`. -8. Probe `/v1/models` only when an endpoint was intentionally exposed. -9. Send SIGINT, enforce a bounded shutdown, capture the final exit state, and +10. Probe `/v1/models` only when an endpoint was intentionally exposed. +11. Send SIGINT, enforce a bounded shutdown, capture the final exit state, and verify listeners/processes disappeared. A metadata-only check, successful install, `--version`, or transient live PID @@ -189,8 +203,12 @@ Require agreement on the requested semantic version across: Also prove: - Native packages own the expected executable path. +- Native packages own the versioned runtime tree and product manifests; the + host remains backend-neutral and no package installs backend libraries beside + the executable. - Images install their native package rather than copying an unrelated binary. -- Homebrew downloads the expected upstream Apple Silicon archive. +- Homebrew installs the host plus the matching runtime under `libexec` from the + expected upstream Apple Silicon product archive. - npm selects the current host's advertised prebuilt addon. - No test invokes a shadowing user-local executable. - No channel resolves to an older release. diff --git a/.skills/distribution-certification/references/format-checks.md b/.skills/distribution-certification/references/format-checks.md index d7a4ab5..05a64cb 100644 --- a/.skills/distribution-certification/references/format-checks.md +++ b/.skills/distribution-certification/references/format-checks.md @@ -25,7 +25,11 @@ For every package: - run `dpkg-deb --info` and `dpkg-deb --contents`; - require package `mesh-llm`, normalized upstream version equal to the requested version, correct Debian architecture, and expected executable ownership; -- validate backend-specific dependency metadata against the artifact name. +- require ownership of + `/usr/local/lib/mesh-llm//native-runtimes/` plus the + product and host-import manifests; +- validate backend-specific dependency metadata against the artifact name; +- reject backend libraries installed beside `/usr/local/bin/mesh-llm`. Before installing, run a simulated local-package transaction and capture the complete apt/dpkg transaction state, including dependencies, package @@ -37,7 +41,9 @@ Install the CPU package through apt/dpkg, then: - query installed version; - invoke the package-owned path, normally `/usr/local/bin/mesh-llm`; -- run `--version` and `runtime list`; +- run `--version`, `--help`, and `runtime list` without GPU passthrough; +- require `runtime list` to discover the package-owned runtime with an empty + user cache and verify the runtime was not copied into that cache; - start isolated client mode using unique API and console ports; - observe readiness and endpoint from a second SSH channel; - send SIGINT and require bounded shutdown. @@ -64,11 +70,15 @@ For every package: - inspect with `pacman -Qip` and `pacman -Qlp`; - require package `mesh-llm`, normalized requested version, `x86_64` or the matrix architecture, expected executable ownership, and backend-consistent - dependencies. + dependencies; +- require ownership of the versioned runtime tree and product/host-import + manifests, with no backend libraries beside the host executable. Install only the designated smoke package when safe. Query it with `pacman -Q`, -then run the exact owned executable's version, runtime list, isolated client -readiness, optional endpoint probe, SIGINT, and bounded shutdown. +then run the exact owned executable's version, help, runtime list with an empty +user cache and no device passthrough, isolated client readiness, optional +endpoint probe, SIGINT, and bounded shutdown. Confirm runtime discovery uses the +package-owned versioned tree without populating the user runtime cache. On cleanup: @@ -103,7 +113,11 @@ Then: - run `brew audit --strict --online` and `brew test`; - resolve the exact executable through `brew --prefix mesh-llm`; - verify its file architecture and `--version`; -- run its `runtime list`; +- verify `libexec/native-runtimes/`, `product-manifest.json`, and + `host-imports.json` are formula-owned; +- run `--help` and `runtime list` without device passthrough and with an empty + user cache, requiring discovery of the formula-owned runtime without copying + it into the cache; - start the exact Cellar/opt binary with isolated HOME/runtime and unique API and console ports; - require readiness, optional `/v1/models` HTTP success, and clean SIGINT. @@ -183,17 +197,21 @@ For every image: 6. Prove native-package installation: - Ubuntu: `dpkg-query`, `dpkg -s`, and `dpkg -L`. - Arch: `pacman -Q`, `pacman -Qi`, and `pacman -Ql`. -7. Require the installed package to own the expected executable path. -8. Run a unique `--rm` version container from - `@sha256:` and require exact MeshLLM version. -9. Start a uniquely named client container with explicit platform, +7. Require the installed package to own the expected executable path, versioned + runtime tree, product manifest, and host-import report. +8. Verify the host-import report rejects no backend imports and no backend + libraries are installed beside the host executable. +9. Run unique `--rm` version, help, and runtime-list containers without device + passthrough from `@sha256:`. Require exact MeshLLM + version and discovery of the package-owned runtime with an empty user cache. +10. Start a uniquely named client container with explicit platform, `--log-format json --no-console client --auto`, and no GPU/device passthrough. -10. Require a ready event and running container. -11. Do not publish ports unless performing an endpoint probe. -12. Send SIGINT, wait with a bound, and capture logs plus final +11. Require a ready event and running container. +12. Do not publish ports unless performing an endpoint probe. +13. Send SIGINT, wait with a bound, and capture logs plus final `Running=false`/exit code. -13. Remove the test container in a trap/finally path. +14. Remove the test container in a trap/finally path. Run AMD64 images under emulation on Apple Silicon when Docker supports it and record native versus emulated execution. If the daemon cannot emulate an diff --git a/.skills/distribution-certification/references/report-contract.md b/.skills/distribution-certification/references/report-contract.md index 4547551..97e5784 100644 --- a/.skills/distribution-certification/references/report-contract.md +++ b/.skills/distribution-certification/references/report-contract.md @@ -27,7 +27,10 @@ Record: - command text, redacted output, and exit code; - artifact URL, release asset ID/timestamp, checksum, and OCI digest; - package metadata/contents/ownership; +- product-manifest, host-import, and selected-runtime identities and digests; - selected executable/addon path; +- adjacent runtime discovery and proof that the isolated user cache stayed + empty; - readiness/status/endpoint evidence; - signal, bounded wait, and final exit state; - pre-existing state and cleanup verification. diff --git a/AGENTS.md b/AGENTS.md index 26cd2bb..dae7aa8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,10 +23,11 @@ Keep work structured around explicit artifacts and avoid one-off build paths. Build shared artifacts first, then fan out only after those artifacts are ready: 1. Resolve the `mesh-llm` release ref once to an immutable source SHA. -2. Build source-independent artifacts once, such as the UI dist. -3. Build native backend artifacts per distro/backend/platform row, such as the llama.cpp ABI directory. -4. Build the final `mesh-llm` binary per distro/backend/platform row from those restored artifacts. -5. Build native package artifacts from the binary and metadata. +2. Resolve and verify one backend-neutral host artifact per OS/architecture. +3. Resolve and verify one native runtime per platform/backend/backend-version. +4. Compose the immutable inputs into a product-v2 bundle; never rebuild the host + for a backend alias. +5. Build native package artifacts from the complete bundle and metadata. 6. Assemble Docker runtime images from the native package artifact for that same row. When adding a new distro or backend, update all affected layers in the same @@ -39,7 +40,7 @@ Docker images should exercise the same package artifacts users receive. The preferred flow is: ```text -binary artifact -> native package artifact -> runtime image installs native package +host artifact + runtime artifact -> product bundle -> native package -> runtime image ``` Do not add a second path that rebuilds `mesh-llm` directly inside the final diff --git a/README.md b/README.md index 95f41c7..1215de2 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,15 @@ This repository is the packaging and distribution control plane for published [` ```text published upstream tag + immutable tag SHA - -> verified release archives -> native packages -> OCI images + -> verified host/runtime product bundles -> native packages -> OCI images -> npm addon lanes -> assembled @mesh-llm/sdk tarball -> clean install test ``` -Application packages and images never rebuild `mesh-llm`; they consume verified upstream binaries. The npm lanes are the sole exception: they check out the immutable release SHA to compile the SDK's N-API addons. `native-runtimes.json` and native runtime archives remain upstream assets. +Application packages and images never rebuild `mesh-llm`; they consume the +verified backend-neutral host and selected native runtime from an upstream +product-v2 bundle. The npm lanes are the sole exception: they check out the +immutable release SHA to compile the SDK's N-API addons, which use the same +dynamic runtime resolver. ## Supported channels diff --git a/TODO.md b/TODO.md index 0bf02f3..8169f98 100644 --- a/TODO.md +++ b/TODO.md @@ -65,22 +65,19 @@ strategy. filters for `ubuntu-cuda-12.9.2` on amd64/arm64 and `ubuntu-cuda-13.1.2` on amd64, and the image-matrix test suite. -- [x] Keep native runtime artifacts out of package-manager outputs. - - Final result: source builds now use `dynamic-native-runtime` by default, - native packages are documented as application packages only, and - `native-runtimes.json`/native runtime archives remain upstream `mesh-llm` - release assets. - - QA: run matrix validation, image-matrix tests, Dockerfile checks for UI, - binary, native-package, and runtime targets, and docs scans for stale - native-runtime bundling language. +- [ ] Compose package-manager outputs from upstream contract-v2 products. + - Final result: archive verification enforces the product schema, checks both + immutable digests, and stages the backend-neutral host plus selected runtime + into Debian/Arch, Homebrew, and OCI outputs without rebuilding either input. + - QA: run upstream archive/schema tests, matrix validation, Homebrew rendering + tests, shell syntax checks, Dockerfile checks, and one package-install smoke + proving ownership of the versioned runtime directory. - [ ] Convert official release packaging to consume upstream release archives. - - Keep source-build paths for dry runs, but official package-manager - publication should start from upstream `package-release.sh` outputs when - those release assets are available. - - QA: package one Ubuntu CPU `.deb`, one Homebrew tarball/formula, and one OCI - image from upstream release archive inputs without rebuilding native - runtimes in this repository. + - Official package-manager publication starts from upstream composed + `package-release` outputs. Source compilation is not a package/image lane. + - QA: package one Ubuntu CPU `.deb`, one Homebrew formula, and one OCI image + from upstream inputs without rebuilding the host or native runtime. - [x] Harden native package quality checks. - `.deb`: run `dpkg-deb --info` and `lintian` where available. diff --git a/docker/Dockerfile.mesh-llm b/docker/Dockerfile.mesh-llm index a73aa5a..7938e04 100644 --- a/docker/Dockerfile.mesh-llm +++ b/docker/Dockerfile.mesh-llm @@ -4,7 +4,7 @@ ARG PACKAGE_BASE_IMAGE=ubuntu:24.04 ARG RUNTIME_BASE_IMAGE=ubuntu:24.04 # The upstream release workflow owns compilation. This stage only converts its -# verified binary into the native package users receive. +# verified host-plus-runtime product bundle into the native package users receive. FROM ${PACKAGE_BASE_IMAGE} AS native-package-builder ARG DISTRO=ubuntu ARG BACKEND=cpu @@ -15,7 +15,7 @@ COPY docker/install-package-build-deps.sh /usr/local/bin/install-package-build-d RUN sh /usr/local/bin/install-package-build-deps.sh "${DISTRO}" WORKDIR /work COPY packaging/native/build-package.sh ./build-package.sh -COPY artifacts/upstream/mesh-llm ./mesh-llm +COPY artifacts/upstream ./mesh-bundle RUN sh ./build-package.sh \ "${DISTRO}" \ "${BACKEND}" \ @@ -23,7 +23,7 @@ RUN sh ./build-package.sh \ "${TARGET_ARCH}" \ "${MESH_LLM_VERSION}" \ /out/packages \ - ./mesh-llm + ./mesh-bundle FROM scratch AS native-package-artifact COPY --from=native-package-builder /out/packages / @@ -37,6 +37,9 @@ ARG BACKEND_VERSION= ARG MESH_LLM_VERSION ARG MESH_LLM_REF ARG MESH_LLM_SOURCE_SHA +ARG MESH_LLM_HOST_SHA +ARG MESH_LLM_RUNTIME_ID +ARG MESH_LLM_RUNTIME_SHA ARG MESH_LLM_REPOSITORY=Mesh-LLM/mesh-llm LABEL org.opencontainers.image.title="mesh-llm" \ org.opencontainers.image.description="Packaged mesh-llm runtime" \ @@ -47,7 +50,10 @@ LABEL org.opencontainers.image.title="mesh-llm" \ io.mesh-llm.upstream.source="https://github.com/${MESH_LLM_REPOSITORY}" \ io.mesh-llm.backend="${BACKEND}" \ io.mesh-llm.backend.version="${BACKEND_VERSION}" \ - io.mesh-llm.release.ref="${MESH_LLM_REF}" + io.mesh-llm.host.sha256="${MESH_LLM_HOST_SHA}" \ + io.mesh-llm.release.ref="${MESH_LLM_REF}" \ + io.mesh-llm.runtime.id="${MESH_LLM_RUNTIME_ID}" \ + io.mesh-llm.runtime.sha256="${MESH_LLM_RUNTIME_SHA}" COPY docker/install-runtime-deps.sh /usr/local/bin/install-runtime-deps.sh COPY docker/install-native-package.sh /usr/local/bin/install-native-package.sh COPY artifacts/native-package /packages diff --git a/docker/qa-runtime-image.sh b/docker/qa-runtime-image.sh index 34d6d75..ae87f12 100755 --- a/docker/qa-runtime-image.sh +++ b/docker/qa-runtime-image.sh @@ -23,17 +23,15 @@ case "$distro" in esac test -x /usr/local/bin/mesh-llm -if [ "$backend" = "cuda" ]; then - # The NVIDIA container runtime injects libcuda.so.1 from the host. Every - # user-space library shipped by the image must already resolve here. - missing="$(ldd /usr/local/bin/mesh-llm | awk '/not found/ { print $1 }')" - [ "$missing" = "libcuda.so.1" ] || { - echo "unexpected CUDA runtime dependency set: ${missing:-none missing}" >&2 - exit 1 - } -else - mesh-llm --version | grep -F "$version" - mesh-llm runtime list +missing="$(ldd /usr/local/bin/mesh-llm | awk '/not found/ { print $1 }')" +[ -z "$missing" ] || { echo "host has unresolved dependencies: $missing" >&2; exit 1; } +if ldd /usr/local/bin/mesh-llm | grep -Eiq 'cuda|cublas|nccl|hip|hsa|vulkan|ggml|llama'; then + echo "backend dependency leaked into the mesh-llm host" >&2 + exit 1 fi +test "$(find "/usr/local/lib/mesh-llm/$version/native-runtimes" -name manifest.json -type f | wc -l)" -eq 1 +test -f "/usr/local/lib/mesh-llm/$version/product-manifest.json" +mesh-llm --version | grep -F "$version" +mesh-llm runtime list printf 'runtime image QA passed for %s/%s\n' "$distro" "$backend" diff --git a/docs/gpu-runbooks.md b/docs/gpu-runbooks.md index c1b26d3..4963858 100644 --- a/docs/gpu-runbooks.md +++ b/docs/gpu-runbooks.md @@ -14,6 +14,15 @@ Vulkan images require the distro Vulkan loader. A loader package failure is down ## Device validation -GitHub-hosted runners validate archive integrity, package installation, and the command surface without GPU devices. CUDA package startup uses NVIDIA's SDK driver stub; the final-image check requires every dependency except the host-injected `libcuda.so.1` to resolve. ROCm and Vulkan command smoke runs without a device. None of these checks prove inference on NVIDIA, AMD, or Vulkan hardware. Hardware qualification should consume the published candidate image on a controlled host and record driver, device, runtime installation, and inference evidence. Do not add self-hosted build runners to this packaging workflow merely to perform device qualification. +GitHub-hosted runners validate archive integrity, package installation, and the +command surface without GPU devices or driver stubs. The host executable is +backend-neutral, so `--version`, `--help`, and `runtime list` must work for +CUDA, ROCm, Vulkan, and CPU images without device passthrough. A direct +`libcuda.so.1`, ROCm, Vulkan, ggml, or llama import from the host is a product +failure. These checks do not prove inference on NVIDIA, AMD, or Vulkan +hardware. Hardware qualification should consume the published candidate image +on a controlled host and record driver, device, runtime selection, and +inference evidence. Do not add self-hosted build runners to this packaging +workflow merely to perform device qualification. If hardware validation fails, first reproduce with the exact versioned image tag and inspect `mesh-llm runtime list`. Retagging or rebuilding the package cannot repair a host-driver or upstream runtime-bundle defect. diff --git a/docs/matrix.md b/docs/matrix.md index 12d2be2..0797971 100644 --- a/docs/matrix.md +++ b/docs/matrix.md @@ -1,6 +1,9 @@ # Packaging matrix -`packaging/images.json` is the only target source of truth. Schema 2 separates the upstream compiled flavor from the downstream package/runtime presentation. +`packaging/images.json` is the only target source of truth. Schema 2 separates +the selected upstream runtime flavor from the downstream package/image +presentation. Every upstream row is a product-v2 archive containing the +OS/architecture-neutral host plus exactly one runtime. Each active row declares its distro, backend display version, `upstream_flavor`, package format/base, runtime base, platforms, support level, and release track. Matrix expansion derives the upstream archive/checksum URLs, deduplicated archive artifact ID, package artifact name, GitHub-hosted runner, and OCI tags. @@ -34,4 +37,9 @@ Linux arm64/x64, and Windows x64. `npm-matrix` expands the enabled lanes; ## Archive deduplication -Ubuntu and Arch rows with the same platform/flavor share one verified upstream artifact. A full matrix currently expands to 11 package rows from 8 Linux archives, plus one macOS archive for Homebrew. That is the main efficiency boundary: compilation happens once upstream, verification once here, and distro packaging fans out afterward. +Ubuntu and Arch rows with the same platform/flavor share one verified composed +product. A full matrix currently expands to 11 package rows from 8 Linux +products, plus one macOS product for Homebrew. Host compilation happens once +per OS/architecture upstream; runtime compilation happens once per runtime +row; this repository verifies composition once and fans out distro packaging +without rebuilding either layer. diff --git a/docs/native-packages.md b/docs/native-packages.md index e593462..8143fef 100644 --- a/docs/native-packages.md +++ b/docs/native-packages.md @@ -1,12 +1,21 @@ # Native packages -The package pipeline is deliberately binary-only: +The package pipeline is deliberately composition-only: ```text -verified upstream mesh-bundle/mesh-llm -> native package -> install QA -> OCI image +verified upstream host + runtime bundle -> native package -> install QA -> OCI image ``` -`scripts/upstream-archive.ts` requires a matching one-line SHA256 sidecar, rejects unsafe or unexpected archive layouts, extracts only `mesh-bundle/mesh-llm`, and records source URL/digest/version/flavor provenance. `packaging/native/build-package.sh` stages that binary and produces exactly one package with version, distro, architecture, backend, and backend version in its filename. +`scripts/upstream-archive.ts` requires a matching one-line SHA256 sidecar, +enforces `schemas/product-v2.schema.json` semantics and a strict archive +allowlist, verifies the host/runtime digests, extracts the complete product +bundle, and records both immutable inputs in provenance. +The producer repository carries an identical product-v2 schema. Contract +changes update both copies together; a release must not proceed with +unexplained schema drift. +`packaging/native/build-package.sh` stages that verified bundle and produces +exactly one package with version, distro, architecture, backend, and backend +version in its filename. Supported emitted formats are `.deb` for Ubuntu and `.pkg.tar.zst` for Arch. APK construction exists as a future format helper but no Alpine row is emitted until upstream provides musl binaries. @@ -14,10 +23,20 @@ All variants use the package identity `mesh-llm`; backend/distro details belong Native metadata declares the user-space loader dependencies needed by the selected backend. Ubuntu CUDA packages depend on the matching toolkit-series CUDA runtime, cuBLAS, and NCCL packages; Ubuntu ROCm depends on hipBLAS, which pulls its ROCm BLAS/runtime closure. The GPU vendor repository is therefore a prerequisite for installing those packages outside the configured vendor base. Host driver libraries and devices are intentionally not package dependencies. -`scripts/native-package-qa.sh` verifies the exact filename and single-package invariant, writes SHA256 manifests, inspects native metadata, installs through the distro package manager in the configured runtime base, and runs `mesh-llm --version` plus `mesh-llm runtime list`. CUDA QA temporarily installs the matching small `cuda-driver-dev` package so the commands can load its vendor-provided `libcuda` stub; the real `libcuda.so.1` remains a host-driver responsibility and is never packaged into the application or final image. - -The Dockerfile's `runtime-qa` stage extends the exact final runtime stage and verifies that it contains the native `mesh-llm` package. CPU, Vulkan, and ROCm execute the command surface. CUDA must resolve every shared library except `libcuda.so.1`, the one library injected by the NVIDIA container runtime on a GPU host. Dry runs emit this stage as BuildKit cache only instead of exporting and loading a duplicate image tarball. This separates offline packaging proof from hardware qualification without hiding an unexpected missing dependency. - -Packages contain the application binary only. Native runtimes and `native-runtimes.json` remain owned and distributed by upstream MeshLLM. +`scripts/native-package-qa.sh` verifies the exact filename and single-package +invariant, writes SHA256 manifests, inspects native metadata, installs through +the distro package manager, proves ownership of the host plus the versioned +runtime directory, and runs `mesh-llm --version` plus `mesh-llm runtime list` +without a GPU device or driver. + +The Dockerfile's `runtime-qa` stage extends the exact final runtime stage. It +verifies package ownership, rejects backend imports or unresolved libraries +from the host executable, and exercises the command surface without device +access. Backend libraries may reference their driver interface only from inside +the native runtime. Hardware-qualified serving is separate additive coverage. + +Packages install the host at `/usr/local/bin/mesh-llm` and the selected runtime +at `/usr/local/lib/mesh-llm//native-runtimes/`, alongside +the product manifest and host import report. Native package repositories are not a current release channel. GitHub Release assets may be published with checksums and SBOMs; apt/apk/pacman repositories remain blocked until the signing requirements in `package-signing.md` are implemented. diff --git a/docs/package-signing.md b/docs/package-signing.md index b544a7e..8d6f84a 100644 --- a/docs/package-signing.md +++ b/docs/package-signing.md @@ -1,9 +1,10 @@ # Package signing policy Unsigned native package files may be attached to GitHub Releases only when they -are accompanied by SHA256 manifests, SBOMs, and GitHub artifact attestation -verification notes. Do not publish unsigned artifacts through apt, apk, pacman, -or Homebrew package repositories. +are accompanied by SHA256 manifests, SBOMs, GitHub artifact-attestation +verification notes, and composition provenance naming the immutable host and +runtime digests. Do not publish unsigned artifacts through apt, apk, pacman, or +Homebrew package repositories. ## Repository signing requirements @@ -30,8 +31,9 @@ each package format: date, and rotation schedule in release documentation before repository launch. - Rotate keys on a documented cadence and immediately after any suspected secret exposure. -- Keep GitHub Release assets, checksums, SBOMs, attestation references, image - digests, and signatures for the full support window of each `mesh-llm` release. +- Keep GitHub Release assets, host/runtime/product checksums, SBOMs, + composition provenance, attestation references, image digests, and signatures + for the full support window of each `mesh-llm` release. ## Release gate diff --git a/docs/packaging-readiness-gaps.md b/docs/packaging-readiness-gaps.md index 2bdb536..ade7ac3 100644 --- a/docs/packaging-readiness-gaps.md +++ b/docs/packaging-readiness-gaps.md @@ -10,7 +10,12 @@ The repository implementation is archive-first: it verifies already-built upstre - [x] Make dry-run execute all validation while forcibly skipping publication. QA: workflow policy and final readiness job. - [x] Remove Blacksmith/self-hosted orchestration. QA: precheck scans workflow/Docker paths for legacy runner/source-build strings. - [x] Correct channel claims: block Alpine/musl and Intel macOS; enable upstream Linux Vulkan and arm64 CUDA 13. QA: matrix tests. -- [x] Make GPU package and image QA accurate on GitHub-hosted runners. QA: CUDA package startup uses the vendor SDK driver stub, final CUDA images report only `libcuda.so.1` as host-injected, the lean ROCm 7.0 image stays within hosted disk, and full dry run [29455769787](https://github.com/Mesh-LLM/mesh-packaging/actions/runs/29455769787) succeeds. +- [ ] Re-certify GPU package and image QA against product-v2 bundles. QA: + backend-neutral hosts pass `--version`, `--help`, and `runtime list` without + devices or driver stubs; packages own versioned runtime trees; a new full dry + run succeeds. The earlier static-host baseline is preserved in + [run 29455769787](https://github.com/Mesh-LLM/mesh-packaging/actions/runs/29455769787) + for historical comparison only. ## Operational work outside this checkout diff --git a/docs/packaging-readiness-scorecard.md b/docs/packaging-readiness-scorecard.md index 5eeabee..8aba899 100644 --- a/docs/packaging-readiness-scorecard.md +++ b/docs/packaging-readiness-scorecard.md @@ -4,12 +4,12 @@ Readiness is evidence-based, not a static percentage. A release is ready only wh | Area | Implemented gate | Remaining operational proof | |---|---|---| -| Upstream accuracy | Published tag, release, exact archive names, checksums, safe layout, version smoke | Run against each new release | +| Upstream accuracy | Published tag/release, exact product names/checksums, strict product-v2 layout, host/runtime digest verification | Run against each new release | | Matrix coverage | Schema validation and explicit active/blocked rows | Review when upstream asset inventory changes | -| Package correctness | Native metadata, checksum, install, version/runtime command smoke | Full dry-run evidence | -| OCI correctness | Image installs the exact package and repeats command smoke | Full dry-run evidence | -| Homebrew | Direct upstream arm64 archive, strict audit, install, test, and canonical tap sync | Validate each release before tap update | -| Efficiency | 8 verified archives fan out to 11 package rows; no source builds | Record duration and cache behavior | +| Package correctness | Native metadata, checksum, host/runtime ownership, no-device version/help/runtime-list smoke | Full dry-run evidence | +| OCI correctness | Image installs the exact package, labels both input digests, and repeats no-device smoke | Full dry-run evidence plus separate hardware qualification | +| Homebrew | Direct upstream arm64 product, formula-owned runtime, strict audit/install/test, canonical tap sync | Validate each release before tap update | +| Efficiency | One host per OS/architecture is composed with 8 verified runtimes and fans out to 11 package rows; no source builds | Record duration, artifact sizes, and cache behavior | | Publish safety | Dry-run override, job-local write permissions, release environment | Configure/approve environment and observe first publish rehearsal | | Automation | Repository dispatch receiver | Provision upstream fine-grained dispatch credential/App | diff --git a/docs/publishing.md b/docs/publishing.md index 2193062..1b7f8a6 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -15,8 +15,10 @@ GitHub Release assets, GHCR, npm, and the [`Mesh-LLM/tap`](https://github.com/Mesh-LLM/homebrew-tap) Homebrew tap are the enabled public channels. Do not create apt, apk, or pacman repositories until signing/trust-root ownership and rollback procedures exist. Homebrew publishes -a formula that references the immutable upstream macOS archive and its -upstream-verified SHA256; it does not repackage that binary. The tap polls this +a formula that references the immutable upstream macOS product archive and its +upstream-verified SHA256. The formula installs the host in `bin`, the selected +runtime under formula-owned `libexec/native-runtimes`, and the product/import +manifests in `libexec`; it does not rebuild either input. The tap polls this repository's latest non-prerelease packaging release, validates and installs the attached `mesh-llm.rb`, and commits it only when it changes. @@ -31,7 +33,10 @@ input from the `v0.74.0` bootstrap release and normalizes the assembled package to `@mesh-llm/sdk`. All other package names are rejected, and subsequent upstream releases declare the canonical name directly. -Moving convenience OCI tags are published alongside immutable version tags. Rollback must never mutate the versioned tag silently: stop the affected row, preserve evidence, and publish a new upstream version or explicit correction record. +Moving convenience OCI tags are published alongside immutable version tags. +Rollback must never mutate a versioned host, runtime, product, package, or image +silently: stop the affected row, preserve both input digests as evidence, and +publish a new upstream version or explicit correction record. This repository is the sole GHCR producer. A successful non-canary `Mesh-LLM/mesh-llm` release with the complete GPU bundle set dispatches diff --git a/docs/release-checklist.md b/docs/release-checklist.md index fb39772..471c169 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -4,14 +4,21 @@ - [ ] The upstream tag and non-draft GitHub Release exist and the tag resolves to one immutable SHA. - [ ] Every generated archive and `.sha256` sidecar exists in that release. -- [ ] Archive verification accepts only `mesh-bundle/mesh-llm` and confirms the extracted Linux payload is an ELF executable. Version startup is proven later inside the matching package/runtime base because GPU binaries require vendor shared libraries. +- [ ] Archive verification accepts only the product-v2 host, host-import report, + product manifest, and exactly one runtime tree. It verifies every recorded + digest and rejects unexpected or traversal-prone entries. - [ ] The full matrix matches the current release asset inventory. Unsupported channels remain disabled rather than inferred. ## Dry-run QA - [ ] A full `dry_run=true` workflow succeeds with publish jobs skipped. -- [ ] Every native package passes metadata inspection, exact filename/checksum checks, package-manager installation, `mesh-llm --version`, and `mesh-llm runtime list`; CUDA command smoke uses only the matching vendor SDK driver stub. -- [ ] Every runtime image installs the matching package artifact. CPU, Vulkan, and ROCm pass command smoke; CUDA resolves every shared dependency except host-injected `libcuda.so.1`. +- [ ] Every native package passes metadata inspection, exact filename/checksum + checks, package-manager installation, `mesh-llm --version`, `--help`, and + `mesh-llm runtime list` without GPU passthrough. The package owns the + versioned runtime tree, and an empty user cache stays empty. +- [ ] Every runtime image installs the matching package artifact. Every backend, + including CUDA, passes the no-device command smoke and the neutral host has no + backend runtime imports. - [ ] The arm64 Homebrew formula installs and tests the upstream Metal archive. - [ ] Every enabled npm addon lane succeeds; the assembled tarball passes `npm publish --dry-run`, installs in a clean project, and loads the host addon. diff --git a/docs/runner-capacity.md b/docs/runner-capacity.md index a65ffa7..3c469ba 100644 --- a/docs/runner-capacity.md +++ b/docs/runner-capacity.md @@ -2,8 +2,20 @@ All automation uses GitHub-hosted runners. Linux amd64 uses `ubuntu-24.04`, Linux arm64 uses `ubuntu-24.04-arm`, and Homebrew uses `macos-15`. No self-hosted or Blacksmith runner contract remains. -The full active matrix has 11 Linux package/image rows but only 8 unique Linux upstream archives. Archive verification is deduplicated before distro fan-out. Compilation, UI generation, and llama.cpp builds happen only in upstream MeshLLM, eliminating the largest former cost and drift source. +The full active matrix has 11 Linux package/image rows but only 8 unique Linux +product archives. Archive, host digest, runtime digest, and product-manifest +verification are deduplicated before distro fan-out. Host compilation, UI +generation, and native-runtime builds happen only in upstream MeshLLM, +eliminating the largest former cost and drift source. Use `variant_filter` and `platform_filter` for review iteration. A production dry run should still exercise every active row because rolling Arch dependencies and vendor runtime bases can drift independently even when the upstream binary is unchanged. BuildKit GitHub cache scopes are per package row to keep package layers reusable without cross-row contamination. Runtime dry runs target `runtime-qa` with `type=cacheonly` and deliberately do not export a GitHub Actions cache: exporting either an image tarball or multi-gigabyte Arch CUDA/ROCm cache layers costs more disk, bandwidth, and cache quota than rebuilding the vendor package layer in place. -The expected cost order is CPU < Vulkan < CUDA < ROCm, driven here by QA/runtime base download and package installation rather than compilation. The ROCm row deliberately uses `rocm/dev-ubuntu-24.04:7.0`; its `complete` sibling is more than 5 GB compressed and exhausts a standard hosted runner during extraction. The first complete v0.73.1 dry run after these optimizations finished all 35 jobs in 9m57s. Record subsequent full-run durations and artifact sizes in release notes until enough history exists to establish budgets. +The expected cost order is CPU < Vulkan < CUDA < ROCm, driven here by +QA/runtime-base download and package installation rather than compilation. All +rows run neutral-host command smoke without a device; optional hardware +qualification belongs on controlled GPU runners and does not replace the +hosted no-device gate. The ROCm row deliberately uses +`rocm/dev-ubuntu-24.04:7.0`; its `complete` sibling is more than 5 GB compressed +and exhausts a standard hosted runner during extraction. The first complete +v0.73.1 static-host dry run finished all 35 jobs in 9m57s; keep it as historical +data and record a new product-v2 baseline before setting current budgets. diff --git a/docs/tagging.md b/docs/tagging.md index 3182d41..092f3ef 100644 --- a/docs/tagging.md +++ b/docs/tagging.md @@ -11,4 +11,8 @@ Examples: `0.73.1-ubuntu-arm64-cuda13.1.2`, `ubuntu-arm64-cuda13.1.2`, and `0.73 Package filenames follow `mesh-llm----[backend-version].`. This makes the package installed in an image traceable without inspecting registry metadata. -There is no generic `latest` GPU tag and no implicit multi-architecture tag. Standard OCI labels record source, version, immutable upstream revision, release ref, backend, and backend version. +There is no generic `latest` GPU tag and no implicit multi-architecture tag. +Standard OCI labels record source, version, immutable upstream revision, +release ref, backend/backend version, neutral-host digest, selected runtime ID, +and runtime digest. Different backend tags for the same OS/architecture must +name the same host digest. diff --git a/packaging/homebrew/Formula/mesh-llm.rb.template b/packaging/homebrew/Formula/mesh-llm.rb.template index 0429d42..063c1cd 100644 --- a/packaging/homebrew/Formula/mesh-llm.rb.template +++ b/packaging/homebrew/Formula/mesh-llm.rb.template @@ -8,11 +8,15 @@ class MeshLlm < Formula depends_on arch: :arm64 def install - # Homebrew strips the archive's single mesh-bundle/ top-level directory. bin.install "mesh-llm" + libexec.install "native-runtimes" + libexec.install "product-manifest.json" + libexec.install "host-imports.json" end test do assert_match version.to_s, shell_output("#{bin}/mesh-llm --version") + assert_match "native runtime", shell_output("#{bin}/mesh-llm runtime list") + assert_predicate libexec/"product-manifest.json", :exist? end end diff --git a/packaging/homebrew/README.md b/packaging/homebrew/README.md index 5664406..48d33d4 100644 --- a/packaging/homebrew/README.md +++ b/packaging/homebrew/README.md @@ -9,10 +9,13 @@ mesh-llm-v-aarch64-apple-darwin.tar.gz -> brew install and brew test on macos-15 ``` -Homebrew strips the archive's single `mesh-bundle/` top-level directory before -the formula installs `mesh-llm`; this repository does not rebuild or re-tar the -binary. Intel is intentionally unsupported until upstream publishes an x86_64 -macOS archive. +Homebrew strips the archive's single `mesh-bundle/` top-level directory. The +formula installs the backend-neutral host into `bin` and the selected Metal +runtime plus product/import manifests into formula-owned `libexec`. Runtime +discovery resolves `libexec/native-runtimes` without a cache download. This +repository does not rebuild or re-tar either immutable input. Intel is +intentionally unsupported until upstream publishes an x86_64 macOS product +bundle. The rendered formula is attached to this repository's package release. [`Mesh-LLM/homebrew-tap`](https://github.com/Mesh-LLM/homebrew-tap) polls the diff --git a/packaging/native/README.md b/packaging/native/README.md index 1ac0b37..7ddf696 100644 --- a/packaging/native/README.md +++ b/packaging/native/README.md @@ -1,5 +1,12 @@ # Native package builder -`build-package.sh` accepts a verified upstream `mesh-llm` binary plus matrix metadata and emits one `.deb`, `.apk`, or `.pkg.tar.zst`. Active rows currently emit Ubuntu `.deb` and Arch `.pkg.tar.zst`; APK is retained only for a future upstream musl archive. +`build-package.sh` accepts a verified upstream product-v2 bundle plus matrix +metadata and emits one `.deb`, `.apk`, or `.pkg.tar.zst`. The package owns both +`/usr/local/bin/mesh-llm` and the selected runtime under +`/usr/local/lib/mesh-llm//native-runtimes`. Active rows currently emit +Ubuntu `.deb` and Arch `.pkg.tar.zst`; APK is retained only for a future +upstream musl archive. -The final OCI target installs the exact emitted package through the distro package manager. Do not add source compilation, direct binary-copy runtime paths, native runtime archives, or `native-runtimes.json` here. +The final OCI target installs the exact emitted package through the distro +package manager. Do not add source compilation or direct binary/runtime copy +paths to the final image. diff --git a/packaging/native/build-package.sh b/packaging/native/build-package.sh index 32ac3e5..be7c671 100644 --- a/packaging/native/build-package.sh +++ b/packaging/native/build-package.sh @@ -7,9 +7,18 @@ backend_version="${3:-}" arch="${4:?arch is required}" version="${5:?version is required}" output_dir="${6:?output dir is required}" -binary="${7:?mesh-llm binary path is required}" +bundle="${7:?verified mesh-bundle directory is required}" +binary="$bundle/mesh-llm" +product_manifest="$bundle/product-manifest.json" +host_imports="$bundle/host-imports.json" test -f "$binary" || { echo "binary not found: $binary" >&2; exit 1; } +test -f "$product_manifest" || { echo "product manifest not found: $product_manifest" >&2; exit 1; } +test -f "$host_imports" || { echo "host import report not found: $host_imports" >&2; exit 1; } +runtime_count="$(find "$bundle/native-runtimes" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" +[ "$runtime_count" = 1 ] || { echo "expected exactly one native runtime in $bundle" >&2; exit 1; } +runtime_dir="$(find "$bundle/native-runtimes" -mindepth 1 -maxdepth 1 -type d | head -n 1)" +test -f "$runtime_dir/manifest.json" || { echo "runtime manifest not found: $runtime_dir/manifest.json" >&2; exit 1; } backend_suffix="$backend" if [ "$backend" != "cpu" ] && [ -n "$backend_version" ]; then @@ -57,8 +66,15 @@ esac package_file="mesh-llm-${version}-${file_component}.${extension}" work_dir="$(mktemp -d)" root_dir="${work_dir}/root" -mkdir -p "$output_dir" "$root_dir/usr/local/bin" +product_root="$root_dir/usr/local/lib/mesh-llm/$version" +mkdir -p "$output_dir" "$root_dir/usr/local/bin" "$product_root/native-runtimes" install -m 0755 "$binary" "$root_dir/usr/local/bin/mesh-llm" +cp -R "$runtime_dir" "$product_root/native-runtimes/" +install -m 0644 "$product_manifest" "$product_root/product-manifest.json" +install -m 0644 "$host_imports" "$product_root/host-imports.json" +if [ -f "$bundle/upstream-provenance.json" ]; then + install -m 0644 "$bundle/upstream-provenance.json" "$product_root/upstream-provenance.json" +fi installed_size_kb="$(du -sk "$root_dir" | awk '{ print $1 }')" installed_size_bytes="$(du -sb "$root_dir" | awk '{ print $1 }')" diff --git a/schemas/product-v2.schema.json b/schemas/product-v2.schema.json new file mode 100644 index 0000000..2e20fd6 --- /dev/null +++ b/schemas/product-v2.schema.json @@ -0,0 +1,86 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://meshllm.ai/schemas/product-v2.schema.json", + "title": "MeshLLM composed product bundle", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "contract", + "mesh_version", + "backend", + "host", + "runtime" + ], + "properties": { + "schema_version": { + "const": 2 + }, + "contract": { + "const": "mesh-llm-product-v2" + }, + "mesh_version": { + "type": "string", + "minLength": 1 + }, + "backend": { + "type": "string", + "minLength": 1 + }, + "host": { + "$ref": "#/$defs/host_artifact" + }, + "runtime": { + "$ref": "#/$defs/runtime_artifact" + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "host_artifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "sha256" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "$ref": "#/$defs/sha256" + } + } + }, + "runtime_artifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "path", + "sha256", + "manifest_sha256" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "path": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "manifest_sha256": { + "$ref": "#/$defs/sha256" + } + } + } + } +} diff --git a/scripts/native-package-qa.sh b/scripts/native-package-qa.sh index c98f297..3591c27 100755 --- a/scripts/native-package-qa.sh +++ b/scripts/native-package-qa.sh @@ -14,9 +14,8 @@ Options: Environment: NATIVE_PACKAGE_QA_LINTIAN=1 Run lintian for .deb packages through a Debian container. -CUDA note: - CUDA command smoke uses the vendor SDK's libcuda stub. The real libcuda.so.1 - is injected by the NVIDIA container runtime and is unavailable on hosted CI. +All command smoke checks run without a GPU device or driver. Backend libraries +belong to the packaged native runtime, never the mesh-llm host executable. EOF exit 2 } @@ -125,13 +124,7 @@ run_package_container() { } # shellcheck disable=SC2016 -runtime_smoke='mesh-llm --version | grep -F "$EXPECTED_VERSION" && mesh-llm runtime list' -if [ "$backend" = "cuda" ]; then - # Use the SDK-provided driver stub to validate loader closure and the command - # surface without pretending a hosted runner has an NVIDIA device or driver. - # shellcheck disable=SC2016 - runtime_smoke='cuda_stub="$(find /usr/local/cuda-* /opt/cuda -path "*/stubs/libcuda.so" -print -quit 2>/dev/null || true)" && [ -n "$cuda_stub" ] && mkdir -p /tmp/mesh-llm-driver-stubs && ln -sf "$cuda_stub" /tmp/mesh-llm-driver-stubs/libcuda.so.1 && export LD_LIBRARY_PATH="/tmp/mesh-llm-driver-stubs${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" && mesh-llm --version | grep -F "$EXPECTED_VERSION" && mesh-llm runtime list' -fi +runtime_smoke='test "$(find "/usr/local/lib/mesh-llm/$EXPECTED_VERSION/native-runtimes" -name manifest.json -type f | wc -l)" -eq 1 && test -f "/usr/local/lib/mesh-llm/$EXPECTED_VERSION/product-manifest.json" && mesh-llm --version | grep -F "$EXPECTED_VERSION" && mesh-llm runtime list' case "$distro" in ubuntu) @@ -148,14 +141,8 @@ case "$distro" in fi if [ "$install" = true ]; then [ -n "$runtime_base_image" ] || { echo "--runtime-base-image is required for install tests" >&2; exit 1; } - qa_dependencies="" - if [ "$backend" = "cuda" ]; then - [ -n "$backend_version" ] || { echo "CUDA install QA requires --backend-version" >&2; exit 1; } - cuda_series="$(printf '%s\n' "$backend_version" | awk -F. '{ print $1 "-" $2 }')" - qa_dependencies="apt-get install -y --no-install-recommends cuda-driver-dev-$cuda_series && " - fi # shellcheck disable=SC2016 - run_package_container "$runtime_base_image" 'apt-get update && '"$qa_dependencies"'apt-get install -y --no-install-recommends "/packages/$PACKAGE_FILE" && '"$runtime_smoke" + run_package_container "$runtime_base_image" 'apt-get update && apt-get install -y --no-install-recommends "/packages/$PACKAGE_FILE" && '"$runtime_smoke" fi ;; alpine) diff --git a/scripts/upstream-archive.ts b/scripts/upstream-archive.ts index 83803e6..19a736b 100755 --- a/scripts/upstream-archive.ts +++ b/scripts/upstream-archive.ts @@ -1,7 +1,7 @@ #!/usr/bin/env -S node --experimental-strip-types import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { chmodSync, copyFileSync, createReadStream, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, cpSync, createReadStream, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; import { tmpdir } from "node:os"; import { pathToFileURL } from "node:url"; @@ -40,11 +40,117 @@ export function validateArchiveEntries(entries: string[]): void { if (entry.startsWith("/") || entry.split("/").includes("..")) { throw new Error(`unsafe archive entry: ${entry}`); } + if (entry !== "mesh-bundle" && entry !== "mesh-bundle/" && !entry.startsWith("mesh-bundle/")) { + throw new Error(`unexpected archive entry outside mesh-bundle: ${entry}`); + } } const files = normalized.filter((entry) => !entry.endsWith("/")); - if (files.length !== 1 || files[0] !== "mesh-bundle/mesh-llm") { - throw new Error(`archive must contain only mesh-bundle/mesh-llm; found: ${files.join(", ") || "no files"}`); + const required = [ + "mesh-bundle/mesh-llm", + "mesh-bundle/product-manifest.json", + "mesh-bundle/host-imports.json", + ]; + for (const path of required) { + if (!files.includes(path)) throw new Error(`archive is missing required product file: ${path}`); + } + const runtimeIds = new Set(); + for (const path of files) { + if (required.includes(path)) continue; + const match = /^mesh-bundle\/native-runtimes\/([^/]+)\/(.+)$/.exec(path); + if (!match) throw new Error(`unexpected product archive entry: ${path}`); + const [, runtimeId, relative] = match; + if (relative !== "manifest.json" && relative !== "README.md" && !relative.startsWith("lib/") && !relative.startsWith("tools/")) { + throw new Error(`unexpected native runtime entry: ${path}`); + } + runtimeIds.add(runtimeId); + } + if (runtimeIds.size !== 1) { + throw new Error(`archive must contain exactly one native runtime; found ${runtimeIds.size}`); + } + const runtimePrefix = `mesh-bundle/native-runtimes/${[...runtimeIds][0]}`; + for (const path of [`${runtimePrefix}/manifest.json`, `${runtimePrefix}/README.md`]) { + if (!files.includes(path)) throw new Error(`archive is missing required native runtime file: ${path}`); + } + if (!files.some((path) => path.startsWith(`${runtimePrefix}/lib/`))) { + throw new Error(`native runtime must contain at least one library under ${runtimePrefix}/lib`); + } +} + +export function validateArchiveEntryTypes(entries: string[]): void { + for (const entry of entries.filter(Boolean)) { + const type = entry[0]; + if (type !== "-" && type !== "d") { + throw new Error(`archive links and special files are not allowed: ${entry}`); + } + } +} + +type ProductManifest = { + schema_version: number; + contract: string; + mesh_version: string; + backend: string; + host: { path: string; sha256: string }; + runtime: { id: string; path: string; sha256: string; manifest_sha256: string }; +}; + +function assertSha256(value: unknown, name: string): asserts value is string { + if (typeof value !== "string" || !/^[0-9a-f]{64}$/.test(value)) { + throw new Error(`${name} must be a lowercase SHA-256 digest`); + } +} + +function assertExactKeys(value: object, expected: string[], name: string): void { + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index])) { + throw new Error(`${name} contains unexpected or missing fields: ${actual.join(", ")}`); + } +} + +export function validateProductManifest(value: unknown): ProductManifest { + if (!value || typeof value !== "object") throw new Error("product manifest must be an object"); + const manifest = value as ProductManifest; + assertExactKeys(manifest, ["schema_version", "contract", "mesh_version", "backend", "host", "runtime"], "product manifest"); + if (manifest.schema_version !== 2 || manifest.contract !== "mesh-llm-product-v2") { + throw new Error("unsupported product manifest contract"); + } + if (typeof manifest.mesh_version !== "string" || !manifest.mesh_version) throw new Error("product manifest mesh_version is required"); + if (typeof manifest.backend !== "string" || !manifest.backend) throw new Error("product manifest backend is required"); + if (!manifest.host || typeof manifest.host !== "object") throw new Error("product manifest host is required"); + assertExactKeys(manifest.host, ["path", "sha256"], "product host"); + if (manifest.host?.path !== "mesh-llm") throw new Error("product manifest host path must be mesh-llm"); + assertSha256(manifest.host?.sha256, "product host sha256"); + if (!manifest.runtime || typeof manifest.runtime !== "object") throw new Error("product manifest runtime is required"); + assertExactKeys(manifest.runtime, ["id", "path", "sha256", "manifest_sha256"], "product runtime"); + if (typeof manifest.runtime?.id !== "string" || !manifest.runtime.id) throw new Error("product runtime id is required"); + if (manifest.runtime.path !== `native-runtimes/${manifest.runtime.id}`) { + throw new Error("product runtime path must match its runtime id"); + } + assertSha256(manifest.runtime.sha256, "product runtime sha256"); + assertSha256(manifest.runtime.manifest_sha256, "product runtime manifest sha256"); + return manifest; +} + +function filesBelow(root: string, current = root): string[] { + return readdirSync(current).flatMap((name) => { + const path = resolve(current, name); + return statSync(path).isDirectory() ? filesBelow(root, path) : [path]; + }); +} + +export function sha256Tree(root: string): string { + const digest = createHash("sha256"); + for (const path of filesBelow(root).sort()) { + const relative = path.slice(root.length + 1).replaceAll("\\", "/"); + const encoded = Buffer.from(relative); + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(encoded.length)); + digest.update(length); + digest.update(encoded); + digest.update(createHash("sha256").update(readFileSync(path)).digest()); } + return digest.digest("hex"); } function runTar(args: string[]): string { @@ -62,22 +168,36 @@ export async function verifyAndExtract(input: Inputs) { if (actual !== expected) throw new Error(`SHA-256 mismatch for ${archiveName}: expected ${expected}, got ${actual}`); validateArchiveEntries(runTar(["-tzf", archive]).split(/\r?\n/)); + validateArchiveEntryTypes(runTar(["-tvzf", archive]).split(/\r?\n/)); mkdirSync(input.outputDir, { recursive: true }); const temporary = mkdtempSync(resolve(tmpdir(), "mesh-llm-upstream-")); try { - runTar(["-xzf", archive, "-C", temporary, "mesh-bundle/mesh-llm"]); + runTar(["-xzf", archive, "-C", temporary, "mesh-bundle"]); + cpSync(resolve(temporary, "mesh-bundle"), input.outputDir, { recursive: true }); const binary = resolve(input.outputDir, "mesh-llm"); - copyFileSync(resolve(temporary, "mesh-bundle/mesh-llm"), binary); chmodSync(binary, 0o755); + const productManifest = validateProductManifest(JSON.parse(readFileSync(resolve(input.outputDir, "product-manifest.json"), "utf8"))); + if (productManifest.mesh_version !== input.version.replace(/^v/, "")) throw new Error("product manifest version does not match requested upstream version"); + if (productManifest.backend !== input.flavor) throw new Error("product manifest backend does not match requested upstream flavor"); + const hostSha256 = await sha256File(binary); + if (hostSha256 !== productManifest.host.sha256) throw new Error("product host digest does not match extracted mesh-llm"); + const runtime = resolve(input.outputDir, productManifest.runtime.path); + const runtimeSha256 = sha256Tree(runtime); + if (runtimeSha256 !== productManifest.runtime.sha256) throw new Error("product runtime digest does not match extracted runtime tree"); + const runtimeManifestSha256 = await sha256File(resolve(runtime, "manifest.json")); + if (runtimeManifestSha256 !== productManifest.runtime.manifest_sha256) throw new Error("product runtime manifest digest does not match"); const provenance = resolve(input.outputDir, "upstream-provenance.json"); writeFileSync(provenance, `${JSON.stringify({ archive: archiveName, flavor: input.flavor, + host_sha256: productManifest.host?.sha256, + runtime_id: productManifest.runtime?.id, + runtime_sha256: productManifest.runtime?.sha256, sha256: actual, source_url: input.sourceUrl, version: input.version, }, null, 2)}\n`); - return { binary, provenance, sha256: actual }; + return { binary, bundle: resolve(input.outputDir), provenance, sha256: actual }; } finally { rmSync(temporary, { recursive: true, force: true }); } diff --git a/tests/upstream-archive.test.ts b/tests/upstream-archive.test.ts index 19713f3..a003621 100644 --- a/tests/upstream-archive.test.ts +++ b/tests/upstream-archive.test.ts @@ -1,10 +1,18 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, resolve } from "node:path"; import { test } from "node:test"; -import { main, parseChecksum, sha256File, validateArchiveEntries, verifyAndExtract } from "../scripts/upstream-archive.ts"; +import { + main, + parseChecksum, + sha256File, + sha256Tree, + validateArchiveEntries, + validateArchiveEntryTypes, + verifyAndExtract, +} from "../scripts/upstream-archive.ts"; async function fixture(t: { after(callback: () => void): void }) { const directory = mkdtempSync(resolve(tmpdir(), "upstream-test-")); @@ -13,11 +21,33 @@ async function fixture(t: { after(callback: () => void): void }) { mkdirSync(bundle, { recursive: true }); writeFileSync(resolve(bundle, "mesh-llm"), "#!/bin/sh\necho mesh-llm 0.73.1\n"); chmodSync(resolve(bundle, "mesh-llm"), 0o755); + writeFileSync(resolve(bundle, "host-imports.json"), "{}\n"); + const runtime = resolve(bundle, "native-runtimes/linux-cpu"); + mkdirSync(resolve(runtime, "lib"), { recursive: true }); + writeFileSync(resolve(runtime, "manifest.json"), "{}\n"); + writeFileSync(resolve(runtime, "README.md"), "runtime\n"); + writeFileSync(resolve(runtime, "lib/libllama.so"), "runtime\n"); + const hostSha256 = await sha256File(resolve(bundle, "mesh-llm")); + const runtimeSha256 = sha256Tree(runtime); + const runtimeManifestSha256 = await sha256File(resolve(runtime, "manifest.json")); + writeFileSync(resolve(bundle, "product-manifest.json"), JSON.stringify({ + backend: "cpu", + contract: "mesh-llm-product-v2", + host: { path: "mesh-llm", sha256: hostSha256 }, + mesh_version: "0.73.1", + runtime: { + id: "linux-cpu", + manifest_sha256: runtimeManifestSha256, + path: "native-runtimes/linux-cpu", + sha256: runtimeSha256, + }, + schema_version: 2, + })); const archive = resolve(directory, "mesh-llm-v0.73.1-x86_64-unknown-linux-gnu.tar.gz"); assert.equal(spawnSync("tar", ["-czf", archive, "-C", resolve(directory, "stage"), "mesh-bundle"]).status, 0); const checksum = `${archive}.sha256`; writeFileSync(checksum, `${await sha256File(archive)} ${basename(archive)}\n`); - return { directory, archive, checksum }; + return { directory, archive, checksum, hostSha256, runtimeSha256 }; } test("verifies checksum, layout, extraction, and provenance", async (t) => { @@ -25,7 +55,17 @@ test("verifies checksum, layout, extraction, and provenance", async (t) => { const outputDir = resolve(data.directory, "output"); const result = await verifyAndExtract({ archive: data.archive, checksum: data.checksum, outputDir, sourceUrl: "https://example.test/archive", version: "0.73.1", flavor: "cpu" }); assert.match(readFileSync(result.binary, "utf8"), /mesh-llm/); - assert.deepEqual(JSON.parse(readFileSync(result.provenance, "utf8")), { archive: basename(data.archive), flavor: "cpu", sha256: result.sha256, source_url: "https://example.test/archive", version: "0.73.1" }); + assert.equal(existsSync(resolve(outputDir, "native-runtimes/linux-cpu/lib/libllama.so")), true); + assert.deepEqual(JSON.parse(readFileSync(result.provenance, "utf8")), { + archive: basename(data.archive), + flavor: "cpu", + host_sha256: data.hostSha256, + runtime_id: "linux-cpu", + runtime_sha256: data.runtimeSha256, + sha256: result.sha256, + source_url: "https://example.test/archive", + version: "0.73.1", + }); }); test("rejects malformed checksums and unsafe layouts", async (t) => { @@ -37,7 +77,37 @@ test("rejects malformed checksums and unsafe layouts", async (t) => { assert.throws(() => parseChecksum(`${"a".repeat(64)} x\n${"b".repeat(64)} y`, basename(data.archive)), /exactly one/); assert.throws(() => validateArchiveEntries(["../bad"]), /unsafe/); assert.throws(() => validateArchiveEntries(["/bad"]), /unsafe/); - assert.throws(() => validateArchiveEntries(["mesh-bundle/", "mesh-bundle/other"]), /must contain only/); + assert.doesNotThrow(() => validateArchiveEntryTypes([ + "drwxr-xr-x 0 user group 0 Jan 1 00:00 mesh-bundle/", + "-rwxr-xr-x 0 user group 1 Jan 1 00:00 mesh-bundle/mesh-llm", + ])); + assert.throws( + () => validateArchiveEntryTypes([ + "lrwxr-xr-x 0 user group 0 Jan 1 00:00 mesh-bundle/link -> ../../outside", + ]), + /links and special files/, + ); + assert.throws(() => validateArchiveEntries(["mesh-bundle/", "mesh-bundle/other"]), /missing required/); + const required = [ + "mesh-bundle/mesh-llm", + "mesh-bundle/product-manifest.json", + "mesh-bundle/host-imports.json", + ]; + assert.throws(() => validateArchiveEntries(required), /exactly one native runtime/); + assert.throws(() => validateArchiveEntries([ + ...required, + "mesh-bundle/native-runtimes/a/manifest.json", + "mesh-bundle/native-runtimes/a/README.md", + "mesh-bundle/native-runtimes/a/lib/a.so", + "mesh-bundle/native-runtimes/b/manifest.json", + ]), /exactly one native runtime/); + assert.throws(() => validateArchiveEntries([ + ...required, + "mesh-bundle/native-runtimes/a/manifest.json", + "mesh-bundle/native-runtimes/a/README.md", + "mesh-bundle/native-runtimes/a/lib/a.so", + "mesh-bundle/extra", + ]), /unexpected product/); writeFileSync(data.checksum, `${"0".repeat(64)} ${basename(data.archive)}\n`); await assert.rejects(() => verifyAndExtract({ archive: data.archive, checksum: data.checksum, outputDir: resolve(data.directory, "bad"), sourceUrl: "x", version: "x", flavor: "x" }), /mismatch/); }); From 2c55a4edd711dfbcc2411bdd040fead427a22d82 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Tue, 28 Jul 2026 23:51:38 -0400 Subject: [PATCH 2/5] address PR comments --- .github/workflows/images-release.yml | 24 +++++++++- docs/gpu-runbooks.md | 17 ++++--- docs/matrix.md | 2 +- packaging/native/build-package.sh | 1 + schemas/product-v2.schema.json | 6 ++- scripts/upstream-archive.ts | 23 ++++++++-- tests/native-package.test.ts | 31 +++++++++++++ tests/upstream-archive.test.ts | 64 +++++++++++++++++++++++++- tests/workflow-provenance.test.ts | 68 ++++++++++++++++++++++++++++ 9 files changed, 220 insertions(+), 16 deletions(-) create mode 100644 tests/native-package.test.ts create mode 100644 tests/workflow-provenance.test.ts diff --git a/.github/workflows/images-release.yml b/.github/workflows/images-release.yml index 923e830..1d41545 100644 --- a/.github/workflows/images-release.yml +++ b/.github/workflows/images-release.yml @@ -296,7 +296,17 @@ jobs: shell: bash run: | set -euo pipefail - provenance="$(find artifacts/native-package -name '*.upstream-provenance.json' -type f -print -quit)" + mapfile -d '' -t provenance_files < <(find artifacts/native-package -name '*.upstream-provenance.json' -type f -print0) + if (( ${#provenance_files[@]} != 1 )); then + echo "expected exactly one upstream provenance file, found ${#provenance_files[@]}" >&2 + exit 1 + fi + provenance="${provenance_files[0]}" + jq -e ' + (.host_sha256 | type == "string" and test("^[0-9a-f]{64}$")) and + (.runtime_id | type == "string" and length > 0) and + (.runtime_sha256 | type == "string" and test("^[0-9a-f]{64}$")) + ' "$provenance" >/dev/null { echo "host_sha=$(jq -r .host_sha256 "$provenance")" echo "runtime_id=$(jq -r .runtime_id "$provenance")" @@ -587,7 +597,17 @@ jobs: shell: bash run: | set -euo pipefail - provenance="$(find artifacts/native-package -name '*.upstream-provenance.json' -type f -print -quit)" + mapfile -d '' -t provenance_files < <(find artifacts/native-package -name '*.upstream-provenance.json' -type f -print0) + if (( ${#provenance_files[@]} != 1 )); then + echo "expected exactly one upstream provenance file, found ${#provenance_files[@]}" >&2 + exit 1 + fi + provenance="${provenance_files[0]}" + jq -e ' + (.host_sha256 | type == "string" and test("^[0-9a-f]{64}$")) and + (.runtime_id | type == "string" and length > 0) and + (.runtime_sha256 | type == "string" and test("^[0-9a-f]{64}$")) + ' "$provenance" >/dev/null { echo "host_sha=$(jq -r .host_sha256 "$provenance")" echo "runtime_id=$(jq -r .runtime_id "$provenance")" diff --git a/docs/gpu-runbooks.md b/docs/gpu-runbooks.md index 4963858..37e9de9 100644 --- a/docs/gpu-runbooks.md +++ b/docs/gpu-runbooks.md @@ -17,12 +17,15 @@ Vulkan images require the distro Vulkan loader. A loader package failure is down GitHub-hosted runners validate archive integrity, package installation, and the command surface without GPU devices or driver stubs. The host executable is backend-neutral, so `--version`, `--help`, and `runtime list` must work for -CUDA, ROCm, Vulkan, and CPU images without device passthrough. A direct -`libcuda.so.1`, ROCm, Vulkan, ggml, or llama import from the host is a product -failure. These checks do not prove inference on NVIDIA, AMD, or Vulkan -hardware. Hardware qualification should consume the published candidate image -on a controlled host and record driver, device, runtime selection, and -inference evidence. Do not add self-hosted build runners to this packaging -workflow merely to perform device qualification. +CUDA, ROCm, Vulkan, and CPU images without device passthrough. +`docker/qa-runtime-image.sh` then requires `ldd` to report no unresolved +dependencies and rejects case-insensitive matches for `cuda`, `cublas`, `nccl`, +`hip`, `hsa`, `vulkan`, `ggml`, or `llama`. Unresolved imports or these +forbidden direct host dependencies are product failures. These checks do not +prove inference on NVIDIA, AMD, or Vulkan hardware. Hardware qualification +should consume the published candidate image on a controlled host and record +driver, device, runtime selection, and inference evidence. Do not add +self-hosted build runners to this packaging workflow merely to perform device +qualification. If hardware validation fails, first reproduce with the exact versioned image tag and inspect `mesh-llm runtime list`. Retagging or rebuilding the package cannot repair a host-driver or upstream runtime-bundle defect. diff --git a/docs/matrix.md b/docs/matrix.md index 0797971..e503a38 100644 --- a/docs/matrix.md +++ b/docs/matrix.md @@ -3,7 +3,7 @@ `packaging/images.json` is the only target source of truth. Schema 2 separates the selected upstream runtime flavor from the downstream package/image presentation. Every upstream row is a product-v2 archive containing the -OS/architecture-neutral host plus exactly one runtime. +backend-neutral host for its specific OS/architecture plus exactly one runtime. Each active row declares its distro, backend display version, `upstream_flavor`, package format/base, runtime base, platforms, support level, and release track. Matrix expansion derives the upstream archive/checksum URLs, deduplicated archive artifact ID, package artifact name, GitHub-hosted runner, and OCI tags. diff --git a/packaging/native/build-package.sh b/packaging/native/build-package.sh index be7c671..44f3967 100644 --- a/packaging/native/build-package.sh +++ b/packaging/native/build-package.sh @@ -15,6 +15,7 @@ host_imports="$bundle/host-imports.json" test -f "$binary" || { echo "binary not found: $binary" >&2; exit 1; } test -f "$product_manifest" || { echo "product manifest not found: $product_manifest" >&2; exit 1; } test -f "$host_imports" || { echo "host import report not found: $host_imports" >&2; exit 1; } +test -d "$bundle/native-runtimes" || { echo "native runtime directory not found: $bundle/native-runtimes" >&2; exit 1; } runtime_count="$(find "$bundle/native-runtimes" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" [ "$runtime_count" = 1 ] || { echo "expected exactly one native runtime in $bundle" >&2; exit 1; } runtime_dir="$(find "$bundle/native-runtimes" -mindepth 1 -maxdepth 1 -type d | head -n 1)" diff --git a/schemas/product-v2.schema.json b/schemas/product-v2.schema.json index 2e20fd6..9c87f5b 100644 --- a/schemas/product-v2.schema.json +++ b/schemas/product-v2.schema.json @@ -68,11 +68,13 @@ "properties": { "id": { "type": "string", - "minLength": 1 + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" }, "path": { "type": "string", - "minLength": 1 + "minLength": 1, + "pattern": "^native-runtimes/[A-Za-z0-9][A-Za-z0-9._-]*$" }, "sha256": { "$ref": "#/$defs/sha256" diff --git a/scripts/upstream-archive.ts b/scripts/upstream-archive.ts index 19a736b..1c54204 100755 --- a/scripts/upstream-archive.ts +++ b/scripts/upstream-archive.ts @@ -1,7 +1,7 @@ #!/usr/bin/env -S node --experimental-strip-types import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { chmodSync, cpSync, createReadStream, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { chmodSync, closeSync, cpSync, createReadStream, mkdirSync, mkdtempSync, openSync, readFileSync, readdirSync, readSync, rmSync, statSync, writeFileSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; import { tmpdir } from "node:os"; import { pathToFileURL } from "node:url"; @@ -15,6 +15,8 @@ type Inputs = { flavor: string; }; +const runtimeIdPattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + export function sha256File(path: string): Promise { return new Promise((resolveDigest, reject) => { const hash = createHash("sha256"); @@ -59,6 +61,7 @@ export function validateArchiveEntries(entries: string[]): void { const match = /^mesh-bundle\/native-runtimes\/([^/]+)\/(.+)$/.exec(path); if (!match) throw new Error(`unexpected product archive entry: ${path}`); const [, runtimeId, relative] = match; + if (!runtimeIdPattern.test(runtimeId)) throw new Error(`unsafe native runtime id: ${runtimeId}`); if (relative !== "manifest.json" && relative !== "README.md" && !relative.startsWith("lib/") && !relative.startsWith("tools/")) { throw new Error(`unexpected native runtime entry: ${path}`); } @@ -124,6 +127,7 @@ export function validateProductManifest(value: unknown): ProductManifest { if (!manifest.runtime || typeof manifest.runtime !== "object") throw new Error("product manifest runtime is required"); assertExactKeys(manifest.runtime, ["id", "path", "sha256", "manifest_sha256"], "product runtime"); if (typeof manifest.runtime?.id !== "string" || !manifest.runtime.id) throw new Error("product runtime id is required"); + if (!runtimeIdPattern.test(manifest.runtime.id)) throw new Error("product runtime id contains unsafe characters"); if (manifest.runtime.path !== `native-runtimes/${manifest.runtime.id}`) { throw new Error("product runtime path must match its runtime id"); } @@ -141,14 +145,26 @@ function filesBelow(root: string, current = root): string[] { export function sha256Tree(root: string): string { const digest = createHash("sha256"); + const buffer = Buffer.allocUnsafe(1024 * 1024); for (const path of filesBelow(root).sort()) { const relative = path.slice(root.length + 1).replaceAll("\\", "/"); const encoded = Buffer.from(relative); const length = Buffer.alloc(8); + const fileDigest = createHash("sha256"); + const file = openSync(path, "r"); + try { + let bytesRead = readSync(file, buffer, 0, buffer.length, null); + while (bytesRead > 0) { + fileDigest.update(buffer.subarray(0, bytesRead)); + bytesRead = readSync(file, buffer, 0, buffer.length, null); + } + } finally { + closeSync(file); + } length.writeBigUInt64BE(BigInt(encoded.length)); digest.update(length); digest.update(encoded); - digest.update(createHash("sha256").update(readFileSync(path)).digest()); + digest.update(fileDigest.digest()); } return digest.digest("hex"); } @@ -169,10 +185,11 @@ export async function verifyAndExtract(input: Inputs) { validateArchiveEntries(runTar(["-tzf", archive]).split(/\r?\n/)); validateArchiveEntryTypes(runTar(["-tvzf", archive]).split(/\r?\n/)); - mkdirSync(input.outputDir, { recursive: true }); const temporary = mkdtempSync(resolve(tmpdir(), "mesh-llm-upstream-")); try { runTar(["-xzf", archive, "-C", temporary, "mesh-bundle"]); + rmSync(input.outputDir, { recursive: true, force: true }); + mkdirSync(input.outputDir, { recursive: true }); cpSync(resolve(temporary, "mesh-bundle"), input.outputDir, { recursive: true }); const binary = resolve(input.outputDir, "mesh-llm"); chmodSync(binary, 0o755); diff --git a/tests/native-package.test.ts b/tests/native-package.test.ts new file mode 100644 index 0000000..c6c29da --- /dev/null +++ b/tests/native-package.test.ts @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { test } from "node:test"; + +test("package builder rejects a missing native runtime directory", (t) => { + const directory = mkdtempSync(resolve(tmpdir(), "native-package-test-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const bundle = resolve(directory, "bundle"); + mkdirSync(bundle); + writeFileSync(resolve(bundle, "mesh-llm"), "binary\n"); + writeFileSync(resolve(bundle, "product-manifest.json"), "{}\n"); + writeFileSync(resolve(bundle, "host-imports.json"), "{}\n"); + + const result = spawnSync("sh", [ + resolve("packaging/native/build-package.sh"), + "ubuntu", + "cpu", + "", + "amd64", + "0.73.1", + resolve(directory, "output"), + bundle, + ], { encoding: "utf8" }); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /native runtime directory not found/); + assert.doesNotMatch(result.stderr, /find:/); +}); diff --git a/tests/upstream-archive.test.ts b/tests/upstream-archive.test.ts index a003621..08570ac 100644 --- a/tests/upstream-archive.test.ts +++ b/tests/upstream-archive.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, resolve } from "node:path"; import { test } from "node:test"; @@ -11,9 +12,29 @@ import { sha256Tree, validateArchiveEntries, validateArchiveEntryTypes, + validateProductManifest, verifyAndExtract, } from "../scripts/upstream-archive.ts"; +function legacySha256Tree(root: string): string { + const digest = createHash("sha256"); + const filesBelow = (current: string): string[] => readdirSync(current).flatMap((name) => { + const path = resolve(current, name); + return statSync(path).isDirectory() ? filesBelow(path) : [path]; + }); + const files = filesBelow(root).sort(); + for (const path of files) { + const relative = path.slice(root.length + 1).replaceAll("\\", "/"); + const encoded = Buffer.from(relative); + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(encoded.length)); + digest.update(length); + digest.update(encoded); + digest.update(createHash("sha256").update(readFileSync(path)).digest()); + } + return digest.digest("hex"); +} + async function fixture(t: { after(callback: () => void): void }) { const directory = mkdtempSync(resolve(tmpdir(), "upstream-test-")); t.after(() => rmSync(directory, { recursive: true, force: true })); @@ -68,6 +89,47 @@ test("verifies checksum, layout, extraction, and provenance", async (t) => { }); }); +test("rejects unsafe runtime IDs before accepting runtime paths", () => { + const sha256 = "a".repeat(64); + assert.throws(() => validateProductManifest({ + backend: "cpu", + contract: "mesh-llm-product-v2", + host: { path: "mesh-llm", sha256 }, + mesh_version: "0.73.1", + runtime: { + id: "../linux-cpu", + manifest_sha256: sha256, + path: "native-runtimes/../linux-cpu", + sha256, + }, + schema_version: 2, + }), /runtime id/); + assert.throws(() => validateArchiveEntries([ + "mesh-bundle/mesh-llm", + "mesh-bundle/product-manifest.json", + "mesh-bundle/host-imports.json", + "mesh-bundle/native-runtimes/linux cpu/manifest.json", + "mesh-bundle/native-runtimes/linux cpu/README.md", + "mesh-bundle/native-runtimes/linux cpu/lib/libllama.so", + ]), /runtime id/); +}); + +test("keeps sha256Tree digest compatible with the original tree format", async (t) => { + const data = await fixture(t); + const runtime = resolve(data.directory, "stage/mesh-bundle/native-runtimes/linux-cpu"); + assert.equal(sha256Tree(runtime), legacySha256Tree(runtime)); +}); + +test("removes stale output contents after successful extraction", async (t) => { + const data = await fixture(t); + const outputDir = resolve(data.directory, "stale-output"); + mkdirSync(outputDir, { recursive: true }); + writeFileSync(resolve(outputDir, "stale.txt"), "stale\n"); + await verifyAndExtract({ archive: data.archive, checksum: data.checksum, outputDir, sourceUrl: "https://example.test/archive", version: "0.73.1", flavor: "cpu" }); + assert.equal(existsSync(resolve(outputDir, "stale.txt")), false); + assert.equal(existsSync(resolve(outputDir, "mesh-llm")), true); +}); + test("rejects malformed checksums and unsafe layouts", async (t) => { const data = await fixture(t); assert.equal(parseChecksum(`${"A".repeat(64)} *${basename(data.archive)}\n`, basename(data.archive)), "a".repeat(64)); diff --git a/tests/workflow-provenance.test.ts b/tests/workflow-provenance.test.ts new file mode 100644 index 0000000..702a9f9 --- /dev/null +++ b/tests/workflow-provenance.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { test } from "node:test"; + +const workflow = readFileSync(resolve(".github/workflows/images-release.yml"), "utf8"); +const productInputSteps = [...workflow.matchAll( + / - name: Read immutable product inputs\n id: product\n shell: bash\n run: \|\n((?: .*\n)+?)(?= - )/g, +)].map((match) => match[1].replace(/^ /gm, "")); + +function runStep(script: string, provenance: readonly object[]): { + readonly outputContents: string; + readonly status: number | null; + readonly stderr: string; +} { + const directory = mkdtempSync(resolve(tmpdir(), "workflow-provenance-test-")); + const artifactDirectory = resolve(directory, "artifacts/native-package"); + mkdirSync(artifactDirectory, { recursive: true }); + provenance.forEach((value, index) => { + writeFileSync(resolve(artifactDirectory, `${index}.upstream-provenance.json`), `${JSON.stringify(value)}\n`); + }); + const output = resolve(directory, "github-output"); + const result = spawnSync("bash", ["-c", script], { + cwd: directory, + encoding: "utf8", + env: { ...process.env, GITHUB_OUTPUT: output }, + }); + const contents = result.status === 0 ? readFileSync(output, "utf8") : ""; + rmSync(directory, { recursive: true, force: true }); + return { outputContents: contents, status: result.status, stderr: result.stderr }; +} + +test("both product input steps emit validated provenance", () => { + assert.equal(productInputSteps.length, 2); + const provenance = { + host_sha256: "a".repeat(64), + runtime_id: "linux-cpu", + runtime_sha256: "b".repeat(64), + }; + for (const script of productInputSteps) { + const result = runStep(script, [provenance]); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.outputContents, `host_sha=${provenance.host_sha256}\nruntime_id=linux-cpu\nruntime_sha=${provenance.runtime_sha256}\n`); + } +}); + +test("both product input steps reject ambiguous or invalid provenance", () => { + assert.equal(productInputSteps.length, 2); + const valid = { + host_sha256: "a".repeat(64), + runtime_id: "linux-cpu", + runtime_sha256: "b".repeat(64), + }; + const invalidCases: readonly (readonly object[])[] = [ + [], + [valid, valid], + [{ ...valid, host_sha256: "invalid" }], + [{ ...valid, runtime_sha256: "B".repeat(64) }], + [{ ...valid, runtime_id: "" }], + ]; + for (const script of productInputSteps) { + for (const provenance of invalidCases) { + assert.notEqual(runStep(script, provenance).status, 0); + } + } +}); From ea96521e2ad23a5521471cbd78684dc6e1885ed0 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 01:00:59 -0400 Subject: [PATCH 3/5] test: certify composed distribution runtimes --- .github/workflows/images-precheck.yml | 2 +- .github/workflows/images-release.yml | 83 +++++++- .skills/distribution-certification/SKILL.md | 6 + .../references/format-checks.md | 6 + AGENTS.md | 12 ++ README.md | 15 +- TODO.md | 13 +- docker/Dockerfile.mesh-llm | 4 +- docker/qa-runtime-image.sh | 8 +- docs/matrix.md | 4 +- docs/native-packages.md | 22 +- docs/packaging-readiness-gaps.md | 8 +- docs/packaging-readiness-scorecard.md | 7 +- .../homebrew/Formula/mesh-llm.rb.template | 59 ++++++ packaging/homebrew/README.md | 7 + schemas/product-v2.schema.json | 3 +- scripts/client-readiness-smoke.sh | 188 ++++++++++++++++++ scripts/native-package-qa.sh | 6 +- scripts/node-sdk-runtime-smoke.cjs | 123 ++++++++++++ scripts/verify-host-invariant.ts | 45 +++++ scripts/verify-product-schema.ts | 38 ++++ tests/client-readiness-smoke.test.ts | 133 +++++++++++++ tests/homebrew-release.test.ts | 13 ++ tests/node-sdk-runtime-smoke.test.ts | 106 ++++++++++ tests/product-contract.test.ts | 63 ++++++ tests/workflow-provenance.test.ts | 2 + 26 files changed, 942 insertions(+), 34 deletions(-) create mode 100644 scripts/client-readiness-smoke.sh create mode 100644 scripts/node-sdk-runtime-smoke.cjs create mode 100644 scripts/verify-host-invariant.ts create mode 100644 scripts/verify-product-schema.ts create mode 100644 tests/client-readiness-smoke.test.ts create mode 100644 tests/node-sdk-runtime-smoke.test.ts create mode 100644 tests/product-contract.test.ts diff --git a/.github/workflows/images-precheck.yml b/.github/workflows/images-precheck.yml index 386c2d4..fd7cbea 100644 --- a/.github/workflows/images-precheck.yml +++ b/.github/workflows/images-precheck.yml @@ -40,7 +40,7 @@ jobs: --test-coverage-lines=100 --test-coverage-branches=100 --test-coverage-functions=100 \ tests/image-matrix.test.ts node --experimental-strip-types --test \ - tests/homebrew-release.test.ts tests/node-sdk-package.test.ts tests/upstream-archive.test.ts + tests/client-readiness-smoke.test.ts tests/homebrew-release.test.ts tests/node-sdk-package.test.ts tests/node-sdk-runtime-smoke.test.ts tests/product-contract.test.ts tests/upstream-archive.test.ts tests/workflow-provenance.test.ts - name: Lint shell scripts run: | sudo apt-get update diff --git a/.github/workflows/images-release.yml b/.github/workflows/images-release.yml index 1d41545..398a60f 100644 --- a/.github/workflows/images-release.yml +++ b/.github/workflows/images-release.yml @@ -184,6 +184,20 @@ jobs: echo "npm_plan=$npm_plan" echo "npm_enabled=$(jq -r .enabled <<<"$npm_plan")" } >> "$GITHUB_OUTPUT" + - name: Verify producer product-v2 schema + env: + MESH_SHA: ${{ steps.meta.outputs.mesh_sha }} + shell: bash + run: | + set -euo pipefail + producer_schema="artifacts/producer-product-v2.schema.json" + mkdir -p artifacts + curl --fail --location --retry 3 --proto '=https' --tlsv1.2 \ + --output "$producer_schema" \ + "https://raw.githubusercontent.com/Mesh-LLM/mesh-llm/${MESH_SHA}/schemas/product-v2.schema.json" + node --experimental-strip-types scripts/verify-product-schema.ts \ + --producer-schema "$producer_schema" \ + --consumer-schema schemas/product-v2.schema.json upstream: name: Verify upstream ${{ matrix.upstream_asset_name }} @@ -211,6 +225,11 @@ jobs: --source-url "${{ matrix.upstream_asset_url }}" \ --version "${{ matrix.mesh_version }}" \ --flavor "${{ matrix.upstream_flavor }}" + mkdir -p artifacts/provenance + jq --arg arch "${{ matrix.arch }}" --arg platform "${{ matrix.platform }}" \ + '. + {arch: $arch, platform: $platform}' \ + artifacts/upstream/upstream-provenance.json \ + > "artifacts/provenance/${{ matrix.upstream_artifact_id }}.json" file artifacts/upstream/mesh-llm | grep -q 'ELF.*executable' - uses: actions/upload-artifact@v7 with: @@ -218,10 +237,38 @@ jobs: path: artifacts/upstream if-no-files-found: error retention-days: 14 + - uses: actions/upload-artifact@v7 + with: + name: mesh-llm-upstream-provenance-${{ matrix.upstream_artifact_id }} + path: artifacts/provenance/${{ matrix.upstream_artifact_id }}.json + if-no-files-found: error + retention-days: 14 + + upstream-host-invariant: + name: Verify one immutable host per platform and architecture + needs: [plan, upstream] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: '24' + - uses: actions/download-artifact@v8 + with: + pattern: mesh-llm-upstream-provenance-* + path: artifacts/upstream-provenance + merge-multiple: true + - name: Reject backend products with different verified host bytes + shell: bash + run: | + set -euo pipefail + mapfile -d '' -t provenance_files < <(find artifacts/upstream-provenance -type f -name '*.json' -print0 | sort -z) + (( ${#provenance_files[@]} > 0 )) || { echo "no upstream provenance records downloaded" >&2; exit 1; } + node --experimental-strip-types scripts/verify-host-invariant.ts "${provenance_files[@]}" native-package: name: Package ${{ matrix.artifact_id }} - needs: [plan, upstream] + needs: [plan, upstream, upstream-host-invariant] strategy: fail-fast: false matrix: ${{ fromJSON(needs.plan.outputs.package_matrix) }} @@ -304,7 +351,7 @@ jobs: provenance="${provenance_files[0]}" jq -e ' (.host_sha256 | type == "string" and test("^[0-9a-f]{64}$")) and - (.runtime_id | type == "string" and length > 0) and + (.runtime_id | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$")) and (.runtime_sha256 | type == "string" and test("^[0-9a-f]{64}$")) ' "$provenance" >/dev/null { @@ -425,14 +472,32 @@ jobs: run: npm test --prefix upstream-source/sdk/node - name: Build Node SDK addon run: npm run build:native --prefix upstream-source/sdk/node - - name: Smoke-test Node SDK addon - working-directory: upstream-source/sdk/node + - name: Pack, fresh-install, and start Node SDK addon + timeout-minutes: 3 shell: bash env: MESH_VERSION: ${{ needs.plan.outputs.mesh_version }} + NODE_SDK_TARGET: ${{ matrix.target }} run: | set -euo pipefail - node --input-type=commonjs -e 'const assert = require("node:assert/strict"); const sdk = require(process.cwd()); assert.equal(sdk.currentMeshVersion(), process.env.MESH_VERSION)' + smoke_root="$(mktemp -d)" + trap 'rm -rf "$smoke_root"' EXIT + mkdir -p "$smoke_root/tarball" "$smoke_root/consumer" + ( + cd upstream-source/sdk/node + npm pack --pack-destination "$smoke_root/tarball" + ) + tarball="$(find "$smoke_root/tarball" -maxdepth 1 -type f -name '*.tgz' -print -quit)" + test -n "$tarball" + ( + cd "$smoke_root/consumer" + npm init --yes >/dev/null + npm install "$tarball" + ) + node scripts/node-sdk-runtime-smoke.cjs \ + --package-root "$smoke_root/consumer" \ + --expected-version "$MESH_VERSION" \ + --target "$NODE_SDK_TARGET" - name: Stage Node SDK addon shell: bash env: @@ -495,6 +560,7 @@ jobs: --expected-version "$MESH_VERSION" \ --targets "$targets" - name: Test, pack, install, and dry-run publish + timeout-minutes: 5 env: NPM_PLAN: ${{ needs.plan.outputs.npm_plan }} MESH_VERSION: ${{ needs.plan.outputs.mesh_version }} @@ -519,7 +585,10 @@ jobs: npm init --yes >/dev/null npm install "../$tarball" if jq -e '.targets | index("linux-x64")' <<<"$NPM_PLAN" >/dev/null; then - node -e "const sdk = require('@mesh-llm/sdk'); if (sdk.currentMeshVersion() !== process.env.MESH_VERSION) process.exit(1)" + node "$GITHUB_WORKSPACE/scripts/node-sdk-runtime-smoke.cjs" \ + --package-root "$PWD" \ + --expected-version "$MESH_VERSION" \ + --target linux-x64-assembled-package fi ) - uses: actions/upload-artifact@v7 @@ -605,7 +674,7 @@ jobs: provenance="${provenance_files[0]}" jq -e ' (.host_sha256 | type == "string" and test("^[0-9a-f]{64}$")) and - (.runtime_id | type == "string" and length > 0) and + (.runtime_id | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$")) and (.runtime_sha256 | type == "string" and test("^[0-9a-f]{64}$")) ' "$provenance" >/dev/null { diff --git a/.skills/distribution-certification/SKILL.md b/.skills/distribution-certification/SKILL.md index 8cb92e0..136a35e 100644 --- a/.skills/distribution-certification/SKILL.md +++ b/.skills/distribution-certification/SKILL.md @@ -187,6 +187,12 @@ Common runtime rule: A metadata-only check, successful install, `--version`, or transient live PID is not runtime certification. +For composed product-v2 channels, also record the producer schema SHA-256 and +prove it is byte-identical to the packaging consumer schema at the immutable +upstream source SHA. Group all selected product provenance records by +OS/architecture and require exactly one host SHA-256 per group before treating +backend rows as aliases of a shared host. + ## Phase 3: cross-channel consistency Require agreement on the requested semantic version across: diff --git a/.skills/distribution-certification/references/format-checks.md b/.skills/distribution-certification/references/format-checks.md index 05a64cb..850fe11 100644 --- a/.skills/distribution-certification/references/format-checks.md +++ b/.skills/distribution-certification/references/format-checks.md @@ -169,6 +169,12 @@ Inside the fresh project: This certifies the SDK/native addon, not a standalone CLI. +Release CI should run the same fresh-project public lifecycle smoke for every +built addon lane, including `darwin-arm64`, before assembling the cross-platform +tarball. It should repeat the smoke for the assembled package on the runner's +host target. This prepublication coverage complements, but does not replace, +the canonical certification above against the exact public registry package. + Remove the project, the per-project npm cache, SDK cache/runtime paths, and process. Do not delete any other npm cache. Retain only redacted evidence and the packed tarball if the evidence policy calls for it. diff --git a/AGENTS.md b/AGENTS.md index dae7aa8..87270a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,11 @@ Build shared artifacts first, then fan out only after those artifacts are ready: 5. Build native package artifacts from the complete bundle and metadata. 6. Assemble Docker runtime images from the native package artifact for that same row. +Before any package/image fan-out, prove every verified product for the same +OS/architecture has the same host SHA-256. The workflow must consume +upstream-produced, verified host/runtime bytes and reject schema drift at the +immutable upstream commit; it must never infer host identity from a tag name. + When adding a new distro or backend, update all affected layers in the same change: matrix data, validation, dependency installers, package metadata, workflows, and docs. @@ -47,6 +52,13 @@ Do not add a second path that rebuilds `mesh-llm` directly inside the final runtime image. If an image needs a binary, it should receive the package or artifact produced by the package pipeline. +Package-install and final-image QA must start `client --auto` without a GPU +device or driver stub, using isolated ports/runtime/cache roots and JSON logs. +They must observe either the JSON `Client ready` message or the structured +`passive_mode`/`status=ready`/`role=client` event, retain a live process, and +prove bounded SIGINT shutdown. `--version` and `runtime list` alone are not +runtime QA. + Only bypass the package artifact when a distro has no supported package format yet, and document that exception in `docs/native-packages.md`. diff --git a/README.md b/README.md index 1215de2..8f3f1c1 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,25 @@ This repository is the packaging and distribution control plane for published [` ```text published upstream tag + immutable tag SHA -> verified host/runtime product bundles -> native packages -> OCI images - -> npm addon lanes -> assembled @mesh-llm/sdk tarball -> clean install test + -> npm addon lanes -> per-lane fresh install/start -> assembled @mesh-llm/sdk + tarball -> clean install/start ``` Application packages and images never rebuild `mesh-llm`; they consume the verified backend-neutral host and selected native runtime from an upstream product-v2 bundle. The npm lanes are the sole exception: they check out the immutable release SHA to compile the SDK's N-API addons, which use the same -dynamic runtime resolver. +dynamic runtime resolver. Every addon lane packs and installs a fresh consumer, +then exercises the public SDK lifecycle (`Node.create`, `start`, `status`, and +`stop`) with isolated runtime state and a bounded normal-exit check. The +assembled package repeats that proof on Linux. This certifies the SDK/native +addon; npm does not publish the standalone `mesh-llm` CLI. + +Before package fan-out, the release workflow byte-checks the product-v2 schema +against the immutable upstream source commit and requires one verified host +SHA-256 for every selected OS/architecture. Native packages, OCI images, and +the Homebrew formula then prove no-driver JSON client readiness and clean +SIGINT shutdown, not merely `--version`. ## Supported channels diff --git a/TODO.md b/TODO.md index 8169f98..36e31a4 100644 --- a/TODO.md +++ b/TODO.md @@ -6,7 +6,9 @@ - [x] Assemble, preflight, and publish the canonical `@mesh-llm/sdk` tarball. QA: local fixture dry runs produce the expected cross-platform tarball, - install it into a clean consumer project, load the host addon, and pass + every addon lane packs and installs into a clean consumer project, and its + public `Node` API completes bounded start/status/finally-stop with normal + process exit. The assembled host package repeats the lifecycle proof before `npm publish --dry-run`; workflow lint proves CI can schedule every lane. - [x] Document npm packaging and hand ownership off from `mesh-llm`. @@ -69,9 +71,12 @@ strategy. - Final result: archive verification enforces the product schema, checks both immutable digests, and stages the backend-neutral host plus selected runtime into Debian/Arch, Homebrew, and OCI outputs without rebuilding either input. - - QA: run upstream archive/schema tests, matrix validation, Homebrew rendering - tests, shell syntax checks, Dockerfile checks, and one package-install smoke - proving ownership of the versioned runtime directory. + - QA: run upstream archive/schema tests (including byte-identical schema + verification at the immutable producer SHA), matrix validation, Homebrew + rendering tests, shell syntax checks, Dockerfile checks, and no-driver + package/image/Homebrew client-readiness smokes proving ownership of the + versioned runtime directory, a live structured client-ready event, and + bounded SIGINT. - [ ] Convert official release packaging to consume upstream release archives. - Official package-manager publication starts from upstream composed diff --git a/docker/Dockerfile.mesh-llm b/docker/Dockerfile.mesh-llm index 7938e04..35b7e39 100644 --- a/docker/Dockerfile.mesh-llm +++ b/docker/Dockerfile.mesh-llm @@ -73,5 +73,7 @@ ARG DISTRO=ubuntu ARG BACKEND=cpu ARG MESH_LLM_VERSION COPY docker/qa-runtime-image.sh /usr/local/bin/qa-runtime-image +COPY scripts/client-readiness-smoke.sh /usr/local/bin/client-readiness-smoke RUN sh /usr/local/bin/qa-runtime-image "${DISTRO}" "${BACKEND}" "${MESH_LLM_VERSION}" && \ - rm /usr/local/bin/qa-runtime-image + MESH_LLM_SMOKE_BIN=/usr/local/bin/mesh-llm-entrypoint sh /usr/local/bin/client-readiness-smoke && \ + rm /usr/local/bin/qa-runtime-image /usr/local/bin/client-readiness-smoke diff --git a/docker/qa-runtime-image.sh b/docker/qa-runtime-image.sh index ae87f12..ee8ae49 100755 --- a/docker/qa-runtime-image.sh +++ b/docker/qa-runtime-image.sh @@ -23,6 +23,7 @@ case "$distro" in esac test -x /usr/local/bin/mesh-llm +test -x /usr/local/bin/mesh-llm-entrypoint missing="$(ldd /usr/local/bin/mesh-llm | awk '/not found/ { print $1 }')" [ -z "$missing" ] || { echo "host has unresolved dependencies: $missing" >&2; exit 1; } if ldd /usr/local/bin/mesh-llm | grep -Eiq 'cuda|cublas|nccl|hip|hsa|vulkan|ggml|llama'; then @@ -31,7 +32,8 @@ if ldd /usr/local/bin/mesh-llm | grep -Eiq 'cuda|cublas|nccl|hip|hsa|vulkan|ggml fi test "$(find "/usr/local/lib/mesh-llm/$version/native-runtimes" -name manifest.json -type f | wc -l)" -eq 1 test -f "/usr/local/lib/mesh-llm/$version/product-manifest.json" -mesh-llm --version | grep -F "$version" -mesh-llm runtime list +/usr/local/bin/mesh-llm --version | grep -F "$version" +/usr/local/bin/mesh-llm runtime list +/usr/local/bin/mesh-llm-entrypoint --version | grep -F "$version" -printf 'runtime image QA passed for %s/%s\n' "$distro" "$backend" +printf 'runtime image command-surface QA passed for %s/%s; client readiness follows\n' "$distro" "$backend" diff --git a/docs/matrix.md b/docs/matrix.md index e503a38..a04db85 100644 --- a/docs/matrix.md +++ b/docs/matrix.md @@ -42,4 +42,6 @@ product. A full matrix currently expands to 11 package rows from 8 Linux products, plus one macOS product for Homebrew. Host compilation happens once per OS/architecture upstream; runtime compilation happens once per runtime row; this repository verifies composition once and fans out distro packaging -without rebuilding either layer. +without rebuilding either layer. Before fan-out, the release workflow groups +verified upstream provenance by platform/architecture and rejects any group +with more than one host SHA-256. diff --git a/docs/native-packages.md b/docs/native-packages.md index 8143fef..fbbb711 100644 --- a/docs/native-packages.md +++ b/docs/native-packages.md @@ -10,9 +10,11 @@ verified upstream host + runtime bundle -> native package -> install QA -> OCI i enforces `schemas/product-v2.schema.json` semantics and a strict archive allowlist, verifies the host/runtime digests, extracts the complete product bundle, and records both immutable inputs in provenance. -The producer repository carries an identical product-v2 schema. Contract -changes update both copies together; a release must not proceed with -unexplained schema drift. +The producer repository carries an identical product-v2 schema. The release +plan downloads that schema at the immutable upstream source SHA and byte-checks +it against this checkout before any archive, package, or image job starts. +Contract changes update both copies in the same cross-repository change; a +release must not proceed with unexplained schema drift. `packaging/native/build-package.sh` stages that verified bundle and produces exactly one package with version, distro, architecture, backend, and backend version in its filename. @@ -27,13 +29,19 @@ Native metadata declares the user-space loader dependencies needed by the select invariant, writes SHA256 manifests, inspects native metadata, installs through the distro package manager, proves ownership of the host plus the versioned runtime directory, and runs `mesh-llm --version` plus `mesh-llm runtime list` -without a GPU device or driver. +without a GPU device or driver. It then uses the shared readiness helper with +unique API/console ports and cache/runtime roots to start +`--log-format json --no-console client --auto`, require either the JSON +`Client ready` message or the structured +`passive_mode`/`status=ready`/`role=client` event while the process is alive, +and require bounded SIGINT shutdown. The Dockerfile's `runtime-qa` stage extends the exact final runtime stage. It verifies package ownership, rejects backend imports or unresolved libraries -from the host executable, and exercises the command surface without device -access. Backend libraries may reference their driver interface only from inside -the native runtime. Hardware-qualified serving is separate additive coverage. +from the host executable, and runs that same no-driver client readiness smoke +without device access. Backend libraries may reference their driver interface +only from inside the native runtime. Hardware-qualified serving is separate +additive coverage. Packages install the host at `/usr/local/bin/mesh-llm` and the selected runtime at `/usr/local/lib/mesh-llm//native-runtimes/`, alongside diff --git a/docs/packaging-readiness-gaps.md b/docs/packaging-readiness-gaps.md index ade7ac3..949fe70 100644 --- a/docs/packaging-readiness-gaps.md +++ b/docs/packaging-readiness-gaps.md @@ -11,9 +11,11 @@ The repository implementation is archive-first: it verifies already-built upstre - [x] Remove Blacksmith/self-hosted orchestration. QA: precheck scans workflow/Docker paths for legacy runner/source-build strings. - [x] Correct channel claims: block Alpine/musl and Intel macOS; enable upstream Linux Vulkan and arm64 CUDA 13. QA: matrix tests. - [ ] Re-certify GPU package and image QA against product-v2 bundles. QA: - backend-neutral hosts pass `--version`, `--help`, and `runtime list` without - devices or driver stubs; packages own versioned runtime trees; a new full dry - run succeeds. The earlier static-host baseline is preserved in + backend-neutral hosts pass `--version`, `--help`, `runtime list`, and an + isolated JSON `client --auto` readiness/clean-SIGINT smoke without devices or + driver stubs; packages own versioned runtime trees; all product rows per + OS/architecture attest the same host SHA-256; and a new full dry run + succeeds. The earlier static-host baseline is preserved in [run 29455769787](https://github.com/Mesh-LLM/mesh-packaging/actions/runs/29455769787) for historical comparison only. diff --git a/docs/packaging-readiness-scorecard.md b/docs/packaging-readiness-scorecard.md index 8aba899..bc0eb9c 100644 --- a/docs/packaging-readiness-scorecard.md +++ b/docs/packaging-readiness-scorecard.md @@ -6,9 +6,10 @@ Readiness is evidence-based, not a static percentage. A release is ready only wh |---|---|---| | Upstream accuracy | Published tag/release, exact product names/checksums, strict product-v2 layout, host/runtime digest verification | Run against each new release | | Matrix coverage | Schema validation and explicit active/blocked rows | Review when upstream asset inventory changes | -| Package correctness | Native metadata, checksum, host/runtime ownership, no-device version/help/runtime-list smoke | Full dry-run evidence | -| OCI correctness | Image installs the exact package, labels both input digests, and repeats no-device smoke | Full dry-run evidence plus separate hardware qualification | -| Homebrew | Direct upstream arm64 product, formula-owned runtime, strict audit/install/test, canonical tap sync | Validate each release before tap update | +| Package correctness | Native metadata, checksum, host/runtime ownership, version/runtime-list checks, and no-device client readiness with bounded shutdown | Full dry-run evidence | +| OCI correctness | Image installs the exact package, labels both input digests, checks the final entrypoint, and reaches client readiness without a device | Full dry-run evidence plus separate hardware qualification | +| Homebrew | Direct upstream arm64 product, formula-owned runtime, strict audit/install/test, and no-device client readiness | Validate each release before tap update | +| npm SDK | Every addon lane is packed into a fresh project and must complete public `Node` start/status/finally-stop with normal process exit; the assembled package repeats the host-lane proof | Validate the exact public package separately during distribution certification | | Efficiency | One host per OS/architecture is composed with 8 verified runtimes and fans out to 11 package rows; no source builds | Record duration, artifact sizes, and cache behavior | | Publish safety | Dry-run override, job-local write permissions, release environment | Configure/approve environment and observe first publish rehearsal | | Automation | Repository dispatch receiver | Provision upstream fine-grained dispatch credential/App | diff --git a/packaging/homebrew/Formula/mesh-llm.rb.template b/packaging/homebrew/Formula/mesh-llm.rb.template index 063c1cd..5c5c015 100644 --- a/packaging/homebrew/Formula/mesh-llm.rb.template +++ b/packaging/homebrew/Formula/mesh-llm.rb.template @@ -18,5 +18,64 @@ class MeshLlm < Formula assert_match version.to_s, shell_output("#{bin}/mesh-llm --version") assert_match "native runtime", shell_output("#{bin}/mesh-llm runtime list") assert_predicate libexec/"product-manifest.json", :exist? + + require "json" + require "timeout" + + smoke_root = testpath/"client-readiness" + log = smoke_root/"client.jsonl" + runtime_root = smoke_root/"runtime" + port = free_port + console = free_port + console = free_port while console == port + pid = fork do + ENV["HOME"] = (smoke_root/"home").to_s + ENV["XDG_CACHE_HOME"] = (smoke_root/"cache").to_s + ENV["XDG_CONFIG_HOME"] = (smoke_root/"config").to_s + ENV["XDG_RUNTIME_DIR"] = runtime_root.to_s + ENV["MESH_LLM_RUNTIME_ROOT"] = runtime_root.to_s + ENV["MESH_LLM_NATIVE_RUNTIME_CACHE_DIR"] = (smoke_root/"native-runtime-cache").to_s + [smoke_root/"home", smoke_root/"cache", smoke_root/"config", runtime_root, + smoke_root/"native-runtime-cache"].each(&:mkpath) + log.open("w") do |file| + $stdout.reopen(file) + $stderr.reopen(file) + exec bin/"mesh-llm", "--log-format", "json", "--port", port.to_s, + "--console", console.to_s, "--no-console", "client", "--auto" + end + end + + ready = false + 45.times do + ready = log.exist? && log.each_line.any? do |line| + event = JSON.parse(line) + event["message"].to_s.include?("Client ready") || + (event["event"] == "passive_mode" && event["status"] == "ready" && event["role"] == "client") + rescue JSON::ParserError + false + end + break if ready && Process.kill(0, pid) + + sleep 1 + rescue Errno::ESRCH + break + end + + begin + assert ready, "mesh-llm client did not reach Client ready: #{log.exist? ? log.read : "no log"}" + assert Process.kill(0, pid), "mesh-llm client exited after readiness" + ensure + begin + Process.kill("INT", pid) if Process.kill(0, pid) + _, status = Timeout.timeout(10) { Process.wait2(pid) } + assert_predicate status, :success?, "mesh-llm client did not shut down cleanly" + rescue Errno::ESRCH, Errno::ECHILD + flunk "mesh-llm client exited before bounded SIGINT shutdown" + rescue Timeout::Error + Process.kill("TERM", pid) rescue nil + Process.wait(pid) rescue nil + flunk "mesh-llm client did not stop within 10 seconds of SIGINT" + end + end end end diff --git a/packaging/homebrew/README.md b/packaging/homebrew/README.md index 48d33d4..abec87d 100644 --- a/packaging/homebrew/README.md +++ b/packaging/homebrew/README.md @@ -17,6 +17,13 @@ repository does not rebuild or re-tar either immutable input. Intel is intentionally unsupported until upstream publishes an x86_64 macOS product bundle. +The formula test also starts the exact Cellar binary in client mode with +isolated HOME/XDG/runtime/native-runtime-cache paths and unique API/console +ports. It requires a JSON client-ready message or the structured +`passive_mode`/`status=ready`/`role=client` event, a live process, and bounded +SIGINT shutdown; this is a no-driver client smoke, not GPU-serving +qualification. + The rendered formula is attached to this repository's package release. [`Mesh-LLM/homebrew-tap`](https://github.com/Mesh-LLM/homebrew-tap) polls the latest non-prerelease package release, validates and installs the formula, and diff --git a/schemas/product-v2.schema.json b/schemas/product-v2.schema.json index 9c87f5b..4788373 100644 --- a/schemas/product-v2.schema.json +++ b/schemas/product-v2.schema.json @@ -49,7 +49,8 @@ "properties": { "path": { "type": "string", - "minLength": 1 + "minLength": 1, + "pattern": "^(?![A-Za-z]:)(?:(?!\\.{1,2}(?:/|$))[^/\\\\]+)(?:/(?:(?!\\.{1,2}(?:/|$))[^/\\\\]+))*$" }, "sha256": { "$ref": "#/$defs/sha256" diff --git a/scripts/client-readiness-smoke.sh b/scripts/client-readiness-smoke.sh new file mode 100644 index 0000000..aaf1f3b --- /dev/null +++ b/scripts/client-readiness-smoke.sh @@ -0,0 +1,188 @@ +#!/bin/sh +# Run from a package-installed image with no GPU device or driver stub. This +# deliberately certifies the host/client path only; hardware-qualified serving +# belongs to separate backend tests. +set -eu + +mesh_llm_bin="${MESH_LLM_SMOKE_BIN:-/usr/local/bin/mesh-llm}" +ready_timeout="${MESH_LLM_SMOKE_READY_TIMEOUT_SECONDS:-45}" +shutdown_timeout="${MESH_LLM_SMOKE_SHUTDOWN_TIMEOUT_SECONDS:-10}" + +case "$ready_timeout:$shutdown_timeout" in + *[!0-9:]*|:*|*:) echo "smoke timeouts must be positive integer seconds" >&2; exit 2 ;; +esac +[ "$ready_timeout" -gt 0 ] || { echo "readiness timeout must be positive" >&2; exit 2; } +[ "$shutdown_timeout" -gt 0 ] || { echo "shutdown timeout must be positive" >&2; exit 2; } +[ -x "$mesh_llm_bin" ] || { echo "mesh-llm smoke executable is not executable: $mesh_llm_bin" >&2; exit 1; } + +smoke_root="$(mktemp -d "${TMPDIR:-/tmp}/mesh-llm-client-smoke.XXXXXX")" +pid="" +watcher_pid="" +guardian_pid="" +log="$smoke_root/client.jsonl" +ready_marker="$smoke_root/ready" +ready_timeout_marker="$smoke_root/ready-timeout" +watcher_cancel_marker="$smoke_root/watcher-cancel" +shutdown_timeout_marker="$smoke_root/shutdown-timeout" +guardian_done_marker="$smoke_root/guardian-done" +# Fresh containers normally have no listeners. Derive two distinct high ports +# anyway so concurrent package QA invocations cannot collide inside a shared +# network namespace. +port_seed=$(( ($$ % 9000) + 20000 )) +api_port="${MESH_LLM_SMOKE_API_PORT:-$port_seed}" +console_port="${MESH_LLM_SMOKE_CONSOLE_PORT:-$((port_seed + 10000))}" + +stop_helper() { + helper_pid="$1" + cancel_marker="$2" + if [ -n "$helper_pid" ]; then + : > "$cancel_marker" + wait "$helper_pid" 2>/dev/null || true + fi +} + +signal_and_wait() { + timeout_marker="$1" + rm -f "$guardian_done_marker" "$timeout_marker" + kill -INT "$pid" 2>/dev/null || true + ( + elapsed=0 + while [ "$elapsed" -lt "$shutdown_timeout" ]; do + sleep 1 + [ ! -e "$guardian_done_marker" ] || exit 0 + elapsed=$((elapsed + 1)) + done + if kill -0 "$pid" 2>/dev/null; then + : > "$timeout_marker" + kill -TERM "$pid" 2>/dev/null || true + sleep 1 + [ -e "$guardian_done_marker" ] || kill -KILL "$pid" 2>/dev/null || true + fi + ) /dev/null 2>&1 & + guardian_pid=$! + set +e + wait "$pid" + child_exit_code=$? + set -e + pid="" + : > "$guardian_done_marker" + stop_helper "$guardian_pid" "$guardian_done_marker" + guardian_pid="" +} + +cleanup() { + trap - EXIT HUP INT TERM USR1 USR2 + stop_helper "$watcher_pid" "$watcher_cancel_marker" + watcher_pid="" + stop_helper "$guardian_pid" "$guardian_done_marker" + guardian_pid="" + if [ -n "$pid" ]; then + cleanup_timeout_marker="$smoke_root/cleanup-timeout" + signal_and_wait "$cleanup_timeout_marker" + if [ -e "$cleanup_timeout_marker" ]; then + echo "client readiness cleanup forced termination after SIGINT timeout" >&2 + fi + fi + rm -rf "$smoke_root" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +mkdir -p \ + "$smoke_root/home" \ + "$smoke_root/cache" \ + "$smoke_root/config" \ + "$smoke_root/state" \ + "$smoke_root/runtime" \ + "$smoke_root/native-runtime-cache" +chmod 700 \ + "$smoke_root/home" \ + "$smoke_root/cache" \ + "$smoke_root/config" \ + "$smoke_root/state" \ + "$smoke_root/runtime" \ + "$smoke_root/native-runtime-cache" + +( + trap - INT TERM + export HOME="$smoke_root/home" + export XDG_CACHE_HOME="$smoke_root/cache" + export XDG_CONFIG_HOME="$smoke_root/config" + export XDG_STATE_HOME="$smoke_root/state" + export XDG_RUNTIME_DIR="$smoke_root/runtime" + export MESH_LLM_RUNTIME_ROOT="$smoke_root/runtime" + export MESH_LLM_NATIVE_RUNTIME_CACHE_DIR="$smoke_root/native-runtime-cache" + exec "$mesh_llm_bin" --log-format json --port "$api_port" --console "$console_port" --no-console client --auto +) >"$log" 2>&1 & +pid=$! + +main_pid=$$ +trap ':' USR1 USR2 +( + elapsed=0 + while [ "$elapsed" -lt "$ready_timeout" ]; do + [ ! -e "$watcher_cancel_marker" ] || exit 0 + if grep -Eq '^[[:space:]]*\{.*"Client ready".*\}[[:space:]]*$' "$log" || + grep -E '"event"[[:space:]]*:[[:space:]]*"passive_mode"' "$log" | + grep -E '"status"[[:space:]]*:[[:space:]]*"ready"' | + grep -Eq '"role"[[:space:]]*:[[:space:]]*"client"'; then + : > "$ready_marker" + kill -USR1 "$main_pid" 2>/dev/null || true + exit 0 + fi + sleep 1 + [ ! -e "$watcher_cancel_marker" ] || exit 0 + elapsed=$((elapsed + 1)) + done + : > "$ready_timeout_marker" + kill -USR2 "$main_pid" 2>/dev/null || true +) /dev/null 2>&1 & +watcher_pid=$! + +set +e +wait "$pid" +readiness_wait_exit=$? +set -e +stop_helper "$watcher_pid" "$watcher_cancel_marker" +watcher_pid="" +trap - USR1 USR2 + +if [ -e "$ready_timeout_marker" ]; then + echo "mesh-llm client did not reach structured readiness while alive within ${ready_timeout}s" >&2 + cat "$log" >&2 || true + exit 1 +fi + +if [ ! -e "$ready_marker" ]; then + pid="" + echo "mesh-llm client exited before readiness with exit code $readiness_wait_exit" >&2 + cat "$log" >&2 || true + exit 1 +fi + +if ! kill -0 "$pid" 2>/dev/null; then + set +e + wait "$pid" + readiness_wait_exit=$? + set -e + pid="" + echo "mesh-llm client exited at readiness with exit code $readiness_wait_exit" >&2 + cat "$log" >&2 || true + exit 1 +fi + +signal_and_wait "$shutdown_timeout_marker" +if [ -e "$shutdown_timeout_marker" ]; then + echo "mesh-llm client did not stop after SIGINT within ${shutdown_timeout}s" >&2 + cat "$log" >&2 || true + exit 1 +fi +if [ "$child_exit_code" -ne 0 ]; then + echo "mesh-llm client exited with $child_exit_code after SIGINT" >&2 + cat "$log" >&2 || true + exit 1 +fi + +printf 'mesh-llm client readiness smoke passed (api=%s console=%s)\n' "$api_port" "$console_port" diff --git a/scripts/native-package-qa.sh b/scripts/native-package-qa.sh index 3591c27..72dba95 100755 --- a/scripts/native-package-qa.sh +++ b/scripts/native-package-qa.sh @@ -111,6 +111,9 @@ command -v docker >/dev/null 2>&1 || { } abs_package_dir="$(cd "$package_dir" && pwd -P)" +script_dir="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +client_smoke_script="$script_dir/client-readiness-smoke.sh" +[ -f "$client_smoke_script" ] || { echo "client readiness smoke script is missing: $client_smoke_script" >&2; exit 1; } run_package_container() { image="$1" @@ -119,12 +122,13 @@ run_package_container() { -e PACKAGE_FILE="$expected_file" \ -e EXPECTED_VERSION="$version" \ -v "$abs_package_dir:/packages:ro" \ + -v "$client_smoke_script:/usr/local/bin/client-readiness-smoke:ro" \ "$image" \ sh -eu -c "$script" } # shellcheck disable=SC2016 -runtime_smoke='test "$(find "/usr/local/lib/mesh-llm/$EXPECTED_VERSION/native-runtimes" -name manifest.json -type f | wc -l)" -eq 1 && test -f "/usr/local/lib/mesh-llm/$EXPECTED_VERSION/product-manifest.json" && mesh-llm --version | grep -F "$EXPECTED_VERSION" && mesh-llm runtime list' +runtime_smoke='test "$(find "/usr/local/lib/mesh-llm/$EXPECTED_VERSION/native-runtimes" -name manifest.json -type f | wc -l)" -eq 1 && test -f "/usr/local/lib/mesh-llm/$EXPECTED_VERSION/product-manifest.json" && mesh-llm --version | grep -F "$EXPECTED_VERSION" && mesh-llm runtime list && sh /usr/local/bin/client-readiness-smoke' case "$distro" in ubuntu) diff --git a/scripts/node-sdk-runtime-smoke.cjs b/scripts/node-sdk-runtime-smoke.cjs new file mode 100644 index 0000000..ab85d7f --- /dev/null +++ b/scripts/node-sdk-runtime-smoke.cjs @@ -0,0 +1,123 @@ +'use strict' + +const assert = require('node:assert/strict') +const { spawn } = require('node:child_process') +const { mkdtempSync, mkdirSync, rmSync } = require('node:fs') +const { tmpdir } = require('node:os') +const path = require('node:path') +const { createRequire } = require('node:module') + +function parseArgs(argv) { + const values = new Map() + for (let index = 0; index < argv.length; index += 2) { + const name = argv[index] + const value = argv[index + 1] + if (!name?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`invalid argument near ${name || ''}`) + } + values.set(name.slice(2), value) + } + const required = (name) => { + const value = values.get(name) + if (!value) throw new Error(`--${name} is required`) + return value + } + const timeoutMs = Number(values.get('timeout-ms') || '90000') + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 100) { + throw new Error('--timeout-ms must be an integer of at least 100') + } + return { + expectedVersion: required('expected-version'), + packageRoot: path.resolve(required('package-root')), + target: required('target'), + timeoutMs + } +} + +function operationTimeout(name, promise, timeoutMs) { + let timer + return Promise.race([ + Promise.resolve(promise), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${name} exceeded ${timeoutMs}ms`)), timeoutMs) + }) + ]).finally(() => clearTimeout(timer)) +} + +async function runChild(options) { + const projectRequire = createRequire(path.join(options.packageRoot, 'package.json')) + const sdk = projectRequire('@mesh-llm/sdk') + assert.equal(sdk.currentMeshVersion(), options.expectedVersion) + assert.equal(typeof sdk.generateOwnerKeypairHex, 'function') + assert.equal(typeof sdk.Node?.create, 'function') + + const smokeRoot = mkdtempSync(path.join(tmpdir(), `mesh-llm-node-sdk-${options.target}-`)) + const cacheDir = path.join(smokeRoot, 'cache') + const runtimeDir = path.join(smokeRoot, 'runtime') + mkdirSync(cacheDir, { recursive: true }) + mkdirSync(runtimeDir, { recursive: true }) + let node + try { + node = sdk.Node.create({ + ownerKeypairHex: sdk.generateOwnerKeypairHex(), + inviteToken: `packaging-smoke-${options.target}-${process.pid}-${Date.now()}`, + cacheDir, + runtimeDir, + servingEnabled: false + }) + await operationTimeout('node.start()', node.start(), 45000) + const status = await operationTimeout('node.status()', node.status(), 15000) + assert.ok(status && typeof status === 'object', 'node.status() must return an object') + process.stdout.write(`${JSON.stringify({ target: options.target, status })}\n`) + } finally { + try { + if (node) await operationTimeout('node.stop()', node.stop(), 15000) + } finally { + rmSync(smokeRoot, { recursive: true, force: true }) + } + } +} + +function supervise(options) { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [__filename, '--child-options', JSON.stringify(options)], { + cwd: options.packageRoot, + env: process.env, + stdio: 'inherit' + }) + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + child.kill('SIGKILL') + }, options.timeoutMs) + child.once('error', (error) => { + clearTimeout(timer) + reject(error) + }) + child.once('exit', (code, signal) => { + clearTimeout(timer) + if (timedOut) { + reject(new Error(`Node SDK smoke did not exit normally within ${options.timeoutMs}ms`)) + } else if (code !== 0 || signal) { + reject(new Error(`Node SDK smoke child exited with code=${code} signal=${signal || 'none'}`)) + } else { + resolve() + } + }) + }) +} + +async function main() { + if (process.argv[2] === '--child-options') { + await runChild(JSON.parse(process.argv[3])) + return + } + const options = parseArgs(process.argv.slice(2)) + await supervise(options) + process.stdout.write(`Node SDK runtime smoke passed for ${options.target}\n`) +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 +}) diff --git a/scripts/verify-host-invariant.ts b/scripts/verify-host-invariant.ts new file mode 100644 index 0000000..1737435 --- /dev/null +++ b/scripts/verify-host-invariant.ts @@ -0,0 +1,45 @@ +#!/usr/bin/env -S node --experimental-strip-types +import { readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +type Provenance = { arch: string; host_sha256: string; platform: string }; +const SHA256 = /^[0-9a-f]{64}$/; + +export function verifyHostInvariant(records: readonly Provenance[]): void { + if (records.length === 0) throw new Error("no verified upstream provenance records were supplied"); + const byPlatform = new Map>(); + for (const record of records) { + if (typeof record.platform !== "string" || !/^[a-z0-9]+\/[a-z0-9_]+$/.test(record.platform)) { + throw new Error(`invalid provenance platform: ${String(record.platform)}`); + } + if (typeof record.arch !== "string" || !/^[a-z0-9_]+$/.test(record.arch)) { + throw new Error(`invalid provenance architecture: ${String(record.arch)}`); + } + if (typeof record.host_sha256 !== "string" || !SHA256.test(record.host_sha256)) { + throw new Error(`invalid host SHA-256 for ${record.platform}/${record.arch}`); + } + const key = `${record.platform}/${record.arch}`; + const digests = byPlatform.get(key) ?? new Set(); + digests.add(record.host_sha256); + byPlatform.set(key, digests); + } + for (const [platform, digests] of byPlatform) { + if (digests.size !== 1) { + throw new Error(`backend product rows for ${platform} contain different host SHA-256 values: ${[...digests].sort().join(", ")}`); + } + } +} + +export function main(argv: string[]): number { + try { + if (argv.length === 0) throw new Error("at least one provenance JSON path is required"); + verifyHostInvariant(argv.map((path) => JSON.parse(readFileSync(path, "utf8")) as Provenance)); + console.log(`verified one host SHA-256 per platform/architecture across ${argv.length} product rows`); + return 0; + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + return 1; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) process.exitCode = main(process.argv.slice(2)); diff --git a/scripts/verify-product-schema.ts b/scripts/verify-product-schema.ts new file mode 100644 index 0000000..7a09fcc --- /dev/null +++ b/scripts/verify-product-schema.ts @@ -0,0 +1,38 @@ +#!/usr/bin/env -S node --experimental-strip-types +import { createHash, timingSafeEqual } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +export function sha256(contents: Buffer): string { + return createHash("sha256").update(contents).digest("hex"); +} + +export function verifyProductSchema(producer: Buffer, consumer: Buffer): { sha256: string } { + JSON.parse(producer.toString("utf8")); + JSON.parse(consumer.toString("utf8")); + if (producer.length !== consumer.length || !timingSafeEqual(producer, consumer)) { + throw new Error(`product-v2 schema drift: producer=${sha256(producer)} consumer=${sha256(consumer)}`); + } + return { sha256: sha256(consumer) }; +} + +function requiredOption(argv: string[], name: string): string { + const index = argv.indexOf(name); + const value = index >= 0 ? argv[index + 1] : undefined; + if (!value || value.startsWith("--")) throw new Error(`${name} is required`); + return value; +} + +export function main(argv: string[]): number { + try { + const producer = readFileSync(requiredOption(argv, "--producer-schema")); + const consumer = readFileSync(requiredOption(argv, "--consumer-schema")); + console.log(JSON.stringify(verifyProductSchema(producer, consumer))); + return 0; + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + return 1; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) process.exitCode = main(process.argv.slice(2)); diff --git a/tests/client-readiness-smoke.test.ts b/tests/client-readiness-smoke.test.ts new file mode 100644 index 0000000..8aba2c4 --- /dev/null +++ b/tests/client-readiness-smoke.test.ts @@ -0,0 +1,133 @@ +import assert from "node:assert/strict"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { spawnSync } from "node:child_process"; +import { test } from "node:test"; + +const smoke = resolve("scripts/client-readiness-smoke.sh"); + +function fakeExecutable( + t: { after(callback: () => void): void }, + body: string, +): { executable: string; marker: string } { + const directory = mkdtempSync(resolve(tmpdir(), "client-readiness-smoke-test-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const executable = resolve(directory, "mesh-llm"); + const marker = resolve(directory, "events"); + writeFileSync(executable, `#!/bin/sh\nset -eu\n${body}`); + chmodSync(executable, 0o755); + return { executable, marker }; +} + +function fakeNodeExecutable( + t: { after(callback: () => void): void }, + body: string, +): { executable: string; marker: string } { + const directory = mkdtempSync(resolve(tmpdir(), "client-readiness-smoke-test-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const executable = resolve(directory, "mesh-llm"); + const marker = resolve(directory, "events"); + writeFileSync(executable, `#!/usr/bin/env node\n${body}`); + chmodSync(executable, 0o755); + return { executable, marker }; +} + +function runSmoke(executable: string, marker: string, readyTimeout = "3", shutdownTimeout = "3") { + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; + return spawnSync("sh", [smoke], { + encoding: "utf8", + timeout: 15_000, + env: { + ...env, + MESH_LLM_SMOKE_BIN: executable, + MESH_LLM_SMOKE_READY_TIMEOUT_SECONDS: readyTimeout, + MESH_LLM_SMOKE_SHUTDOWN_TIMEOUT_SECONDS: shutdownTimeout, + SMOKE_MARKER: marker, + }, + }); +} + +function markerPids(marker: string): number[] { + return readFileSync(marker, "utf8").trim().split("\n").map((line) => Number(line.split(":")[1])); +} + +function assertProcessAbsent(pid: number): void { + assert.throws(() => process.kill(pid, 0), (error: NodeJS.ErrnoException) => error.code === "ESRCH"); +} + +test("client readiness smoke requires JSON readiness, a live process, and clean SIGINT", { concurrency: false }, (t) => { + const fixture = fakeNodeExecutable(t, ` +const fs = require('node:fs') +fs.writeFileSync(process.env.SMOKE_MARKER, \`start:\${process.pid}\\n\`) +process.on('SIGINT', () => { + fs.appendFileSync(process.env.SMOKE_MARKER, \`int:\${process.pid}\\n\`) + process.exit(0) +}) +console.log('{"event":"passive_mode","message":"Client daemon ready; local model loading is disabled","role":"client","status":"ready"}') +setInterval(() => {}, 1000) +`); + const result = runSmoke(fixture.executable, fixture.marker); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + assert.match(result.stdout, /client readiness smoke passed/); + const pids = markerPids(fixture.marker); + assert.deepEqual(pids, [pids[0], pids[0]], "the exec-owned runtime PID must receive SIGINT"); + assertProcessAbsent(pids[0]); +}); + +test("client readiness smoke fails an executable that exits before ready", { concurrency: false }, (t) => { + const fixture = fakeExecutable(t, ` +printf 'start:%s\\n' "$$" > "$SMOKE_MARKER" +exit 42 +`); + const startedAt = performance.now(); + const result = runSmoke(fixture.executable, fixture.marker); + const elapsed = performance.now() - startedAt; + assert.notEqual(result.status, 0); + assert.match(result.stderr, /exited before readiness with exit code 42/); + assert.ok(elapsed < 2_500, `early exit took ${elapsed}ms to detect`); + assertProcessAbsent(markerPids(fixture.marker)[0]); +}); + +test("client readiness smoke requires structured fields on one JSONL record", { concurrency: false }, (t) => { + const fixture = fakeNodeExecutable(t, ` +const fs = require('node:fs') +fs.writeFileSync(process.env.SMOKE_MARKER, \`start:\${process.pid}\\n\`) +process.on('SIGINT', () => { + fs.appendFileSync(process.env.SMOKE_MARKER, \`int:\${process.pid}\\n\`) + process.exit(0) +}) +console.log('{"event":"passive_mode","status":"starting","role":"server"}') +console.log('{"event":"unrelated","status":"ready","role":"server"}') +console.log('{"event":"unrelated","status":"starting","role":"client"}') +setInterval(() => {}, 1000) +`); + const result = runSmoke(fixture.executable, fixture.marker, "1", "3"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not reach structured readiness/); + assertProcessAbsent(markerPids(fixture.marker)[0]); +}); + +test("client readiness smoke bounds SIGINT and force-cleans an unresponsive runtime", { concurrency: false }, (t) => { + const fixture = fakeNodeExecutable(t, ` +const fs = require('node:fs') +fs.writeFileSync(process.env.SMOKE_MARKER, \`start:\${process.pid}\\n\`) +process.on('SIGINT', () => {}) +process.on('SIGTERM', () => {}) +console.log('{"message":"Client ready"}') +setInterval(() => {}, 1000) +`); + const result = runSmoke(fixture.executable, fixture.marker, "3", "1"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not stop after SIGINT within 1s/); + assertProcessAbsent(markerPids(fixture.marker)[0]); +}); + +test("runtime image QA covers both direct binary and final entrypoint command paths", { concurrency: false }, () => { + const dockerfile = readFileSync(resolve("docker/Dockerfile.mesh-llm"), "utf8"); + const imageQa = readFileSync(resolve("docker/qa-runtime-image.sh"), "utf8"); + assert.match(dockerfile, /MESH_LLM_SMOKE_BIN=\/usr\/local\/bin\/mesh-llm-entrypoint sh \/usr\/local\/bin\/client-readiness-smoke/); + assert.match(imageQa, /\/usr\/local\/bin\/mesh-llm --version/); + assert.match(imageQa, /\/usr\/local\/bin\/mesh-llm-entrypoint --version/); +}); diff --git a/tests/homebrew-release.test.ts b/tests/homebrew-release.test.ts index 3c4eb3d..265d27e 100644 --- a/tests/homebrew-release.test.ts +++ b/tests/homebrew-release.test.ts @@ -31,3 +31,16 @@ test("validates inputs and CLI", (t) => { assert.equal(main(["unexpected"]), 1); assert.equal(main(["--version", "0.73.1"]), 1); }); + +test("formula test certifies isolated no-driver client readiness", () => { + const template = readFileSync("packaging/homebrew/Formula/mesh-llm.rb.template", "utf8"); + for (const snippet of [ + "MESH_LLM_NATIVE_RUNTIME_CACHE_DIR", + "MESH_LLM_RUNTIME_ROOT", + '"--log-format", "json"', + '"--no-console", "client", "--auto"', + "Client ready", + 'Process.kill("INT", pid)', + "Timeout.timeout(10)", + ]) assert.match(template, new RegExp(snippet.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))); +}); diff --git a/tests/node-sdk-runtime-smoke.test.ts b/tests/node-sdk-runtime-smoke.test.ts new file mode 100644 index 0000000..41322b5 --- /dev/null +++ b/tests/node-sdk-runtime-smoke.test.ts @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { spawnSync } from "node:child_process"; +import { test } from "node:test"; + +const smoke = resolve("scripts/node-sdk-runtime-smoke.cjs"); + +type FixtureBehavior = "normal" | "leak" | "start-failure"; + +function project( + t: { after(callback: () => void): void }, + behavior: FixtureBehavior = "normal", +): { root: string; marker: string } { + const root = mkdtempSync(resolve(tmpdir(), "node-sdk-runtime-smoke-test-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const packageRoot = resolve(root, "node_modules/@mesh-llm/sdk"); + const marker = resolve(root, "events.jsonl"); + mkdirSync(packageRoot, { recursive: true }); + writeFileSync(resolve(root, "package.json"), '{"private":true}\n'); + writeFileSync(resolve(packageRoot, "package.json"), '{"name":"@mesh-llm/sdk","main":"index.js"}\n'); + writeFileSync(resolve(packageRoot, "index.js"), ` + const fs = require('node:fs') + const append = (event) => fs.appendFileSync(process.env.SMOKE_MARKER, JSON.stringify(event) + '\\n') + module.exports = { + currentMeshVersion: () => '1.2.3', + generateOwnerKeypairHex: () => 'owner-keypair', + Node: { create(options) { + if (!options.cacheDir || !options.runtimeDir || options.servingEnabled !== false) throw new Error('invalid isolation') + if (!options.inviteToken.startsWith('packaging-smoke-test-target-')) throw new Error('invite token is not unique') + append({ event: 'create', servingEnabled: options.servingEnabled }) + return { + async start() { + append({ event: 'start' }) + ${behavior === "start-failure" ? "throw new Error('start partially initialized')" : ""} + }, + async status() { append({ event: 'status' }); return { connected: false, peerCount: 0 } }, + async stop() { + append({ event: 'stop' }) + ${behavior === "leak" ? "setInterval(() => {}, 1000)" : ""} + } + } + } } + } + `); + return { root, marker }; +} + +test("runtime smoke starts, reads status, stops, and exits normally", (t) => { + const fixture = project(t); + const result = spawnSync(process.execPath, [smoke, + "--package-root", fixture.root, + "--expected-version", "1.2.3", + "--target", "test-target", + "--timeout-ms", "3000", + ], { encoding: "utf8", timeout: 5000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + assert.match(result.stdout, /runtime smoke passed/); + assert.deepEqual(readFileSync(fixture.marker, "utf8").trim().split("\n").map((line) => JSON.parse(line).event), [ + "create", "start", "status", "stop", + ]); +}); + +test("runtime smoke fails a native addon that prevents normal process exit", (t) => { + const fixture = project(t, "leak"); + const result = spawnSync(process.execPath, [smoke, + "--package-root", fixture.root, + "--expected-version", "1.2.3", + "--target", "test-target", + "--timeout-ms", "400", + ], { encoding: "utf8", timeout: 3000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not exit normally/); +}); + +test("runtime smoke attempts stop when start partially initializes and rejects", (t) => { + const fixture = project(t, "start-failure"); + const result = spawnSync(process.execPath, [smoke, + "--package-root", fixture.root, + "--expected-version", "1.2.3", + "--target", "test-target", + "--timeout-ms", "3000", + ], { encoding: "utf8", timeout: 5000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /start partially initialized/); + assert.deepEqual(readFileSync(fixture.marker, "utf8").trim().split("\n").map((line) => JSON.parse(line).event), [ + "create", "start", "stop", + ]); +}); + +test("runtime smoke validates required arguments", () => { + const result = spawnSync(process.execPath, [smoke], { encoding: "utf8" }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /--expected-version is required/); +}); + +test("release workflow fresh-installs and starts every addon lane and the assembled package", () => { + const workflow = readFileSync(resolve(".github/workflows/images-release.yml"), "utf8"); + assert.match(workflow, /name: Pack, fresh-install, and start Node SDK addon/); + assert.match(workflow, /NODE_SDK_TARGET: \$\{\{ matrix\.target \}\}/); + assert.match(workflow, /npm install "\$tarball"/); + assert.match(workflow, /node scripts\/node-sdk-runtime-smoke\.cjs/); + assert.match(workflow, /--target "\$NODE_SDK_TARGET"/); + assert.match(workflow, /--target linux-x64-assembled-package/); +}); diff --git a/tests/product-contract.test.ts b/tests/product-contract.test.ts new file mode 100644 index 0000000..43dd301 --- /dev/null +++ b/tests/product-contract.test.ts @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { test } from "node:test"; +import { verifyHostInvariant } from "../scripts/verify-host-invariant.ts"; +import { verifyProductSchema } from "../scripts/verify-product-schema.ts"; + +const sha = (character: string) => character.repeat(64); + +test("product schema verifier accepts exact bytes and rejects schema drift", () => { + const schema = Buffer.from('{"schema_version":2}\n'); + assert.deepEqual(verifyProductSchema(schema, Buffer.from(schema)), { + sha256: "f3d8a19f6b3bc52dbb1fb06e455e04c1a252308544786a30a6d21df70a476ddc", + }); + assert.throws(() => verifyProductSchema(schema, Buffer.from('{"schema_version":3}\n')), /schema drift/); + assert.throws(() => verifyProductSchema(Buffer.from("not json"), schema), /Unexpected token/); +}); + +test("host invariant rejects backend products with different hosts for one platform", () => { + assert.doesNotThrow(() => verifyHostInvariant([ + { platform: "linux/amd64", arch: "amd64", host_sha256: sha("a") }, + { platform: "linux/amd64", arch: "amd64", host_sha256: sha("a") }, + { platform: "linux/arm64", arch: "arm64", host_sha256: sha("b") }, + ])); + assert.throws(() => verifyHostInvariant([ + { platform: "linux/amd64", arch: "amd64", host_sha256: sha("a") }, + { platform: "linux/amd64", arch: "amd64", host_sha256: sha("b") }, + ]), /different host SHA-256/); + assert.throws(() => verifyHostInvariant([{ platform: "linux/amd64", arch: "amd64", host_sha256: "bad" }]), /invalid host/); +}); + +test("release workflow verifies the producer schema and gates package rows on host identity", () => { + const contents = readFileSync(resolve(".github/workflows/images-release.yml"), "utf8"); + assert.match(contents, /Verify producer product-v2 schema/); + assert.match(contents, /verify-product-schema\.ts/); + assert.match(contents, /upstream-host-invariant:/); + assert.match(contents, /needs: \[plan, upstream, upstream-host-invariant\]/); +}); + +test("schema verifier CLI reports mismatched producer bytes", (t) => { + const directory = mkdtempSync(resolve(tmpdir(), "product-contract-test-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const producer = resolve(directory, "producer.json"); + const consumer = resolve(directory, "consumer.json"); + writeFileSync(producer, "{}\n"); + writeFileSync(consumer, "{}\n"); + assert.equal(spawnSync("node", ["--experimental-strip-types", "scripts/verify-product-schema.ts", "--producer-schema", producer, "--consumer-schema", consumer], { encoding: "utf8" }).status, 0); + writeFileSync(consumer, "{ }\n"); + assert.notEqual(spawnSync("node", ["--experimental-strip-types", "scripts/verify-product-schema.ts", "--producer-schema", producer, "--consumer-schema", consumer], { encoding: "utf8" }).status, 0); +}); + +test("host invariant CLI rejects invalid provenance", (t) => { + const directory = mkdtempSync(resolve(tmpdir(), "host-invariant-test-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const first = resolve(directory, "first.json"); + const second = resolve(directory, "second.json"); + writeFileSync(first, JSON.stringify({ platform: "linux/amd64", arch: "amd64", host_sha256: sha("a") })); + writeFileSync(second, JSON.stringify({ platform: "linux/amd64", arch: "amd64", host_sha256: sha("b") })); + assert.notEqual(spawnSync("node", ["--experimental-strip-types", "scripts/verify-host-invariant.ts", first, second], { encoding: "utf8" }).status, 0); + assert.notEqual(spawnSync("node", ["--experimental-strip-types", "scripts/verify-host-invariant.ts"], { encoding: "utf8" }).status, 0); +}); diff --git a/tests/workflow-provenance.test.ts b/tests/workflow-provenance.test.ts index 702a9f9..e57e875 100644 --- a/tests/workflow-provenance.test.ts +++ b/tests/workflow-provenance.test.ts @@ -59,6 +59,8 @@ test("both product input steps reject ambiguous or invalid provenance", () => { [{ ...valid, host_sha256: "invalid" }], [{ ...valid, runtime_sha256: "B".repeat(64) }], [{ ...valid, runtime_id: "" }], + [{ ...valid, runtime_id: "linux-cpu\nMESH_LLM_HOST_SHA=forged" }], + [{ ...valid, runtime_id: "linux/cpu" }], ]; for (const script of productInputSteps) { for (const provenance of invalidCases) { From 0c2157b36c34ff238cc6861004479d7e55f878e4 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 03:53:50 -0400 Subject: [PATCH 4/5] fix: harden distribution runtime certification --- TODO.md | 6 +++ docker/qa-runtime-image.sh | 8 ++- docs/release-checklist.md | 17 +++++-- scripts/client-readiness-smoke.sh | 71 ++++++++++----------------- scripts/node-sdk-runtime-smoke.cjs | 59 +++++++++++++++++----- tests/client-readiness-smoke.test.ts | 13 +++++ tests/node-sdk-runtime-smoke.test.ts | 73 ++++++++++++++++++++++++++-- 7 files changed, 180 insertions(+), 67 deletions(-) diff --git a/TODO.md b/TODO.md index 36e31a4..b3206f7 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,11 @@ # Production Readiness TODO +- [x] Harden composed-runtime certification teardown and dependency inspection. + QA: shell syntax accepts `docker/qa-runtime-image.sh`; focused client-readiness + and Node SDK smoke tests prove fail-closed captured `ldd` output, SIGTERM-first + Node stop/temp cleanup, bounded SIGKILL fallback, and the updated release + checklist requires real runtime readiness and clean shutdown. + - [x] Model npm addon builds as independently toggleable packaging lanes. QA: configuration validation and 100% coverage matrix tests prove enabled, disabled, filtered, and empty npm matrix behavior. diff --git a/docker/qa-runtime-image.sh b/docker/qa-runtime-image.sh index ee8ae49..faafc27 100755 --- a/docker/qa-runtime-image.sh +++ b/docker/qa-runtime-image.sh @@ -24,9 +24,13 @@ esac test -x /usr/local/bin/mesh-llm test -x /usr/local/bin/mesh-llm-entrypoint -missing="$(ldd /usr/local/bin/mesh-llm | awk '/not found/ { print $1 }')" +if ! ldd_output="$(ldd /usr/local/bin/mesh-llm 2>&1)"; then + echo "ldd failed to inspect the mesh-llm host: $ldd_output" >&2 + exit 1 +fi +missing="$(printf '%s\n' "$ldd_output" | awk '/not found/ { print $1 }')" [ -z "$missing" ] || { echo "host has unresolved dependencies: $missing" >&2; exit 1; } -if ldd /usr/local/bin/mesh-llm | grep -Eiq 'cuda|cublas|nccl|hip|hsa|vulkan|ggml|llama'; then +if printf '%s\n' "$ldd_output" | grep -Eiq 'cuda|cublas|nccl|hip|hsa|vulkan|ggml|llama'; then echo "backend dependency leaked into the mesh-llm host" >&2 exit 1 fi diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 471c169..7035475 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -15,14 +15,23 @@ - [ ] Every native package passes metadata inspection, exact filename/checksum checks, package-manager installation, `mesh-llm --version`, `--help`, and `mesh-llm runtime list` without GPU passthrough. The package owns the - versioned runtime tree, and an empty user cache stays empty. + versioned runtime tree, and an empty user cache stays empty. The exact + package-owned executable then reaches structured JSON client readiness while + alive and stops cleanly on SIGINT within the bounded timeout. - [ ] Every runtime image installs the matching package artifact. Every backend, including CUDA, passes the no-device command smoke and the neutral host has no - backend runtime imports. + backend runtime imports. The final entrypoint reaches structured JSON client + readiness while the container remains alive, stops cleanly within the bounded + timeout, and leaves no test container or listener behind. - [ ] The arm64 Homebrew formula installs and tests the upstream Metal archive. + The exact Cellar binary reaches structured JSON client readiness while alive + and completes bounded clean shutdown with isolated runtime and cache state. - [ ] Every enabled npm addon lane succeeds; the assembled tarball passes - `npm publish --dry-run`, installs in a clean project, and loads the host addon. -- [ ] The final readiness manifest reports success for plan, upstream, native packages, runtime images, and Homebrew. + `npm publish --dry-run`, installs in a clean project, and completes public + `Node.create`, `start`, `status`, and `finally`-guarded `stop` with normal + process exit and verified temporary-state cleanup. +- [ ] The final readiness manifest reports success for plan, upstream, native + packages, runtime images, Homebrew, and every enabled npm addon/preflight lane. ## Publication diff --git a/scripts/client-readiness-smoke.sh b/scripts/client-readiness-smoke.sh index aaf1f3b..ff9034e 100644 --- a/scripts/client-readiness-smoke.sh +++ b/scripts/client-readiness-smoke.sh @@ -17,12 +17,8 @@ esac smoke_root="$(mktemp -d "${TMPDIR:-/tmp}/mesh-llm-client-smoke.XXXXXX")" pid="" -watcher_pid="" guardian_pid="" log="$smoke_root/client.jsonl" -ready_marker="$smoke_root/ready" -ready_timeout_marker="$smoke_root/ready-timeout" -watcher_cancel_marker="$smoke_root/watcher-cancel" shutdown_timeout_marker="$smoke_root/shutdown-timeout" guardian_done_marker="$smoke_root/guardian-done" # Fresh containers normally have no listeners. Derive two distinct high ports @@ -71,9 +67,7 @@ signal_and_wait() { } cleanup() { - trap - EXIT HUP INT TERM USR1 USR2 - stop_helper "$watcher_pid" "$watcher_cancel_marker" - watcher_pid="" + trap - EXIT HUP INT TERM stop_helper "$guardian_pid" "$guardian_done_marker" guardian_pid="" if [ -n "$pid" ]; then @@ -118,50 +112,37 @@ chmod 700 \ ) >"$log" 2>&1 & pid=$! -main_pid=$$ -trap ':' USR1 USR2 -( - elapsed=0 - while [ "$elapsed" -lt "$ready_timeout" ]; do - [ ! -e "$watcher_cancel_marker" ] || exit 0 - if grep -Eq '^[[:space:]]*\{.*"Client ready".*\}[[:space:]]*$' "$log" || - grep -E '"event"[[:space:]]*:[[:space:]]*"passive_mode"' "$log" | - grep -E '"status"[[:space:]]*:[[:space:]]*"ready"' | - grep -Eq '"role"[[:space:]]*:[[:space:]]*"client"'; then - : > "$ready_marker" - kill -USR1 "$main_pid" 2>/dev/null || true - exit 0 - fi - sleep 1 - [ ! -e "$watcher_cancel_marker" ] || exit 0 - elapsed=$((elapsed + 1)) - done - : > "$ready_timeout_marker" - kill -USR2 "$main_pid" 2>/dev/null || true -) /dev/null 2>&1 & -watcher_pid=$! - -set +e -wait "$pid" -readiness_wait_exit=$? -set -e -stop_helper "$watcher_pid" "$watcher_cancel_marker" -watcher_pid="" -trap - USR1 USR2 +readiness_reached=false +elapsed=0 +while [ "$elapsed" -lt "$ready_timeout" ]; do + if grep -Eq '^[[:space:]]*\{.*"Client ready".*\}[[:space:]]*$' "$log" || + grep -E '"event"[[:space:]]*:[[:space:]]*"passive_mode"' "$log" | + grep -E '"status"[[:space:]]*:[[:space:]]*"ready"' | + grep -Eq '"role"[[:space:]]*:[[:space:]]*"client"'; then + readiness_reached=true + break + fi + if ! kill -0 "$pid" 2>/dev/null; then + set +e + wait "$pid" + readiness_wait_exit=$? + set -e + pid="" + echo "mesh-llm client exited before readiness with exit code $readiness_wait_exit" >&2 + cat "$log" >&2 || true + exit 1 + fi + sleep 1 + elapsed=$((elapsed + 1)) +done -if [ -e "$ready_timeout_marker" ]; then +if [ "$readiness_reached" != true ]; then + signal_and_wait "$shutdown_timeout_marker" echo "mesh-llm client did not reach structured readiness while alive within ${ready_timeout}s" >&2 cat "$log" >&2 || true exit 1 fi -if [ ! -e "$ready_marker" ]; then - pid="" - echo "mesh-llm client exited before readiness with exit code $readiness_wait_exit" >&2 - cat "$log" >&2 || true - exit 1 -fi - if ! kill -0 "$pid" 2>/dev/null; then set +e wait "$pid" diff --git a/scripts/node-sdk-runtime-smoke.cjs b/scripts/node-sdk-runtime-smoke.cjs index ab85d7f..ed4db7c 100644 --- a/scripts/node-sdk-runtime-smoke.cjs +++ b/scripts/node-sdk-runtime-smoke.cjs @@ -26,11 +26,16 @@ function parseArgs(argv) { if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 100) { throw new Error('--timeout-ms must be an integer of at least 100') } + const shutdownGraceMs = Number(values.get('shutdown-grace-ms') || '20000') + if (!Number.isSafeInteger(shutdownGraceMs) || shutdownGraceMs < 100) { + throw new Error('--shutdown-grace-ms must be an integer of at least 100') + } return { expectedVersion: required('expected-version'), packageRoot: path.resolve(required('package-root')), target: required('target'), - timeoutMs + timeoutMs, + shutdownGraceMs } } @@ -50,13 +55,37 @@ async function runChild(options) { assert.equal(sdk.currentMeshVersion(), options.expectedVersion) assert.equal(typeof sdk.generateOwnerKeypairHex, 'function') assert.equal(typeof sdk.Node?.create, 'function') + assert.equal(typeof options.smokeRoot, 'string') - const smokeRoot = mkdtempSync(path.join(tmpdir(), `mesh-llm-node-sdk-${options.target}-`)) + const smokeRoot = options.smokeRoot const cacheDir = path.join(smokeRoot, 'cache') const runtimeDir = path.join(smokeRoot, 'runtime') mkdirSync(cacheDir, { recursive: true }) mkdirSync(runtimeDir, { recursive: true }) let node + let cleanupPromise + const cleanup = () => { + if (!cleanupPromise) { + cleanupPromise = (async () => { + try { + if (node) await operationTimeout('node.stop()', node.stop(), 15000) + } finally { + rmSync(smokeRoot, { recursive: true, force: true }) + } + })() + } + return cleanupPromise + } + const terminate = () => { + void cleanup().then( + () => process.exit(0), + (error) => { + console.error(error instanceof Error ? error.message : String(error)) + process.exit(1) + } + ) + } + process.once('SIGTERM', terminate) try { node = sdk.Node.create({ ownerKeypairHex: sdk.generateOwnerKeypairHex(), @@ -70,32 +99,40 @@ async function runChild(options) { assert.ok(status && typeof status === 'object', 'node.status() must return an object') process.stdout.write(`${JSON.stringify({ target: options.target, status })}\n`) } finally { - try { - if (node) await operationTimeout('node.stop()', node.stop(), 15000) - } finally { - rmSync(smokeRoot, { recursive: true, force: true }) - } + await cleanup() } } function supervise(options) { return new Promise((resolve, reject) => { - const child = spawn(process.execPath, [__filename, '--child-options', JSON.stringify(options)], { + const smokeRoot = mkdtempSync(path.join(tmpdir(), `mesh-llm-node-sdk-${options.target}-`)) + const childOptions = { ...options, smokeRoot } + const child = spawn(process.execPath, [__filename, '--child-options', JSON.stringify(childOptions)], { cwd: options.packageRoot, env: process.env, stdio: 'inherit' }) let timedOut = false + let killTimer + const clearTimers = () => { + clearTimeout(timer) + if (killTimer) clearTimeout(killTimer) + } const timer = setTimeout(() => { timedOut = true - child.kill('SIGKILL') + child.kill('SIGTERM') + killTimer = setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL') + }, options.shutdownGraceMs) }, options.timeoutMs) child.once('error', (error) => { - clearTimeout(timer) + clearTimers() + rmSync(smokeRoot, { recursive: true, force: true }) reject(error) }) child.once('exit', (code, signal) => { - clearTimeout(timer) + clearTimers() + rmSync(smokeRoot, { recursive: true, force: true }) if (timedOut) { reject(new Error(`Node SDK smoke did not exit normally within ${options.timeoutMs}ms`)) } else if (code !== 0 || signal) { diff --git a/tests/client-readiness-smoke.test.ts b/tests/client-readiness-smoke.test.ts index 8aba2c4..8cbdc50 100644 --- a/tests/client-readiness-smoke.test.ts +++ b/tests/client-readiness-smoke.test.ts @@ -124,10 +124,23 @@ setInterval(() => {}, 1000) assertProcessAbsent(markerPids(fixture.marker)[0]); }); +test("client readiness smoke polls readiness without shell-signal wakeups", { concurrency: false }, () => { + const source = readFileSync(smoke, "utf8"); + assert.match(source, /readiness_reached=false/); + assert.match(source, /if ! kill -0 "\$pid" 2>\/dev\/null; then/); + assert.doesNotMatch(source, /USR[12]/); + assert.doesNotMatch(source, /watcher_pid/); +}); + test("runtime image QA covers both direct binary and final entrypoint command paths", { concurrency: false }, () => { const dockerfile = readFileSync(resolve("docker/Dockerfile.mesh-llm"), "utf8"); const imageQa = readFileSync(resolve("docker/qa-runtime-image.sh"), "utf8"); assert.match(dockerfile, /MESH_LLM_SMOKE_BIN=\/usr\/local\/bin\/mesh-llm-entrypoint sh \/usr\/local\/bin\/client-readiness-smoke/); assert.match(imageQa, /\/usr\/local\/bin\/mesh-llm --version/); assert.match(imageQa, /\/usr\/local\/bin\/mesh-llm-entrypoint --version/); + assert.ok(imageQa.includes('if ! ldd_output="$(ldd /usr/local/bin/mesh-llm 2>&1)"; then')); + assert.match(imageQa, /ldd failed to inspect the mesh-llm host/); + assert.equal((imageQa.match(/ldd \/usr\/local\/bin\/mesh-llm/g) ?? []).length, 1); + assert.match(imageQa, /printf '%s\\n' "\$ldd_output" \| awk/); + assert.match(imageQa, /printf '%s\\n' "\$ldd_output" \| grep/); }); diff --git a/tests/node-sdk-runtime-smoke.test.ts b/tests/node-sdk-runtime-smoke.test.ts index 41322b5..ab09acd 100644 --- a/tests/node-sdk-runtime-smoke.test.ts +++ b/tests/node-sdk-runtime-smoke.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; import { spawnSync } from "node:child_process"; @@ -7,7 +7,9 @@ import { test } from "node:test"; const smoke = resolve("scripts/node-sdk-runtime-smoke.cjs"); -type FixtureBehavior = "normal" | "leak" | "start-failure"; +type FixtureBehavior = "normal" | "leak" | "start-failure" | "start-hang" | "stop-hang"; + +type FixtureEvent = { event: string; smokeRoot?: string }; function project( t: { after(callback: () => void): void }, @@ -29,16 +31,18 @@ function project( Node: { create(options) { if (!options.cacheDir || !options.runtimeDir || options.servingEnabled !== false) throw new Error('invalid isolation') if (!options.inviteToken.startsWith('packaging-smoke-test-target-')) throw new Error('invite token is not unique') - append({ event: 'create', servingEnabled: options.servingEnabled }) + append({ event: 'create', servingEnabled: options.servingEnabled, smokeRoot: require('node:path').dirname(options.cacheDir) }) return { async start() { append({ event: 'start' }) ${behavior === "start-failure" ? "throw new Error('start partially initialized')" : ""} + ${behavior === "start-hang" ? "await new Promise(() => {})" : ""} }, async status() { append({ event: 'status' }); return { connected: false, peerCount: 0 } }, async stop() { append({ event: 'stop' }) ${behavior === "leak" ? "setInterval(() => {}, 1000)" : ""} + ${behavior === "stop-hang" ? "await new Promise(() => {})" : ""} } } } } @@ -47,6 +51,16 @@ function project( return { root, marker }; } +function fixtureEvents(marker: string): FixtureEvent[] { + return readFileSync(marker, "utf8").trim().split("\n").map((line) => JSON.parse(line)); +} + +function assertSmokeRootRemoved(marker: string): void { + const smokeRoot = fixtureEvents(marker)[0]?.smokeRoot; + assert.equal(typeof smokeRoot, "string"); + assert.equal(existsSync(smokeRoot!), false, `smoke root still exists: ${smokeRoot}`); +} + test("runtime smoke starts, reads status, stops, and exits normally", (t) => { const fixture = project(t); const result = spawnSync(process.execPath, [smoke, @@ -57,9 +71,10 @@ test("runtime smoke starts, reads status, stops, and exits normally", (t) => { ], { encoding: "utf8", timeout: 5000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); assert.match(result.stdout, /runtime smoke passed/); - assert.deepEqual(readFileSync(fixture.marker, "utf8").trim().split("\n").map((line) => JSON.parse(line).event), [ + assert.deepEqual(fixtureEvents(fixture.marker).map(({ event }) => event), [ "create", "start", "status", "stop", ]); + assertSmokeRootRemoved(fixture.marker); }); test("runtime smoke fails a native addon that prevents normal process exit", (t) => { @@ -72,6 +87,10 @@ test("runtime smoke fails a native addon that prevents normal process exit", (t) ], { encoding: "utf8", timeout: 3000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); assert.notEqual(result.status, 0); assert.match(result.stderr, /did not exit normally/); + assert.deepEqual(fixtureEvents(fixture.marker).map(({ event }) => event), [ + "create", "start", "status", "stop", + ]); + assertSmokeRootRemoved(fixture.marker); }); test("runtime smoke attempts stop when start partially initializes and rejects", (t) => { @@ -84,9 +103,47 @@ test("runtime smoke attempts stop when start partially initializes and rejects", ], { encoding: "utf8", timeout: 5000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); assert.notEqual(result.status, 0); assert.match(result.stderr, /start partially initialized/); - assert.deepEqual(readFileSync(fixture.marker, "utf8").trim().split("\n").map((line) => JSON.parse(line).event), [ + assert.deepEqual(fixtureEvents(fixture.marker).map(({ event }) => event), [ "create", "start", "stop", ]); + assertSmokeRootRemoved(fixture.marker); +}); + +test("runtime smoke handles SIGTERM by stopping a partially started Node and removing temp state", (t) => { + const fixture = project(t, "start-hang"); + const result = spawnSync(process.execPath, [smoke, + "--package-root", fixture.root, + "--expected-version", "1.2.3", + "--target", "test-target", + "--timeout-ms", "400", + "--shutdown-grace-ms", "1500", + ], { encoding: "utf8", timeout: 3000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not exit normally/); + assert.deepEqual(fixtureEvents(fixture.marker).map(({ event }) => event), [ + "create", "start", "stop", + ]); + assertSmokeRootRemoved(fixture.marker); +}); + +test("runtime smoke bounds SIGTERM cleanup before SIGKILL and parent temp cleanup", (t) => { + const fixture = project(t, "stop-hang"); + const startedAt = performance.now(); + const result = spawnSync(process.execPath, [smoke, + "--package-root", fixture.root, + "--expected-version", "1.2.3", + "--target", "test-target", + "--timeout-ms", "400", + "--shutdown-grace-ms", "200", + ], { encoding: "utf8", timeout: 3000, env: { ...process.env, SMOKE_MARKER: fixture.marker } }); + const elapsed = performance.now() - startedAt; + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not exit normally/); + assert.ok(elapsed < 2000, `forced cleanup took ${elapsed}ms`); + assert.deepEqual(fixtureEvents(fixture.marker).map(({ event }) => event), [ + "create", "start", "status", "stop", + ]); + assertSmokeRootRemoved(fixture.marker); }); test("runtime smoke validates required arguments", () => { @@ -95,6 +152,12 @@ test("runtime smoke validates required arguments", () => { assert.match(result.stderr, /--expected-version is required/); }); +test("runtime smoke supervises graceful termination before forced termination", () => { + const source = readFileSync(smoke, "utf8"); + assert.match(source, /process\.once\('SIGTERM', terminate\)/); + assert.ok(source.indexOf("child.kill('SIGTERM')") < source.indexOf("child.kill('SIGKILL')")); +}); + test("release workflow fresh-installs and starts every addon lane and the assembled package", () => { const workflow = readFileSync(resolve(".github/workflows/images-release.yml"), "utf8"); assert.match(workflow, /name: Pack, fresh-install, and start Node SDK addon/); From 5756edcb0820571162f56607b6188d4ae60826d6 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 29 Jul 2026 04:00:08 -0400 Subject: [PATCH 5/5] fix: accept reordered readiness events --- scripts/client-readiness-smoke.sh | 19 +++++++++++++++---- tests/client-readiness-smoke.test.ts | 4 +++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/scripts/client-readiness-smoke.sh b/scripts/client-readiness-smoke.sh index ff9034e..463e5cd 100644 --- a/scripts/client-readiness-smoke.sh +++ b/scripts/client-readiness-smoke.sh @@ -112,13 +112,24 @@ chmod 700 \ ) >"$log" 2>&1 & pid=$! +readiness_in_log() { + awk ' + /^[[:space:]]*\{.*\}[[:space:]]*$/ { + client_ready = $0 ~ /"Client ready"/ + structured_ready = \ + $0 ~ /"event"[[:space:]]*:[[:space:]]*"passive_mode"/ && \ + $0 ~ /"status"[[:space:]]*:[[:space:]]*"ready"/ && \ + $0 ~ /"role"[[:space:]]*:[[:space:]]*"client"/ + if (client_ready || structured_ready) found = 1 + } + END { exit found ? 0 : 1 } + ' "$log" +} + readiness_reached=false elapsed=0 while [ "$elapsed" -lt "$ready_timeout" ]; do - if grep -Eq '^[[:space:]]*\{.*"Client ready".*\}[[:space:]]*$' "$log" || - grep -E '"event"[[:space:]]*:[[:space:]]*"passive_mode"' "$log" | - grep -E '"status"[[:space:]]*:[[:space:]]*"ready"' | - grep -Eq '"role"[[:space:]]*:[[:space:]]*"client"'; then + if readiness_in_log; then readiness_reached=true break fi diff --git a/tests/client-readiness-smoke.test.ts b/tests/client-readiness-smoke.test.ts index 8cbdc50..f8efd1c 100644 --- a/tests/client-readiness-smoke.test.ts +++ b/tests/client-readiness-smoke.test.ts @@ -65,7 +65,7 @@ process.on('SIGINT', () => { fs.appendFileSync(process.env.SMOKE_MARKER, \`int:\${process.pid}\\n\`) process.exit(0) }) -console.log('{"event":"passive_mode","message":"Client daemon ready; local model loading is disabled","role":"client","status":"ready"}') +console.log('{"role":"client","status":"ready","message":"Client daemon ready; local model loading is disabled","event":"passive_mode"}') setInterval(() => {}, 1000) `); const result = runSmoke(fixture.executable, fixture.marker); @@ -127,7 +127,9 @@ setInterval(() => {}, 1000) test("client readiness smoke polls readiness without shell-signal wakeups", { concurrency: false }, () => { const source = readFileSync(smoke, "utf8"); assert.match(source, /readiness_reached=false/); + assert.match(source, /readiness_in_log/); assert.match(source, /if ! kill -0 "\$pid" 2>\/dev\/null; then/); + assert.doesNotMatch(source, /grep -E '"event"/); assert.doesNotMatch(source, /USR[12]/); assert.doesNotMatch(source, /watcher_pid/); });