Consume and verify product-v2 release bundles

.github/workflows/images-precheck.yml

@@ -40,7 +40,7 @@ jobs:
             --test-coverage-lines=100 --test-coverage-branches=100 --test-coverage-functions=100 \
             tests/image-matrix.test.ts
           node --experimental-strip-types --test \
-            tests/homebrew-release.test.ts tests/node-sdk-package.test.ts tests/upstream-archive.test.ts
+            tests/client-readiness-smoke.test.ts tests/homebrew-release.test.ts tests/node-sdk-package.test.ts tests/node-sdk-runtime-smoke.test.ts tests/product-contract.test.ts tests/upstream-archive.test.ts tests/workflow-provenance.test.ts
       - name: Lint shell scripts
         run: |
           sudo apt-get update
@@ -57,8 +57,11 @@ jobs:
       - uses: docker/setup-buildx-action@v4
       - name: Check package-first Dockerfile targets
         run: |
-          mkdir -p artifacts/upstream artifacts/native-package
+          mkdir -p artifacts/upstream/native-runtimes/test-runtime/lib artifacts/native-package
           touch artifacts/upstream/mesh-llm
+          touch artifacts/upstream/product-manifest.json artifacts/upstream/host-imports.json
+          touch artifacts/upstream/native-runtimes/test-runtime/manifest.json
+          touch artifacts/upstream/native-runtimes/test-runtime/lib/libllama.so
           touch artifacts/native-package/mesh-llm-0.73.1-ubuntu-amd64-cpu.deb
           chmod +x artifacts/upstream/mesh-llm
           docker buildx build --check --platform linux/amd64 --target native-package-artifact --file docker/Dockerfile.mesh-llm \

.github/workflows/images-release.yml

@@ -184,6 +184,20 @@ jobs:
             echo "npm_plan=$npm_plan"
             echo "npm_enabled=$(jq -r .enabled <<<"$npm_plan")"
           } >> "$GITHUB_OUTPUT"
+      - name: Verify producer product-v2 schema
+        env:
+          MESH_SHA: ${{ steps.meta.outputs.mesh_sha }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          producer_schema="artifacts/producer-product-v2.schema.json"
+          mkdir -p artifacts
+          curl --fail --location --retry 3 --proto '=https' --tlsv1.2 \
+            --output "$producer_schema" \
+            "https://raw.githubusercontent.com/Mesh-LLM/mesh-llm/${MESH_SHA}/schemas/product-v2.schema.json"
+          node --experimental-strip-types scripts/verify-product-schema.ts \
+            --producer-schema "$producer_schema" \
+            --consumer-schema schemas/product-v2.schema.json
 
   upstream:
     name: Verify upstream ${{ matrix.upstream_asset_name }}
@@ -211,17 +225,50 @@ jobs:
             --source-url "${{ matrix.upstream_asset_url }}" \
             --version "${{ matrix.mesh_version }}" \
             --flavor "${{ matrix.upstream_flavor }}"
+          mkdir -p artifacts/provenance
+          jq --arg arch "${{ matrix.arch }}" --arg platform "${{ matrix.platform }}" \
+            '. + {arch: $arch, platform: $platform}' \
+            artifacts/upstream/upstream-provenance.json \
+            > "artifacts/provenance/${{ matrix.upstream_artifact_id }}.json"
           file artifacts/upstream/mesh-llm | grep -q 'ELF.*executable'
       - uses: actions/upload-artifact@v7
         with:
           name: ${{ matrix.upstream_artifact_id }}
           path: artifacts/upstream
           if-no-files-found: error
           retention-days: 14
+      - uses: actions/upload-artifact@v7
+        with:
+          name: mesh-llm-upstream-provenance-${{ matrix.upstream_artifact_id }}
+          path: artifacts/provenance/${{ matrix.upstream_artifact_id }}.json
+          if-no-files-found: error
+          retention-days: 14
+
+  upstream-host-invariant:
+    name: Verify one immutable host per platform and architecture
+    needs: [plan, upstream]
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-node@v7
+        with:
+          node-version: '24'
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: mesh-llm-upstream-provenance-*
+          path: artifacts/upstream-provenance
+          merge-multiple: true
+      - name: Reject backend products with different verified host bytes
+        shell: bash
+        run: |
+          set -euo pipefail
+          mapfile -d '' -t provenance_files < <(find artifacts/upstream-provenance -type f -name '*.json' -print0 | sort -z)
+          (( ${#provenance_files[@]} > 0 )) || { echo "no upstream provenance records downloaded" >&2; exit 1; }
+          node --experimental-strip-types scripts/verify-host-invariant.ts "${provenance_files[@]}"
 
   native-package:
     name: Package ${{ matrix.artifact_id }}
-    needs: [plan, upstream]
+    needs: [plan, upstream, upstream-host-invariant]
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(needs.plan.outputs.package_matrix) }}
@@ -233,7 +280,7 @@ jobs:
           name: ${{ matrix.upstream_artifact_id }}
           path: artifacts/upstream
       - uses: docker/setup-buildx-action@v4
-      - name: Build native package from verified upstream binary
+      - name: Build native package from verified upstream product bundle
         uses: docker/build-push-action@v7
         with:
           context: .
@@ -291,6 +338,27 @@ jobs:
           name: ${{ matrix.native_package_artifact_name }}
           path: artifacts/native-package
       - uses: docker/setup-buildx-action@v4
+      - name: Read immutable product inputs
+        id: product
+        shell: bash
+        run: |
+          set -euo pipefail
+          mapfile -d '' -t provenance_files < <(find artifacts/native-package -name '*.upstream-provenance.json' -type f -print0)
+          if (( ${#provenance_files[@]} != 1 )); then
+            echo "expected exactly one upstream provenance file, found ${#provenance_files[@]}" >&2
+            exit 1
+          fi
+          provenance="${provenance_files[0]}"
+          jq -e '
+            (.host_sha256 | type == "string" and test("^[0-9a-f]{64}$")) and
+            (.runtime_id | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$")) and
+            (.runtime_sha256 | type == "string" and test("^[0-9a-f]{64}$"))
+          ' "$provenance" >/dev/null
+          {
+            echo "host_sha=$(jq -r .host_sha256 "$provenance")"
+            echo "runtime_id=$(jq -r .runtime_id "$provenance")"
+            echo "runtime_sha=$(jq -r .runtime_sha256 "$provenance")"
+          } >> "$GITHUB_OUTPUT"
       - name: Build and QA package-installed runtime image
         uses: docker/build-push-action@v7
         with:
@@ -307,6 +375,9 @@ jobs:
             MESH_LLM_VERSION=${{ matrix.mesh_version }}
             MESH_LLM_REF=${{ needs.plan.outputs.mesh_ref }}
             MESH_LLM_SOURCE_SHA=${{ needs.plan.outputs.mesh_sha }}
+            MESH_LLM_HOST_SHA=${{ steps.product.outputs.host_sha }}
+            MESH_LLM_RUNTIME_ID=${{ steps.product.outputs.runtime_id }}
+            MESH_LLM_RUNTIME_SHA=${{ steps.product.outputs.runtime_sha }}
 
   homebrew:
     name: Verify Homebrew formula from upstream macOS archive
@@ -401,14 +472,32 @@ jobs:
         run: npm test --prefix upstream-source/sdk/node
       - name: Build Node SDK addon
         run: npm run build:native --prefix upstream-source/sdk/node
-      - name: Smoke-test Node SDK addon
-        working-directory: upstream-source/sdk/node
+      - name: Pack, fresh-install, and start Node SDK addon
+        timeout-minutes: 3
         shell: bash
         env:
           MESH_VERSION: ${{ needs.plan.outputs.mesh_version }}
+          NODE_SDK_TARGET: ${{ matrix.target }}
         run: |
           set -euo pipefail
-          node --input-type=commonjs -e 'const assert = require("node:assert/strict"); const sdk = require(process.cwd()); assert.equal(sdk.currentMeshVersion(), process.env.MESH_VERSION)'
+          smoke_root="$(mktemp -d)"
+          trap 'rm -rf "$smoke_root"' EXIT
+          mkdir -p "$smoke_root/tarball" "$smoke_root/consumer"
+          (
+            cd upstream-source/sdk/node
+            npm pack --pack-destination "$smoke_root/tarball"
+          )
+          tarball="$(find "$smoke_root/tarball" -maxdepth 1 -type f -name '*.tgz' -print -quit)"
+          test -n "$tarball"
+          (
+            cd "$smoke_root/consumer"
+            npm init --yes >/dev/null
+            npm install "$tarball"
+          )
+          node scripts/node-sdk-runtime-smoke.cjs \
+            --package-root "$smoke_root/consumer" \
+            --expected-version "$MESH_VERSION" \
+            --target "$NODE_SDK_TARGET"
       - name: Stage Node SDK addon
         shell: bash
         env:
@@ -471,6 +560,7 @@ jobs:
             --expected-version "$MESH_VERSION" \
             --targets "$targets"
       - name: Test, pack, install, and dry-run publish
+        timeout-minutes: 5
         env:
           NPM_PLAN: ${{ needs.plan.outputs.npm_plan }}
           MESH_VERSION: ${{ needs.plan.outputs.mesh_version }}
@@ -495,7 +585,10 @@ jobs:
             npm init --yes >/dev/null
             npm install "../$tarball"
             if jq -e '.targets | index("linux-x64")' <<<"$NPM_PLAN" >/dev/null; then
-              node -e "const sdk = require('@mesh-llm/sdk'); if (sdk.currentMeshVersion() !== process.env.MESH_VERSION) process.exit(1)"
+              node "$GITHUB_WORKSPACE/scripts/node-sdk-runtime-smoke.cjs" \
+                --package-root "$PWD" \
+                --expected-version "$MESH_VERSION" \
+                --target linux-x64-assembled-package
             fi
           )
       - uses: actions/upload-artifact@v7
@@ -568,6 +661,27 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ github.token }}
+      - name: Read immutable product inputs
+        id: product
+        shell: bash
+        run: |
+          set -euo pipefail
+          mapfile -d '' -t provenance_files < <(find artifacts/native-package -name '*.upstream-provenance.json' -type f -print0)
+          if (( ${#provenance_files[@]} != 1 )); then
+            echo "expected exactly one upstream provenance file, found ${#provenance_files[@]}" >&2
+            exit 1
+          fi
+          provenance="${provenance_files[0]}"
+          jq -e '
+            (.host_sha256 | type == "string" and test("^[0-9a-f]{64}$")) and
+            (.runtime_id | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$")) and
+            (.runtime_sha256 | type == "string" and test("^[0-9a-f]{64}$"))
+          ' "$provenance" >/dev/null
+          {
+            echo "host_sha=$(jq -r .host_sha256 "$provenance")"
+            echo "runtime_id=$(jq -r .runtime_id "$provenance")"
+            echo "runtime_sha=$(jq -r .runtime_sha256 "$provenance")"
+          } >> "$GITHUB_OUTPUT"
       - id: push
         uses: docker/build-push-action@v7
         with:
@@ -587,6 +701,9 @@ jobs:
             MESH_LLM_VERSION=${{ matrix.mesh_version }}
             MESH_LLM_REF=${{ needs.plan.outputs.mesh_ref }}
             MESH_LLM_SOURCE_SHA=${{ needs.plan.outputs.mesh_sha }}
+            MESH_LLM_HOST_SHA=${{ steps.product.outputs.host_sha }}
+            MESH_LLM_RUNTIME_ID=${{ steps.product.outputs.runtime_id }}
+            MESH_LLM_RUNTIME_SHA=${{ steps.product.outputs.runtime_sha }}
       - uses: actions/attest-build-provenance@v4
         with:
           subject-name: ${{ env.IMAGE_NAME }}

.skills/distribution-certification/SKILL.md

@@ -130,24 +130,34 @@ Snapshot the packaging release before other tests:
 4. Download the aggregate checksum, aggregate provenance, formula, every native
    package, every package sidecar, and every expected row SBOM/upstream
    provenance file.
-5. Verify every downloaded asset byte-for-byte against the server-reported
+5. Download each upstream MeshLLM product archive and require the product-v2
+   layout: one backend-neutral `mesh-llm` host, `product-manifest.json`,
+   `host-imports.json`, and exactly one selected runtime below
+   `native-runtimes/<runtime-id>/manifest.json`.
+6. Verify every downloaded asset byte-for-byte against the server-reported
    digest where the release surface provides one, then verify every native
    package against both its sidecar and aggregate checksum.
-6. Run the aggregate checksum across every file it names, not just native
+7. Run the aggregate checksum across every file it names, not just native
    packages. A stale formula or metadata checksum is a release failure.
-7. Compare asset timestamps/digests with the release snapshot before and after
+8. Compare asset timestamps/digests with the release snapshot before and after
    checksum verification. Reject moved tags, replaced assets, stale aggregate
    checksums, and any checksum generated before the asset bytes it claims.
-8. Download the immutable packaging tag source and derive the expected matrix
+9. Download the immutable packaging tag source and derive the expected matrix
    using its checked-in matrix and tag-generation code. Do not build product
    code.
-9. Confirm one SBOM and one upstream-provenance file for every enabled native
+10. Confirm one SBOM and one upstream-provenance file for every enabled native
    row. Cryptographically verify SBOM and provenance subjects against the exact
    artifact digests they claim, and reject unrelated or stale subjects.
    Explicitly record disabled/unsupported rows.
-10. Compare tagged channel identity—especially npm package scope—with the
+11. Compare tagged channel identity—especially npm package scope—with the
    artifact actually published.
 
+For each product archive, validate every digest recorded by
+`product-manifest.json`, verify that the host dependency report rejects no
+backend runtime imports, and confirm that the selected runtime ID, platform,
+backend, and MeshLLM version match the release row. Reject absolute build paths
+or additional top-level payload files.
+
 Preserve both redacted machine-readable inventory and a concise human-readable
 validation log.
 
@@ -160,19 +170,29 @@ Common runtime rule:
 1. Resolve the package-owned or channel-owned executable directly.
 2. Run `--version` and require exact MeshLLM semantic version equality.
 3. Run `runtime list` where the channel exposes the CLI.
-4. Isolate `HOME`, XDG paths, cache directory, and runtime directory.
-5. Reserve both an API port and a console port. Pass both explicitly even when
+4. Require `runtime list` to discover the channel-owned adjacent runtime while
+   the user cache is empty; it must not copy that runtime into the cache.
+5. Run `--version`, `--help`, and `runtime list` without GPU/device passthrough.
+   A backend driver loader failure is a product failure.
+6. Isolate `HOME`, XDG paths, cache directory, and runtime directory.
+7. Reserve both an API port and a console port. Pass both explicitly even when
    using `--no-console`; some versions may still initialize the web server.
-6. Start client mode with `--log-format json --no-console --auto`.
-7. Require the process/container to remain alive and emit a real ready event
+8. Start client mode with `--log-format json --no-console --auto`.
+9. Require the process/container to remain alive and emit a real ready event
    such as `Client ready`.
-8. Probe `/v1/models` only when an endpoint was intentionally exposed.
-9. Send SIGINT, enforce a bounded shutdown, capture the final exit state, and
+10. Probe `/v1/models` only when an endpoint was intentionally exposed.
+11. Send SIGINT, enforce a bounded shutdown, capture the final exit state, and
    verify listeners/processes disappeared.
 
 A metadata-only check, successful install, `--version`, or transient live PID
 is not runtime certification.
 
+For composed product-v2 channels, also record the producer schema SHA-256 and
+prove it is byte-identical to the packaging consumer schema at the immutable
+upstream source SHA. Group all selected product provenance records by
+OS/architecture and require exactly one host SHA-256 per group before treating
+backend rows as aliases of a shared host.
+
 ## Phase 3: cross-channel consistency
 
 Require agreement on the requested semantic version across:
@@ -189,8 +209,12 @@ Require agreement on the requested semantic version across:
 Also prove:
 
 - Native packages own the expected executable path.
+- Native packages own the versioned runtime tree and product manifests; the
+  host remains backend-neutral and no package installs backend libraries beside
+  the executable.
 - Images install their native package rather than copying an unrelated binary.
-- Homebrew downloads the expected upstream Apple Silicon archive.
+- Homebrew installs the host plus the matching runtime under `libexec` from the
+  expected upstream Apple Silicon product archive.
 - npm selects the current host's advertised prebuilt addon.
 - No test invokes a shadowing user-local executable.
 - No channel resolves to an older release.