diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 022425c87..368518e3d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -312,11 +312,11 @@ jobs: python_api_wheels=(nemo_relay-*.whl) cli_npm=(nemo-relay-bin-npm-*.tgz) node_npm=(nemo-relay-node-npm-*.tgz) - expect_count 5 "CLI binary" "${cli_binaries[@]}" - expect_count 5 "CLI wheel" "${cli_wheels[@]}" - expect_count 5 "Python API wheel" "${python_api_wheels[@]}" - expect_count 6 "CLI npm" "${cli_npm[@]}" - expect_count 6 "Node npm" "${node_npm[@]}" + expect_count 7 "CLI binary" "${cli_binaries[@]}" + expect_count 7 "CLI wheel" "${cli_wheels[@]}" + expect_count 7 "Python API wheel" "${python_api_wheels[@]}" + expect_count 8 "CLI npm" "${cli_npm[@]}" + expect_count 8 "Node npm" "${node_npm[@]}" mapfile -t distribution_assets < <( find . -maxdepth 1 -type f \ @@ -517,7 +517,7 @@ jobs: run: | set -euo pipefail version="${{ github.ref_name }}" - for pkg in node-packages/nemo-relay-node-npm-{linux-x64,linux-arm64,darwin-arm64,win32-x64,win32-arm64}-${version}.tgz; do + for pkg in node-packages/nemo-relay-node-npm-{linux-x64,linux-arm64,linux-x64-musl,linux-arm64-musl,darwin-arm64,win32-x64,win32-arm64}-${version}.tgz; do name="$(tar xOf "$pkg" package/package.json | node -p 'JSON.parse(require("fs").readFileSync(0, "utf8")).name')" if npm view "${name}@${version}" version --registry https://registry.npmjs.org >/dev/null 2>&1; then echo "${name} ${version} already exists on npm; skipping" @@ -536,7 +536,7 @@ jobs: run: | set -euo pipefail version="${{ github.ref_name }}" - for pkg in cli-packages/nemo-relay-bin-npm-{linux-x64,linux-arm64,darwin-arm64,win32-x64,win32-arm64}-${version}.tgz; do + for pkg in cli-packages/nemo-relay-bin-npm-{linux-x64,linux-arm64,linux-x64-musl,linux-arm64-musl,darwin-arm64,win32-x64,win32-arm64}-${version}.tgz; do name="$(tar xOf "$pkg" package/package.json | node -p 'JSON.parse(require("fs").readFileSync(0, "utf8")).name')" if npm view "${name}@${version}" version --registry https://registry.npmjs.org >/dev/null 2>&1; then echo "${name} ${version} already exists on npm; skipping" diff --git a/.github/workflows/ci_node.yml b/.github/workflows/ci_node.yml index 98c8c0597..30fdfcef4 100644 --- a/.github/workflows/ci_node.yml +++ b/.github/workflows/ci_node.yml @@ -220,6 +220,151 @@ jobs: path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/npm/*.tgz if-no-files-found: error + PackageMusllinux: + name: Package (musllinux-${{ matrix.node_arch }}) + needs: [Test] + if: ${{ inputs.run_package && !cancelled() && needs.Test.result == 'success' }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - node_arch: x64 + platform: linux-musl-amd64 + runner: ubuntu-latest + container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a + - node_arch: arm64 + platform: linux-musl-arm64 + runner: ubuntu-24.04-arm + container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - name: Derive Node package version + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -euo pipefail + version="$(node -e 'const fs = require("fs"); const pkg = JSON.parse(fs.readFileSync("crates/node/package.json", "utf8")); if (!pkg.version) { throw new Error("crates/node/package.json missing version field"); } console.log(pkg.version);')" + if [[ "${{ inputs.ref_type }}" == "tag" ]]; then + version="${{ inputs.ref_name }}" + else + version="${version}+${GITHUB_SHA::8}" + fi + printf 'NEMO_RELAY_NODE_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" + + - name: Build musllinux Node package + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -euo pipefail + output_dir="${NEMO_RELAY_CI_WORKSPACE_TMP}/npm-musllinux" + mkdir -p "$output_dir" + docker run --rm \ + --volume "${NEMO_RELAY_CI_WORKSPACE}:${NEMO_RELAY_CI_WORKSPACE}" \ + --workdir "${NEMO_RELAY_CI_WORKSPACE}" \ + --env NEMO_RELAY_RUST_VERSION="${{ steps.ci-config.outputs.rust_version }}" \ + --env NEMO_RELAY_NODE_OUTPUT_DIR="$output_dir" \ + --env NEMO_RELAY_NODE_VERSION="${NEMO_RELAY_NODE_PACKAGE_VERSION}" \ + --env NEMO_RELAY_NODE_PLATFORM="${{ matrix.platform }}" \ + "${{ matrix.container_image }}" \ + sh -ceu ' + apk add --no-cache build-base cmake curl python3 + curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain "$NEMO_RELAY_RUST_VERSION" + export PATH="$HOME/.cargo/bin:$PATH" + node scripts/package_node_musllinux.mjs \ + --version "$NEMO_RELAY_NODE_VERSION" \ + --platform "$NEMO_RELAY_NODE_PLATFORM" \ + --out "$NEMO_RELAY_NODE_OUTPUT_DIR" + ' + + - name: Upload musllinux Node npm package artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: node-npm-package-${{ matrix.platform }} + path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/npm-musllinux/*.tgz + if-no-files-found: error + + PackageMusllinuxSmoke: + name: Package smoke (musllinux-${{ matrix.node_arch }}) + needs: [Package, PackageMusllinux] + if: ${{ inputs.run_package && !cancelled() && needs.Package.result == 'success' && needs.PackageMusllinux.result == 'success' }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - node_arch: x64 + platform: linux-musl-amd64 + package_name: nemo-relay-node-linux-x64-musl + runner: ubuntu-latest + container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a + - node_arch: arm64 + platform: linux-musl-arm64 + package_name: nemo-relay-node-linux-arm64-musl + runner: ubuntu-24.04-arm + container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a + + steps: + - name: Download Node metapackage artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: node-npm-package-linux-amd64 + path: node-metapackage/ + + - name: Download matching musllinux Node native package artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: node-npm-package-${{ matrix.platform }} + path: node-native/ + + - name: Install and load musllinux split Node package + run: | + set -euo pipefail + docker run --rm \ + --volume "${{ github.workspace }}:${{ github.workspace }}" \ + --workdir "${{ github.workspace }}" \ + --env NEMO_RELAY_NODE_PACKAGE_NAME="${{ matrix.package_name }}" \ + --env NEMO_RELAY_WORKSPACE="${{ github.workspace }}" \ + "${{ matrix.container_image }}" \ + sh -ceu ' + smoke_dir="$(mktemp -d)" + trap "rm -rf $smoke_dir" EXIT + cd "$smoke_dir" + npm init --yes >/dev/null + metapackage="$(find "$NEMO_RELAY_WORKSPACE/node-metapackage" -name "nemo-relay-node-npm-[0-9]*.tgz" -print -quit)" + native="$(find "$NEMO_RELAY_WORKSPACE/node-native" -name "*.tgz" -print -quit)" + test -n "$metapackage" + test -n "$native" + npm install --ignore-scripts "$native" "$metapackage" + node -e " + const assert = require(\"node:assert/strict\"); + const manifest = require(process.env.NEMO_RELAY_NODE_PACKAGE_NAME + \"/package.json\"); + assert.deepEqual(manifest.libc, [\"musl\"]); + for (const entrypoint of [ + \"nemo-relay-node\", + \"nemo-relay-node/typed\", + \"nemo-relay-node/plugin\", + \"nemo-relay-node/adaptive\", + \"nemo-relay-node/observability\", + \"nemo-relay-node/pii_redaction\", + \"nemo-relay-node/model_pricing\", + ]) { + require(entrypoint); + } + " + ' + PackageOpenClaw: name: Package OpenClaw plugin needs: [Test] diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 8a4d0828a..e119706f6 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -163,6 +163,120 @@ jobs: working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: uv cache prune --ci + Package-musllinux: + name: Package (musllinux-${{ matrix.architecture }}) + needs: [Test] + if: ${{ inputs.run_package && !cancelled() && needs.Test.result == 'success' }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - architecture: x86_64 + runner: ubuntu-latest + image: quay.io/pypa/musllinux_1_2_x86_64@sha256:7b54360b191356838f6e41d4b9820c69be4f3d8dc8e7bca26f2684d48e5c8633 + - architecture: aarch64 + runner: ubuntu-24.04-arm + image: quay.io/pypa/musllinux_1_2_aarch64@sha256:274a947b4d5d745b56b868b0d052641e0798a652e3e65755987edf60547d1268 + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - name: Derive packaging tool versions + run: | + set -euo pipefail + maturin_version="$(awk ' + $0 == "name = \"maturin\"" { found = 1; next } + found && /^version = / { + gsub(/"/, "", $3) + print $3 + exit + } + ' uv.lock)" + if [[ -z "$maturin_version" ]]; then + echo "Error: failed to read maturin version from uv.lock" >&2 + exit 1 + fi + version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n1)" + if [[ -z "$version" ]]; then + echo "Error: failed to read workspace version from Cargo.toml" >&2 + exit 1 + fi + if [[ "${{ inputs.ref_type }}" == "tag" ]]; then + version="${{ inputs.ref_name }}" + else + version="${version}+${GITHUB_SHA::8}" + fi + { + printf 'NEMO_RELAY_MATURIN_VERSION=%s\n' "$maturin_version" + printf 'NEMO_RELAY_PACKAGE_VERSION=%s\n' "$version" + } >> "$GITHUB_ENV" + + - name: Create packaging output directory + run: | + set -euo pipefail + mkdir -p "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/musllinux-wheels" + + - name: Build and smoke test musllinux wheel + env: + MUSLLINUX_IMAGE: ${{ matrix.image }} + MUSLLINUX_INTERPRETER: /opt/python/cp311-cp311/bin/python + MUSLLINUX_WHEEL_DIRECTORY: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/musllinux-wheels + NEMO_RELAY_RUST_VERSION: ${{ steps.ci-config.outputs.rust_version }} + run: | + set -euo pipefail + docker run --rm \ + --volume "${{ env.NEMO_RELAY_CI_WORKSPACE }}:${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + --workdir "${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + --env MUSLLINUX_INTERPRETER \ + --env MUSLLINUX_WHEEL_DIRECTORY \ + --env NEMO_RELAY_MATURIN_VERSION \ + --env NEMO_RELAY_PACKAGE_VERSION \ + --env NEMO_RELAY_RUST_VERSION \ + "$MUSLLINUX_IMAGE" \ + /bin/sh -ec ' + apk add --no-cache curl + curl --proto "=https" --tlsv1.2 --silent --show-error --fail https://sh.rustup.rs | \ + sh -s -- -y --profile minimal --default-toolchain "$NEMO_RELAY_RUST_VERSION" + export PATH="$HOME/.cargo/bin:$PATH" + "$MUSLLINUX_INTERPRETER" -m pip install --no-cache-dir "maturin==$NEMO_RELAY_MATURIN_VERSION" + export PATH="$(dirname "$MUSLLINUX_INTERPRETER"):$PATH" + "$MUSLLINUX_INTERPRETER" scripts/package_python_musllinux.py \ + --version "$NEMO_RELAY_PACKAGE_VERSION" \ + --out "$MUSLLINUX_WHEEL_DIRECTORY" \ + --interpreter "$MUSLLINUX_INTERPRETER" + wheel="$(find "$MUSLLINUX_WHEEL_DIRECTORY" -maxdepth 1 -name "*.whl" -print -quit)" + test -n "$wheel" + "$MUSLLINUX_INTERPRETER" -m venv /tmp/nemo-relay-musllinux-venv + /tmp/nemo-relay-musllinux-venv/bin/python -m pip install --no-deps "$wheel" + /tmp/nemo-relay-musllinux-venv/bin/python -c "import nemo_relay" + ' + + - name: Verify wheel platform tag + run: | + set -euo pipefail + shopt -s nullglob + wheels=("${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/musllinux-wheels"/*-cp311-abi3-musllinux_1_2_${{ matrix.architecture }}.whl) + if [[ "${#wheels[@]}" -ne 1 ]]; then + echo "Error: expected one cp311-abi3 musllinux wheel for ${{ matrix.architecture }}, found ${#wheels[@]}" >&2 + exit 1 + fi + + - name: Upload wheel artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheel-musllinux-${{ matrix.architecture }} + path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/musllinux-wheels/*.whl + if-no-files-found: error + Package: name: Package (${{ matrix.platform }}) needs: [Test] diff --git a/.github/workflows/ci_rust.yml b/.github/workflows/ci_rust.yml index 16cc4f4d3..c76ee9d30 100644 --- a/.github/workflows/ci_rust.yml +++ b/.github/workflows/ci_rust.yml @@ -198,19 +198,39 @@ jobs: include: - platform: linux-amd64 runner: ubuntu-latest - target: x86_64-unknown-linux-musl + target: x86_64-unknown-linux-gnu + build_image: quay.io/pypa/manylinux2014_x86_64@sha256:35baef377f64c2ae2e7ed647917ecd090c50f6e8b06fd605012661c2e954cc92 + runtime_image: quay.io/pypa/manylinux2014_x86_64@sha256:35baef377f64c2ae2e7ed647917ecd090c50f6e8b06fd605012661c2e954cc92 - platform: linux-arm64 + runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + build_image: quay.io/pypa/manylinux2014_aarch64@sha256:63f73ecd2be3ad7cf66a402c72c1594b3f9140e263ab6838896e93a8fbf53794 + runtime_image: quay.io/pypa/manylinux2014_aarch64@sha256:63f73ecd2be3ad7cf66a402c72c1594b3f9140e263ab6838896e93a8fbf53794 + - platform: linux-musl-amd64 + runner: ubuntu-latest + target: x86_64-unknown-linux-musl + build_image: '' + runtime_image: quay.io/pypa/musllinux_1_2_x86_64@sha256:7b54360b191356838f6e41d4b9820c69be4f3d8dc8e7bca26f2684d48e5c8633 + - platform: linux-musl-arm64 runner: ubuntu-24.04-arm target: aarch64-unknown-linux-musl + build_image: '' + runtime_image: quay.io/pypa/musllinux_1_2_aarch64@sha256:274a947b4d5d745b56b868b0d052641e0798a652e3e65755987edf60547d1268 - platform: macos-arm64 runner: macos-15 target: aarch64-apple-darwin + build_image: '' + runtime_image: '' - platform: windows-amd64 runner: windows-2022 target: x86_64-pc-windows-msvc + build_image: '' + runtime_image: '' - platform: windows-arm64 runner: windows-11-arm target: aarch64-pc-windows-msvc + build_image: '' + runtime_image: '' steps: - name: Checkout @@ -249,11 +269,40 @@ jobs: sudo apt-get -o Acquire::Retries=5 install -y musl-tools - name: Build CLI release binary + if: ${{ matrix.build_image == '' }} working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: | set -e cargo build --release --target "${{ matrix.target }}" -p nemo-relay-cli + - name: Build CLI release binary in manylinux + if: ${{ matrix.build_image != '' }} + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -euo pipefail + docker run --rm \ + --volume "${{ env.NEMO_RELAY_CI_WORKSPACE }}:${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + --workdir "${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + --env NEMO_RELAY_RUST_VERSION="${{ steps.ci-config.outputs.rust_version }}" \ + "${{ matrix.build_image }}" \ + /bin/bash -ec ' + curl --proto "=https" --tlsv1.2 --silent --show-error --fail https://sh.rustup.rs | \ + sh -s -- -y --profile minimal --default-toolchain "$NEMO_RELAY_RUST_VERSION" + export PATH="$HOME/.cargo/bin:$PATH" + cargo build --release --target "${{ matrix.target }}" -p nemo-relay-cli + ' + + - name: Verify CLI binary in its Linux runtime container + if: ${{ matrix.runtime_image != '' }} + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -euo pipefail + docker run --rm \ + --volume "${{ env.NEMO_RELAY_CI_WORKSPACE }}:${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + --workdir "${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + "${{ matrix.runtime_image }}" \ + "${{ env.NEMO_RELAY_CI_WORKSPACE }}/target/${{ matrix.target }}/release/nemo-relay" --version + - name: Stage CLI binary artifact working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: | @@ -306,6 +355,7 @@ jobs: python scripts/package-cli-bin.py "${args[@]}" - name: Install and run CLI wheel + if: ${{ !endsWith(matrix.target, '-musl') }} working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: | set -euo pipefail @@ -321,6 +371,24 @@ jobs: "$venv_python" -m pip install --force-reinstall --no-deps "${wheels[0]}" "$venv_cli" --version + - name: Install and run CLI wheel on musllinux + if: ${{ endsWith(matrix.target, '-musl') }} + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -euo pipefail + docker run --rm \ + --volume "${{ env.NEMO_RELAY_CI_WORKSPACE }}:${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + --workdir "${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + --env NEMO_RELAY_CI_WORKSPACE_TMP \ + "${{ matrix.runtime_image }}" \ + /bin/sh -ec ' + wheel="$(find "$NEMO_RELAY_CI_WORKSPACE_TMP/cli-packages" -maxdepth 1 -name "*.whl" -print -quit)" + test -n "$wheel" + /opt/python/cp311-cp311/bin/python -m venv /tmp/nemo-relay-cli-wheel-venv + /tmp/nemo-relay-cli-wheel-venv/bin/python -m pip install --force-reinstall --no-deps "$wheel" + /tmp/nemo-relay-cli-wheel-venv/bin/nemo-relay --version + ' + - name: Upload CLI binary artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/RELEASING.md b/RELEASING.md index 4bde75f50..55286e1ca 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -32,7 +32,7 @@ The release pipeline publishes these package surfaces from a tag push: |---|---| | crates.io | `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, `nemo-relay-cli` | | PyPI | `nemo-relay`, `nemo-relay-plugin`, `nemo-relay-cli-bin` | -| npm | `nemo-relay-node` and its five platform packages, `nemo-relay-openclaw`, `nemo-relay-cli-bin`, and its five platform packages | +| npm | `nemo-relay-node` and its seven platform packages, `nemo-relay-openclaw`, `nemo-relay-cli-bin`, and its seven platform packages | | GitHub Releases | CLI binaries, `nemo-relay` and `nemo-relay-cli-bin` wheels, CLI and Node npm tarballs, and checksums | | Fern | The documentation site | @@ -148,10 +148,12 @@ Before you create a release tag, confirm the following: `nemo-relay-cli-bin` - npm trusted publishers are configured for `nemo-relay-node`, `nemo-relay-node-linux-x64-gnu`, `nemo-relay-node-linux-arm64-gnu`, + `nemo-relay-node-linux-x64-musl`, `nemo-relay-node-linux-arm64-musl`, `nemo-relay-node-darwin-arm64`, `nemo-relay-node-win32-x64-msvc`, and `nemo-relay-node-win32-arm64-msvc` - npm trusted publishers are configured for `nemo-relay-cli-bin` and `nemo-relay-cli-bin-linux-x64`, `nemo-relay-cli-bin-linux-arm64`, + `nemo-relay-cli-bin-linux-x64-musl`, `nemo-relay-cli-bin-linux-arm64-musl`, `nemo-relay-cli-bin-darwin-arm64`, `nemo-relay-cli-bin-win32-x64`, and `nemo-relay-cli-bin-win32-arm64` 5. The GitHub Release entry is ready to become the only canonical release-notes @@ -254,12 +256,16 @@ The release pipeline then: validation. 3. Builds publishable package artifacts with the exact tag version: - `package-rust` packs the published Rust crates for local validation. - - `package-node` packs one native npm Node.js package per supported platform - and creates the JavaScript and TypeScript metapackage once. + - `package-node` packs one native npm Node.js package per supported platform, + including GNU and musl Linux packages for x86_64 and ARM64, and creates the + JavaScript and TypeScript metapackage once. - `package-openclaw` packs the npm OpenClaw plugin package. - - `package-python` builds platform `nemo-relay` wheels. + - `package-python` builds platform `nemo-relay` wheels, including + `musllinux_1_2_x86_64` and `musllinux_1_2_aarch64` wheels. - `package-python-plugin` builds the `nemo-relay-plugin` wheel. - - The Rust CLI matrix packages each prebuilt binary as a + - The Rust CLI matrix builds GNU Linux binaries in manylinux containers and + musl Linux binaries natively, validates each in its matching container, and + packages each prebuilt binary as a `nemo-relay-cli-bin` wheel and npm platform package, and creates the npm launcher package once. - The distribution release-asset job uploads the CLI binaries, `nemo-relay` @@ -318,8 +324,8 @@ NVIDIA Artifactory publication for the same tag: npm trusted publishing has its own registry-side constraints: - Each npm package can only have one trusted publisher configured at a time. -- Configure trusted publishers for `nemo-relay-node`, all five Node platform - packages, `nemo-relay-openclaw`, `nemo-relay-cli-bin`, and all five CLI +- Configure trusted publishers for `nemo-relay-node`, all seven Node platform + packages, `nemo-relay-openclaw`, `nemo-relay-cli-bin`, and all seven CLI platform packages before pushing a release tag. - npm trusted publishing currently supports GitHub-hosted runners, not self-hosted runners. @@ -353,8 +359,8 @@ After the release is live, verify: are visible on crates.io. 2. The `nemo-relay` and `nemo-relay-cli-bin` wheels are visible on PyPI, and `pip install "nemo-relay[cli]"` exposes `nemo-relay`. -3. The `nemo-relay-node`, its five platform packages, `nemo-relay-openclaw`, - `nemo-relay-cli-bin`, and its five platform packages are visible on npm. +3. The `nemo-relay-node`, its seven platform packages, `nemo-relay-openclaw`, + `nemo-relay-cli-bin`, and its seven platform packages are visible on npm. 4. The Unix and Windows installers resolve the new stable tag and verify matching CLI release asset checksums on their supported platforms. 5. The Fern documentation site shows the expected version and release notes. diff --git a/scripts/package-cli-bin.py b/scripts/package-cli-bin.py index 8ef2b9ff8..15d5eefaa 100755 --- a/scripts/package-cli-bin.py +++ b/scripts/package-cli-bin.py @@ -35,6 +35,7 @@ class Platform: npm_cpu: str wheel_platforms: tuple[str, ...] executable: str + libc: str | None = None @property def npm_package(self) -> str: @@ -46,20 +47,40 @@ def npm_package(self) -> str: platform.target: platform for platform in ( Platform( - "x86_64-unknown-linux-musl", + "x86_64-unknown-linux-gnu", "linux-x64", "linux", "x64", - ("manylinux_2_17_x86_64", "musllinux_1_2_x86_64"), + ("manylinux_2_17_x86_64",), "nemo-relay", + "glibc", ), Platform( - "aarch64-unknown-linux-musl", + "aarch64-unknown-linux-gnu", "linux-arm64", "linux", "arm64", - ("manylinux_2_17_aarch64", "musllinux_1_2_aarch64"), + ("manylinux_2_17_aarch64",), + "nemo-relay", + "glibc", + ), + Platform( + "x86_64-unknown-linux-musl", + "linux-x64-musl", + "linux", + "x64", + ("musllinux_1_2_x86_64",), "nemo-relay", + "musl", + ), + Platform( + "aarch64-unknown-linux-musl", + "linux-arm64-musl", + "linux", + "arm64", + ("musllinux_1_2_aarch64",), + "nemo-relay", + "musl", ), Platform( "aarch64-apple-darwin", @@ -179,7 +200,7 @@ def add_tar_bytes(archive: tarfile.TarFile, path: str, content: bytes, mode: int def build_npm_platform(binary: Path, platform: Platform, version: str, output: Path) -> Path: """Build an OS- and CPU-constrained npm package containing the CLI binary.""" - filename = f"nemo-relay-bin-npm-{platform.npm_os}-{platform.npm_cpu}-{version}.tgz" + filename = f"nemo-relay-bin-npm-{platform.npm_suffix}-{version}.tgz" destination = output / filename manifest = { "name": platform.npm_package, @@ -191,6 +212,8 @@ def build_npm_platform(binary: Path, platform: Platform, version: str, output: P "license": LICENSE, "repository": {"type": "git", "url": f"git+{REPOSITORY}.git"}, } + if platform.libc is not None: + manifest["libc"] = [platform.libc] with tarfile.open(destination, "w:gz") as archive: add_tar_bytes(archive, "package/package.json", json.dumps(manifest, indent=2).encode() + b"\n") add_tar_bytes( @@ -205,13 +228,14 @@ def build_npm_platform(binary: Path, platform: Platform, version: str, output: P def launcher_source() -> bytes: """Return the Node.js launcher that selects the installed native package.""" - mapping = { - f"{platform.npm_os}-{platform.npm_cpu}": { + mapping: dict[str, dict[str, dict[str, str]]] = {} + for platform in PLATFORMS.values(): + key = f"{platform.npm_os}-{platform.npm_cpu}" + libc = platform.libc or "default" + mapping.setdefault(key, {})[libc] = { "package": platform.npm_package, "executable": platform.executable, } - for platform in PLATFORMS.values() - } return ( "#!/usr/bin/env node\n" "// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n" @@ -220,7 +244,10 @@ def launcher_source() -> bytes: "const path = require('node:path');\n\n" f"const platforms = {json.dumps(mapping, indent=2)};\n" "const key = `${process.platform}-${process.arch}`;\n" - "const selected = platforms[key];\n" + "const libc = process.platform === 'linux'\n" + " ? (process.report?.getReport?.().header.glibcVersionRuntime ? 'glibc' : 'musl')\n" + " : 'default';\n" + "const selected = platforms[key]?.[libc] ?? platforms[key]?.default;\n" "if (!selected) {\n" " console.error(`nemo-relay-cli-bin does not support ${process.platform}/${process.arch}`);\n" " process.exit(1);\n" diff --git a/scripts/package-node-bin.py b/scripts/package-node-bin.py index 809420d53..ddfb5a32c 100755 --- a/scripts/package-node-bin.py +++ b/scripts/package-node-bin.py @@ -39,6 +39,8 @@ def package_name(self) -> str: for platform in ( Platform("linux-amd64", "linux-x64-gnu", "linux", "x64", "nemo-relay.linux-x64-gnu.node", "glibc"), Platform("linux-arm64", "linux-arm64-gnu", "linux", "arm64", "nemo-relay.linux-arm64-gnu.node", "glibc"), + Platform("linux-musl-amd64", "linux-x64-musl", "linux", "x64", "nemo-relay.linux-x64-musl.node", "musl"), + Platform("linux-musl-arm64", "linux-arm64-musl", "linux", "arm64", "nemo-relay.linux-arm64-musl.node", "musl"), Platform("macos-arm64", "darwin-arm64", "darwin", "arm64", "nemo-relay.darwin-arm64.node"), Platform("windows-amd64", "win32-x64-msvc", "win32", "x64", "nemo-relay.win32-x64-msvc.node"), Platform("windows-arm64", "win32-arm64-msvc", "win32", "arm64", "nemo-relay.win32-arm64-msvc.node"), @@ -90,7 +92,10 @@ def build_native_package(node_dir: Path, platform: Platform, version: str, outpu binary = node_dir / platform.binary if not binary.is_file(): raise FileNotFoundError(f"Node native binary does not exist: {binary}") - destination = output / f"nemo-relay-node-npm-{platform.npm_os}-{platform.npm_cpu}-{version}.tgz" + artifact_suffix = f"{platform.npm_os}-{platform.npm_cpu}" + if platform.libc == "musl": + artifact_suffix += "-musl" + destination = output / f"nemo-relay-node-npm-{artifact_suffix}-{version}.tgz" with tarfile.open(destination, "w:gz") as archive: add_tar_bytes(archive, "package/package.json", json.dumps(manifest, indent=2).encode() + b"\n") add_tar_bytes(archive, f"package/{platform.binary}", binary.read_bytes(), mode=0o755) diff --git a/scripts/package_node_musllinux.mjs b/scripts/package_node_musllinux.mjs new file mode 100755 index 000000000..8428370e7 --- /dev/null +++ b/scripts/package_node_musllinux.mjs @@ -0,0 +1,79 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; + +const ignoredDirectories = new Set([".git", ".venv", "node_modules", "target", "tmp"]); + +function command(name, args, cwd) { + const result = spawnSync(name, args, { cwd, stdio: "inherit" }); + if (result.status !== 0) { + throw new Error(`${name} ${args.join(" ")} failed with exit code ${result.status}`); + } +} + +function argumentsFrom(args) { + let version; + let output; + let platform; + for (let index = 0; index < args.length; index += 1) { + if (args[index] === "--version") { + version = args[++index]; + } else if (args[index] === "--out") { + output = args[++index]; + } else if (args[index] === "--platform") { + platform = args[++index]; + } else { + throw new Error(`Unexpected argument: ${args[index]}`); + } + } + if (!version || !output || !platform) { + throw new Error("Usage: package_node_musllinux.mjs --version VERSION --platform PLATFORM --out DIRECTORY"); + } + return { output: resolve(output), platform, version }; +} + +function main() { + const { output, platform, version } = argumentsFrom(process.argv.slice(2)); + const repository = process.cwd(); + const temporaryDirectory = mkdtempSync(join(tmpdir(), "nemo-relay-node-musllinux-")); + const sourceDirectory = join(temporaryDirectory, "source"); + + try { + mkdirSync(output, { recursive: true }); + cpSync(repository, sourceDirectory, { + filter: (source) => !ignoredDirectories.has(basename(source)), + recursive: true, + }); + command("npm", ["install", "--workspace=nemo-relay-node", "--ignore-scripts"], sourceDirectory); + command("npm", ["run", "--workspace=nemo-relay-node", "build"], sourceDirectory); + command( + "python3", + [ + "scripts/package-node-bin.py", + "--node-dir", + "crates/node", + "--platform", + platform, + "--version", + version, + "--output-dir", + output, + ], + sourceDirectory, + ); + + const packages = readdirSync(output).filter((entry) => entry.endsWith(".tgz")); + if (packages.length !== 1) { + throw new Error(`Expected one npm package artifact in ${output}, found ${packages.length}`); + } + } finally { + rmSync(temporaryDirectory, { force: true, recursive: true }); + } +} + +main(); diff --git a/scripts/package_python_musllinux.py b/scripts/package_python_musllinux.py new file mode 100755 index 000000000..fdee52747 --- /dev/null +++ b/scripts/package_python_musllinux.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Build a musllinux wheel without modifying the checked-out source tree.""" + +from __future__ import annotations + +import argparse +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +VERSION_PATTERN = re.compile( + r"^(?P\d+\.\d+\.\d+)" + r"(?:-(?Palpha|beta|rc)(?:\.(?P\d+))?)?" + r"(?:\+(?P[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$" +) + + +def semver_to_pep440(version: str) -> str: + """Translate the release version used by Cargo into a PEP 440 version.""" + match = VERSION_PATTERN.fullmatch(version) + if not match: + raise ValueError( + "Unsupported Python package version format. Expected SemVer with optional " + "alpha/beta/rc prerelease and optional build metadata." + ) + + pep440 = match.group("release") + pre_label = match.group("pre_label") + if pre_label: + pre_map = {"alpha": "a", "beta": "b", "rc": "rc"} + pep440 += f"{pre_map[pre_label]}{match.group('pre_num') or '0'}" + + local = match.group("local") + if local: + normalized_local = ".".join(part.lower() for part in re.split(r"[._-]+", local) if part) + if not normalized_local: + raise ValueError("Python package local version metadata cannot be empty") + pep440 += f"+{normalized_local}" + + return pep440 + + +def materialize_python_version(source: Path, version: str) -> None: + """Set the explicit Python package version in a disposable source copy.""" + pyproject = source / "pyproject.toml" + text = pyproject.read_text() + if 'dynamic = ["version"]' not in text: + raise ValueError("Failed to find dynamic version field in pyproject.toml") + pyproject.write_text(text.replace('dynamic = ["version"]', f'version = "{version}"', 1)) + + +def copy_source(source: Path, destination: Path) -> None: + """Copy only source inputs needed by Maturin into the disposable workspace.""" + shutil.copytree( + source, + destination, + ignore=shutil.ignore_patterns(".git", ".venv", "target", "tmp", "__pycache__"), + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True, help="Raw SemVer release version") + parser.add_argument("--out", required=True, type=Path, help="Directory for the built wheel") + parser.add_argument("--interpreter", required=True, help="CPython executable used to build the ABI3 wheel") + args = parser.parse_args() + + source = Path.cwd().resolve() + output = args.out.resolve() + output.mkdir(parents=True, exist_ok=True) + version = semver_to_pep440(args.version) + + with tempfile.TemporaryDirectory(prefix="nemo-relay-musllinux-") as temporary_directory: + build_source = Path(temporary_directory) / "source" + copy_source(source, build_source) + materialize_python_version(build_source, version) + subprocess.run( + [ + "maturin", + "build", + "--release", + "--compatibility", + "musllinux_1_2", + "--interpreter", + args.interpreter, + "--out", + str(output), + ], + check=True, + cwd=build_source, + ) + + wheels = list(output.glob("*.whl")) + if len(wheels) != 1: + raise RuntimeError(f"Expected one musllinux wheel in {output}, found {len(wheels)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_package_cli_bin.py b/scripts/tests/test_package_cli_bin.py index 972e43255..6d352cd46 100644 --- a/scripts/tests/test_package_cli_bin.py +++ b/scripts/tests/test_package_cli_bin.py @@ -32,18 +32,21 @@ def required_member(archive: tarfile.TarFile, name: str) -> IO[bytes]: class PackageCliBinTests(unittest.TestCase): - def test_packages_linux_binary_for_python_and_npm(self) -> None: + def test_packages_linux_binaries_for_python_and_npm(self) -> None: with tempfile.TemporaryDirectory() as temporary: output = Path(temporary) binary = output / "nemo-relay" binary.write_bytes(b"test-binary") - platform = PACKAGE_CLI_BIN.PLATFORMS["x86_64-unknown-linux-musl"] + gnu_platform = PACKAGE_CLI_BIN.PLATFORMS["x86_64-unknown-linux-gnu"] + musl_platform = PACKAGE_CLI_BIN.PLATFORMS["x86_64-unknown-linux-musl"] previous_directory = Path.cwd() try: os.chdir(output) - wheel = PACKAGE_CLI_BIN.build_wheel(binary, platform, "0.7.0-rc.1", output) - native = PACKAGE_CLI_BIN.build_npm_platform(binary, platform, "0.7.0-rc.1", output) + wheel = PACKAGE_CLI_BIN.build_wheel(binary, gnu_platform, "0.7.0-rc.1", output) + native = PACKAGE_CLI_BIN.build_npm_platform(binary, gnu_platform, "0.7.0-rc.1", output) + musl_wheel = PACKAGE_CLI_BIN.build_wheel(binary, musl_platform, "0.7.0-rc.1", output) + musl_native = PACKAGE_CLI_BIN.build_npm_platform(binary, musl_platform, "0.7.0-rc.1", output) launcher = PACKAGE_CLI_BIN.build_npm_launcher("0.7.0-rc.1", output) finally: os.chdir(previous_directory) @@ -56,18 +59,26 @@ def test_packages_linux_binary_for_python_and_npm(self) -> None: self.assertTrue(stat.S_ISREG(script_mode)) self.assertEqual(stat.S_IMODE(script_mode), 0o755) wheel_metadata = archive.read(next(name for name in names if name.endswith("/WHEEL"))) - self.assertIn(b"Tag: py3-none-musllinux_1_2_x86_64", wheel_metadata) + self.assertIn(b"Tag: py3-none-manylinux_2_17_x86_64", wheel_metadata) + + self.assertIn("0.7.0rc1-py3-none-musllinux_1_2_x86_64", musl_wheel.name) self.assertEqual(native.name, "nemo-relay-bin-npm-linux-x64-0.7.0-rc.1.tgz") with tarfile.open(native) as archive: manifest = json.load(required_member(archive, "package/package.json")) self.assertEqual(manifest["os"], ["linux"]) self.assertEqual(manifest["cpu"], ["x64"]) + self.assertEqual(manifest["libc"], ["glibc"]) self.assertEqual( required_member(archive, "package/bin/nemo-relay").read(), b"test-binary", ) + self.assertEqual(musl_native.name, "nemo-relay-bin-npm-linux-x64-musl-0.7.0-rc.1.tgz") + with tarfile.open(musl_native) as archive: + manifest = json.load(required_member(archive, "package/package.json")) + self.assertEqual(manifest["libc"], ["musl"]) + self.assertEqual(launcher.name, "nemo-relay-bin-npm-0.7.0-rc.1.tgz") with tarfile.open(launcher) as archive: manifest = json.load(required_member(archive, "package/package.json")) @@ -76,6 +87,10 @@ def test_packages_linux_binary_for_python_and_npm(self) -> None: manifest["optionalDependencies"]["nemo-relay-cli-bin-linux-x64"], "0.7.0-rc.1", ) + self.assertEqual( + manifest["optionalDependencies"]["nemo-relay-cli-bin-linux-x64-musl"], + "0.7.0-rc.1", + ) launcher_path = output / "launcher/package/bin/nemo-relay.js" launcher_path.parent.mkdir(parents=True) launcher_path.write_bytes(required_member(archive, "package/bin/nemo-relay.js").read())