diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 802f9c77f..75b778b86 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -400,6 +400,16 @@ jobs: name: plugin-wheel path: dist/ + - name: Verify Python API wheel artifacts + run: | + set -euo pipefail + shopt -s nullglob + wheels=(dist/nemo_relay-*.whl) + if [ "${#wheels[@]}" -ne 7 ]; then + echo "Error: expected 7 Python API wheel artifacts, found ${#wheels[@]}" >&2 + exit 1 + fi + - name: Publish to PyPI with trusted publishing uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: diff --git a/.github/workflows/ci_node.yml b/.github/workflows/ci_node.yml index d15e01569..c767e884e 100644 --- a/.github/workflows/ci_node.yml +++ b/.github/workflows/ci_node.yml @@ -215,6 +215,90 @@ jobs: path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/npm/*.tgz if-no-files-found: error + PackageMusllinux: + name: Package (musllinux-${{ matrix.node_arch }}) + needs: [Test] + if: ${{ inputs.run_package && !cancelled() && needs.Test.result == 'success' }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - node_arch: x64 + runner: ubuntu-latest + container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a + - node_arch: arm64 + runner: ubuntu-24.04-arm + container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - name: Derive Node package version + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -euo pipefail + version="$(node -e 'const fs = require("fs"); const pkg = JSON.parse(fs.readFileSync("crates/node/package.json", "utf8")); if (!pkg.version) { throw new Error("crates/node/package.json missing version field"); } console.log(pkg.version);')" + if [[ "${{ inputs.ref_type }}" == "tag" ]]; then + version="${{ inputs.ref_name }}" + else + version="${version}+${GITHUB_SHA::8}" + fi + printf 'NEMO_RELAY_NODE_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" + + - name: Build and smoke-test musllinux Node package + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -euo pipefail + output_dir="${NEMO_RELAY_CI_WORKSPACE_TMP}/npm-musllinux" + mkdir -p "$output_dir" + docker run --rm \ + --volume "${NEMO_RELAY_CI_WORKSPACE}:${NEMO_RELAY_CI_WORKSPACE}" \ + --workdir "${NEMO_RELAY_CI_WORKSPACE}" \ + --env NEMO_RELAY_RUST_VERSION="${{ steps.ci-config.outputs.rust_version }}" \ + --env NEMO_RELAY_NODE_OUTPUT_DIR="$output_dir" \ + --env NEMO_RELAY_NODE_VERSION="${NEMO_RELAY_NODE_PACKAGE_VERSION}" \ + "${{ matrix.container_image }}" \ + sh -ceu ' + apk add --no-cache build-base curl + curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain "$NEMO_RELAY_RUST_VERSION" + export PATH="$HOME/.cargo/bin:$PATH" + node scripts/package_node_musllinux.mjs --version "$NEMO_RELAY_NODE_VERSION" --out "$NEMO_RELAY_NODE_OUTPUT_DIR" + package="$(find "$NEMO_RELAY_NODE_OUTPUT_DIR" -maxdepth 1 -name "*.tgz" -print -quit)" + test -n "$package" + smoke_dir="$(mktemp -d)" + trap "rm -rf $smoke_dir" EXIT + cd "$smoke_dir" + npm install --ignore-scripts "$package" + node -e "require(\"nemo-relay-node\")" + ' + + - name: Verify musllinux N-API binary + run: | + set -euo pipefail + shopt -s nullglob + packages=("${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/npm-musllinux"/*.tgz) + if [ "${#packages[@]}" -ne 1 ]; then + echo "Error: expected one npm package artifact, found ${#packages[@]}" >&2 + exit 1 + fi + tar tzf "${packages[0]}" | grep -Fx "package/nemo-relay.linux-${{ matrix.node_arch }}-musl.node" + + - name: Upload musllinux npm package artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: npm-linux-musl-${{ matrix.node_arch == 'x64' && 'amd64' || 'arm64' }} + path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/npm-musllinux/*.tgz + if-no-files-found: error + PackageOpenClaw: name: Package OpenClaw plugin needs: [Test] @@ -284,8 +368,8 @@ jobs: Consolidate: name: Consolidate - needs: [Package] - if: ${{ inputs.run_package && !cancelled() && needs.Package.result == 'success' }} + needs: [Package, PackageMusllinux] + if: ${{ inputs.run_package && !cancelled() && needs.Package.result == 'success' && needs.PackageMusllinux.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -303,6 +387,18 @@ jobs: name: npm-linux-arm64 path: npm-linux-arm64/ + - name: Download npm linux-musl-amd64 artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: npm-linux-musl-amd64 + path: npm-linux-musl-amd64/ + + - name: Download npm linux-musl-arm64 artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: npm-linux-musl-arm64 + path: npm-linux-musl-arm64/ + - name: Download npm macos-arm64 artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -326,7 +422,8 @@ jobs: set -euo pipefail mkdir -p combined first=true - for dir in npm-linux-amd64 npm-linux-arm64 npm-macos-arm64 npm-windows-amd64 npm-windows-arm64; do + dirs=(npm-linux-amd64 npm-linux-arm64 npm-linux-musl-amd64 npm-linux-musl-arm64 npm-macos-arm64 npm-windows-amd64 npm-windows-arm64) + for dir in "${dirs[@]}"; do shopt -s nullglob tgzs=("${dir}"/*.tgz) shopt -u nullglob diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 8a4d0828a..b6b14e31d 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -163,6 +163,125 @@ jobs: working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: uv cache prune --ci + Package-musllinux: + name: Package (musllinux-${{ matrix.architecture }}) + needs: [Test] + if: >- + ${{ + inputs.run_package && + !cancelled() && + needs.Test.result == 'success' + }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - architecture: x86_64 + runner: ubuntu-latest + image: quay.io/pypa/musllinux_1_2_x86_64@sha256:7b54360b191356838f6e41d4b9820c69be4f3d8dc8e7bca26f2684d48e5c8633 + - architecture: aarch64 + runner: ubuntu-24.04-arm + image: quay.io/pypa/musllinux_1_2_aarch64@sha256:274a947b4d5d745b56b868b0d052641e0798a652e3e65755987edf60547d1268 + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - name: Derive packaging tool versions + run: | + set -euo pipefail + maturin_version="$(awk ' + $0 == "name = \"maturin\"" { found = 1; next } + found && /^version = / { + gsub(/"/, "", $3) + print $3 + exit + } + ' uv.lock)" + if [[ -z "$maturin_version" ]]; then + echo "Error: failed to read maturin version from uv.lock" >&2 + exit 1 + fi + version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n1)" + if [[ -z "$version" ]]; then + echo "Error: failed to read workspace version from Cargo.toml" >&2 + exit 1 + fi + if [[ "${{ inputs.ref_type }}" == "tag" ]]; then + version="${{ inputs.ref_name }}" + else + version="${version}+${GITHUB_SHA::8}" + fi + { + printf 'NEMO_RELAY_MATURIN_VERSION=%s\n' "$maturin_version" + printf 'NEMO_RELAY_PACKAGE_VERSION=%s\n' "$version" + } >> "$GITHUB_ENV" + + - name: Create packaging output directory + run: | + set -euo pipefail + mkdir -p "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/musllinux-wheels" + + - name: Build and smoke test musllinux wheel + env: + MUSLLINUX_IMAGE: ${{ matrix.image }} + MUSLLINUX_INTERPRETER: /opt/python/cp311-cp311/bin/python + MUSLLINUX_WHEEL_DIRECTORY: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/musllinux-wheels + NEMO_RELAY_RUST_VERSION: ${{ steps.ci-config.outputs.rust_version }} + run: | + set -euo pipefail + docker run --rm \ + --volume "${{ env.NEMO_RELAY_CI_WORKSPACE }}:${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + --workdir "${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + --env MUSLLINUX_INTERPRETER \ + --env MUSLLINUX_WHEEL_DIRECTORY \ + --env NEMO_RELAY_MATURIN_VERSION \ + --env NEMO_RELAY_PACKAGE_VERSION \ + --env NEMO_RELAY_RUST_VERSION \ + "$MUSLLINUX_IMAGE" \ + /bin/sh -ec ' + apk add --no-cache curl + curl --proto "=https" --tlsv1.2 --silent --show-error --fail https://sh.rustup.rs | \ + sh -s -- -y --profile minimal --default-toolchain "$NEMO_RELAY_RUST_VERSION" + export PATH="$HOME/.cargo/bin:$PATH" + "$MUSLLINUX_INTERPRETER" -m pip install --no-cache-dir "maturin==$NEMO_RELAY_MATURIN_VERSION" + export PATH="$(dirname "$MUSLLINUX_INTERPRETER"):$PATH" + "$MUSLLINUX_INTERPRETER" scripts/package_python_musllinux.py \ + --version "$NEMO_RELAY_PACKAGE_VERSION" \ + --out "$MUSLLINUX_WHEEL_DIRECTORY" \ + --interpreter "$MUSLLINUX_INTERPRETER" + wheel="$(find "$MUSLLINUX_WHEEL_DIRECTORY" -maxdepth 1 -name "*.whl" -print -quit)" + test -n "$wheel" + "$MUSLLINUX_INTERPRETER" -m venv /tmp/nemo-relay-musllinux-venv + /tmp/nemo-relay-musllinux-venv/bin/python -m pip install --no-deps "$wheel" + /tmp/nemo-relay-musllinux-venv/bin/python -c "import nemo_relay" + ' + + - name: Verify wheel platform tag + run: | + set -euo pipefail + shopt -s nullglob + wheels=("${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/musllinux-wheels"/*-cp311-abi3-musllinux_1_2_${{ matrix.architecture }}.whl) + if [ "${#wheels[@]}" -ne 1 ]; then + echo "Error: expected one cp311-abi3 musllinux wheel for ${{ matrix.architecture }}, found ${#wheels[@]}" >&2 + exit 1 + fi + + - name: Upload wheel artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheel-musllinux-${{ matrix.architecture }} + path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/musllinux-wheels/*.whl + if-no-files-found: error + Package: name: Package (${{ matrix.platform }}) needs: [Test] diff --git a/.github/workflows/ci_rust.yml b/.github/workflows/ci_rust.yml index bd554bb34..a1eca2bf6 100644 --- a/.github/workflows/ci_rust.yml +++ b/.github/workflows/ci_rust.yml @@ -199,18 +199,23 @@ jobs: - platform: linux-amd64 runner: ubuntu-latest target: x86_64-unknown-linux-musl + musllinux_image: quay.io/pypa/musllinux_1_2_x86_64@sha256:7b54360b191356838f6e41d4b9820c69be4f3d8dc8e7bca26f2684d48e5c8633 - platform: linux-arm64 runner: ubuntu-24.04-arm target: aarch64-unknown-linux-musl + musllinux_image: quay.io/pypa/musllinux_1_2_aarch64@sha256:274a947b4d5d745b56b868b0d052641e0798a652e3e65755987edf60547d1268 - platform: macos-arm64 runner: macos-15 target: aarch64-apple-darwin + musllinux_image: '' - platform: windows-amd64 runner: windows-2022 target: x86_64-pc-windows-msvc + musllinux_image: '' - platform: windows-arm64 runner: windows-11-arm target: aarch64-pc-windows-msvc + musllinux_image: '' steps: - name: Checkout @@ -254,6 +259,17 @@ jobs: set -e cargo build --release --target "${{ matrix.target }}" -p nemo-relay-cli + - name: Verify CLI binary on musllinux + if: ${{ matrix.musllinux_image != '' }} + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -euo pipefail + docker run --rm \ + --volume "${{ env.NEMO_RELAY_CI_WORKSPACE }}:${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + --workdir "${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ + "${{ matrix.musllinux_image }}" \ + "${{ env.NEMO_RELAY_CI_WORKSPACE }}/target/${{ matrix.target }}/release/nemo-relay" --version + - name: Stage CLI binary artifact working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: | diff --git a/RELEASING.md b/RELEASING.md index 8830dfe68..092671f89 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -239,12 +239,15 @@ The release pipeline then: validation. 3. Builds publishable package artifacts with the exact tag version: - `package-rust` packs the published Rust crates for local validation. - - `package-node` packs the npm Node.js package. + - `package-node` packs the npm Node.js package with GNU and musl Linux + N-API binaries for x86_64 and ARM64. - `package-openclaw` packs the npm OpenClaw plugin package. - - `package-python` builds platform `nemo-relay` wheels. + - `package-python` builds platform `nemo-relay` wheels, including + `musllinux_1_2_x86_64` and `musllinux_1_2_aarch64` wheels. - `package-python-plugin` builds the `nemo-relay-plugin` wheel. - - The CLI release-asset job uploads each platform `nemo-relay` binary and - includes those binaries in `SHA256SUMS`. + - The CLI release-asset job uploads each platform `nemo-relay` binary, + validates the Linux binaries on musllinux, and includes those binaries in + `SHA256SUMS`. 4. Publishes packages from the top-level workflow after the reusable packaging jobs complete: - `publish-rust` stamps Cargo workspace versions from the release tag, then diff --git a/scripts/package_node_musllinux.mjs b/scripts/package_node_musllinux.mjs new file mode 100755 index 000000000..300db1fe2 --- /dev/null +++ b/scripts/package_node_musllinux.mjs @@ -0,0 +1,79 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; + +const ignoredDirectories = new Set([".git", ".venv", "node_modules", "target", "tmp"]); + +function command(name, args, cwd) { + const result = spawnSync(name, args, { cwd, stdio: "inherit" }); + if (result.status !== 0) { + throw new Error(`${name} ${args.join(" ")} failed with exit code ${result.status}`); + } +} + +function argumentsFrom(args) { + let version; + let output; + for (let index = 0; index < args.length; index += 1) { + if (args[index] === "--version") { + version = args[++index]; + } else if (args[index] === "--out") { + output = args[++index]; + } else { + throw new Error(`Unexpected argument: ${args[index]}`); + } + } + if (!version || !output) { + throw new Error("Usage: package_node_musllinux.mjs --version VERSION --out DIRECTORY"); + } + return { output: resolve(output), version }; +} + +function setPackageVersion(sourceDirectory, version) { + const packagePath = join(sourceDirectory, "crates", "node", "package.json"); + const packageJson = JSON.parse(readFileSync(packagePath, "utf8")); + packageJson.version = version; + writeFileSync(packagePath, `${JSON.stringify(packageJson, null, 2)}\n`); + + const lockPath = join(sourceDirectory, "package-lock.json"); + const lock = JSON.parse(readFileSync(lockPath, "utf8")); + lock.packages["crates/node"].version = version; + writeFileSync(lockPath, `${JSON.stringify(lock, null, 2)}\n`); +} + +function main() { + const { output, version } = argumentsFrom(process.argv.slice(2)); + const repository = process.cwd(); + const temporaryDirectory = mkdtempSync(join(tmpdir(), "nemo-relay-node-musllinux-")); + const sourceDirectory = join(temporaryDirectory, "source"); + + try { + mkdirSync(output, { recursive: true }); + cpSync(repository, sourceDirectory, { + filter: (source) => !ignoredDirectories.has(basename(source)), + recursive: true, + }); + setPackageVersion(sourceDirectory, version); + command("npm", ["install", "--workspace=nemo-relay-node", "--ignore-scripts"], sourceDirectory); + command("npm", ["run", "--workspace=nemo-relay-node", "build"], sourceDirectory); + command( + "npm", + ["pack", "--workspace=nemo-relay-node", "--pack-destination", output], + sourceDirectory, + ); + + const packages = readdirSync(output).filter((entry) => entry.endsWith(".tgz")); + if (packages.length !== 1) { + throw new Error(`Expected one npm package artifact in ${output}, found ${packages.length}`); + } + } finally { + rmSync(temporaryDirectory, { force: true, recursive: true }); + } +} + +main(); diff --git a/scripts/package_python_musllinux.py b/scripts/package_python_musllinux.py new file mode 100755 index 000000000..fdee52747 --- /dev/null +++ b/scripts/package_python_musllinux.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Build a musllinux wheel without modifying the checked-out source tree.""" + +from __future__ import annotations + +import argparse +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +VERSION_PATTERN = re.compile( + r"^(?P\d+\.\d+\.\d+)" + r"(?:-(?Palpha|beta|rc)(?:\.(?P\d+))?)?" + r"(?:\+(?P[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$" +) + + +def semver_to_pep440(version: str) -> str: + """Translate the release version used by Cargo into a PEP 440 version.""" + match = VERSION_PATTERN.fullmatch(version) + if not match: + raise ValueError( + "Unsupported Python package version format. Expected SemVer with optional " + "alpha/beta/rc prerelease and optional build metadata." + ) + + pep440 = match.group("release") + pre_label = match.group("pre_label") + if pre_label: + pre_map = {"alpha": "a", "beta": "b", "rc": "rc"} + pep440 += f"{pre_map[pre_label]}{match.group('pre_num') or '0'}" + + local = match.group("local") + if local: + normalized_local = ".".join(part.lower() for part in re.split(r"[._-]+", local) if part) + if not normalized_local: + raise ValueError("Python package local version metadata cannot be empty") + pep440 += f"+{normalized_local}" + + return pep440 + + +def materialize_python_version(source: Path, version: str) -> None: + """Set the explicit Python package version in a disposable source copy.""" + pyproject = source / "pyproject.toml" + text = pyproject.read_text() + if 'dynamic = ["version"]' not in text: + raise ValueError("Failed to find dynamic version field in pyproject.toml") + pyproject.write_text(text.replace('dynamic = ["version"]', f'version = "{version}"', 1)) + + +def copy_source(source: Path, destination: Path) -> None: + """Copy only source inputs needed by Maturin into the disposable workspace.""" + shutil.copytree( + source, + destination, + ignore=shutil.ignore_patterns(".git", ".venv", "target", "tmp", "__pycache__"), + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True, help="Raw SemVer release version") + parser.add_argument("--out", required=True, type=Path, help="Directory for the built wheel") + parser.add_argument("--interpreter", required=True, help="CPython executable used to build the ABI3 wheel") + args = parser.parse_args() + + source = Path.cwd().resolve() + output = args.out.resolve() + output.mkdir(parents=True, exist_ok=True) + version = semver_to_pep440(args.version) + + with tempfile.TemporaryDirectory(prefix="nemo-relay-musllinux-") as temporary_directory: + build_source = Path(temporary_directory) / "source" + copy_source(source, build_source) + materialize_python_version(build_source, version) + subprocess.run( + [ + "maturin", + "build", + "--release", + "--compatibility", + "musllinux_1_2", + "--interpreter", + args.interpreter, + "--out", + str(output), + ], + check=True, + cwd=build_source, + ) + + wheels = list(output.glob("*.whl")) + if len(wheels) != 1: + raise RuntimeError(f"Expected one musllinux wheel in {output}, found {len(wheels)}") + + +if __name__ == "__main__": + main()