diff --git a/.github/ci-path-filters.yml b/.github/ci-path-filters.yml index a3b494ca3..812b9af1b 100644 --- a/.github/ci-path-filters.yml +++ b/.github/ci-path-filters.yml @@ -59,7 +59,10 @@ rust_package: - 'crates/ffi/nemo_relay.h' - 'crates/ffi/src/**' - 'justfile' + - 'packages/cli-bin/**' + - 'python/cli-bin/**' - 'rust-toolchain.toml' + - 'scripts/package-cli-bin.py' node_package: - 'Cargo.lock' @@ -79,8 +82,11 @@ node_package: - 'justfile' - 'package.json' - 'package-lock.json' + - 'packages/cli-bin/**' - 'pyproject.toml' - 'rust-toolchain.toml' + - 'scripts/package-cli-bin.py' + - 'scripts/package-node-bin.py' - 'uv.lock' python_package: @@ -96,8 +102,10 @@ python_package: - 'crates/python/src/**' - 'justfile' - 'pyproject.toml' + - 'python/cli-bin/**' - 'python/nemo_relay/**' - 'rust-toolchain.toml' + - 'scripts/package-cli-bin.py' - 'uv.lock' python_integration_langchain: @@ -124,7 +132,9 @@ dependencies: - 'integrations/**/package.json' - 'package.json' - 'package-lock.json' + - 'packages/cli-bin/**' - 'pyproject.toml' + - 'python/cli-bin/**' - 'scripts/licensing/**' - 'uv.lock' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 802f9c77f..022425c87 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -257,8 +257,8 @@ jobs: exit 1 fi - release-cli-artifacts: - name: Release CLI Artifacts + release-distribution-artifacts: + name: Release Distribution Artifacts needs: [prepare, ci_required] if: ${{ needs.prepare.outputs.publish_packages == 'true' && needs.ci_required.result == 'success' }} runs-on: ubuntu-latest @@ -269,28 +269,77 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Download CLI binary artifacts + - name: Download CLI distribution artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: cli-* merge-multiple: true path: release-assets/ - - name: Generate CLI release checksums + - name: Download Node distribution artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: node-npm-package-* + merge-multiple: true + path: release-assets/ + + - name: Download Python API wheel artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: wheel-* + merge-multiple: true + path: release-assets/ + + - name: Generate distribution checksums run: | set -euo pipefail ( cd release-assets - cli_assets=(nemo-relay-cli-*) - sha256sum "${cli_assets[@]}" > SHA256SUMS + expect_count() { + local expected="$1" + local label="$2" + shift 2 + local actual="$#" + if [ "$actual" -ne "$expected" ]; then + echo "Error: expected ${expected} ${label} artifacts, found ${actual}" >&2 + exit 1 + fi + } + + cli_binaries=(nemo-relay-cli-*) + cli_wheels=(nemo_relay_cli_bin-*.whl) + python_api_wheels=(nemo_relay-*.whl) + cli_npm=(nemo-relay-bin-npm-*.tgz) + node_npm=(nemo-relay-node-npm-*.tgz) + expect_count 5 "CLI binary" "${cli_binaries[@]}" + expect_count 5 "CLI wheel" "${cli_wheels[@]}" + expect_count 5 "Python API wheel" "${python_api_wheels[@]}" + expect_count 6 "CLI npm" "${cli_npm[@]}" + expect_count 6 "Node npm" "${node_npm[@]}" + + mapfile -t distribution_assets < <( + find . -maxdepth 1 -type f \ + ! -name 'SHA256SUMS' \ + ! -name '*.sha256' \ + -printf '%f\n' | + sort + ) + if [ "${#distribution_assets[@]}" -eq 0 ]; then + echo "Error: no distribution artifacts were collected" >&2 + exit 1 + fi + sha256sum "${distribution_assets[@]}" > SHA256SUMS sha256sum --check SHA256SUMS - while read -r digest filename; do - printf '%s %s\n' "$digest" "$filename" > "${filename}.sha256" - done < SHA256SUMS + for filename in nemo-relay-cli-*; do + if [[ "$filename" == *.tgz || "$filename" == *.whl || "$filename" == *.sha256 ]]; then + continue + fi + sha256sum "$filename" > "${filename}.sha256" + done - printf 'CLI release assets:\n' + printf 'Distribution release assets:\n' printf ' release-assets/%s\n' * ) @@ -298,7 +347,7 @@ jobs: uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0 with: draft: true - prerelease: ${{ contains(github.ref_name, '-beta.') || contains(github.ref_name, '-rc.') }} + prerelease: ${{ contains(github.ref_name, '-alpha.') || contains(github.ref_name, '-beta.') || contains(github.ref_name, '-rc.') }} overwrite_files: true fail_on_unmatched_files: true files: release-assets/* @@ -400,6 +449,13 @@ jobs: name: plugin-wheel path: dist/ + - name: Download CLI wheel artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: cli-python-wheel-* + merge-multiple: true + path: dist/ + - name: Publish to PyPI with trusted publishing uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: @@ -428,11 +484,12 @@ jobs: node-version: ${{ steps.ci-config.outputs.node_version }} registry-url: "https://registry.npmjs.org" - - name: Download consolidated Node package artifact + - name: Download split Node package artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: npm-consolidated - path: . + pattern: node-npm-package-* + merge-multiple: true + path: node-packages/ - name: Download OpenClaw plugin artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -440,6 +497,13 @@ jobs: name: openclaw-npm path: openclaw-package/ + - name: Download CLI npm artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: cli-npm-package-* + merge-multiple: true + path: cli-packages/ + - name: Select npm dist-tag run: | set -e @@ -453,14 +517,39 @@ jobs: run: | set -euo pipefail version="${{ github.ref_name }}" + for pkg in node-packages/nemo-relay-node-npm-{linux-x64,linux-arm64,darwin-arm64,win32-x64,win32-arm64}-${version}.tgz; do + name="$(tar xOf "$pkg" package/package.json | node -p 'JSON.parse(require("fs").readFileSync(0, "utf8")).name')" + if npm view "${name}@${version}" version --registry https://registry.npmjs.org >/dev/null 2>&1; then + echo "${name} ${version} already exists on npm; skipping" + continue + fi + npm publish "$pkg" --access public --tag "${NEMO_RELAY_NPM_DIST_TAG}" + done + metapackage="node-packages/nemo-relay-node-npm-${version}.tgz" if npm view "nemo-relay-node@${version}" version --registry https://registry.npmjs.org >/dev/null 2>&1; then echo "nemo-relay-node ${version} already exists on npm; skipping" - exit 0 + else + npm publish "$metapackage" --access public --tag "${NEMO_RELAY_NPM_DIST_TAG}" + fi + + - name: Publish CLI packages to npm + run: | + set -euo pipefail + version="${{ github.ref_name }}" + for pkg in cli-packages/nemo-relay-bin-npm-{linux-x64,linux-arm64,darwin-arm64,win32-x64,win32-arm64}-${version}.tgz; do + name="$(tar xOf "$pkg" package/package.json | node -p 'JSON.parse(require("fs").readFileSync(0, "utf8")).name')" + if npm view "${name}@${version}" version --registry https://registry.npmjs.org >/dev/null 2>&1; then + echo "${name} ${version} already exists on npm; skipping" + continue + fi + npm publish "$pkg" --access public --tag "${NEMO_RELAY_NPM_DIST_TAG}" + done + launcher="cli-packages/nemo-relay-bin-npm-${version}.tgz" + if npm view "nemo-relay-cli-bin@${version}" version --registry https://registry.npmjs.org >/dev/null 2>&1; then + echo "nemo-relay-cli-bin ${version} already exists on npm; skipping" + else + npm publish "$launcher" --access public --tag "${NEMO_RELAY_NPM_DIST_TAG}" fi - unzip -q ./consolidated.zip -d combined - echo "Platform binaries included:" - ls -la combined/package/*.node - npm publish ./combined/package --access public --tag "${NEMO_RELAY_NPM_DIST_TAG}" - name: Publish OpenClaw plugin package to npm run: | diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index e6b087386..ab2cdc6fd 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -220,3 +220,10 @@ jobs: fi pre-commit run --from-ref "$base_ref" --to-ref HEAD --show-diff-on-failure fi + + - name: Test binary package assembly + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: >- + uv run --no-project python -m unittest + scripts.tests.test_package_cli_bin + scripts.tests.test_package_node_bin diff --git a/.github/workflows/ci_node.yml b/.github/workflows/ci_node.yml index d15e01569..98c8c0597 100644 --- a/.github/workflows/ci_node.yml +++ b/.github/workflows/ci_node.yml @@ -172,6 +172,10 @@ jobs: with: node-version: ${{ steps.ci-config.outputs.node_version }} + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: ${{ steps.ci-config.outputs.default_python_version }} + - uses: taiki-e/install-action@c070f87102a1c75b3183910f391c1cb887fe13c8 # v2.77.6 with: tool: just@${{ steps.ci-config.outputs.just_version }} @@ -204,14 +208,15 @@ jobs: just \ --set ci true \ --set output_dir "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}" \ + --set node_platform "${{ matrix.platform }}" \ --set ref_name "${NEMO_RELAY_PACKAGE_VERSION}" \ package-node - - name: Upload npm package artifact + - name: Upload split Node npm package artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ inputs.run_package }} with: - name: npm-${{ matrix.platform }} + name: node-npm-package-${{ matrix.platform }} path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/npm/*.tgz if-no-files-found: error @@ -282,79 +287,79 @@ jobs: path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/openclaw/*.tgz if-no-files-found: error - Consolidate: - name: Consolidate + PackageSmoke: + name: Package smoke (${{ matrix.platform }}) needs: [Package] if: ${{ inputs.run_package && !cancelled() && needs.Package.result == 'success' }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} timeout-minutes: 30 permissions: contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: ubuntu-latest + - platform: linux-arm64 + runner: ubuntu-24.04-arm + - platform: macos-arm64 + runner: macos-15 + - platform: windows-amd64 + runner: windows-2022 + - platform: windows-arm64 + runner: windows-11-arm steps: - - name: Download npm linux-amd64 artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - name: npm-linux-amd64 - path: npm-linux-amd64/ + persist-credentials: false - - name: Download npm linux-arm64 artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: npm-linux-arm64 - path: npm-linux-arm64/ + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions - - name: Download npm macos-arm64 artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 with: - name: npm-macos-arm64 - path: npm-macos-arm64/ + node-version: ${{ steps.ci-config.outputs.node_version }} - - name: Download npm windows-amd64 artifact + - name: Download Node metapackage artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: npm-windows-amd64 - path: npm-windows-amd64/ + name: node-npm-package-linux-amd64 + path: node-metapackage/ - - name: Download npm windows-arm64 artifact + - name: Download matching Node native package artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: npm-windows-arm64 - path: npm-windows-arm64/ + name: node-npm-package-${{ matrix.platform }} + path: node-native/ - - name: Consolidate Node package directory - run: | - set -euo pipefail - mkdir -p combined - first=true - for dir in npm-linux-amd64 npm-linux-arm64 npm-macos-arm64 npm-windows-amd64 npm-windows-arm64; do - shopt -s nullglob - tgzs=("${dir}"/*.tgz) - shopt -u nullglob - if [ "${#tgzs[@]}" -eq 0 ]; then - echo "Error: no npm package artifact found in ${dir}" >&2 - exit 1 - fi - for tgz in "${tgzs[@]}"; do - if [ "${first}" = true ]; then - tar xzf "${tgz}" -C combined - first=false - else - tar xzf "${tgz}" -C combined --wildcards '*.node' - fi - done - done - echo "Platform binaries included:" - ls -la combined/package/*.node - - - name: Archive consolidated Node package + - name: Install and load split Node package run: | set -euo pipefail - cd combined - zip -rq "${{ github.workspace }}/consolidated.zip" package - - - name: Upload consolidated Node package artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: npm-consolidated - path: ${{ github.workspace }}/consolidated.zip - if-no-files-found: error + mkdir package-smoke + cd package-smoke + npm init --yes >/dev/null + metapackage=(../node-metapackage/nemo-relay-node-npm-[0-9]*.tgz) + native=(../node-native/nemo-relay-node-npm-*-*-[0-9]*.tgz) + npm install --ignore-scripts "${native[0]}" "${metapackage[0]}" + node - <<'NODE' + const assert = require('node:assert/strict'); + const { readdirSync } = require('node:fs'); + const nativePackages = readdirSync('node_modules').filter( + (name) => name.startsWith('nemo-relay-node-') && name !== 'nemo-relay-node', + ); + assert.equal(nativePackages.length, 1); + for (const entrypoint of [ + 'nemo-relay-node', + 'nemo-relay-node/typed', + 'nemo-relay-node/plugin', + 'nemo-relay-node/adaptive', + 'nemo-relay-node/observability', + 'nemo-relay-node/pii_redaction', + 'nemo-relay-node/model_pricing', + ]) { + require(entrypoint); + } + NODE diff --git a/.github/workflows/ci_rust.yml b/.github/workflows/ci_rust.yml index bd554bb34..16cc4f4d3 100644 --- a/.github/workflows/ci_rust.yml +++ b/.github/workflows/ci_rust.yml @@ -278,9 +278,66 @@ jobs: mkdir -p "${NEMO_RELAY_CI_WORKSPACE_TMP}/cli" cp "$source" "${NEMO_RELAY_CI_WORKSPACE_TMP}/cli/${asset}" + - name: Package CLI binary for PyPI and npm + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -euo pipefail + target="${{ matrix.target }}" + binary="nemo-relay" + if [ "${{ runner.os }}" = "Windows" ]; then + binary="${binary}.exe" + fi + source="${NEMO_RELAY_CI_WORKSPACE}/target/${target}/release/${binary}" + version="${{ github.ref_name }}" + if [ "${{ github.ref_type }}" != "tag" ]; then + version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n1)+${GIT_COMMIT::8}" + fi + rm -rf "${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-packages" + mkdir -p "${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-packages" + args=( + --binary "$source" + --target "$target" + --version "$version" + --output-dir "${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-packages" + ) + if [ "${{ matrix.platform }}" = "linux-amd64" ]; then + args+=(--npm-launcher) + fi + python scripts/package-cli-bin.py "${args[@]}" + + - name: Install and run CLI wheel + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -euo pipefail + wheels=("${NEMO_RELAY_CI_WORKSPACE_TMP}"/cli-packages/*.whl) + python -m venv "${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-wheel-venv" + if [ "${{ runner.os }}" = "Windows" ]; then + venv_python="${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-wheel-venv/Scripts/python.exe" + venv_cli="${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-wheel-venv/Scripts/nemo-relay.exe" + else + venv_python="${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-wheel-venv/bin/python" + venv_cli="${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-wheel-venv/bin/nemo-relay" + fi + "$venv_python" -m pip install --force-reinstall --no-deps "${wheels[0]}" + "$venv_cli" --version + - name: Upload CLI binary artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: cli-${{ matrix.platform }} + name: cli-binary-${{ matrix.platform }} path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/cli/* if-no-files-found: error + + - name: Upload CLI wheel artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-python-wheel-${{ matrix.platform }} + path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/cli-packages/*.whl + if-no-files-found: error + + - name: Upload CLI npm artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-npm-package-${{ matrix.platform }} + path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/cli-packages/*.tgz + if-no-files-found: error diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4e4c087af..416fdca4c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,7 +68,7 @@ collect:github-artifacts: fi export GH_TOKEN="${NEMO_RELAY_CI_GITHUB_TOKEN}" - mkdir -p collected/wheels collected/node downloaded + mkdir -p collected/wheels collected/node collected/cli-npm downloaded tag="${CI_COMMIT_TAG:-}" if [ -z "$tag" ]; then @@ -132,23 +132,26 @@ collect:github-artifacts: fi gh run download "$run_id" --repo "$NEMO_RELAY_CI_GITHUB_REPOSITORY" --pattern 'wheel-*' --dir downloaded/wheels + gh run download "$run_id" --repo "$NEMO_RELAY_CI_GITHUB_REPOSITORY" --pattern 'cli-python-wheel-*' --dir downloaded/cli-wheels + gh run download "$run_id" --repo "$NEMO_RELAY_CI_GITHUB_REPOSITORY" --pattern 'cli-npm-package-*' --dir downloaded/cli-npm gh run download "$run_id" --repo "$NEMO_RELAY_CI_GITHUB_REPOSITORY" --name 'plugin-wheel' --dir downloaded/wheels - gh run download "$run_id" --repo "$NEMO_RELAY_CI_GITHUB_REPOSITORY" --name npm-consolidated --dir downloaded/node + gh run download "$run_id" --repo "$NEMO_RELAY_CI_GITHUB_REPOSITORY" --pattern 'node-npm-package-*' --dir downloaded/node find downloaded/wheels -type f -name '*.whl' -exec cp {} collected/wheels/ \; - node_zip="$(find downloaded/node -type f -name consolidated.zip -print -quit)" - if [ -z "$node_zip" ]; then - echo "Error: collected GitHub npm-consolidated artifact did not contain consolidated.zip." >&2 - exit 1 - fi - cp "$node_zip" collected/node/consolidated.zip + find downloaded/cli-wheels -type f -name '*.whl' -exec cp {} collected/wheels/ \; + find downloaded/cli-npm -type f -name '*.tgz' -exec cp {} collected/cli-npm/ \; + find downloaded/node -type f -name '*.tgz' -exec cp {} collected/node/ \; if ! ls collected/wheels/*.whl >/dev/null 2>&1; then echo "Error: collected GitHub wheel artifacts did not contain any .whl files." >&2 exit 1 fi - if [ ! -f collected/node/consolidated.zip ]; then - echo "Error: collected GitHub npm-consolidated artifact did not contain consolidated.zip." >&2 + if ! ls collected/node/*.tgz >/dev/null 2>&1; then + echo "Error: collected GitHub Node artifacts did not contain any .tgz files." >&2 + exit 1 + fi + if ! ls collected/cli-npm/*.tgz >/dev/null 2>&1; then + echo "Error: collected GitHub CLI npm artifacts did not contain any .tgz files." >&2 exit 1 fi @@ -166,7 +169,8 @@ collect:github-artifacts: expire_in: 7 days paths: - collected/wheels/*.whl - - collected/node/consolidated.zip + - collected/node/*.tgz + - collected/cli-npm/*.tgz - collected/github-run.json publish:artifactory:wheels: @@ -302,8 +306,6 @@ publish:artifactory:npm: needs: - job: collect:github-artifacts artifacts: true - before_script: - - apt-get update -qq && apt-get install -y --no-install-recommends unzip ca-certificates && rm -rf /var/lib/apt/lists/* script: - | set -eu @@ -312,14 +314,8 @@ publish:artifactory:npm: echo "Error: uploading npm packages to Artifactory requires NEMO_RELAY_CI_ARTIFACTORY_USER, NEMO_RELAY_CI_ARTIFACTORY_KEY, and NEMO_RELAY_CI_ARTIFACTORY_NPM_URL." >&2 exit 1 fi - if [ ! -f collected/node/consolidated.zip ]; then - echo "Error: no collected consolidated Node package artifact found." >&2 - exit 1 - fi - mkdir -p collected/node/consolidated - unzip -q collected/node/consolidated.zip -d collected/node/consolidated - if [ ! -d collected/node/consolidated/package ]; then - echo "Error: consolidated Node artifact did not contain a package directory." >&2 + if ! ls collected/node/*.tgz >/dev/null 2>&1; then + echo "Error: no collected split Node package artifacts found." >&2 exit 1 fi @@ -329,4 +325,13 @@ publish:artifactory:npm: npm config set registry "$registry_url" npm config set "${registry_key}:_auth" "$npm_auth" - npm publish --tag dev collected/node/consolidated/package + for platform in linux-x64 linux-arm64 darwin-arm64 win32-x64 win32-arm64; do + package="collected/node/nemo-relay-node-npm-${platform}-${CI_COMMIT_TAG}.tgz" + npm publish --tag dev "$package" + done + npm publish --tag dev "collected/node/nemo-relay-node-npm-${CI_COMMIT_TAG}.tgz" + for platform in linux-x64 linux-arm64 darwin-arm64 win32-x64 win32-arm64; do + package="collected/cli-npm/nemo-relay-bin-npm-${platform}-${CI_COMMIT_TAG}.tgz" + npm publish --tag dev "$package" + done + npm publish --tag dev "collected/cli-npm/nemo-relay-bin-npm-${CI_COMMIT_TAG}.tgz" diff --git a/README.md b/README.md index 237fec0bb..1036653b6 100644 --- a/README.md +++ b/README.md @@ -45,14 +45,32 @@ trajectory file, you have concrete data to inspect, debug, and build on. ### Local Agent Trajectory This walkthrough shows an end-to-end quick success setup. Install the -`nemo-relay-cli`, turn on local exporters, run Codex, Claude Code, or Hermes +NeMo Relay CLI, turn on local exporters, run Codex, Claude Code, or Hermes through Relay, and check that Relay wrote both raw events and normalized trajectories. #### 1. Install the CLI -Run the installer for your platform: +Install the prebuilt CLI from PyPI: + +```bash +pip install nemo-relay-cli-bin +``` + +Install the prebuilt CLI from npm: + +```bash +npm install --global nemo-relay-cli-bin +``` + +Python API users can install the matching CLI through the optional extra: + +```bash +pip install "nemo-relay[cli]" +``` + +Alternatively, run the installer for your platform: ```bash curl -fsSL https://raw.githubusercontent.com/NVIDIA/NeMo-Relay/main/install.sh | sh diff --git a/RELEASING.md b/RELEASING.md index 8830dfe68..4bde75f50 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -31,9 +31,9 @@ The release pipeline publishes these package surfaces from a tag push: | Ecosystem | Published Surface | |---|---| | crates.io | `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, `nemo-relay-cli` | -| PyPI | `nemo-relay`, `nemo-relay-plugin` | -| npm | `nemo-relay-node`, `nemo-relay-openclaw` | -| GitHub Releases | CLI binaries and `SHA256SUMS` | +| PyPI | `nemo-relay`, `nemo-relay-plugin`, `nemo-relay-cli-bin` | +| npm | `nemo-relay-node` and its five platform packages, `nemo-relay-openclaw`, `nemo-relay-cli-bin`, and its five platform packages | +| GitHub Releases | CLI binaries, `nemo-relay` and `nemo-relay-cli-bin` wheels, CLI and Node npm tarballs, and checksums | | Fern | The documentation site | Go remains source-first. There is no separate Go package-manager publication @@ -61,6 +61,9 @@ NeMo Relay versions are anchored on the workspace SemVer in the repository root lock entries and must be updated with it. - `integrations/openclaw/package.json` carries the base npm version for the OpenClaw plugin package and must stay aligned with the same release version. +- `packages/cli-bin/package.json` and `python/cli-bin/pyproject.toml` carry the + base CLI package versions in their registry-specific SemVer and PEP 440 + spellings. The `nemo-relay[cli]` extra must pin the exact PyPI version. - The Python package version is derived at packaging time. `pyproject.toml` stays `dynamic = ["version"]` in the repository, and the packaging recipe writes a concrete version into `pyproject.toml` and `crates/python/Cargo.toml` @@ -141,6 +144,16 @@ Before you create a release tag, confirm the following: [`.github/workflows/ci.yaml`](.github/workflows/ci.yaml) workflow - GitHub Actions `id-token: write` access is available for the top-level npm publish job - GitHub Actions `id-token: write` access for the top-level PyPI publish job + - Pending or existing PyPI trusted publishing is configured for + `nemo-relay-cli-bin` + - npm trusted publishers are configured for `nemo-relay-node`, + `nemo-relay-node-linux-x64-gnu`, `nemo-relay-node-linux-arm64-gnu`, + `nemo-relay-node-darwin-arm64`, `nemo-relay-node-win32-x64-msvc`, and + `nemo-relay-node-win32-arm64-msvc` + - npm trusted publishers are configured for `nemo-relay-cli-bin` and + `nemo-relay-cli-bin-linux-x64`, `nemo-relay-cli-bin-linux-arm64`, + `nemo-relay-cli-bin-darwin-arm64`, `nemo-relay-cli-bin-win32-x64`, and + `nemo-relay-cli-bin-win32-arm64` 5. The GitHub Release entry is ready to become the only canonical release-notes surface. @@ -167,6 +180,8 @@ The helper updates: 4. [`integrations/openclaw/package.json`](integrations/openclaw/package.json) and the `integrations/openclaw` entry in the root [`package-lock.json`](package-lock.json) to the same release version. +5. The Python and npm `nemo-relay-cli-bin` metadata, including the exact + `nemo-relay[cli]` dependency version. Review docs and snippets that mention explicit versions, including: - [`README.md`](README.md) @@ -239,12 +254,23 @@ The release pipeline then: validation. 3. Builds publishable package artifacts with the exact tag version: - `package-rust` packs the published Rust crates for local validation. - - `package-node` packs the npm Node.js package. + - `package-node` packs one native npm Node.js package per supported platform + and creates the JavaScript and TypeScript metapackage once. - `package-openclaw` packs the npm OpenClaw plugin package. - `package-python` builds platform `nemo-relay` wheels. - `package-python-plugin` builds the `nemo-relay-plugin` wheel. - - The CLI release-asset job uploads each platform `nemo-relay` binary and - includes those binaries in `SHA256SUMS`. + - The Rust CLI matrix packages each prebuilt binary as a + `nemo-relay-cli-bin` wheel and npm platform package, and creates the npm + launcher package once. + - The distribution release-asset job uploads the CLI binaries, `nemo-relay` + API wheels, CLI wheels, CLI npm packages, and split Node npm packages. + `SHA256SUMS` covers every attached distribution artifact, and raw CLI + binaries also receive individual `.sha256` files for installer + compatibility. + GitHub-facing npm artifacts use + `nemo-relay-bin-npm[--]-.tgz` for the CLI and + `nemo-relay-node-npm[--]-.tgz` for Node.js; their + registry package names remain in the tarball manifests. 4. Publishes packages from the top-level workflow after the reusable packaging jobs complete: - `publish-rust` stamps Cargo workspace versions from the release tag, then @@ -254,11 +280,12 @@ The release pipeline then: `nemo-relay-switchyard`, `nemo-relay-ffi`, and `nemo-relay-cli` through trusted publishing from the top-level workflow - - `publish-python` downloads both the `nemo-relay` and `nemo-relay-plugin` - wheel artifacts and uploads them to PyPI with trusted publishing from the - top-level workflow - - `publish-npm` publishes the Node.js and OpenClaw plugin npm packages - through npm trusted publishing from the top-level workflow + - `publish-python` downloads the `nemo-relay`, `nemo-relay-plugin`, and + `nemo-relay-cli-bin` wheel artifacts and uploads them to PyPI with trusted + publishing from the top-level workflow + - `publish-npm` publishes the Node.js and CLI native packages before their + metapackages, then publishes the OpenClaw package, through npm trusted + publishing from the top-level workflow - Stable tags publish to the npm `latest` dist-tag - Prerelease tags such as `0.1.0-rc.1` publish to the npm `next` dist-tag so they do not become the default upgrade target @@ -291,9 +318,9 @@ NVIDIA Artifactory publication for the same tag: npm trusted publishing has its own registry-side constraints: - Each npm package can only have one trusted publisher configured at a time. -- Because this repository publishes `nemo-relay-node` and `nemo-relay-openclaw`, - configure trusted publishers for both packages - before pushing a release tag. +- Configure trusted publishers for `nemo-relay-node`, all five Node platform + packages, `nemo-relay-openclaw`, `nemo-relay-cli-bin`, and all five CLI + platform packages before pushing a release tag. - npm trusted publishing currently supports GitHub-hosted runners, not self-hosted runners. @@ -324,9 +351,10 @@ After the release is live, verify: `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and `nemo-relay-cli` crates are visible on crates.io. -2. The `nemo-relay` wheel is visible on PyPI. -3. The `nemo-relay-node` and `nemo-relay-openclaw` packages - are visible on npm. +2. The `nemo-relay` and `nemo-relay-cli-bin` wheels are visible on PyPI, and + `pip install "nemo-relay[cli]"` exposes `nemo-relay`. +3. The `nemo-relay-node`, its five platform packages, `nemo-relay-openclaw`, + `nemo-relay-cli-bin`, and its five platform packages are visible on npm. 4. The Unix and Windows installers resolve the new stable tag and verify matching CLI release asset checksums on their supported platforms. 5. The Fern documentation site shows the expected version and release notes. diff --git a/crates/cli/README.md b/crates/cli/README.md index 443d3003d..274b8c9fa 100644 --- a/crates/cli/README.md +++ b/crates/cli/README.md @@ -62,7 +62,25 @@ The CLI provides these capabilities: ## Installation Options -Cargo: +Install the prebuilt CLI from PyPI: + +```bash +pip install nemo-relay-cli-bin +``` + +Install the prebuilt CLI from npm: + +```bash +npm install --global nemo-relay-cli-bin +``` + +Install the Python API and matching CLI with the optional extra: + +```bash +pip install "nemo-relay[cli]" +``` + +Build and install the CLI from crates.io with Cargo: ```bash cargo install nemo-relay-cli diff --git a/docs/getting-started/installation.mdx b/docs/getting-started/installation.mdx index c6583da3c..edaa85180 100644 --- a/docs/getting-started/installation.mdx +++ b/docs/getting-started/installation.mdx @@ -33,6 +33,24 @@ coding-agent hook and LLM gateway observability. Install the latest stable release: + +```bash +pip install nemo-relay-cli-bin +``` + +To install the Python API and matching CLI together: + +```bash +pip install "nemo-relay[cli]" +``` + + + +```bash +npm install --global nemo-relay-cli-bin +``` + + ```bash curl -fsSL https://raw.githubusercontent.com/NVIDIA/NeMo-Relay/main/install.sh | sh @@ -47,9 +65,10 @@ irm https://raw.githubusercontent.com/NVIDIA/NeMo-Relay/main/install.ps1 | iex -These commands intentionally download the installer from `main` and install the -latest stable CLI release. The version of this documentation page does not -select the CLI version; set `NEMO_RELAY_VERSION` to install a specific release. +The shell installer commands intentionally download the installer from `main` +and install the latest stable CLI release. The version of this documentation +page does not select the CLI version; set `NEMO_RELAY_VERSION` to install a +specific release. The Unix installer downloads the matching GitHub Release binary to `$HOME/.local/bin`. The PowerShell installer downloads `nemo-relay.exe` to @@ -158,9 +177,9 @@ new download before replacing the binary in the selected installation directory. ### Alternative Installation Methods -Use the installers on a supported platform when you want a prebuilt release -asset. Use Cargo on unsupported platforms or when you prefer to build from -source: +The PyPI, npm, and installer methods all install the same release binary on +supported platforms. Use Cargo on unsupported platforms or when you prefer to +build from source: ```bash cargo install nemo-relay-cli diff --git a/justfile b/justfile index c64a671ab..fefb0560d 100644 --- a/justfile +++ b/justfile @@ -13,6 +13,8 @@ output_dir := "" ref_name := "" # Linux package artifacts target this minimum glibc version for compatibility. linux_glibc_version := "2.17" +# Supported Node package platform key. CI sets this from its package matrix. +node_platform := "" bash_helpers := ''' set -euo pipefail @@ -562,6 +564,7 @@ set_node_package_versions() { local version="$1" set_npm_package_version crates/node/package.json package-lock.json "$version" crates/node set_npm_package_version integrations/openclaw/package.json package-lock.json "$version" integrations/openclaw + set_npm_package_version packages/cli-bin/package.json package-lock.json "$version" packages/cli-bin set_npm_package_dependency_version integrations/openclaw/package.json package-lock.json integrations/openclaw nemo-relay-node "$version" } @@ -689,6 +692,35 @@ else: if updated != text: path.write_text(updated) print("crates/python/Cargo.toml uses the workspace version") + +path = Path("python/cli-bin/pyproject.toml") +text = path.read_text() +updated, count = re.subn( + r'^version = "(.*)"$', + f'version = "{version}"', + text, + count=1, + flags=re.MULTILINE, +) +if count != 1: + raise SystemExit("Failed to update version in python/cli-bin/pyproject.toml") +if updated != text: + path.write_text(updated) +print(f"python/cli-bin/pyproject.toml version updated to {version}") + +path = Path("pyproject.toml") +text = path.read_text() +updated, count = re.subn( + r'("nemo-relay-cli-bin==)[^"]+(")', + rf'\g<1>{version}\g<2>', + text, + count=1, +) +if count != 1: + raise SystemExit("Failed to update the nemo-relay CLI extra version") +if updated != text: + path.write_text(updated) +print(f"nemo-relay CLI extra updated to {version}") PY } @@ -1530,6 +1562,11 @@ package-node: # If `ref_name` is empty, append the current short HEAD SHA to the version. # If `ref_name` is set, write it as the exact package version before packing. linux_glibc_version="{{ linux_glibc_version }}" + node_platform="{{ node_platform }}" + python_executable="python" + if ! command -v "$python_executable" >/dev/null 2>&1; then + python_executable="python3" + fi output_dir="{{ output_dir }}" cd "$NEMO_RELAY_REPO_ROOT" package_dir="$(prepare_package_dir npm)" @@ -1557,7 +1594,34 @@ package-node: fi npm install --workspace=nemo-relay-node --ignore-scripts npm run --workspace=nemo-relay-node "${build_args[@]}" - npm pack --workspace=nemo-relay-node --pack-destination "$package_dir" + if [[ -z "$node_platform" ]]; then + case "$(uname -s)-$(uname -m)" in + Linux-x86_64) node_platform="linux-amd64" ;; + Linux-aarch64|Linux-arm64) node_platform="linux-arm64" ;; + Darwin-arm64) node_platform="macos-arm64" ;; + MINGW*|MSYS*|CYGWIN*) + if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then + node_platform="windows-arm64" + else + node_platform="windows-amd64" + fi + ;; + *) + echo "Error: unsupported Node package host $(uname -s)/$(uname -m)" >&2 + exit 1 + ;; + esac + fi + package_args=( + --node-dir crates/node + --platform "$node_platform" + --version "$package_version" + --output-dir "$package_dir" + ) + if [[ "$node_platform" == "linux-amd64" ]]; then + package_args+=(--metapackage) + fi + "$python_executable" scripts/package-node-bin.py "${package_args[@]}" shopt -s nullglob packages=("$package_dir"/*.tgz) if ((${#packages[@]} == 0)); then @@ -1674,3 +1738,19 @@ package-python-plugin: echo "Error: No Python plugin wheels found in $package_dir" exit 1 fi + +# Package a prebuilt CLI binary for PyPI and npm. +package-cli-bin binary target version package_dir npm_launcher="false": + #!/usr/bin/env bash + set -euo pipefail + cd "$NEMO_RELAY_REPO_ROOT" + args=( + --binary "{{ binary }}" + --target "{{ target }}" + --version "{{ version }}" + --output-dir "{{ package_dir }}" + ) + if [[ "{{ npm_launcher }}" == "true" ]]; then + args+=(--npm-launcher) + fi + uv run --no-project python scripts/package-cli-bin.py "${args[@]}" diff --git a/package-lock.json b/package-lock.json index 475fe716e..ead50344b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,8 @@ "name": "nemo-relay-workspace", "workspaces": [ "crates/node", - "integrations/openclaw" + "integrations/openclaw", + "packages/cli-bin" ], "devDependencies": { "fern-api": "5.57.0", @@ -928,6 +929,10 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/nemo-relay-cli-bin": { + "resolved": "packages/cli-bin", + "link": true + }, "node_modules/nemo-relay-node": { "resolved": "crates/node", "link": true @@ -5129,6 +5134,11 @@ "funding": { "url": "https://github.com/sponsors/eemeli" } + }, + "packages/cli-bin": { + "name": "nemo-relay-cli-bin", + "version": "0.7.0", + "license": "Apache-2.0" } } } diff --git a/package.json b/package.json index cf8803a04..9c8d01d4e 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ }, "workspaces": [ "crates/node", - "integrations/openclaw" + "integrations/openclaw", + "packages/cli-bin" ], "devDependencies": { "fern-api": "5.57.0", diff --git a/packages/cli-bin/package.json b/packages/cli-bin/package.json new file mode 100644 index 000000000..43cc3afba --- /dev/null +++ b/packages/cli-bin/package.json @@ -0,0 +1,7 @@ +{ + "name": "nemo-relay-cli-bin", + "version": "0.7.0", + "description": "Prebuilt NeMo Relay command-line interface.", + "private": true, + "license": "Apache-2.0" +} diff --git a/pyproject.toml b/pyproject.toml index cb833ab4e..98678156d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,10 @@ test = [ ] [project.optional-dependencies] +cli = [ + "nemo-relay-cli-bin==0.7.0", +] + langchain = [ "langchain>=1.3.14,<2.0.0", "langchain-core", @@ -106,6 +110,7 @@ default-groups = ["dev", "test"] package = true [tool.uv.sources] +nemo-relay-cli-bin = { path = "python/cli-bin" } nemo-relay-plugin = { workspace = true } [tool.uv.workspace] diff --git a/python/cli-bin/pyproject.toml b/python/cli-bin/pyproject.toml new file mode 100644 index 000000000..1f430da2d --- /dev/null +++ b/python/cli-bin/pyproject.toml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +requires = ["uv_build>=0.9,<0.12"] +build-backend = "uv_build" + +[project] +name = "nemo-relay-cli-bin" +version = "0.7.0" +description = "Prebuilt NeMo Relay command-line interface." +requires-python = ">=3.11" +license = "Apache-2.0" + +[tool.uv.build-backend] +module-name = "nemo_relay_cli_bin" +module-root = "src" diff --git a/python/cli-bin/src/nemo_relay_cli_bin/__init__.py b/python/cli-bin/src/nemo_relay_cli_bin/__init__.py new file mode 100644 index 000000000..a1ced58ab --- /dev/null +++ b/python/cli-bin/src/nemo_relay_cli_bin/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Development placeholder for the platform-specific CLI wheel distribution.""" diff --git a/scripts/package-cli-bin.py b/scripts/package-cli-bin.py new file mode 100755 index 000000000..8ef2b9ff8 --- /dev/null +++ b/scripts/package-cli-bin.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Package a prebuilt NeMo Relay CLI binary for PyPI and npm.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import io +import json +import os +import stat +import tarfile +import zipfile +from dataclasses import dataclass +from pathlib import Path + +PACKAGE_NAME = "nemo-relay-cli-bin" +SUMMARY = "Prebuilt NeMo Relay command-line interface." +LICENSE = "Apache-2.0" +REPOSITORY = "https://github.com/NVIDIA/NeMo-Relay" +ROOT = Path(__file__).resolve().parent.parent + + +@dataclass(frozen=True) +class Platform: + """Describe one supported CLI distribution platform.""" + + target: str + npm_suffix: str + npm_os: str + npm_cpu: str + wheel_platforms: tuple[str, ...] + executable: str + + @property + def npm_package(self) -> str: + """Return the npm package name for this platform.""" + return f"{PACKAGE_NAME}-{self.npm_suffix}" + + +PLATFORMS = { + platform.target: platform + for platform in ( + Platform( + "x86_64-unknown-linux-musl", + "linux-x64", + "linux", + "x64", + ("manylinux_2_17_x86_64", "musllinux_1_2_x86_64"), + "nemo-relay", + ), + Platform( + "aarch64-unknown-linux-musl", + "linux-arm64", + "linux", + "arm64", + ("manylinux_2_17_aarch64", "musllinux_1_2_aarch64"), + "nemo-relay", + ), + Platform( + "aarch64-apple-darwin", + "darwin-arm64", + "darwin", + "arm64", + ("macosx_11_0_arm64",), + "nemo-relay", + ), + Platform( + "x86_64-pc-windows-msvc", + "win32-x64", + "win32", + "x64", + ("win_amd64",), + "nemo-relay.exe", + ), + Platform( + "aarch64-pc-windows-msvc", + "win32-arm64", + "win32", + "arm64", + ("win_arm64",), + "nemo-relay.exe", + ), + ) +} + + +def wheel_version(version: str) -> str: + """Translate the repository SemVer spelling to PEP 440.""" + import re + + match = re.fullmatch( + r"(?P\d+\.\d+\.\d+)" + r"(?:-(?P