diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 5c4fa3d7569..2b72110bd7c 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -188,10 +188,12 @@ reviews: - `hermes-discord-e2e` — Hermes Discord config schema + placeholder isolation - `hermes-slack-e2e` — Hermes Slack policy + Python placeholder egress + - `openshell-gateway-upgrade-e2e` — stale Linux Docker-driver gateway + process restart after OpenShell upgrade To run selectively: ``` - gh workflow run nightly-e2e.yaml --ref -f jobs=cloud-e2e,sandbox-operations-e2e,rebuild-openclaw-e2e,messaging-compatible-endpoint-e2e,hermes-discord-e2e,hermes-slack-e2e + gh workflow run nightly-e2e.yaml --ref -f jobs=cloud-e2e,sandbox-operations-e2e,rebuild-openclaw-e2e,messaging-compatible-endpoint-e2e,hermes-discord-e2e,hermes-slack-e2e,openshell-gateway-upgrade-e2e ``` - path: "src/nemoclaw.ts" @@ -425,6 +427,20 @@ reviews: - path: "nemoclaw-blueprint/openclaw-plugins/kimi-inference-compat/**" instructions: *e2e-kimi-inference-compat + - path: "test/e2e/test-openshell-gateway-upgrade.sh" + instructions: | + This script validates the old OpenShell install upgrade guard for + Linux Docker-driver gateway processes. + + **E2E test recommendation:** + - `openshell-gateway-upgrade-e2e` — stale gateway process restart after + OpenShell upgrade + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=openshell-gateway-upgrade-e2e + ``` + - path: ".github/workflows/nightly-e2e.yaml" instructions: | This is the nightly E2E workflow definition. Changes here affect diff --git a/.github/actions/resolve-hermes-base-image/action.yaml b/.github/actions/resolve-hermes-base-image/action.yaml index cacbb88d27d..da6d5f8a9b5 100644 --- a/.github/actions/resolve-hermes-base-image/action.yaml +++ b/.github/actions/resolve-hermes-base-image/action.yaml @@ -10,10 +10,53 @@ runs: - name: Resolve Hermes sandbox base image shell: bash run: | - if docker pull ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest 2>/dev/null; then - echo "HERMES_BASE_IMAGE=ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest" >> "$GITHUB_ENV" - else - echo "::warning::GHCR Hermes base image not available, building locally" - docker build -f agents/hermes/Dockerfile.base -t nemoclaw-hermes-base-local . - echo "HERMES_BASE_IMAGE=nemoclaw-hermes-base-local" >> "$GITHUB_ENV" + set -euo pipefail + + image="ghcr.io/nvidia/nemoclaw/hermes-sandbox-base" + min_glibc="2.39" + + glibc_version() { + docker run --rm --entrypoint /usr/bin/ldd "$1" --version 2>/dev/null \ + | sed -nE 's/.*GLIBC ([0-9]+\.[0-9]+).*/\1/p; s/.* ([0-9]+\.[0-9]+)$/\1/p' \ + | head -n 1 + } + + glibc_ok() { + local have="$1" + [[ -n "$have" ]] && [[ "$(printf '%s\n%s\n' "$min_glibc" "$have" | sort -V | head -n 1)" == "$min_glibc" ]] + } + + try_image() { + local ref="$1" version + if ! docker pull "$ref" >/dev/null 2>&1; then + return 1 + fi + version="$(glibc_version "$ref" || true)" + if ! glibc_ok "$version"; then + echo "::warning::Hermes sandbox base image ${ref} has glibc ${version:-unknown}; need >= ${min_glibc}" + return 1 + fi + echo "HERMES_BASE_IMAGE=${ref}" >> "$GITHUB_ENV" + return 0 + } + + candidates=() + if [[ -n "${GITHUB_SHA:-}" ]]; then + candidates+=("${image}:${GITHUB_SHA:0:8}" "${image}:${GITHUB_SHA:0:7}") + fi + candidates+=("${image}:latest") + + for ref in "${candidates[@]}"; do + if try_image "$ref"; then + exit 0 + fi + done + + echo "::warning::No compatible GHCR Hermes sandbox base image found, building locally" + docker build -f agents/hermes/Dockerfile.base -t nemoclaw-hermes-base-local . + version="$(glibc_version nemoclaw-hermes-base-local || true)" + if ! glibc_ok "$version"; then + echo "::error::Local Hermes sandbox base image has glibc ${version:-unknown}; need >= ${min_glibc}" + exit 1 fi + echo "HERMES_BASE_IMAGE=nemoclaw-hermes-base-local" >> "$GITHUB_ENV" diff --git a/.github/actions/resolve-sandbox-base-image/action.yaml b/.github/actions/resolve-sandbox-base-image/action.yaml index dc44380ad2d..52e2f7a9886 100644 --- a/.github/actions/resolve-sandbox-base-image/action.yaml +++ b/.github/actions/resolve-sandbox-base-image/action.yaml @@ -10,10 +10,53 @@ runs: - name: Resolve sandbox base image shell: bash run: | - if docker pull ghcr.io/nvidia/nemoclaw/sandbox-base:latest 2>/dev/null; then - echo "BASE_IMAGE=ghcr.io/nvidia/nemoclaw/sandbox-base:latest" >> "$GITHUB_ENV" - else - echo "::warning::GHCR base image not available, building locally" - docker build -f Dockerfile.base -t nemoclaw-sandbox-base-local . - echo "BASE_IMAGE=nemoclaw-sandbox-base-local" >> "$GITHUB_ENV" + set -euo pipefail + + image="ghcr.io/nvidia/nemoclaw/sandbox-base" + min_glibc="2.39" + + glibc_version() { + docker run --rm --entrypoint /usr/bin/ldd "$1" --version 2>/dev/null \ + | sed -nE 's/.*GLIBC ([0-9]+\.[0-9]+).*/\1/p; s/.* ([0-9]+\.[0-9]+)$/\1/p' \ + | head -n 1 + } + + glibc_ok() { + local have="$1" + [[ -n "$have" ]] && [[ "$(printf '%s\n%s\n' "$min_glibc" "$have" | sort -V | head -n 1)" == "$min_glibc" ]] + } + + try_image() { + local ref="$1" version + if ! docker pull "$ref" >/dev/null 2>&1; then + return 1 + fi + version="$(glibc_version "$ref" || true)" + if ! glibc_ok "$version"; then + echo "::warning::Sandbox base image ${ref} has glibc ${version:-unknown}; need >= ${min_glibc}" + return 1 + fi + echo "BASE_IMAGE=${ref}" >> "$GITHUB_ENV" + return 0 + } + + candidates=() + if [[ -n "${GITHUB_SHA:-}" ]]; then + candidates+=("${image}:${GITHUB_SHA:0:8}" "${image}:${GITHUB_SHA:0:7}") + fi + candidates+=("${image}:latest") + + for ref in "${candidates[@]}"; do + if try_image "$ref"; then + exit 0 + fi + done + + echo "::warning::No compatible GHCR sandbox base image found, building locally" + docker build -f Dockerfile.base -t nemoclaw-sandbox-base-local . + version="$(glibc_version nemoclaw-sandbox-base-local || true)" + if ! glibc_ok "$version"; then + echo "::error::Local sandbox base image has glibc ${version:-unknown}; need >= ${min_glibc}" + exit 1 fi + echo "BASE_IMAGE=nemoclaw-sandbox-base-local" >> "$GITHUB_ENV" diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index f0ed6e0f18f..42864536fdf 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -65,10 +65,12 @@ jobs: - name: Extract metadata id: meta uses: docker/metadata-action@v6 + env: + DOCKER_METADATA_SHORT_SHA_LENGTH: 8 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | - type=raw,value=latest + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} type=sha,prefix=,format=short - name: Validate OpenClaw version input @@ -113,10 +115,12 @@ jobs: - name: Extract metadata id: meta uses: docker/metadata-action@v6 + env: + DOCKER_METADATA_SHORT_SHA_LENGTH: 8 with: images: ${{ env.REGISTRY }}/nvidia/nemoclaw/hermes-sandbox-base tags: | - type=raw,value=latest + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} type=sha,prefix=,format=short - name: Build and push diff --git a/.github/workflows/docker-pin-check.yaml b/.github/workflows/docker-pin-check.yaml index f359d4fe381..affc1b25ced 100644 --- a/.github/workflows/docker-pin-check.yaml +++ b/.github/workflows/docker-pin-check.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # # Weekly check that the pinned Dockerfile base-image digest is still current. -# Fails with an actionable message when a newer node:22-slim is available. +# Fails with an actionable message when a newer node:22-trixie-slim is available. name: docker-pin-check @@ -28,3 +28,4 @@ jobs: run: | bash scripts/update-docker-pin.sh --check DOCKERFILE=Dockerfile.base bash scripts/update-docker-pin.sh --check + DOCKERFILE=agents/hermes/Dockerfile.base bash scripts/update-docker-pin.sh --check diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index e89b2c2321a..f5516028c75 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -19,6 +19,9 @@ # Discord + Slack coverage with cross-talk assertions. See issue #1903. # sandbox-survival-e2e Sandbox survival across gateway restarts (onboard, inference, # gateway stop/start, verify sandbox + workspace + inference). +# openshell-gateway-upgrade-e2e +# Validates stale Linux Docker-driver OpenShell gateway +# processes are restarted after an OpenShell upgrade. # hermes-e2e Hermes Agent E2E — install → onboard --agent hermes → health # probe → live inference. Validates the multi-agent architecture. # hermes-discord-e2e Hermes Discord onboarding — validates the top-level Hermes @@ -58,6 +61,7 @@ on: messaging-compatible-endpoint-e2e, kimi-inference-compat-e2e, token-rotation-e2e, sandbox-survival-e2e, + openshell-gateway-upgrade-e2e, issue-2478-crash-loop-recovery-e2e, hermes-e2e, hermes-discord-e2e, hermes-slack-e2e, sandbox-operations-e2e, inference-routing-e2e, network-policy-e2e, deployment-services-e2e, diagnostics-e2e, @@ -1126,6 +1130,45 @@ jobs: /tmp/nemoclaw-e2e-upgrade-install.log if-no-files-found: ignore + # ── OpenShell gateway upgrade E2E ──────────────────────────── + # Reproduces the old-install upgrade edge case for Linux Docker-driver + # gateways: a healthy gateway process with stale supervisor/runtime env must + # be restarted rather than reused after the current OpenShell install. + openshell-gateway-upgrade-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',openshell-gateway-upgrade-e2e,')) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: "22" + + - name: Run OpenShell gateway upgrade E2E test + env: + GITHUB_TOKEN: ${{ github.token }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: bash test/e2e/test-openshell-gateway-upgrade.sh + + - name: Upload gateway upgrade logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: openshell-gateway-upgrade-logs + path: | + /tmp/nemoclaw-e2e-openshell-gateway-upgrade.log + /tmp/nemoclaw-e2e-openshell-gateway-start.log + /tmp/nemoclaw-e2e-openshell-gateway-process.log + if-no-files-found: ignore + # ── Hermes rebuild upgrade E2E ────────────────────────────── # Same upgrade scenario as OpenClaw but for Hermes Agent. rebuild-hermes-e2e: @@ -1209,12 +1252,14 @@ jobs: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-double-install" run: bash install.sh --non-interactive --yes-i-accept-third-party-software - name: Run double onboard E2E test env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_E2E_INSTALL_SANDBOX_NAME: "e2e-double-install" run: | [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" @@ -1246,12 +1291,14 @@ jobs: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-repair-install" run: bash install.sh --non-interactive --yes-i-accept-third-party-software - name: Run onboard repair E2E test env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_E2E_INSTALL_SANDBOX_NAME: "e2e-repair-install" run: | [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" @@ -1689,6 +1736,7 @@ jobs: shields-config-e2e, rebuild-openclaw-e2e, upgrade-stale-sandbox-e2e, + openshell-gateway-upgrade-e2e, rebuild-hermes-e2e, rebuild-hermes-stale-base-e2e, double-onboard-e2e, @@ -1776,6 +1824,7 @@ jobs: shields-config-e2e, rebuild-openclaw-e2e, upgrade-stale-sandbox-e2e, + openshell-gateway-upgrade-e2e, rebuild-hermes-e2e, rebuild-hermes-stale-base-e2e, double-onboard-e2e, @@ -1911,6 +1960,7 @@ jobs: shields-config-e2e, rebuild-openclaw-e2e, upgrade-stale-sandbox-e2e, + openshell-gateway-upgrade-e2e, rebuild-hermes-e2e, rebuild-hermes-stale-base-e2e, double-onboard-e2e, diff --git a/.github/workflows/pr-self-hosted.yaml b/.github/workflows/pr-self-hosted.yaml index 6cc94c71f6e..9e92ba4e381 100644 --- a/.github/workflows/pr-self-hosted.yaml +++ b/.github/workflows/pr-self-hosted.yaml @@ -43,15 +43,8 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Pull base image from GHCR (fall back to local build) - run: | - if docker pull ghcr.io/nvidia/nemoclaw/sandbox-base:latest 2>/dev/null; then - echo "BASE_IMAGE=ghcr.io/nvidia/nemoclaw/sandbox-base:latest" >> "$GITHUB_ENV" - else - echo "::warning::GHCR base image not available, building locally" - docker build -f Dockerfile.base -t nemoclaw-sandbox-base-local . - echo "BASE_IMAGE=nemoclaw-sandbox-base-local" >> "$GITHUB_ENV" - fi + - name: Resolve sandbox base image + uses: ./.github/actions/resolve-sandbox-base-image - name: Build production image run: docker build --build-arg BASE_IMAGE=${{ env.BASE_IMAGE }} -t nemoclaw-production . @@ -85,15 +78,8 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Pull base image from GHCR (fall back to local build) - run: | - if docker pull ghcr.io/nvidia/nemoclaw/sandbox-base:latest 2>/dev/null; then - echo "BASE_IMAGE=ghcr.io/nvidia/nemoclaw/sandbox-base:latest" >> "$GITHUB_ENV" - else - echo "::warning::GHCR base image not available, building locally" - docker build -f Dockerfile.base -t nemoclaw-sandbox-base-local . - echo "BASE_IMAGE=nemoclaw-sandbox-base-local" >> "$GITHUB_ENV" - fi + - name: Resolve sandbox base image + uses: ./.github/actions/resolve-sandbox-base-image - name: Build production image on arm64 run: docker build --build-arg BASE_IMAGE=${{ env.BASE_IMAGE }} -t nemoclaw-production-arm64 . diff --git a/.github/workflows/wsl-e2e.yaml b/.github/workflows/wsl-e2e.yaml index 63c2df27918..7721e6d119f 100644 --- a/.github/workflows/wsl-e2e.yaml +++ b/.github/workflows/wsl-e2e.yaml @@ -122,6 +122,10 @@ jobs: $script = @' set -euo pipefail export DEBIAN_FRONTEND=noninteractive + printf '%s\n' \ + 'Acquire::ForceIPv4 "true";' \ + 'Acquire::Retries "5";' \ + >/etc/apt/apt.conf.d/99github-actions-network apt-get update apt-get install -y bash ca-certificates curl git jq lsb-release make python3 python3-pip rsync tar unzip xz-utils '@ diff --git a/Dockerfile b/Dockerfile index 0a4e2f252f5..e8f2914c655 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ARG BASE_IMAGE=ghcr.io/nvidia/nemoclaw/sandbox-base:latest # Stage 1: Build TypeScript plugin from source -FROM node:22-slim@sha256:4f77a690f2f8946ab16fe1e791a3ac0667ae1c3575c3e4d0d4589e9ed5bfaf3d AS builder +FROM node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d AS builder ENV NPM_CONFIG_AUDIT=false \ NPM_CONFIG_FUND=false \ NPM_CONFIG_UPDATE_NOTIFIER=false @@ -26,22 +26,32 @@ RUN npm ci && npm run build FROM ${BASE_IMAGE} # Harden: remove unnecessary build tools and network probes from base image (#830) -# Protect procps before autoremove — the GHCR base may predate the procps -# addition, leaving it absent or auto-marked. apt-mark + conditional install -# guarantees ps/top/kill are present regardless of base image staleness. -# Ref: #2343 +# Protect runtime tools before autoremove — the GHCR base may predate the +# procps/e2fsprogs additions, leaving ps/chattr absent or auto-marked. The +# conditional install keeps stale bases usable while fresh bases skip apt. +# Refs: #2343, shields-up chattr hardening # hadolint ignore=DL3001 -RUN apt-mark manual procps 2>/dev/null || true \ - && (apt-get remove --purge -y gcc gcc-12 g++ g++-12 cpp cpp-12 make \ - netcat-openbsd netcat-traditional ncat 2>/dev/null || true) \ - && apt-get autoremove --purge -y \ - && if ! command -v ps >/dev/null 2>&1; then \ - apt-get update && apt-get install -y --no-install-recommends procps=2:4.0.2-3 \ - && rm -rf /var/lib/apt/lists/*; \ - else \ - rm -rf /var/lib/apt/lists/*; \ - fi \ - && ps --version +RUN set -eu; \ + apt-mark manual procps e2fsprogs 2>/dev/null || true; \ + (apt-get remove --purge -y gcc gcc-12 g++ g++-12 cpp cpp-12 make \ + netcat-openbsd netcat-traditional ncat 2>/dev/null || true); \ + apt-get autoremove --purge -y; \ + needs_ps=0; \ + needs_chattr=0; \ + if ! command -v ps >/dev/null 2>&1; then needs_ps=1; fi; \ + if ! command -v chattr >/dev/null 2>&1; then needs_chattr=1; fi; \ + if [ "$needs_ps" = "1" ] || [ "$needs_chattr" = "1" ]; then \ + apt-get update; \ + if [ "$needs_ps" = "1" ]; then \ + apt-get install -y --no-install-recommends procps=2:4.0.4-9; \ + fi; \ + if [ "$needs_chattr" = "1" ]; then \ + apt-get install -y --no-install-recommends e2fsprogs=1.47.2-3+b10; \ + fi; \ + fi; \ + rm -rf /var/lib/apt/lists/*; \ + ps --version; \ + command -v chattr >/dev/null # Copy built plugin and blueprint into the sandbox diff --git a/Dockerfile.base b/Dockerfile.base index 1b853d32ff5..e7a3cc5aaf7 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,6 +1,6 @@ # NemoClaw sandbox base image — expensive, rarely-changing layers. # -# Contains: node:22-slim, apt packages, gosu, user/group setup, +# Contains: node:22-trixie-slim, apt packages, gosu, user/group setup, # .openclaw directory structure, OpenClaw CLI, and PyYAML. # # Built on main merges and pushed to GHCR. The production Dockerfile @@ -12,9 +12,9 @@ # structural (users, directories, symlinks) that doesn't depend on # NemoClaw application code. Specifically: # -# node:22-slim — pinned by sha256 digest, checked weekly by +# node:22-trixie-slim — pinned by sha256 digest, checked weekly by # docker-pin-check.yaml -# apt packages — pinned to exact Debian bookworm versions +# apt packages — pinned to exact Debian trixie versions # gosu 1.19 — pinned release + per-arch sha256 checksum # gateway/sandbox — OS users and groups; names and UIDs are a # users stable contract with OpenShell @@ -36,7 +36,7 @@ # 1. OpenClaw CLI version bump — update OPENCLAW_VERSION default below, or override via --build-arg / workflow_dispatch # 2. New apt package needed — add it to the apt-get install list # 3. gosu upgrade — update URL, checksum, and version -# 4. node:22-slim digest rotated — update-docker-pin.sh updates both +# 4. node:22-trixie-slim digest rotated — update-docker-pin.sh updates all # Dockerfile and Dockerfile.base # 5. New .openclaw subdirectory — add mkdir below # 6. PyYAML or other pip dep bump — change the version below @@ -48,31 +48,32 @@ # by OpenClaw CLI version bumps or the weekly docker-pin-check. # ──────────────────────────────────────────────────────────────────────── -FROM node:22-slim@sha256:4f77a690f2f8946ab16fe1e791a3ac0667ae1c3575c3e4d0d4589e9ed5bfaf3d +FROM node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - python3=3.11.2-1+b1 \ - python3-pip=23.0.1+dfsg-1 \ - python3-venv=3.11.2-1+b1 \ - curl=7.88.1-10+deb12u14 \ - git=1:2.39.5-0+deb12u3 \ - gnupg=2.2.40-1.1+deb12u2 \ - ca-certificates=20230311+deb12u1 \ - iproute2=6.1.0-3 \ - iptables=1.8.9-2 \ - libcap2-bin=1:2.66-4+deb12u2+b2 \ - procps=2:4.0.2-3 \ - dos2unix=7.4.3-1 \ - jq=1.6-2.1+deb12u1 \ - vim-tiny=2:9.0.1378-2+deb12u2 \ - openssh-sftp-server=1:9.2p1-2+deb12u9 \ + python3=3.13.5-1 \ + python3-pip=25.1.1+dfsg-1 \ + python3-venv=3.13.5-1 \ + curl=8.14.1-2+deb13u2 \ + git=1:2.47.3-0+deb13u1 \ + gnupg=2.4.7-21+deb13u1 \ + ca-certificates=20250419 \ + iproute2=6.15.0-1 \ + iptables=1.8.11-2 \ + libcap2-bin=1:2.75-10+b8 \ + procps=2:4.0.4-9 \ + e2fsprogs=1.47.2-3+b10 \ + "dos2unix=7.5.2-1*" \ + jq=1.7.1-6+deb13u1 \ + vim-tiny=2:9.1.1230-2 \ + openssh-sftp-server=1:10.0p1-7+deb13u2 \ && rm -rf /var/lib/apt/lists/* # gosu for privilege separation (gateway vs sandbox user). # Install from GitHub release with checksum verification instead of -# Debian bookworm's ancient 1.14 (2020). Pinned to 1.19 (2025-09). +# Debian's packaged gosu can lag upstream. Pinned to 1.19 (2025-09). # hadolint ignore=DL4006 RUN arch="$(dpkg --print-architecture)" \ && case "$arch" in \ diff --git a/agents/hermes/Dockerfile.base b/agents/hermes/Dockerfile.base index 4f67d9f09b1..c4fa2b612d5 100644 --- a/agents/hermes/Dockerfile.base +++ b/agents/hermes/Dockerfile.base @@ -3,7 +3,7 @@ # # Hermes sandbox base image — expensive, rarely-changing layers. # -# Contains: node:22-slim (OpenShell needs Node), apt packages, gosu, +# Contains: node:22-trixie-slim (OpenShell needs Node), apt packages, gosu, # user/group setup, .hermes directory structure, Hermes CLI, and the # dependencies for NemoClaw-supported Hermes integrations. # @@ -14,11 +14,11 @@ # 1. Hermes version bump — change the HERMES_VERSION below # 2. New apt package needed — add it to the apt-get install list # 3. gosu upgrade — update URL, checksum, and version -# 4. node:22-slim digest rot — update-docker-pin.sh updates both +# 4. node:22-trixie-slim digest rot — update-docker-pin.sh updates all # 5. New .hermes subdirectory — add mkdir below # ──────────────────────────────────────────────────────────────── -FROM node:22-slim@sha256:4f77a690f2f8946ab16fe1e791a3ac0667ae1c3575c3e4d0d4589e9ed5bfaf3d +FROM node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d ENV DEBIAN_FRONTEND=noninteractive @@ -30,22 +30,23 @@ ARG HERMES_UV_EXTRAS="messaging web" ARG UV_VERSION=0.11.8 RUN apt-get update && apt-get install -y --no-install-recommends \ - python3=3.11.2-1+b1 \ - python3-pip=23.0.1+dfsg-1 \ - python3-venv=3.11.2-1+b1 \ - curl=7.88.1-10+deb12u14 \ - git=1:2.39.5-0+deb12u3 \ - gnupg=2.2.40-1.1+deb12u2 \ - ca-certificates=20230311+deb12u1 \ - iproute2=6.1.0-3 \ - iptables=1.8.9-2 \ - libcap2-bin=1:2.66-4+deb12u2+b2 \ - procps=2:4.0.2-3 \ - openssh-sftp-server=1:9.2p1-2+deb12u9 \ - socat=1.7.4.4-2 \ - dos2unix=7.4.3-1 \ - jq=1.6-2.1+deb12u1 \ - vim-tiny=2:9.0.1378-2+deb12u2 \ + python3=3.13.5-1 \ + python3-pip=25.1.1+dfsg-1 \ + python3-venv=3.13.5-1 \ + curl=8.14.1-2+deb13u2 \ + git=1:2.47.3-0+deb13u1 \ + gnupg=2.4.7-21+deb13u1 \ + ca-certificates=20250419 \ + iproute2=6.15.0-1 \ + iptables=1.8.11-2 \ + libcap2-bin=1:2.75-10+b8 \ + procps=2:4.0.4-9 \ + e2fsprogs=1.47.2-3+b10 \ + openssh-sftp-server=1:10.0p1-7+deb13u2 \ + socat=1.8.0.3-1 \ + "dos2unix=7.5.2-1*" \ + jq=1.7.1-6+deb13u1 \ + vim-tiny=2:9.1.1230-2 \ && rm -rf /var/lib/apt/lists/* # gosu for privilege separation (gateway vs sandbox user). diff --git a/agents/hermes/policy-additions.yaml b/agents/hermes/policy-additions.yaml index a47dbc033bf..5dbc8bc123f 100644 --- a/agents/hermes/policy-additions.yaml +++ b/agents/hermes/policy-additions.yaml @@ -63,7 +63,7 @@ network_policies: - allow: { method: GET, path: "/v1/models/**" } binaries: - { path: /usr/local/bin/hermes } - - { path: /usr/bin/python3.11 } + - { path: /usr/bin/python3* } - { path: /opt/hermes/.venv/bin/python } github: @@ -156,7 +156,7 @@ network_policies: - allow: { method: POST, path: "/**" } binaries: - { path: /usr/local/bin/hermes } - - { path: /usr/bin/python3.11 } + - { path: /usr/bin/python3* } - { path: /opt/hermes/.venv/bin/python } # ── PyPI — needed for pip install (skill/plugin deps) ───────── @@ -177,7 +177,7 @@ network_policies: - allow: { method: GET, path: "/**" } binaries: - { path: /usr/local/bin/pip3 } - - { path: /usr/bin/python3.11 } + - { path: /usr/bin/python3* } - { path: /opt/hermes/.venv/bin/python } # ── Messaging — pre-allowed for agent notifications ─────────── @@ -194,7 +194,7 @@ network_policies: - allow: { method: GET, path: "/file/bot*/**" } binaries: - { path: /usr/local/bin/node } - - { path: /usr/bin/python3.11 } + - { path: /usr/bin/python3* } - { path: /opt/hermes/.venv/bin/python } discord: @@ -232,7 +232,7 @@ network_policies: - allow: { method: GET, path: "/**" } binaries: - { path: /usr/local/bin/node } - - { path: /usr/bin/python3.11 } + - { path: /usr/bin/python3* } - { path: /opt/hermes/.venv/bin/python } slack: @@ -269,5 +269,5 @@ network_policies: access: full binaries: - { path: /usr/local/bin/hermes } - - { path: /usr/bin/python3.11 } + - { path: /usr/bin/python3* } - { path: /opt/hermes/.venv/bin/python } diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index 0eaf2fbd5e9..8b437d8418f 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -330,6 +330,15 @@ export no_proxy="$_NO_PROXY_VAL" export NEMOCLAW_DISCORD_FACADE_URL="http://127.0.0.1:${DISCORD_FACADE_PORT}" export PYTHONPATH="/opt/nemoclaw-hermes-discord-preload${PYTHONPATH:+:${PYTHONPATH}}" +# OpenShell injects SSL_CERT_FILE/CURL_CA_BUNDLE for its L7 proxy CA. Persist +# them into connect-session shells so Python Slack probes and Hermes tools trust +# the same proxy CA that the entrypoint received at startup. +if [ -n "${SSL_CERT_FILE:-}" ] && [ -f "${SSL_CERT_FILE}" ]; then + export CURL_CA_BUNDLE="${CURL_CA_BUNDLE:-$SSL_CERT_FILE}" + export REQUESTS_CA_BUNDLE="${REQUESTS_CA_BUNDLE:-$SSL_CERT_FILE}" + export GIT_SSL_CAINFO="${GIT_SSL_CAINFO:-$SSL_CERT_FILE}" +fi + # Resolve sandbox home dir early — used by proxy-env writing and # install_configure_guard before the non-root/root branch below. if [ "$(id -u)" -eq 0 ]; then @@ -359,6 +368,12 @@ export DISCORD_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}" export NEMOCLAW_DISCORD_FACADE_URL="http://127.0.0.1:${DISCORD_FACADE_PORT}" export PYTHONPATH="/opt/nemoclaw-hermes-discord-preload\${PYTHONPATH:+:\${PYTHONPATH}}" PROXYEOF + for _ca_env_name in SSL_CERT_FILE CURL_CA_BUNDLE REQUESTS_CA_BUNDLE GIT_SSL_CAINFO; do + _ca_env_value="${!_ca_env_name:-}" + if [ -n "$_ca_env_value" ]; then + printf 'export %s=%q\n' "$_ca_env_name" "$_ca_env_value" + fi + done } | emit_sandbox_sourced_file "$_PROXY_ENV_FILE" # ── Legacy layout migration ────────────────────────────────────── @@ -512,6 +527,107 @@ migrate_legacy_layout() { echo "[migration] Completed ${label} layout migration (${data_dir} removed)" >&2 } +refresh_hermes_provider_placeholders() { + local env_file="${HERMES_DIR}/.env" + local hash_file="${HERMES_HASH_FILE}" + local compat_hash="${HERMES_DIR}/.config-hash" + [ -f "$env_file" ] || return 0 + + local keys="TELEGRAM_BOT_TOKEN DISCORD_BOT_TOKEN SLACK_BOT_TOKEN SLACK_APP_TOKEN" + local has_scoped_placeholder=0 + local key value + for key in $keys; do + value="${!key:-}" + case "$value" in + openshell:resolve:env:*) has_scoped_placeholder=1 ;; + esac + done + [ "$has_scoped_placeholder" -eq 1 ] || return 0 + + if [ -L "$env_file" ] || [ -L "$hash_file" ] || { [ -e "$compat_hash" ] && [ -L "$compat_hash" ]; }; then + echo "[SECURITY] Refusing Hermes provider placeholder refresh — config or hash path is a symlink" >&2 + return 1 + fi + + if [ "$(id -u)" -eq 0 ]; then + chown root:sandbox "$env_file" || return 1 + chmod 640 "$env_file" || return 1 + chmod u+w "$hash_file" || return 1 + [ ! -f "$compat_hash" ] || chmod u+w "$compat_hash" 2>/dev/null || true + elif [ ! -w "$env_file" ] || [ ! -w "$hash_file" ]; then + echo "[config] Hermes provider placeholders supplied by OpenShell runtime env; .env refresh skipped without write access" >&2 + return 0 + fi + + local _write_rc=0 + NEMOCLAW_PROVIDER_PLACEHOLDER_KEYS="$keys" \ + python3 - "$env_file" <<'PYPLACEHOLDERS' || _write_rc=$? +import os +import sys + +env_file = sys.argv[1] +prefix = "openshell:resolve:env:" +keys = os.environ.get("NEMOCLAW_PROVIDER_PLACEHOLDER_KEYS", "").split() +replacements = {} + +for key in keys: + value = os.environ.get(key, "") + if value.startswith(prefix): + replacements[key] = value + +if not replacements: + sys.exit(0) + +with open(env_file, encoding="utf-8") as f: + lines = f.readlines() + +changed = False +updated = [] +for line in lines: + stripped = line.rstrip("\n") + replaced = False + for key, value in replacements.items(): + if stripped.startswith(f"{key}="): + new_line = f"{key}={value}\n" + updated.append(new_line) + changed = changed or new_line != line + replaced = True + break + if not replaced: + updated.append(line) + +if not changed: + sys.exit(0) + +with open(env_file, "w", encoding="utf-8") as f: + f.writelines(updated) + +print("refreshed=" + ",".join(sorted(replacements))) +PYPLACEHOLDERS + + if [ "$_write_rc" -eq 0 ]; then + if sha256sum "${HERMES_DIR}/config.yaml" "${HERMES_DIR}/.env" >"$hash_file"; then + chown root:root "$hash_file" 2>/dev/null || true + chmod 444 "$hash_file" 2>/dev/null || true + if [ -f "$compat_hash" ]; then + sha256sum "${HERMES_DIR}/config.yaml" "${HERMES_DIR}/.env" >"$compat_hash" || _write_rc=$? + chown sandbox:sandbox "$compat_hash" 2>/dev/null || true + chmod 600 "$compat_hash" 2>/dev/null || true + fi + echo "[config] Refreshed Hermes provider placeholders from OpenShell runtime env" >&2 + else + _write_rc=$? + fi + fi + + if [ "$(id -u)" -eq 0 ]; then + chown sandbox:sandbox "$env_file" 2>/dev/null || true + chmod 640 "$env_file" 2>/dev/null || true + fi + + [ "$_write_rc" -eq 0 ] || return "$_write_rc" +} + # ── Main ───────────────────────────────────────────────────────── # Migrate legacy symlink layout before anything else reads .hermes @@ -529,6 +645,7 @@ if [ "$(id -u)" -ne 0 ]; then echo "[SECURITY] Config integrity check failed — refusing to start (non-root mode)" >&2 exit 1 fi + refresh_hermes_provider_placeholders install_configure_guard configure_messaging_channels @@ -579,6 +696,7 @@ fi # ── Root path (full privilege separation via gosu) ───────────── verify_config_integrity "${HERMES_DIR}" "${HERMES_HASH_FILE}" +refresh_hermes_provider_placeholders install_configure_guard configure_messaging_channels diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 33b3adeab4f..734c65a5e4b 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -65,7 +65,7 @@ The wizard creates an OpenShell gateway, registers inference providers, builds t Use this command for new installs and for recreating a sandbox after changes to policy or configuration. ```console -$ nemoclaw onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--agent ] [--control-ui-port ] [--yes | -y] [--yes-i-accept-third-party-software] +$ nemoclaw onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--sandbox-gpu | --no-sandbox-gpu] [--sandbox-gpu-device ] [--agent ] [--control-ui-port ] [--yes | -y] [--yes-i-accept-third-party-software] ``` :::{warning} @@ -251,6 +251,8 @@ $ nemoclaw onboard --from ./Dockerfile.custom When `nemoclaw onboard` detects an NVIDIA GPU on the host (`nvidia-smi` succeeds), it enables OpenShell GPU passthrough at both the gateway and sandbox level by default. Use `--no-gpu` to opt out when you want host-side inference providers only and do not need direct GPU access inside the sandbox. Use `--gpu` to require GPU passthrough and fail fast if an NVIDIA GPU is not detected. +Use `--sandbox-gpu` or `--no-sandbox-gpu` to control only direct NVIDIA GPU access inside the sandbox. +Use `--sandbox-gpu-device ` to pass a specific OpenShell GPU device selector to `openshell sandbox create`. Prerequisites: @@ -887,7 +889,7 @@ The `nemoclaw setup` command is deprecated. Use `nemoclaw onboard` instead. ::: -This command remains as a compatibility alias to `nemoclaw onboard` and accepts the same flags: `--non-interactive`, `--resume`, `--fresh`, `--recreate-sandbox`, `--gpu` / `--no-gpu`, `--from`, `--name`, `--agent`, `--control-ui-port`, `--yes` / `-y`, `--yes-i-accept-third-party-software`. +This command remains as a compatibility alias to `nemoclaw onboard` and accepts the same flags: `--non-interactive`, `--resume`, `--fresh`, `--recreate-sandbox`, `--gpu` / `--no-gpu`, `--from`, `--name`, `--sandbox-gpu` / `--no-sandbox-gpu`, `--sandbox-gpu-device`, `--agent`, `--control-ui-port`, `--yes` / `-y`, `--yes-i-accept-third-party-software`. ```console $ nemoclaw setup @@ -900,7 +902,7 @@ The `nemoclaw setup-spark` command is deprecated. Use the standard installer and run `nemoclaw onboard` instead, because current OpenShell releases handle the older DGX Spark cgroup behavior. ::: -This command remains as a compatibility alias to `nemoclaw onboard` and accepts the same flags: `--non-interactive`, `--resume`, `--fresh`, `--recreate-sandbox`, `--gpu` / `--no-gpu`, `--from`, `--name`, `--agent`, `--control-ui-port`, `--yes` / `-y`, `--yes-i-accept-third-party-software`. +This command remains as a compatibility alias to `nemoclaw onboard` and accepts the same flags: `--non-interactive`, `--resume`, `--fresh`, `--recreate-sandbox`, `--gpu` / `--no-gpu`, `--from`, `--name`, `--sandbox-gpu` / `--no-sandbox-gpu`, `--sandbox-gpu-device`, `--agent`, `--control-ui-port`, `--yes` / `-y`, `--yes-i-accept-third-party-software`. ```console $ nemoclaw setup-spark @@ -1067,6 +1069,11 @@ These flags toggle optional behaviors during onboarding; set them before running | `NEMOCLAW_OVERLAY_SNAPSHOTTER` | snapshotter name | Selects the containerd overlay snapshotter for sandbox builds. Empty (default) preserves containerd's choice. | | `NEMOCLAW_SKIP_TELEGRAM_REACHABILITY` | `1` to enable | Skips the Telegram bot reachability probe during onboard (useful in restricted networks). | | `NEMOCLAW_CONFIG_ACCEPT_NEW_PATH` | `1` to enable | Accepts a new sandbox config path without an interactive prompt when the stored path differs from the discovered one. | +| `NEMOCLAW_SANDBOX_GPU` | `auto`, `1`, or `0` | Controls sandbox GPU passthrough during onboarding. `auto` enables GPU passthrough when an NVIDIA GPU is detected, `1` requires GPU passthrough, and `0` forces CPU-only sandbox creation. | +| `NEMOCLAW_SANDBOX_GPU_DEVICE` | OpenShell GPU device selector | Selects the GPU device passed with `openshell sandbox create --gpu-device`. Setting this value enables sandbox GPU passthrough unless `NEMOCLAW_SANDBOX_GPU=0` is also set, which is rejected. | +| `NEMOCLAW_OPENSHELL_GATEWAY_BIN` | path | Advanced override for the `openshell-gateway` binary used by the Linux Docker-driver gateway. Defaults to the binary next to `openshell`, then common install paths. | +| `NEMOCLAW_OPENSHELL_SANDBOX_BIN` | path | Advanced override for the `openshell-sandbox` binary passed to the Linux Docker-driver gateway supervisor. Defaults to the binary next to `openshell`, then common install paths. | +| `NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR` | path | Advanced override for the Linux Docker-driver gateway pid file and SQLite state directory. Defaults to `~/.local/state/nemoclaw/openshell-docker-gateway`. | ### Probe Timeouts diff --git a/nemoclaw-blueprint/blueprint.yaml b/nemoclaw-blueprint/blueprint.yaml index 2ea05cd1521..086d0ff309b 100644 --- a/nemoclaw-blueprint/blueprint.yaml +++ b/nemoclaw-blueprint/blueprint.yaml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 version: "0.1.0" -min_openshell_version: "0.0.32" -max_openshell_version: "0.0.36" +min_openshell_version: "0.0.37" +max_openshell_version: "0.0.37" min_openclaw_version: "2026.4.24" # Mirrors the components.sandbox.image manifest digest below. Lets a # downstream consumer (or release tooling) verify the blueprint declares diff --git a/nemoclaw-blueprint/scripts/slack-token-rewriter.js b/nemoclaw-blueprint/scripts/slack-token-rewriter.js index 9eac77e956a..c0d1caf40e4 100644 --- a/nemoclaw-blueprint/scripts/slack-token-rewriter.js +++ b/nemoclaw-blueprint/scripts/slack-token-rewriter.js @@ -2,18 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 // // slack-token-rewriter.js — translates the Bolt-compatible placeholder -// (xoxb|xapp)-OPENSHELL-RESOLVE-ENV-VAR into the canonical -// openshell:resolve:env:VAR form on outbound HTTP, so Slack tokens travel -// the same OpenShell substitution path Discord / Telegram / Brave already -// use without any real token touching openclaw.json. +// (xoxb|xapp)-OPENSHELL-RESOLVE-ENV-VAR into the active OpenShell +// openshell:resolve:env:* placeholder on outbound HTTP, so Slack tokens +// travel the same OpenShell substitution path Discord / Telegram / Brave +// already use without any real token touching openclaw.json. // // Why this preload exists: // Slack's Bolt SDK validates token shape (^xoxb-[A-Za-z0-9_-]+$ / // ^xapp-…$) at App construction, before any HTTP call leaves the -// process — so the canonical openshell:resolve:env:VAR placeholder is -// rejected synchronously and the gateway crashes. We emit a Bolt-shape -// placeholder into openclaw.json (which Bolt accepts), then translate -// it back to canonical form here, just before the bytes hit the wire, +// process — so OpenShell's openshell:resolve:env:* placeholder is rejected +// synchronously and the gateway crashes. We emit a Bolt-shape placeholder +// into openclaw.json (which Bolt accepts), then translate it back to the +// current OpenShell placeholder here, just before the bytes hit the wire, // where OpenShell's L7 proxy substitutes the real token from env. // // Wraps http.request / https.request — every Node HTTP client bottoms @@ -26,7 +26,9 @@ // token in both Authorization and the urlencoded body. // // Invariants: -// - No env reads. Translation is purely structural. +// - Env reads are used only when they contain OpenShell placeholder values. +// Raw env values are ignored so real tokens never enter outbound request +// objects through this preload. // - Mutates options/headers in place. axios reuses the headers object // after request creation, so cloning would break the request lifecycle. // - Idempotent. The output (openshell:resolve:env:VAR) does not match @@ -50,11 +52,29 @@ var BOLT_PLACEHOLDER = /\b(?:xoxb|xapp)-OPENSHELL-RESOLVE-ENV-([A-Z_][A-Z0-9_]*)\b/g; var FAST_PATH = 'OPENSHELL-RESOLVE-ENV-'; + var OPENSHELL_PLACEHOLDER_PREFIX = 'openshell:resolve:env:'; + + function placeholderForEnvKey(envKey) { + var value = ''; + try { + if (typeof process !== 'undefined' && process && process.env) { + value = process.env[envKey]; + } + } catch (_) { + value = ''; + } + if (typeof value === 'string' && value.indexOf(OPENSHELL_PLACEHOLDER_PREFIX) === 0) { + return value; + } + return OPENSHELL_PLACEHOLDER_PREFIX + envKey; + } function rewriteString(s) { if (typeof s !== 'string') return s; if (s.indexOf(FAST_PATH) === -1) return s; - return s.replace(BOLT_PLACEHOLDER, 'openshell:resolve:env:$1'); + return s.replace(BOLT_PLACEHOLDER, function (_match, envKey) { + return placeholderForEnvKey(envKey); + }); } function rewriteHeaders(headers) { diff --git a/package.json b/package.json index e128d58cc7d..cb46b634d7d 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "lint:fix": "npx @biomejs/biome lint --write . && npm run checks", "lint:ts": "cd nemoclaw && npm run check", "format": "npx @biomejs/biome format --write .", - "format:check": "npx @biomejs/biome format --check .", + "format:check": "npx @biomejs/biome format .", "format:ts": "cd nemoclaw && npm run lint:fix && npm run format", "check:installer-hash": "bash scripts/check-installer-hash.sh", "typecheck": "tsc -p jsconfig.json", diff --git a/scripts/brev-launchable-ci-cpu.sh b/scripts/brev-launchable-ci-cpu.sh index 2078817427e..8f0f6a82778 100755 --- a/scripts/brev-launchable-ci-cpu.sh +++ b/scripts/brev-launchable-ci-cpu.sh @@ -12,7 +12,7 @@ # 2. Node.js 22 (nodesource) # 3. OpenShell CLI binary (pinned release) # 4. NemoClaw repo cloned with npm deps installed and TS plugin built -# 5. Docker images pre-pulled (sandbox-base, openshell/cluster, node:22-slim) +# 5. Docker images pre-pulled (sandbox-base, openshell/supervisor, node:22-trixie-slim) # # What this does NOT install (intentionally): # - code-server (not needed for automated CI) @@ -28,7 +28,7 @@ # curl -fsSL https://raw.githubusercontent.com/NVIDIA/NemoClaw//scripts/brev-launchable-ci-cpu.sh | bash # # Environment overrides: -# OPENSHELL_VERSION — OpenShell CLI release tag (default: v0.0.36) +# OPENSHELL_VERSION — OpenShell CLI release tag (default: v0.0.37) # NEMOCLAW_REF — NemoClaw git ref to clone (default: main) # NEMOCLAW_CLONE_DIR — Where to clone NemoClaw (default: ~/NemoClaw) # SKIP_DOCKER_PULL — Set to 1 to skip Docker image pre-pulls @@ -40,7 +40,7 @@ set -euo pipefail # ── Configuration ──────────────────────────────────────────────────── -OPENSHELL_VERSION="${OPENSHELL_VERSION:-v0.0.36}" +OPENSHELL_VERSION="${OPENSHELL_VERSION:-v0.0.37}" NEMOCLAW_REF="${NEMOCLAW_REF:-main}" TARGET_USER="${SUDO_USER:-$(id -un)}" TARGET_HOME="$(getent passwd "$TARGET_USER" | cut -d: -f6)" @@ -53,7 +53,7 @@ SENTINEL="/var/run/nemoclaw-launchable-ready" # timeouts when pulled during CI runs. DOCKER_IMAGES=( "ghcr.io/nvidia/nemoclaw/sandbox-base:latest" - "node:22-slim" + "node:22-trixie-slim" ) # ── Suppress apt noise ─────────────────────────────────────────────── @@ -250,20 +250,20 @@ DOCKER_PULL_PID="" if [[ "${SKIP_DOCKER_PULL:-0}" != "1" ]]; then info "Pre-pulling Docker images in background..." ( - CLUSTER_TAG="${OPENSHELL_VERSION#v}" # v0.0.20 → 0.0.20 - CLUSTER_IMAGE="ghcr.io/nvidia/openshell/cluster:${CLUSTER_TAG}" + SUPERVISOR_TAG="${OPENSHELL_VERSION#v}" # v0.0.37 → 0.0.37 + SUPERVISOR_IMAGE="ghcr.io/nvidia/openshell/supervisor:${SUPERVISOR_TAG}" # Pull all images in parallel - for image in "${DOCKER_IMAGES[@]}" "$CLUSTER_IMAGE"; do + for image in "${DOCKER_IMAGES[@]}" "$SUPERVISOR_IMAGE"; do sg docker -c "docker pull $image" 2>&1 | tail -1 & done wait - # If pinned cluster tag failed, try :latest - if ! sg docker -c "docker image inspect $CLUSTER_IMAGE" >/dev/null 2>&1; then - warn " Could not pull $CLUSTER_IMAGE — trying :latest" - sg docker -c "docker pull ghcr.io/nvidia/openshell/cluster:latest" 2>&1 | tail -1 \ - || warn " Failed to pull openshell/cluster (will be pulled at test time)" + # If pinned supervisor tag failed, try :latest + if ! sg docker -c "docker image inspect $SUPERVISOR_IMAGE" >/dev/null 2>&1; then + warn " Could not pull $SUPERVISOR_IMAGE — trying :latest" + sg docker -c "docker pull ghcr.io/nvidia/openshell/supervisor:latest" 2>&1 | tail -1 \ + || warn " Failed to pull openshell/supervisor (will be pulled at test time)" fi ) & DOCKER_PULL_PID=$! diff --git a/scripts/install-openshell.sh b/scripts/install-openshell.sh index 8a7a17c24d1..0e75b4d0b5c 100755 --- a/scripts/install-openshell.sh +++ b/scripts/install-openshell.sh @@ -33,15 +33,33 @@ esac info "Detected $OS_LABEL ($ARCH_LABEL)" -# Minimum version required for Landlock filesystem policy enforcement -# (NVIDIA/OpenShell#810 fixes the drop_privileges/Landlock ordering bug -# that caused /sandbox to remain writable on 0.0.26). -MIN_VERSION="0.0.32" +# Minimum version required for the released Docker-driver gateway/sandbox +# binaries and the GPU filesystem policy fixes NemoClaw depends on. +MIN_VERSION="0.0.37" # Maximum version validated for this NemoClaw release. Newer OpenShell builds # may change sandbox semantics; upgrade NemoClaw before upgrading past this. -MAX_VERSION="0.0.36" +MAX_VERSION="0.0.37" # Pin fresh installs to this version instead of pulling "latest". PIN_VERSION="$MAX_VERSION" +DEV_MIN_VERSION="0.0.37" + +CHANNEL="${NEMOCLAW_OPENSHELL_CHANNEL:-auto}" +case "$CHANNEL" in + stable | dev | auto) ;; + *) fail "NEMOCLAW_OPENSHELL_CHANNEL must be one of: stable, dev, auto" ;; +esac + +if [ "$CHANNEL" = "auto" ]; then + RESOLVED_CHANNEL="stable" +else + RESOLVED_CHANNEL="$CHANNEL" +fi + +if [ "$RESOLVED_CHANNEL" = "dev" ]; then + RELEASE_TAG="dev" +else + RELEASE_TAG="v${PIN_VERSION}" +fi version_gte() { # Returns 0 (true) if $1 >= $2 — portable, no sort -V (BSD compat) @@ -57,20 +75,41 @@ version_gte() { return 0 } +linux_driver_bins_present() { + if [ "$OS" != "Linux" ]; then + return 0 + fi + command -v openshell-gateway >/dev/null 2>&1 && command -v openshell-sandbox >/dev/null 2>&1 +} + if command -v openshell >/dev/null 2>&1; then - INSTALLED_VERSION="$(openshell --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" + INSTALLED_VERSION_OUTPUT="$(openshell --version 2>&1 || true)" + INSTALLED_VERSION="$(printf '%s\n' "$INSTALLED_VERSION_OUTPUT" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" [ -n "$INSTALLED_VERSION" ] || INSTALLED_VERSION="0.0.0" - if version_gte "$INSTALLED_VERSION" "$MIN_VERSION"; then - if ! version_gte "$MAX_VERSION" "$INSTALLED_VERSION"; then - fail "openshell $INSTALLED_VERSION is above the maximum ($MAX_VERSION) supported by this NemoClaw release. Upgrade NemoClaw first." + if [ "$RESOLVED_CHANNEL" = "dev" ]; then + if version_gte "$INSTALLED_VERSION" "$DEV_MIN_VERSION" && printf '%s\n' "$INSTALLED_VERSION_OUTPUT" | grep -qi 'dev'; then + info "openshell already installed: $INSTALLED_VERSION_OUTPUT (dev channel)" + exit 0 + fi + warn "openshell $INSTALLED_VERSION is not the required dev-channel Docker-driver build — upgrading..." + else + if version_gte "$INSTALLED_VERSION" "$MIN_VERSION"; then + if ! version_gte "$MAX_VERSION" "$INSTALLED_VERSION"; then + fail "openshell $INSTALLED_VERSION is above the maximum ($MAX_VERSION) supported by this NemoClaw release. Upgrade NemoClaw first." + fi + if ! linux_driver_bins_present; then + warn "openshell $INSTALLED_VERSION is missing Docker-driver binaries — reinstalling pinned OpenShell ${PIN_VERSION}..." + else + info "openshell already installed: $INSTALLED_VERSION (>= $MIN_VERSION, <= $MAX_VERSION)" + exit 0 + fi + else + warn "openshell $INSTALLED_VERSION is below minimum $MIN_VERSION — upgrading..." fi - info "openshell already installed: $INSTALLED_VERSION (>= $MIN_VERSION, <= $MAX_VERSION)" - exit 0 fi - warn "openshell $INSTALLED_VERSION is below minimum $MIN_VERSION — upgrading..." fi -info "Installing openshell CLI..." +info "Installing OpenShell from release '$RELEASE_TAG'..." case "$OS" in Darwin) @@ -87,26 +126,48 @@ case "$OS" in ;; esac +declare -a ASSETS=("$ASSET") +declare -a CHECKSUM_FILES=("openshell-checksums-sha256.txt") +if [ "$OS" = "Linux" ]; then + case "$ARCH_LABEL" in + x86_64) + ASSETS+=("openshell-gateway-x86_64-unknown-linux-gnu.tar.gz") + ASSETS+=("openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz") + ;; + aarch64) + ASSETS+=("openshell-gateway-aarch64-unknown-linux-gnu.tar.gz") + ASSETS+=("openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz") + ;; + esac + CHECKSUM_FILES+=("openshell-gateway-checksums-sha256.txt") + CHECKSUM_FILES+=("openshell-sandbox-checksums-sha256.txt") +fi + tmpdir="$(mktemp -d)" trap 'rm -rf "$tmpdir"' EXIT -CHECKSUM_FILE="openshell-checksums-sha256.txt" download_with_curl() { - curl -fsSL "https://github.com/NVIDIA/OpenShell/releases/download/v${PIN_VERSION}/$ASSET" \ - -o "$tmpdir/$ASSET" - curl -fsSL "https://github.com/NVIDIA/OpenShell/releases/download/v${PIN_VERSION}/$CHECKSUM_FILE" \ - -o "$tmpdir/$CHECKSUM_FILE" + local name + for name in "${ASSETS[@]}" "${CHECKSUM_FILES[@]}"; do + curl -fsSL "https://github.com/NVIDIA/OpenShell/releases/download/${RELEASE_TAG}/$name" \ + -o "$tmpdir/$name" + done } if command -v gh >/dev/null 2>&1; then - if GH_PROMPT_DISABLED=1 GH_TOKEN="${GH_TOKEN:-${GITHUB_TOKEN:-}}" gh release download "v${PIN_VERSION}" --repo NVIDIA/OpenShell \ - --pattern "$ASSET" --dir "$tmpdir" 2>/dev/null \ - && GH_PROMPT_DISABLED=1 GH_TOKEN="${GH_TOKEN:-${GITHUB_TOKEN:-}}" gh release download "v${PIN_VERSION}" --repo NVIDIA/OpenShell \ - --pattern "$CHECKSUM_FILE" --dir "$tmpdir" 2>/dev/null; then + gh_ok=1 + for name in "${ASSETS[@]}" "${CHECKSUM_FILES[@]}"; do + if ! GH_PROMPT_DISABLED=1 GH_TOKEN="${GH_TOKEN:-${GITHUB_TOKEN:-}}" gh release download "$RELEASE_TAG" --repo NVIDIA/OpenShell \ + --pattern "$name" --dir "$tmpdir" --clobber 2>/dev/null; then + gh_ok=0 + break + fi + done + if [ "$gh_ok" = "1" ]; then : # gh succeeded else warn "gh CLI download failed (auth may not be configured) — falling back to curl" - rm -f "$tmpdir/$ASSET" "$tmpdir/$CHECKSUM_FILE" + rm -f "$tmpdir"/* download_with_curl fi else @@ -114,24 +175,47 @@ else fi info "Verifying SHA-256 checksum..." -(cd "$tmpdir" && grep -F "$ASSET" "$CHECKSUM_FILE" | shasum -a 256 -c -) \ - || fail "SHA-256 checksum verification failed for $ASSET" +for i in "${!ASSETS[@]}"; do + asset_name="${ASSETS[$i]}" + checksum_file="${CHECKSUM_FILES[$i]}" + (cd "$tmpdir" && grep -F "$asset_name" "$checksum_file" | shasum -a 256 -c -) \ + || fail "SHA-256 checksum verification failed for $asset_name" +done -tar xzf "$tmpdir/$ASSET" -C "$tmpdir" +for asset_name in "${ASSETS[@]}"; do + tar xzf "$tmpdir/$asset_name" -C "$tmpdir" +done target_dir="/usr/local/bin" +install_bins() { + local dir="$1" + install -m 755 "$tmpdir/openshell" "$dir/openshell" + if [ -x "$tmpdir/openshell-gateway" ]; then + install -m 755 "$tmpdir/openshell-gateway" "$dir/openshell-gateway" + fi + if [ -x "$tmpdir/openshell-sandbox" ]; then + install -m 755 "$tmpdir/openshell-sandbox" "$dir/openshell-sandbox" + fi +} + if [ -w "$target_dir" ]; then - install -m 755 "$tmpdir/openshell" "$target_dir/openshell" + install_bins "$target_dir" elif [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] || [ ! -t 0 ]; then target_dir="${XDG_BIN_HOME:-$HOME/.local/bin}" mkdir -p "$target_dir" - install -m 755 "$tmpdir/openshell" "$target_dir/openshell" + install_bins "$target_dir" warn "Installed openshell to $target_dir/openshell (user-local path)" warn "For future shells, run: export PATH=\"$target_dir:\$PATH\"" warn "Add that export to your shell profile, or open a new shell before using openshell directly." else sudo install -m 755 "$tmpdir/openshell" "$target_dir/openshell" + if [ -x "$tmpdir/openshell-gateway" ]; then + sudo install -m 755 "$tmpdir/openshell-gateway" "$target_dir/openshell-gateway" + fi + if [ -x "$tmpdir/openshell-sandbox" ]; then + sudo install -m 755 "$tmpdir/openshell-sandbox" "$target_dir/openshell-sandbox" + fi fi info "$("$target_dir/openshell" --version 2>&1 || echo openshell) installed" diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index 02a6163bf15..baad49ee9b2 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -331,6 +331,34 @@ restore_openclaw_config_after_write() { lock_config_after_write "$config_file" "$hash_file" } +ensure_mutable_openclaw_config_hash() { + local config_dir="/sandbox/.openclaw" + local config_file="${config_dir}/openclaw.json" + local hash_file="${config_dir}/.config-hash" + + [ -f "$config_file" ] || return 0 + if [ -L "$config_dir" ] || [ -L "$config_file" ] || [ -L "$hash_file" ]; then + printf '[SECURITY] Refusing mutable config hash refresh — config directory or file path is a symlink\n' >&2 + return 1 + fi + + # Locked/shields-up mode treats .config-hash as a root-owned trust anchor. + # verify_config_integrity_if_locked already fails closed when that anchor is + # missing, so only synthesize/refresh the mutable-default hash. + if [ "$(openclaw_config_dir_owner "$config_dir")" = "root" ]; then + return 0 + fi + + if ! (cd "$config_dir" && sha256sum openclaw.json >"$hash_file"); then + printf '[SECURITY] Failed to refresh mutable OpenClaw config hash\n' >&2 + return 1 + fi + if [ "$(id -u)" -eq 0 ]; then + chown sandbox:sandbox "$hash_file" 2>/dev/null || true + fi + chmod 660 "$hash_file" 2>/dev/null || true +} + # ── Runtime model/provider override ────────────────────────────── # Patches openclaw.json at startup when NEMOCLAW_MODEL_OVERRIDE is set, # allowing model or provider changes without rebuilding the sandbox image. @@ -639,14 +667,99 @@ PYCORS [ "$_write_rc" -eq 0 ] || return "$_write_rc" } +# OpenShell provider snapshots can expose revision-scoped placeholders such as +# openshell:resolve:env:v11_DISCORD_BOT_TOKEN in the child environment. Refresh +# baked canonical placeholders in openclaw.json after the integrity check so +# token egress keeps working across provider attach/refresh generations without +# ever writing a raw credential to disk. +refresh_openclaw_provider_placeholders() { + local config_file="/sandbox/.openclaw/openclaw.json" + local hash_file="/sandbox/.openclaw/.config-hash" + [ -f "$config_file" ] || return 0 + + local keys="TELEGRAM_BOT_TOKEN DISCORD_BOT_TOKEN SLACK_BOT_TOKEN SLACK_APP_TOKEN BRAVE_API_KEY" + local has_scoped_placeholder=0 + local key value + for key in $keys; do + value="${!key:-}" + case "$value" in + openshell:resolve:env:*) has_scoped_placeholder=1 ;; + esac + done + [ "$has_scoped_placeholder" -eq 1 ] || return 0 + + if [ -L "$config_file" ] || [ -L "$hash_file" ]; then + printf '[SECURITY] Refusing provider placeholder refresh — config or hash path is a symlink\n' >&2 + return 1 + fi + + prepare_openclaw_config_for_write "$config_file" "$hash_file" + local _write_rc=0 + + NEMOCLAW_PROVIDER_PLACEHOLDER_KEYS="$keys" \ + python3 - "$config_file" <<'PYPLACEHOLDERS' || _write_rc=$? +import json +import os +import sys + +config_file = sys.argv[1] +prefix = "openshell:resolve:env:" +keys = os.environ.get("NEMOCLAW_PROVIDER_PLACEHOLDER_KEYS", "").split() +replacements = {} + +for key in keys: + value = os.environ.get(key, "") + if value.startswith(prefix): + replacements[f"{prefix}{key}"] = value + +if not replacements: + sys.exit(0) + +with open(config_file, encoding="utf-8") as f: + config = json.load(f) + +def rewrite(value): + if isinstance(value, str): + for old, new in replacements.items(): + value = value.replace(old, new) + return value + if isinstance(value, list): + return [rewrite(item) for item in value] + if isinstance(value, dict): + return {k: rewrite(v) for k, v in value.items()} + return value + +updated = rewrite(config) +if updated == config: + sys.exit(0) + +with open(config_file, "w", encoding="utf-8") as f: + json.dump(updated, f, indent=2) + f.write("\n") + +print("refreshed=" + ",".join(sorted(replacements))) +PYPLACEHOLDERS + + if [ "$_write_rc" -eq 0 ]; then + if (cd /sandbox/.openclaw && sha256sum openclaw.json >"$hash_file"); then + printf '[config] Refreshed provider placeholders from OpenShell runtime env\n' >&2 + else + _write_rc=$? + fi + fi + + restore_openclaw_config_after_write "$config_file" "$hash_file" + [ "$_write_rc" -eq 0 ] || return "$_write_rc" +} + # ── Slack token rewriter (Bolt-shape → canonical placeholder) ──── # Installs a Node preload that translates the Bolt-compatible placeholder # (xoxb|xapp)-OPENSHELL-RESOLVE-ENV-VAR — emitted into openclaw.json by -# generate-openclaw-config.py — into the canonical openshell:resolve:env:VAR -# form on outbound HTTP. OpenShell's L7 proxy then substitutes the real -# token from env on the wire, the same path Discord/Telegram/Brave already -# take. No real Slack token ever touches openclaw.json, /tmp, or any other -# disk surface readable by the sandbox uid. +# generate-openclaw-config.py — into the active openshell:resolve:env:* +# placeholder on outbound HTTP. OpenShell's L7 proxy then substitutes the +# real token from env on the wire, the same path Discord/Telegram/Brave +# already take. No real Slack token ever touches openclaw.json, /tmp, or +# any other disk surface readable by the sandbox uid. # # Ref: https://github.com/NVIDIA/NemoClaw/issues/2085 @@ -663,7 +776,7 @@ install_slack_token_rewriter() { return 0 fi - printf '[channels] Installing Slack token rewriter (Bolt-shape → canonical)\n' >&2 + printf '[channels] Installing Slack token rewriter (Bolt-shape → OpenShell placeholder)\n' >&2 emit_sandbox_sourced_file "$_SLACK_REWRITER_SCRIPT" <"$_SLACK_REWRITER_SOURCE" @@ -1659,6 +1772,8 @@ if [ "$(id -u)" -ne 0 ]; then apply_model_override reconcile_agent_model_with_provider apply_cors_override + refresh_openclaw_provider_placeholders + ensure_mutable_openclaw_config_hash export_gateway_token write_runtime_shell_env ensure_runtime_shell_env_shim @@ -1752,6 +1867,8 @@ normalize_mutable_config_perms apply_model_override reconcile_agent_model_with_provider apply_cors_override +refresh_openclaw_provider_placeholders +ensure_mutable_openclaw_config_hash export_gateway_token write_runtime_shell_env ensure_runtime_shell_env_shim diff --git a/scripts/update-docker-pin.sh b/scripts/update-docker-pin.sh index 079992a8f1e..82113af1686 100755 --- a/scripts/update-docker-pin.sh +++ b/scripts/update-docker-pin.sh @@ -2,9 +2,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Updates the pinned sha256 digest for node:22-slim in the Dockerfile. -# Queries Docker Hub for the current linux/amd64 manifest digest and -# rewrites every FROM line that references node:22-slim. +# Updates the pinned sha256 digest for node:22-trixie-slim in the Dockerfile. +# Queries Docker Hub for the current multi-arch image-index digest and +# rewrites every FROM line that references node:22-trixie-slim. # # Usage: # scripts/update-docker-pin.sh # update Dockerfile in repo root @@ -21,16 +21,16 @@ case "${1:-}" in esac IMAGE="node" -TAG="22-slim" +TAG="22-trixie-slim" DOCKERFILE="${DOCKERFILE:-Dockerfile}" REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" DOCKERFILE_PATH="${REPO_ROOT}/${DOCKERFILE}" # --------------------------------------------------------------------------- -# Resolve the current multi-arch manifest digest for linux/amd64 +# Resolve the current multi-arch image-index digest # --------------------------------------------------------------------------- resolve_latest_digest() { - local token manifest + local token digest # Step 1: get an auth token for the Docker Hub library repo token=$(curl -fsSL --retry 3 --retry-delay 1 --retry-all-errors \ @@ -43,26 +43,17 @@ resolve_latest_digest() { exit 1 fi - # Step 2: fetch the manifest list and pick the linux/amd64 digest - manifest=$(curl -fsSL --retry 3 --retry-delay 1 --retry-all-errors \ + # Step 2: fetch the tag headers and use Docker-Content-Digest for the index. + digest=$(curl -fsSIL --retry 3 --retry-delay 1 --retry-all-errors \ --connect-timeout 10 --max-time 30 \ -H "Authorization: Bearer ${token}" \ -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.index.v1+json" \ - "https://registry-1.docker.io/v2/library/${IMAGE}/manifests/${TAG}") - - local digest - digest=$(echo "$manifest" | python3 -c " -import sys, json -data = json.load(sys.stdin) -for m in data.get('manifests', []): - p = m.get('platform', {}) - if p.get('os') == 'linux' and p.get('architecture') == 'amd64' and not p.get('variant'): - print(m['digest']) - break -") + "https://registry-1.docker.io/v2/library/${IMAGE}/manifests/${TAG}" \ + | tr -d '\r' \ + | awk -F': ' 'tolower($1) == "docker-content-digest" { print $2; exit }') if [[ -z "$digest" ]]; then - echo "ERROR: could not resolve digest for ${IMAGE}:${TAG}" >&2 + echo "ERROR: could not resolve image-index digest for ${IMAGE}:${TAG}" >&2 exit 1 fi diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index dad92144fa9..82fa747ed84 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { CLI_NAME } from "../../cli/branding"; @@ -57,6 +58,48 @@ export type CleanupSandboxServicesDeps = { const NEMOCLAW_GATEWAY_NAME = "nemoclaw"; const DASHBOARD_FORWARD_PORT = String(DASHBOARD_PORT); +function dockerDriverGatewayPidFile(): string { + const configured = process.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR; + const stateDir = configured && configured.trim() + ? path.resolve(configured.trim()) + : path.join(os.homedir(), ".local", "state", "nemoclaw", "openshell-docker-gateway"); + return path.join(stateDir, "openshell-gateway.pid"); +} + +function isDockerDriverGatewayPid(pid: number): boolean { + try { + const cmdline = fs.readFileSync(`/proc/${pid}/cmdline`, "utf-8").replace(/\0/g, " "); + return cmdline.includes("openshell-gateway"); + } catch { + return false; + } +} + +function stopDockerDriverGatewayProcess(): void { + const pidFile = dockerDriverGatewayPidFile(); + let pid: number | null = null; + try { + pid = Number.parseInt(fs.readFileSync(pidFile, "utf-8").trim(), 10); + } catch { + return; + } + if (!Number.isInteger(pid) || pid <= 0) { + fs.rmSync(pidFile, { force: true }); + return; + } + if (!isDockerDriverGatewayPid(pid)) { + fs.rmSync(pidFile, { force: true }); + return; + } + try { + process.kill(pid, "SIGTERM"); + } catch { + fs.rmSync(pidFile, { force: true }); + return; + } + fs.rmSync(pidFile, { force: true }); +} + function cleanupGatewayAfterLastSandbox(): void { const { runOpenshell } = require("../../adapters/openshell/runtime") as { runOpenshell: (args: string[], opts?: Record) => { status: number | null }; @@ -69,7 +112,21 @@ function cleanupGatewayAfterLastSandbox(): void { ignoreError: true, stdio: ["ignore", "ignore", "ignore"], }); - runOpenshell(["gateway", "destroy", "-g", NEMOCLAW_GATEWAY_NAME], { ignoreError: true }); + if (process.platform === "linux") { + stopDockerDriverGatewayProcess(); + const removeResult = runOpenshell(["gateway", "remove", NEMOCLAW_GATEWAY_NAME], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + if (removeResult.status !== 0) { + runOpenshell(["gateway", "destroy", "-g", NEMOCLAW_GATEWAY_NAME], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + } + } else { + runOpenshell(["gateway", "destroy", "-g", NEMOCLAW_GATEWAY_NAME], { ignoreError: true }); + } dockerRemoveVolumesByPrefix(`openshell-cluster-${NEMOCLAW_GATEWAY_NAME}`, { ignoreError: true, }); @@ -373,8 +430,12 @@ export async function destroySandbox( if (shouldCleanupGateway) { cleanupGatewayAfterLastSandbox(); } else { + const gatewayRemovalHint = + process.platform === "linux" + ? `openshell gateway remove ${NEMOCLAW_GATEWAY_NAME}` + : `openshell gateway destroy -g ${NEMOCLAW_GATEWAY_NAME}`; console.log( - ` Shared NemoClaw gateway preserved. Re-run 'openshell gateway destroy --name ${NEMOCLAW_GATEWAY_NAME}' to remove it,`, + ` Shared NemoClaw gateway preserved. Re-run '${gatewayRemovalHint}' to remove it,`, ); console.log( ` or pass '--cleanup-gateway' / set NEMOCLAW_CLEANUP_GATEWAY=1 next time. (#2166)`, diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 05bd897270c..eca054be4d5 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { CLI_NAME } from "../../cli/branding"; import { dockerCapture, dockerInspect } from "../../adapters/docker"; +import { stripAnsi } from "../../adapters/openshell/client"; import { parseLiveSandboxNames } from "../../runtime-recovery"; import { ROOT, run, shellQuote, validateName } from "../../runner"; import { captureOpenshell, getOpenshellBinary } from "../../adapters/openshell/runtime"; @@ -83,7 +84,14 @@ function renderSnapshotTable( // Query the running src pod's image reference via `kubectl` inside the // gateway container. Returns null on any failure. -function resolveSrcPodImage(srcName: string): string | null { +function resolveSrcPodImage(srcName: string, srcEntry?: SandboxEntry | { name: string }): string | null { + const registeredImage = (srcEntry as { imageTag?: string | null } | undefined)?.imageTag; + const registeredDriver = (srcEntry as { openshellDriver?: string | null } | undefined) + ?.openshellDriver; + if (registeredDriver === "docker" && registeredImage) { + return registeredImage; + } + const gatewayContainer = `openshell-cluster-${NEMOCLAW_GATEWAY_NAME}`; try { const output = dockerCapture( @@ -122,7 +130,7 @@ async function autoCreateSandboxFromSource( const basePolicy = path.join(ROOT, "nemoclaw-blueprint", "policies", "openclaw-sandbox.yaml"); const openshellBin = getOpenshellBinary(); - const fromImage = resolveSrcPodImage(srcName); + const fromImage = resolveSrcPodImage(srcName, srcEntry); if (!fromImage) { console.error(` Cannot auto-create '${dstName}': could not resolve '${srcName}' pod image.`); console.error(` Create '${dstName}' manually with '${CLI_NAME} onboard'.`); @@ -174,7 +182,7 @@ async function autoCreateSandboxFromSource( // Set up DNS proxy in the new pod (same step onboard runs after sandbox create). const dnsScript = path.join(ROOT, "scripts", "setup-dns-proxy.sh"); - if (fs.existsSync(dnsScript)) { + if ((srcEntry as { openshellDriver?: string | null }).openshellDriver !== "docker" && fs.existsSync(dnsScript)) { run(["bash", dnsScript, NEMOCLAW_GATEWAY_NAME, dstName], { ignoreError: true }); } @@ -197,7 +205,17 @@ async function autoCreateSandboxFromSource( // Returns true only when the gateway Docker container is confirmed running. // `openshell sandbox list` reads a local registry and exits 0 even when the // gateway is stopped (#2673), so we probe the container directly instead. -function probeGatewayRunning(): boolean { +function probeDockerDriverGatewayRunning(): boolean { + const status = captureOpenshell(["status"], { ignoreError: true, timeout: 10000 }); + const clean = stripAnsi(status.output || ""); + return status.status === 0 && /^\s*Status:\s*Connected\b/im.test(clean); +} + +function probeGatewayRunning(sandboxName?: string): boolean { + const entry = sandboxName ? registry.getSandbox(sandboxName) : null; + if (entry?.openshellDriver === "docker") { + return probeDockerDriverGatewayRunning(); + } const container = `openshell-cluster-${NEMOCLAW_GATEWAY_NAME}`; const result = dockerInspect( ["--type", "container", "--format", "{{.State.Running}}", container], @@ -211,7 +229,7 @@ export async function runSandboxSnapshot(sandboxName: string, subArgs: string[]) switch (subcommand) { case "create": { const opts = parseSnapshotCreateFlags(subArgs.slice(1)); - if (!probeGatewayRunning()) { + if (!probeGatewayRunning(sandboxName)) { console.error(" Failed to query live sandbox state from OpenShell."); process.exit(1); } @@ -280,7 +298,7 @@ export async function runSandboxSnapshot(sandboxName: string, subArgs: string[]) parsed.targetSandbox === sandboxName ? sandboxName : validateName(parsed.targetSandbox, "target sandbox name"); - if (!probeGatewayRunning()) { + if (!probeGatewayRunning(sandboxName)) { console.error(" Failed to query live sandbox state from OpenShell."); process.exit(1); } diff --git a/src/lib/actions/sandbox/status.ts b/src/lib/actions/sandbox/status.ts index ef907317d10..50c40509067 100644 --- a/src/lib/actions/sandbox/status.ts +++ b/src/lib/actions/sandbox/status.ts @@ -114,7 +114,16 @@ export async function showSandboxStatus(sandboxName: string): Promise { if (lookup.state !== "present") { console.log(" Inference: not verified (gateway/sandbox state not verified)"); } - console.log(` GPU: ${sb.gpuEnabled ? "yes" : "no"}`); + const hostGpu = sb.hostGpuDetected ? "yes" : "no"; + const sandboxGpuEnabled = sb.sandboxGpuEnabled ?? (sb.gpuEnabled === true); + const sandboxGpu = sandboxGpuEnabled ? "enabled" : "disabled"; + const sandboxGpuMode = sb.sandboxGpuMode ? ` (${sb.sandboxGpuMode})` : ""; + const sandboxGpuDevice = sb.sandboxGpuDevice ? ` device=${sb.sandboxGpuDevice}` : ""; + const openshellDriver = sb.openshellDriver || "unknown"; + const openshellVersion = sb.openshellVersion || "unknown"; + console.log(` Host GPU: ${hostGpu}`); + console.log(` Sandbox GPU: ${sandboxGpu}${sandboxGpuMode}${sandboxGpuDevice}`); + console.log(` OpenShell: ${openshellVersion} (${openshellDriver})`); console.log(` Policies: ${(sb.policies || []).join(", ") || "none"}`); // Active session indicator diff --git a/src/lib/agent/base-image.test.ts b/src/lib/agent/base-image.test.ts index f239e9e9f83..3edbc236f71 100644 --- a/src/lib/agent/base-image.test.ts +++ b/src/lib/agent/base-image.test.ts @@ -7,6 +7,7 @@ import type { AgentDefinition } from "./defs"; type AgentOnboardModule = typeof import("../../../dist/lib/agent/onboard"); type DockerImageModule = typeof import("../../../dist/lib/adapters/docker/image"); type DockerInspectModule = typeof import("../../../dist/lib/adapters/docker/inspect"); +type SandboxBaseImageModule = typeof import("../../../dist/lib/sandbox-base-image"); /** * Build a minimal Hermes agent manifest for base-image provisioning tests. @@ -53,6 +54,7 @@ function withMockedDocker( ensureAgentBaseImage: AgentOnboardModule["ensureAgentBaseImage"]; dockerBuildMock: ReturnType; dockerImageInspectMock: ReturnType; + resolveSandboxBaseImageMock: ReturnType; root: string; }) => T, ): T { @@ -61,17 +63,28 @@ function withMockedDocker( // eslint-disable-next-line @typescript-eslint/no-require-imports const dockerInspectModule = require("../../../dist/lib/adapters/docker/inspect") as DockerInspectModule; // eslint-disable-next-line @typescript-eslint/no-require-imports + const sandboxBaseImageModule = require("../../../dist/lib/sandbox-base-image") as SandboxBaseImageModule; + // eslint-disable-next-line @typescript-eslint/no-require-imports const runnerModule = require("../../../dist/lib/runner") as { ROOT: string }; const originalDockerBuild = dockerImageModule.dockerBuild; const originalDockerImageInspect = dockerInspectModule.dockerImageInspect; + const originalResolveSandboxBaseImage = sandboxBaseImageModule.resolveSandboxBaseImage; const agentOnboardModulePath = require.resolve("../../../dist/lib/agent/onboard"); delete require.cache[agentOnboardModulePath]; const dockerBuildMock = vi.fn().mockReturnValue({ status: 0 }); const dockerImageInspectMock = vi.fn(); + const resolveSandboxBaseImageMock = vi.fn().mockReturnValue({ + ref: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:compatible", + digest: null, + source: "source-sha", + glibcVersion: process.platform === "linux" ? "2.41" : null, + }); dockerImageModule.dockerBuild = dockerBuildMock as DockerImageModule["dockerBuild"]; dockerInspectModule.dockerImageInspect = dockerImageInspectMock as DockerInspectModule["dockerImageInspect"]; + sandboxBaseImageModule.resolveSandboxBaseImage = + resolveSandboxBaseImageMock as SandboxBaseImageModule["resolveSandboxBaseImage"]; try { // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -80,11 +93,13 @@ function withMockedDocker( ensureAgentBaseImage: agentOnboardModule.ensureAgentBaseImage, dockerBuildMock, dockerImageInspectMock, + resolveSandboxBaseImageMock, root: runnerModule.ROOT, }); } finally { dockerImageModule.dockerBuild = originalDockerBuild; dockerInspectModule.dockerImageInspect = originalDockerImageInspect; + sandboxBaseImageModule.resolveSandboxBaseImage = originalResolveSandboxBaseImage; delete require.cache[agentOnboardModulePath]; } } @@ -94,29 +109,45 @@ describe("agent base image provisioning", () => { vi.restoreAllMocks(); }); - it("reuses an existing agent base image during normal onboarding", () => { - withMockedDocker(({ ensureAgentBaseImage, dockerBuildMock, dockerImageInspectMock }) => { - dockerImageInspectMock.mockReturnValue({ status: 0 }); - - const result = ensureAgentBaseImage(makeAgent()); - - expect(result).toEqual({ - imageTag: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest", - built: false, - }); - expect(dockerImageInspectMock).toHaveBeenCalledWith( - "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest", - { - ignoreError: true, - suppressOutput: true, - }, - ); - expect(dockerBuildMock).not.toHaveBeenCalled(); - }); + it("reuses a compatible resolved agent base image during normal onboarding", () => { + withMockedDocker( + ({ + ensureAgentBaseImage, + dockerBuildMock, + dockerImageInspectMock, + resolveSandboxBaseImageMock, + root, + }) => { + const result = ensureAgentBaseImage(makeAgent()); + + expect(result).toEqual({ + imageTag: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:compatible", + built: false, + }); + expect(resolveSandboxBaseImageMock).toHaveBeenCalledWith( + expect.objectContaining({ + imageName: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base", + dockerfilePath: "/test/root/agents/hermes/Dockerfile.base", + envVar: "NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF", + label: "Hermes Agent sandbox base image", + requireOpenshellSandboxAbi: process.platform === "linux", + rootDir: root, + }), + ); + expect(dockerImageInspectMock).not.toHaveBeenCalled(); + expect(dockerBuildMock).not.toHaveBeenCalled(); + }, + ); }); it("rebuilds an agent base image when rebuild flow forces local Dockerfile.base refresh", () => { - withMockedDocker(({ ensureAgentBaseImage, dockerBuildMock, dockerImageInspectMock, root }) => { + withMockedDocker(({ + ensureAgentBaseImage, + dockerBuildMock, + dockerImageInspectMock, + resolveSandboxBaseImageMock, + root, + }) => { dockerImageInspectMock.mockReturnValue({ status: 0 }); const result = ensureAgentBaseImage(makeAgent(), { forceBaseImageRebuild: true }); @@ -125,6 +156,7 @@ describe("agent base image provisioning", () => { imageTag: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest", built: true, }); + expect(resolveSandboxBaseImageMock).not.toHaveBeenCalled(); expect(dockerImageInspectMock).not.toHaveBeenCalled(); expect(dockerBuildMock).toHaveBeenCalledWith( "/test/root/agents/hermes/Dockerfile.base", @@ -136,19 +168,34 @@ describe("agent base image provisioning", () => { }); it("throws when a forced agent base image rebuild fails", () => { - withMockedDocker(({ ensureAgentBaseImage, dockerBuildMock }) => { + withMockedDocker(({ ensureAgentBaseImage, dockerBuildMock, resolveSandboxBaseImageMock }) => { dockerBuildMock.mockReturnValue({ status: 23 }); expect(() => ensureAgentBaseImage(makeAgent(), { forceBaseImageRebuild: true })).toThrow( "Failed to build Hermes Agent base image (exit 23)", ); + expect(resolveSandboxBaseImageMock).not.toHaveBeenCalled(); }); }); - it("builds an agent base image when no cached image exists", () => { - withMockedDocker(({ ensureAgentBaseImage, dockerBuildMock, dockerImageInspectMock }) => { + it("builds an agent base image when no resolved image or cached image exists on non-Linux hosts", () => { + withMockedDocker(({ + ensureAgentBaseImage, + dockerBuildMock, + dockerImageInspectMock, + resolveSandboxBaseImageMock, + }) => { + resolveSandboxBaseImageMock.mockReturnValue(null); dockerImageInspectMock.mockReturnValue({ status: 1 }); + if (process.platform === "linux") { + expect(() => ensureAgentBaseImage(makeAgent())).toThrow( + "No compatible Hermes Agent sandbox base image found", + ); + expect(dockerBuildMock).not.toHaveBeenCalled(); + return; + } + const result = ensureAgentBaseImage(makeAgent()); expect(result.built).toBe(true); diff --git a/src/lib/agent/onboard.ts b/src/lib/agent/onboard.ts index 2c5bed6ec9c..2d223057a7f 100644 --- a/src/lib/agent/onboard.ts +++ b/src/lib/agent/onboard.ts @@ -15,6 +15,11 @@ import { getProviderSelectionConfig } from "../inference/config"; import type { JsonObject as LooseObject, JsonValue as LooseValue } from "../core/json-types"; import * as onboardSession from "../state/onboard-session"; import { ROOT, redact, run, shellQuote } from "../runner"; +import { + buildLocalBaseTag, + resolveSandboxBaseImage, + SANDBOX_BASE_TAG, +} from "../sandbox-base-image"; import { sleepSeconds } from "../core/wait"; import { type AgentDefinition, loadAgent, resolveAgentName } from "./defs"; @@ -63,19 +68,49 @@ export function ensureAgentBaseImage( return { imageTag: null, built: false }; } - const baseImageTag = `ghcr.io/nvidia/nemoclaw/${agent.name}-sandbox-base:latest`; + const baseImageName = `ghcr.io/nvidia/nemoclaw/${agent.name}-sandbox-base`; + const baseImageTag = `${baseImageName}:${SANDBOX_BASE_TAG}`; const forceBaseImageRebuild = opts.forceBaseImageRebuild === true; - const inspectResult = forceBaseImageRebuild - ? null - : dockerImageInspect(baseImageTag, { - ignoreError: true, - suppressOutput: true, - }); - if (forceBaseImageRebuild || inspectResult?.status !== 0) { - const message = forceBaseImageRebuild - ? ` Rebuilding ${agent.displayName} base image...` - : ` Building ${agent.displayName} base image (first time only)...`; - console.log(message); + if (forceBaseImageRebuild) { + console.log(` Rebuilding ${agent.displayName} base image...`); + const buildResult = dockerBuild(baseDockerfile, baseImageTag, ROOT, { + ignoreError: true, + stdio: ["ignore", "inherit", "inherit"], + }); + if (buildResult.error || buildResult.status !== 0) { + const detail = buildResult.error + ? `: ${buildResult.error.message}` + : ` (exit ${buildResult.status ?? "unknown"})`; + throw new Error(`Failed to build ${agent.displayName} base image${detail}`); + } + console.log(` \u2713 Base image built: ${baseImageTag}`); + return { imageTag: baseImageTag, built: true }; + } + + const resolved = resolveSandboxBaseImage({ + imageName: baseImageName, + dockerfilePath: baseDockerfile, + localTag: buildLocalBaseTag(`nemoclaw-${agent.name}-sandbox-base-local`, ROOT), + envVar: `NEMOCLAW_${agent.name.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_SANDBOX_BASE_IMAGE_REF`, + label: `${agent.displayName} sandbox base image`, + requireOpenshellSandboxAbi: process.platform === "linux", + rootDir: ROOT, + }); + if (resolved && !forceBaseImageRebuild) { + console.log(` Using ${agent.displayName} base image: ${resolved.ref}`); + return { imageTag: resolved.ref, built: false }; + } + if (!resolved && process.platform === "linux" && !forceBaseImageRebuild) { + throw new Error( + `No compatible ${agent.displayName} sandbox base image found for ${baseImageName}`, + ); + } + const inspectResult = dockerImageInspect(baseImageTag, { + ignoreError: true, + suppressOutput: true, + }); + if (inspectResult?.status !== 0) { + console.log(` Building ${agent.displayName} base image (first time only)...`); const buildResult = dockerBuild(baseDockerfile, baseImageTag, ROOT, { ignoreError: true, stdio: ["ignore", "inherit", "inherit"], @@ -111,7 +146,7 @@ export function createAgentSandbox( throw new Error(`${agent.displayName} is missing a sandbox Dockerfile`); } - ensureAgentBaseImage(agent, opts); + const { imageTag: baseImageRef } = ensureAgentBaseImage(agent, opts); const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-build-")); fs.cpSync(ROOT, buildCtx, { @@ -123,6 +158,13 @@ export function createAgentSandbox( }); const stagedDockerfile = path.join(buildCtx, "Dockerfile"); fs.copyFileSync(agentDockerfile, stagedDockerfile); + if (baseImageRef) { + const dockerfile = fs.readFileSync(stagedDockerfile, "utf8"); + fs.writeFileSync( + stagedDockerfile, + dockerfile.replace(/^ARG BASE_IMAGE=.*$/m, `ARG BASE_IMAGE=${baseImageRef}`), + ); + } console.log(` Using ${agent.displayName} Dockerfile: ${agentDockerfile}`); return { buildCtx, stagedDockerfile }; diff --git a/src/lib/commands/onboard.test.ts b/src/lib/commands/onboard.test.ts index bfcbff58cef..ea7608afd6b 100644 --- a/src/lib/commands/onboard.test.ts +++ b/src/lib/commands/onboard.test.ts @@ -46,6 +46,27 @@ describe("onboard oclif command", () => { expect(runOnboardAction).toHaveBeenCalledWith(["--non-interactive", "--yes"]); }); + it("forwards sandbox GPU flags to legacy onboard parsing", async () => { + await OnboardCliCommand.run( + [ + "--non-interactive", + "--yes", + "--sandbox-gpu", + "--sandbox-gpu-device", + "nvidia.com/gpu=0", + ], + rootDir, + ); + + expect(runOnboardAction).toHaveBeenCalledWith([ + "--non-interactive", + "--sandbox-gpu", + "--sandbox-gpu-device", + "nvidia.com/gpu=0", + "--yes", + ]); + }); + it("forwards --no-gpu to the legacy onboard action", async () => { await OnboardCliCommand.run(["--non-interactive", "--no-gpu"], rootDir); diff --git a/src/lib/commands/onboard/common.ts b/src/lib/commands/onboard/common.ts index 9fa25f0dd55..9b78fb21083 100644 --- a/src/lib/commands/onboard/common.ts +++ b/src/lib/commands/onboard/common.ts @@ -8,7 +8,7 @@ import { NOTICE_ACCEPT_FLAG } from "../../onboard/usage-notice"; const acceptFlagName = NOTICE_ACCEPT_FLAG.replace(/^--/, ""); export const onboardUsage = [ - `onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--agent ] [--control-ui-port ] [--yes | -y] [${NOTICE_ACCEPT_FLAG}]`, + `onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--sandbox-gpu | --no-sandbox-gpu] [--sandbox-gpu-device ] [--agent ] [--control-ui-port ] [--yes | -y] [${NOTICE_ACCEPT_FLAG}]`, ]; export const onboardExamples = [ @@ -17,6 +17,7 @@ export const onboardExamples = [ "<%= config.bin %> onboard --resume", "<%= config.bin %> onboard --fresh", "<%= config.bin %> onboard --from ./Dockerfile --name alpha", + "<%= config.bin %> onboard --sandbox-gpu --sandbox-gpu-device nvidia.com/gpu=0", `<%= config.bin %> onboard --non-interactive --yes --name alpha ${NOTICE_ACCEPT_FLAG}`, ]; @@ -29,6 +30,9 @@ export type OnboardFlags = { "no-gpu"?: boolean; from?: string; name?: string; + "sandbox-gpu"?: boolean; + "no-sandbox-gpu"?: boolean; + "sandbox-gpu-device"?: string; agent?: string; "control-ui-port"?: number; yes?: boolean; @@ -58,6 +62,15 @@ export function buildOnboardFlags(): Record { }), from: Flags.string({ description: "Path to a Dockerfile to use as the sandbox image source" }), name: Flags.string({ description: "Sandbox name" }), + "sandbox-gpu": Flags.boolean({ + description: "Enable direct NVIDIA GPU access inside the sandbox", + }), + "no-sandbox-gpu": Flags.boolean({ + description: "Force CPU sandbox behavior", + }), + "sandbox-gpu-device": Flags.string({ + description: "OpenShell GPU device selector to pass to sandbox create", + }), agent: Flags.string({ description: "Agent runtime to onboard" }), "control-ui-port": Flags.integer({ description: "Host port for the local control UI", @@ -82,6 +95,11 @@ export function toLegacyOnboardArgs(flags: OnboardFlags): string[] { if (flags["no-gpu"]) args.push("--no-gpu"); if (flags.from !== undefined) args.push("--from", flags.from); if (flags.name !== undefined) args.push("--name", flags.name); + if (flags["sandbox-gpu"]) args.push("--sandbox-gpu"); + if (flags["no-sandbox-gpu"]) args.push("--no-sandbox-gpu"); + if (flags["sandbox-gpu-device"] !== undefined) { + args.push("--sandbox-gpu-device", flags["sandbox-gpu-device"]); + } if (flags.agent !== undefined) args.push("--agent", flags.agent); if (flags["control-ui-port"] !== undefined) { args.push("--control-ui-port", String(flags["control-ui-port"])); diff --git a/src/lib/inventory-commands.test.ts b/src/lib/inventory-commands.test.ts index 71683eb86ef..7f070c455bd 100644 --- a/src/lib/inventory-commands.test.ts +++ b/src/lib/inventory-commands.test.ts @@ -75,6 +75,12 @@ describe("inventory commands", () => { model: "configured-alpha", provider: "configured-provider", gpuEnabled: true, + hostGpuDetected: false, + sandboxGpuEnabled: true, + sandboxGpuMode: null, + sandboxGpuDevice: null, + openshellDriver: null, + openshellVersion: null, policies: ["pypi"], agent: "openclaw", isDefault: true, @@ -149,7 +155,7 @@ describe("inventory commands", () => { expect(lines).toContain(" Recovered 1 sandbox entry from the live OpenShell gateway."); expect(lines).toContain(" alpha *"); expect(lines).toContain( - " agent: openclaw model: nvidia/nemotron-3-super-120b-a12b provider: nvidia-prod GPU policies: pypi", + " agent: openclaw model: nvidia/nemotron-3-super-120b-a12b provider: nvidia-prod sandbox GPU policies: pypi", ); }); @@ -175,7 +181,7 @@ describe("inventory commands", () => { }); expect(lines).toContain( - " agent: hermes model: nvidia/nemotron-3-super-120b-a12b provider: nvidia-prod CPU policies: none", + " agent: hermes model: nvidia/nemotron-3-super-120b-a12b provider: nvidia-prod CPU sandbox policies: none", ); }); @@ -208,11 +214,11 @@ describe("inventory commands", () => { // Default sandbox reflects live gateway state, with an onboarded drift note. expect(lines).toContain( - " agent: openclaw model: live-model provider: live-provider GPU policies: none", + " agent: openclaw model: live-model provider: live-provider sandbox GPU policies: none", ); // Stale stored row for the default sandbox must not leak through. expect(lines).not.toContain( - " agent: openclaw model: configured-alpha provider: configured-provider GPU policies: none", + " agent: openclaw model: configured-alpha provider: configured-provider sandbox GPU policies: none", ); expect(lines).toContain( " (onboarded: model=configured-alpha, provider=configured-provider)", @@ -220,7 +226,7 @@ describe("inventory commands", () => { // Non-default sandbox keeps its stored config — the gateway only applies // to whichever sandbox is currently connected. expect(lines).toContain( - " agent: openclaw model: configured-beta provider: beta-provider CPU policies: none", + " agent: openclaw model: configured-beta provider: beta-provider CPU sandbox policies: none", ); }); @@ -245,7 +251,7 @@ describe("inventory commands", () => { }); expect(lines).toContain( - " agent: openclaw model: configured-alpha provider: configured-provider GPU policies: none", + " agent: openclaw model: configured-alpha provider: configured-provider sandbox GPU policies: none", ); expect(lines.some((l) => l.includes("onboarded"))).toBe(false); }); @@ -271,7 +277,7 @@ describe("inventory commands", () => { }); expect(lines).toContain( - " agent: openclaw model: configured-alpha provider: configured-provider GPU policies: none", + " agent: openclaw model: configured-alpha provider: configured-provider sandbox GPU policies: none", ); expect(lines.some((l) => l.includes("onboarded"))).toBe(false); }); @@ -298,7 +304,7 @@ describe("inventory commands", () => { }); expect(lines).toContain( - " agent: openclaw model: live-model provider: configured-provider GPU policies: none", + " agent: openclaw model: live-model provider: configured-provider sandbox GPU policies: none", ); expect(lines).toContain(" (onboarded: model=configured-alpha)"); }); @@ -325,7 +331,7 @@ describe("inventory commands", () => { }); expect(lines).toContain( - " agent: openclaw model: configured-alpha provider: live-provider GPU policies: none", + " agent: openclaw model: configured-alpha provider: live-provider sandbox GPU policies: none", ); expect(lines).toContain(" (onboarded: provider=configured-provider)"); }); diff --git a/src/lib/inventory-commands.ts b/src/lib/inventory-commands.ts index 77eac8f7b31..60367bded7f 100644 --- a/src/lib/inventory-commands.ts +++ b/src/lib/inventory-commands.ts @@ -10,6 +10,12 @@ export interface SandboxEntry { model?: string | null; provider?: string | null; gpuEnabled?: boolean; + hostGpuDetected?: boolean; + sandboxGpuEnabled?: boolean; + sandboxGpuMode?: string | null; + sandboxGpuDevice?: string | null; + openshellDriver?: string | null; + openshellVersion?: string | null; policies?: string[] | null; providerCredentialHashes?: Record | null; messagingChannels?: string[] | null; @@ -51,6 +57,12 @@ export interface SandboxInventoryRow { model: string | null; provider: string | null; gpuEnabled: boolean; + hostGpuDetected: boolean; + sandboxGpuEnabled: boolean; + sandboxGpuMode: string | null; + sandboxGpuDevice: string | null; + openshellDriver: string | null; + openshellVersion: string | null; policies: string[]; agent: string | null; dashboardPort?: number | null; @@ -95,6 +107,12 @@ export interface StatusSandboxRow { model: string | null; provider: string | null; gpuEnabled: boolean; + hostGpuDetected: boolean; + sandboxGpuEnabled: boolean; + sandboxGpuMode: string | null; + sandboxGpuDevice: string | null; + openshellDriver: string | null; + openshellVersion: string | null; policies: string[]; agent: string | null; dashboardPort?: number | null; @@ -129,12 +147,22 @@ function buildSandboxInventoryRow( getActiveSessionCount?: (sandboxName: string) => number | null, ): SandboxInventoryRow { const activeSessionCount = getActiveSessionCount ? getActiveSessionCount(sandbox.name) : null; + const sandboxGpuEnabled = + typeof sandbox.sandboxGpuEnabled === "boolean" + ? sandbox.sandboxGpuEnabled + : sandbox.gpuEnabled === true; return { name: sandbox.name, model: sandbox.model || null, provider: sandbox.provider || null, gpuEnabled: sandbox.gpuEnabled === true, + hostGpuDetected: sandbox.hostGpuDetected === true, + sandboxGpuEnabled, + sandboxGpuMode: safeStatusString(sandbox.sandboxGpuMode || null), + sandboxGpuDevice: safeStatusString(sandbox.sandboxGpuDevice || null), + openshellDriver: safeStatusString(sandbox.openshellDriver || null), + openshellVersion: safeStatusString(sandbox.openshellVersion || null), policies: Array.isArray(sandbox.policies) ? sandbox.policies : [], agent: sandbox.agent || null, ...(sandbox.dashboardPort != null ? { dashboardPort: sandbox.dashboardPort } : {}), @@ -225,7 +253,7 @@ export function renderSandboxInventoryText( const modelDrifted = !!(useLive && liveInference.model && liveInference.model !== sandbox.model); const providerDrifted = !!(useLive && liveInference.provider && liveInference.provider !== sandbox.provider); - const gpu = sandbox.gpuEnabled ? "GPU" : "CPU"; + const gpu = sandbox.sandboxGpuEnabled ? "sandbox GPU" : "CPU sandbox"; const presets = sandbox.policies.length > 0 ? sandbox.policies.join(", ") : "none"; const connected = sandbox.connected ? " ●" : ""; const agent = sandbox.agent || "openclaw"; @@ -267,11 +295,21 @@ function buildStatusSandboxRow( typeof sandbox.dashboardPort === "number" && Number.isFinite(sandbox.dashboardPort) ? sandbox.dashboardPort : null; + const sandboxGpuEnabled = + typeof sandbox.sandboxGpuEnabled === "boolean" + ? sandbox.sandboxGpuEnabled + : sandbox.gpuEnabled === true; return { name: safeStatusString(sandbox.name) || sandbox.name, model: safeStatusString(liveModel || sandbox.model || null), provider: safeStatusString(liveProvider || sandbox.provider || null), gpuEnabled: sandbox.gpuEnabled === true, + hostGpuDetected: sandbox.hostGpuDetected === true, + sandboxGpuEnabled, + sandboxGpuMode: safeStatusString(sandbox.sandboxGpuMode || null), + sandboxGpuDevice: safeStatusString(sandbox.sandboxGpuDevice || null), + openshellDriver: safeStatusString(sandbox.openshellDriver || null), + openshellVersion: safeStatusString(sandbox.openshellVersion || null), policies: Array.isArray(sandbox.policies) ? sandbox.policies .filter((policy): policy is string => typeof policy === "string") diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index c61a7e1f3fb..b55c0672628 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -60,16 +60,22 @@ const { dockerContainerInspectFormat, dockerExecArgv, dockerImageInspect, - dockerImageInspectFormat, dockerInfo, dockerInfoFormat, dockerInspect, - dockerPull, dockerRemoveVolumesByPrefix, dockerRm, dockerRmi, dockerStop, } = docker; +const sandboxBaseImage: typeof import("./sandbox-base-image") = require("./sandbox-base-image"); +const { + OPENCLAW_SANDBOX_BASE_IMAGE: SANDBOX_BASE_IMAGE, + SANDBOX_BASE_TAG, + defaultOpenclawBaseDockerfile, + buildLocalBaseTag, + resolveSandboxBaseImage, +} = sandboxBaseImage; const errnoUtils: typeof import("./core/errno") = require("./core/errno"); const { isErrnoException } = errnoUtils; @@ -311,6 +317,7 @@ const validationRecovery: typeof import("./validation-recovery") = require("./va const webSearch: typeof import("./inference/web-search") = require("./inference/web-search"); import type { AgentDefinition } from "./agent/defs"; +import type { GatewayReuseState } from "./state/gateway"; import type { CurlProbeResult } from "./http-probe"; import type { GatewayInference, ProviderSelectionConfig } from "./inference/config"; import type { GpuInfo, ValidationResult } from "./inference/local"; @@ -433,6 +440,8 @@ type OnboardOptions = { fresh?: boolean; fromDockerfile?: string | null; sandboxName?: string | null; + sandboxGpu?: "enable" | "disable" | null; + sandboxGpuDevice?: string | null; acceptThirdPartySoftware?: boolean; agent?: string | null; controlUiPort?: number | null; @@ -837,6 +846,8 @@ function getBlueprintMaxOpenshellVersion(rootDir = ROOT): string | null { return getBlueprintVersionField("max_openshell_version", rootDir); } +type OpenshellChannel = "stable" | "dev" | "auto"; + /** * Load a named inference profile and router config from blueprint.yaml. * Returns null if the blueprint or profile is missing. @@ -1342,49 +1353,272 @@ async function reconcileModelRouter(): Promise { }); } -// ── Base image digest resolution ──────────────────────────────── -// Pulls the sandbox-base image from GHCR and inspects it to get the -// actual repo digest. This avoids the registry mismatch that broke -// e2e tests in #1937 — the digest always comes from the same registry -// we're pinning to. See #1904. +function getOpenshellChannel(env: NodeJS.ProcessEnv = process.env): OpenshellChannel { + const raw = String(env.NEMOCLAW_OPENSHELL_CHANNEL || "auto") + .trim() + .toLowerCase(); + if (raw === "stable" || raw === "dev" || raw === "auto") return raw; + return "auto"; +} + +function shouldUseOpenshellDevChannel( + _platform: NodeJS.Platform = process.platform, + env: NodeJS.ProcessEnv = process.env, +): boolean { + const channel = getOpenshellChannel(env); + return channel === "dev"; +} -const SANDBOX_BASE_IMAGE = "ghcr.io/nvidia/nemoclaw/sandbox-base"; -const SANDBOX_BASE_TAG = "latest"; +function isOpenshellDevVersion(versionOutput: string | null | undefined): boolean { + return /\bdev[0-9.]*/i.test(String(versionOutput || "")); +} -/** - * Pull sandbox-base:latest from GHCR and resolve its repo digest. - * Returns { digest, ref } on success, or null when the pull or - * inspect fails (offline, GHCR outage, local-only build). - */ -function pullAndResolveBaseImageDigest(): { digest: string; ref: string } | null { - const imageWithTag = `${SANDBOX_BASE_IMAGE}:${SANDBOX_BASE_TAG}`; - const pullResult = dockerPull(imageWithTag, { ignoreError: true, suppressOutput: true }); - if (pullResult.status !== 0) { - // Pull failed — caller should fall back to unpin :latest - return null; +function shouldAllowOpenshellAboveBlueprintMax( + versionOutput: string | null | undefined, + platform: NodeJS.Platform = process.platform, + env: NodeJS.ProcessEnv = process.env, +): boolean { + return shouldUseOpenshellDevChannel(platform, env) && isOpenshellDevVersion(versionOutput); +} + +type SandboxGpuMode = "auto" | "1" | "0"; +type SandboxGpuFlag = "enable" | "disable" | null; + +type SandboxGpuConfig = { + mode: SandboxGpuMode; + hostGpuDetected: boolean; + sandboxGpuEnabled: boolean; + sandboxGpuDevice: string | null; + errors: string[]; +}; + +type ResumeSandboxGpuOverrides = { + flag: SandboxGpuFlag; + device: string | null; +}; + +function isNvidiaGpuDetected(gpu: ReturnType): boolean { + return Boolean(gpu && gpu.type === "nvidia"); +} + +function normalizeSandboxGpuMode(value: string | null | undefined): SandboxGpuMode | null { + const raw = String(value || "") + .trim() + .toLowerCase(); + if (!raw) return null; + if (raw === "auto") return "auto"; + if (raw === "1" || raw === "true" || raw === "yes" || raw === "on") return "1"; + if (raw === "0" || raw === "false" || raw === "no" || raw === "off") return "0"; + return null; +} + +function resolveSandboxGpuConfig( + gpu: ReturnType, + options: { + flag?: SandboxGpuFlag; + device?: string | null; + env?: NodeJS.ProcessEnv; + } = {}, +): SandboxGpuConfig { + const env = options.env ?? process.env; + const errors: string[] = []; + const envModeRaw = env.NEMOCLAW_SANDBOX_GPU; + const envMode = normalizeSandboxGpuMode(envModeRaw); + if (envModeRaw !== undefined && envMode === null) { + errors.push("NEMOCLAW_SANDBOX_GPU must be one of: auto, 1, 0."); } - const inspectOutput = dockerImageInspectFormat("{{json .RepoDigests}}", imageWithTag, { - ignoreError: true, - }); - if (!inspectOutput) return null; + let mode: SandboxGpuMode = envMode ?? "auto"; + if (options.flag === "enable") mode = "1"; + if (options.flag === "disable") mode = "0"; + + const device = (options.device ?? env.NEMOCLAW_SANDBOX_GPU_DEVICE ?? "").trim() || null; + if (device && mode === "0") { + errors.push("NEMOCLAW_SANDBOX_GPU_DEVICE cannot be used when sandbox GPU mode is 0."); + } + if (device && options.flag !== "disable" && envMode !== "0") { + mode = "1"; + } + + const hostGpuDetected = isNvidiaGpuDetected(gpu); + if (mode === "1" && !hostGpuDetected) { + errors.push("Sandbox GPU was requested, but no NVIDIA GPU was detected on the host."); + } + + return { + mode, + hostGpuDetected, + sandboxGpuEnabled: mode === "1" || (mode === "auto" && hostGpuDetected), + sandboxGpuDevice: device, + errors, + }; +} + +function resolveSandboxGpuFlagFromOptions( + opts: Pick, +): SandboxGpuFlag { + const requestedGpuPassthrough = opts.gpu === true; + const optedOutGpuPassthrough = opts.noGpu === true; + const sandboxGpuFlag = opts.sandboxGpu ?? null; + if (requestedGpuPassthrough && optedOutGpuPassthrough) { + console.error(" --gpu and --no-gpu cannot both be set."); + process.exit(1); + } + if ( + (requestedGpuPassthrough && sandboxGpuFlag === "disable") || + (optedOutGpuPassthrough && sandboxGpuFlag === "enable") + ) { + console.error(" --gpu/--no-gpu conflict with the sandbox GPU flags."); + process.exit(1); + } + if (sandboxGpuFlag) return sandboxGpuFlag; + if (requestedGpuPassthrough) return "enable"; + if (optedOutGpuPassthrough) return "disable"; + return null; +} + +function getResumeSandboxGpuOverrides( + entry: Pick | null | undefined, + sessionGpuPassthrough: boolean | undefined, +): ResumeSandboxGpuOverrides { + const recordedMode = normalizeSandboxGpuMode(entry?.sandboxGpuMode); + if (recordedMode === "1") { + return { flag: "enable", device: entry?.sandboxGpuDevice || null }; + } + if (recordedMode === "0") { + return { flag: "disable", device: null }; + } + if (recordedMode === "auto") { + return { flag: null, device: null }; + } + if (sessionGpuPassthrough === true) { + return { flag: "enable", device: entry?.sandboxGpuDevice || null }; + } + return { flag: null, device: null }; +} + +function buildSandboxGpuCreateArgs(config: SandboxGpuConfig): string[] { + if (!config.sandboxGpuEnabled) return []; + const args = ["--gpu"]; + if (config.sandboxGpuDevice) { + args.push("--gpu-device", config.sandboxGpuDevice); + } + return args; +} - // RepoDigests is a JSON array like ["ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:abc..."]. - // Filter to the entry matching our registry — index ordering is not guaranteed. - let repoDigests; +function parseDockerCdiSpecDirs(value: string | null | undefined): string[] { + const raw = String(value || "").trim(); + if (!raw || raw === "") return []; try { - repoDigests = JSON.parse(inspectOutput || "[]"); + const parsed = JSON.parse(raw); + return Array.isArray(parsed) + ? parsed.map((entry) => String(entry || "").trim()).filter(Boolean) + : []; } catch { - return null; + return raw + .split(/[\s,]+/) + .map((entry) => entry.trim()) + .filter(Boolean); } - const repoDigest = Array.isArray(repoDigests) - ? repoDigests.find((entry) => entry.startsWith(`${SANDBOX_BASE_IMAGE}@sha256:`)) - : null; - if (!repoDigest) return null; +} + +function getDockerCdiSpecDirs(): string[] { + return parseDockerCdiSpecDirs( + dockerInfoFormat("{{json .CDISpecDirs}}", { ignoreError: true }), + ); +} + +function isLikelyNvidiaCdiSpecFile(filePath: string): boolean { + if (!/\.(json|ya?ml)$/i.test(filePath)) return false; + let stat: import("fs").Stats; + try { + stat = fs.statSync(filePath); + } catch { + return false; + } + if (!stat.isFile()) return false; + + let content = ""; + try { + content = fs.readFileSync(filePath, "utf-8"); + } catch { + return false; + } + return /nvidia\.com\/gpu|nvidia-container|libcuda|cuda/i.test(content); +} + +function findReadableNvidiaCdiSpecFiles(dirs: string[]): string[] { + const specs: string[] = []; + for (const dir of dirs) { + let entries: string[]; + try { + entries = fs.readdirSync(dir); + } catch { + continue; + } + for (const entry of entries) { + const candidate = path.join(dir, entry); + if (isLikelyNvidiaCdiSpecFile(candidate)) specs.push(candidate); + } + } + return specs.sort(); +} + +function sandboxGpuRemediationLines(): string[] { + return [ + "Install/configure NVIDIA Container Toolkit CDI, then restart Docker:", + " sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml", + " sudo systemctl restart docker", + "Or force CPU sandbox behavior with NEMOCLAW_SANDBOX_GPU=0.", + ]; +} + +function validateSandboxGpuPreflight(config: SandboxGpuConfig): void { + if (config.errors.length > 0) { + console.error(""); + for (const error of config.errors) console.error(` ✗ ${error}`); + process.exit(1); + } + if (!config.sandboxGpuEnabled) return; + if (!isLinuxDockerDriverGatewayEnabled()) return; + + const cdiSpecDirs = getDockerCdiSpecDirs(); + const cdiSpecFiles = findReadableNvidiaCdiSpecFiles(cdiSpecDirs); + if (cdiSpecFiles.length === 0) { + console.error(""); + console.error(" ✗ Docker CDI GPU support was not detected."); + for (const line of sandboxGpuRemediationLines()) { + console.error(` ${line}`); + } + process.exit(1); + } + console.log(` ✓ Docker CDI GPU support detected (${cdiSpecFiles.join(", ")})`); +} + +// ── Base image resolution ─────────────────────────────────────── +// Pulls candidate sandbox-base images from GHCR and inspects them to get the +// actual repo digest when available. This avoids the registry mismatch that +// broke e2e tests in #1937 while still allowing PR branches to use a source-SHA +// base image or local build before latest has been rebuilt. See #1904. - const digest = repoDigest.slice(repoDigest.indexOf("@") + 1); - const ref = `${SANDBOX_BASE_IMAGE}@${digest}`; - return { digest, ref }; +/** + * Resolve a compatible sandbox-base image and pin it to a repo digest when + * possible. PR-branch validation first tries a source-SHA tag, then latest, + * and finally a local Dockerfile.base build when the OpenShell Docker driver + * requires a newer glibc than the published image provides. + */ +function pullAndResolveBaseImageDigest( + options: { requireOpenshellSandboxAbi?: boolean } = {}, +): { digest: string | null; ref: string; source?: string; glibcVersion?: string | null } | null { + return resolveSandboxBaseImage({ + imageName: SANDBOX_BASE_IMAGE, + dockerfilePath: defaultOpenclawBaseDockerfile(ROOT), + localTag: buildLocalBaseTag("nemoclaw-sandbox-base-local", ROOT), + envVar: "NEMOCLAW_SANDBOX_BASE_IMAGE_REF", + label: "OpenClaw sandbox base image", + requireOpenshellSandboxAbi: options.requireOpenshellSandboxAbi === true, + rootDir: ROOT, + }); } function getStableGatewayImageRef(versionOutput: string | null = null): string | null { @@ -1901,6 +2135,114 @@ const CREATE_TIME_POLICY_PRESETS_BY_CHANNEL: Record = { slack: ["slack"], }; +const PROC_COMM_READ_WRITE_PATH = "/proc/self/task/*/comm"; + +function buildDirectGpuPolicyYaml(basePolicy: string): string { + const YAML = require("yaml"); + const parsed = YAML.parse(basePolicy); + if (!parsed || typeof parsed !== "object") { + throw new Error("Cannot prepare direct GPU sandbox policy; base policy is not a YAML mapping."); + } + parsed.filesystem_policy = parsed.filesystem_policy || {}; + const fsPolicy = parsed.filesystem_policy; + fsPolicy.read_only = Array.isArray(fsPolicy.read_only) + ? fsPolicy.read_only.map((entry: unknown) => String(entry)) + : []; + if (!fsPolicy.read_only.includes("/proc")) { + fsPolicy.read_only.push("/proc"); + } + const readWrite = Array.isArray(fsPolicy.read_write) + ? fsPolicy.read_write.map((entry: unknown) => String(entry)) + : []; + fsPolicy.read_write = readWrite.filter((entry: string) => entry !== "/proc"); + if (!fsPolicy.read_write.includes(PROC_COMM_READ_WRITE_PATH)) { + fsPolicy.read_write.push(PROC_COMM_READ_WRITE_PATH); + } + return YAML.stringify(parsed); +} + +const PROC_COMM_WRITE_PROBE = ` +set -eu +tid="$(ls /proc/self/task | head -n 1)" +old="$(cat "/proc/self/task/\${tid}/comm" 2>/dev/null || true)" +printf nemoclaw-gpu >"/proc/self/task/\${tid}/comm" +if [ -n "$old" ]; then printf "%s" "$old" >"/proc/self/task/\${tid}/comm" || true; fi +`; + +const CUDA_INIT_PROBE = ` +python3 - <<'PY' +import ctypes +lib = ctypes.CDLL("libcuda.so.1") +rc = lib.cuInit(0) +print(f"cuInit(0)={rc}") +raise SystemExit(0 if rc == 0 else 1) +PY +`; + +function buildDirectSandboxGpuProofCommands( + sandboxName: string, +): { label: string; args: string[] }[] { + return [ + { + label: "nvidia-smi", + args: ["sandbox", "exec", "-n", sandboxName, "--", "nvidia-smi"], + }, + { + label: "/proc/self/task//comm write", + args: ["sandbox", "exec", "-n", sandboxName, "--", "sh", "-lc", PROC_COMM_WRITE_PROBE], + }, + { + label: "cuInit(0) via libcuda.so.1", + args: ["sandbox", "exec", "-n", sandboxName, "--", "sh", "-lc", CUDA_INIT_PROBE], + }, + ]; +} + +function verifyDirectSandboxGpu(sandboxName: string): void { + console.log(" Verifying direct sandbox GPU access..."); + for (const proof of buildDirectSandboxGpuProofCommands(sandboxName)) { + const result = runOpenshell(proof.args, { + ignoreError: true, + suppressOutput: true, + timeout: 30_000, + }); + if (result.status === 0) { + console.log(` ✓ GPU proof passed: ${proof.label}`); + continue; + } + const diagnostic = compactText(redact(`${result.stderr || ""} ${result.stdout || ""}`)); + console.error(` ✗ GPU proof failed: ${proof.label}`); + if (diagnostic) console.error(` ${diagnostic.slice(0, 300)}`); + for (const line of sandboxGpuRemediationLines()) { + console.error(` ${line}`); + } + const statusText = String(result.status || 1); + const diagnosticSuffix = diagnostic ? `: ${diagnostic.slice(0, 300)}` : ""; + throw new Error(`GPU proof failed: ${proof.label} (status ${statusText})${diagnosticSuffix}`); + } +} + +function prepareDirectGpuSandboxPolicy(basePolicyPath: string): InitialSandboxPolicy { + const basePolicy = fs.readFileSync(basePolicyPath, "utf-8"); + const policyPath = secureTempFile("nemoclaw-gpu-policy", ".yaml"); + fs.writeFileSync(policyPath, buildDirectGpuPolicyYaml(basePolicy), { + encoding: "utf-8", + mode: 0o600, + }); + return { + policyPath, + appliedPresets: [], + cleanup: () => { + try { + cleanupTempDir(policyPath, "nemoclaw-gpu-policy"); + return true; + } catch { + return false; + } + }, + }; +} + function getNetworkPolicyNames(policyContent: string): Set | null { try { // Lazy require: yaml is already a dependency via the policy helpers. @@ -1923,7 +2265,11 @@ function getNetworkPolicyNames(policyContent: string): Set | null { function prepareInitialSandboxCreatePolicy( basePolicyPath: string, activeMessagingChannels: string[], + options: { directGpu?: boolean } = {}, ): InitialSandboxPolicy { + const directGpuPolicy = options.directGpu ? prepareDirectGpuSandboxPolicy(basePolicyPath) : null; + const effectiveBasePolicyPath = directGpuPolicy?.policyPath || basePolicyPath; + const cleanupFns = directGpuPolicy?.cleanup ? [directGpuPolicy.cleanup] : []; const requestedCreateTimePresets = [ ...new Set( activeMessagingChannels.flatMap( @@ -1931,15 +2277,25 @@ function prepareInitialSandboxCreatePolicy( ), ), ]; + const combinedCleanup = + cleanupFns.length > 0 ? () => cleanupFns.map((cleanup) => cleanup()).every(Boolean) : undefined; if (requestedCreateTimePresets.length === 0) { - return { policyPath: basePolicyPath, appliedPresets: [] }; + return { + policyPath: effectiveBasePolicyPath, + appliedPresets: [], + cleanup: combinedCleanup, + }; } - const basePolicy = fs.readFileSync(basePolicyPath, "utf-8"); + const basePolicy = fs.readFileSync(effectiveBasePolicyPath, "utf-8"); const basePolicyNames = getNetworkPolicyNames(basePolicy); if (basePolicyNames === null) { - return { policyPath: basePolicyPath, appliedPresets: [] }; + return { + policyPath: effectiveBasePolicyPath, + appliedPresets: [], + cleanup: combinedCleanup, + }; } const existingCreateTimePresets = requestedCreateTimePresets.filter((preset) => basePolicyNames.has(preset), @@ -1948,7 +2304,11 @@ function prepareInitialSandboxCreatePolicy( (preset) => !basePolicyNames.has(preset), ); if (createTimePresets.length === 0) { - return { policyPath: basePolicyPath, appliedPresets: existingCreateTimePresets }; + return { + policyPath: effectiveBasePolicyPath, + appliedPresets: existingCreateTimePresets, + cleanup: combinedCleanup, + }; } const mergedPolicy = policies.mergePresetNamesIntoPolicy(basePolicy, createTimePresets); @@ -1960,18 +2320,19 @@ function prepareInitialSandboxCreatePolicy( const policyPath = secureTempFile("nemoclaw-initial-policy", ".yaml"); fs.writeFileSync(policyPath, mergedPolicy.policy, { encoding: "utf-8", mode: 0o600 }); + cleanupFns.push(() => { + try { + cleanupTempDir(policyPath, "nemoclaw-initial-policy"); + return true; + } catch { + return false; + } + }); return { policyPath, appliedPresets: [...existingCreateTimePresets, ...mergedPolicy.appliedPresets], - cleanup: () => { - try { - cleanupTempDir(policyPath, "nemoclaw-initial-policy"); - return true; - } catch { - return false; - } - }, + cleanup: () => cleanupFns.map((cleanup) => cleanup()).every(Boolean), }; } @@ -3368,17 +3729,131 @@ function sleep(seconds: number): void { sleepSeconds(seconds); } -function destroyGateway() { - const destroyResult = runOpenshell(["gateway", "destroy", "-g", GATEWAY_NAME], { +function runQuietOpenshell(args: string[]) { + return runOpenshell(args, { ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + suppressOutput: true, + }); +} + +function removeDockerDriverGatewayRegistration(): boolean { + const removeResult = runQuietOpenshell(["gateway", "remove", GATEWAY_NAME]); + if (removeResult.status === 0) return true; + + // OpenShell dev builds before NVIDIA/OpenShell#1221 used `gateway destroy` + // for local metadata cleanup. Post-#1221 builds removed lifecycle verbs and + // use `gateway remove` instead, so keep both forms quiet and best-effort. + const destroyResult = runQuietOpenshell(["gateway", "destroy", "-g", GATEWAY_NAME]); + return destroyResult.status === 0; +} + +function terminateDockerDriverGatewayProcess(pid: number): boolean { + if (!isPidAlive(pid)) { + return false; + } + + try { + process.kill(pid, "SIGTERM"); + for (let i = 0; i < 10; i += 1) { + if (!isPidAlive(pid)) break; + sleep(1); + } + if (isPidAlive(pid)) process.kill(pid, "SIGKILL"); + return true; + } catch { + return false; + } +} + +function stopDockerDriverGatewayProcess(): boolean { + const pid = getDockerDriverGatewayPid(); + if (pid === null || !isPidAlive(pid)) { + fs.rmSync(getDockerDriverGatewayPidFile(), { force: true }); + return false; + } + if (!isDockerDriverGatewayProcess(pid, resolveOpenShellGatewayBinary())) { + fs.rmSync(getDockerDriverGatewayPidFile(), { force: true }); + return false; + } + + const stopped = terminateDockerDriverGatewayProcess(pid); + fs.rmSync(getDockerDriverGatewayPidFile(), { force: true }); + return stopped; +} + +function restartDockerDriverGatewayProcessForDrift(pid: number, reason: string): void { + console.log(` Existing OpenShell Docker-driver gateway is stale (${reason}); restarting...`); + terminateDockerDriverGatewayProcess(pid); + fs.rmSync(getDockerDriverGatewayPidFile(), { force: true }); +} + +async function refreshDockerDriverGatewayReuseState( + gatewayReuseState: GatewayReuseState, +): Promise { + if (!isLinuxDockerDriverGatewayEnabled() || gatewayReuseState !== "healthy") { + return gatewayReuseState; + } + const gatewayBin = resolveOpenShellGatewayBinary(); + const desiredEnv = getDockerDriverGatewayEnv( + runCaptureOpenshell(["--version"], { ignoreError: true }), + ); + const pid = getDockerDriverGatewayPid(); + if (pid !== null && isDockerDriverGatewayProcessAlive()) { + const drift = getDockerDriverGatewayRuntimeDrift(pid, desiredEnv, gatewayBin); + if (drift) { + console.log( + ` Existing OpenShell Docker-driver gateway is stale (${drift.reason}); it will be recreated.`, + ); + return "stale"; + } + return gatewayReuseState; + } + + const portCheck = await checkPortAvailable(GATEWAY_PORT); + const dockerGatewayPid = getDockerDriverGatewayPortListenerPid(portCheck, { + gatewayBin, }); + if (dockerGatewayPid !== null) { + const drift = getDockerDriverGatewayRuntimeDrift(dockerGatewayPid, desiredEnv, gatewayBin); + rememberDockerDriverGatewayPid(dockerGatewayPid); + if (drift) { + console.log( + ` Existing OpenShell Docker-driver gateway is stale (${drift.reason}); it will be recreated.`, + ); + return "stale"; + } + return "healthy"; + } + + // `openshell status` already proved the selected gateway is reachable. If + // the port probe cannot identify the owning PID, avoid tearing down a live + // gateway solely because the pid file is stale. + if (!portCheck.ok && !portCheck.pid) return "healthy"; + + return "stale"; +} + +function destroyGateway(): boolean { + const dockerDriver = isLinuxDockerDriverGatewayEnabled(); + if (dockerDriver) { + stopDockerDriverGatewayProcess(); + } + + const gatewayRemoved = dockerDriver + ? removeDockerDriverGatewayRegistration() + : runOpenshell(["gateway", "destroy", "-g", GATEWAY_NAME], { + ignoreError: true, + }).status === 0; + // Clear the local registry so `nemoclaw list` stays consistent with OpenShell state. (#532) - if (destroyResult.status === 0) { + if (gatewayRemoved) { registry.clearAll(); } - // openshell gateway destroy doesn't remove Docker volumes, which leaves - // corrupted cluster state that breaks the next gateway start. Clean them up. + // Legacy OpenShell gateway cleanup doesn't remove Docker volumes, which + // leaves corrupted cluster state that breaks the next gateway start. dockerRemoveVolumesByPrefix(`openshell-cluster-${GATEWAY_NAME}`, { ignoreError: true }); + return gatewayRemoved; } type FinalGatewayStartFailureOptions = { @@ -3434,6 +3909,8 @@ function handleFinalGatewayStartFailure({ printError(" openshell doctor check"); printError(""); printError(" If gateway cleanup did not complete, run:"); + printError(` openshell gateway remove ${GATEWAY_NAME}`); + printError(` # For OpenShell releases that still expose lifecycle commands:`); printError(` openshell gateway destroy -g ${GATEWAY_NAME}`); printError( ` docker volume ls -q --filter "name=openshell-cluster-${GATEWAY_NAME}" | xargs -r docker volume rm`, @@ -3525,6 +4002,392 @@ function getGatewayLocalEndpoint(): string { return `https://127.0.0.1:${GATEWAY_PORT}`; } +function isLinuxDockerDriverGatewayEnabled( + platform: NodeJS.Platform = process.platform, +): boolean { + return platform === "linux"; +} + +function getDockerDriverGatewayEndpoint(): string { + return `http://127.0.0.1:${GATEWAY_PORT}`; +} + +function getDockerDriverGatewayStateDir(): string { + const configured = process.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR; + if (configured && configured.trim()) return path.resolve(configured.trim()); + return path.join(os.homedir(), ".local", "state", "nemoclaw", "openshell-docker-gateway"); +} + +function getDockerDriverGatewayPidFile(): string { + return path.join(getDockerDriverGatewayStateDir(), "openshell-gateway.pid"); +} + +function resolveSiblingBinary(binaryName: string): string | null { + const openshellBin = OPENSHELL_BIN || resolveOpenshell(); + if (typeof openshellBin !== "string" || openshellBin.length === 0) return null; + const sibling = path.join(path.dirname(openshellBin), binaryName); + if (fs.existsSync(sibling)) return sibling; + return null; +} + +function resolveOpenShellGatewayBinary(): string | null { + const configured = process.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN; + if (configured && configured.trim()) return path.resolve(configured.trim()); + const sibling = resolveSiblingBinary("openshell-gateway"); + if (sibling) return sibling; + for (const candidate of [ + path.join(os.homedir(), ".local", "bin", "openshell-gateway"), + "/usr/local/bin/openshell-gateway", + "/usr/bin/openshell-gateway", + ]) { + if (fs.existsSync(candidate)) return candidate; + } + return null; +} + +function resolveOpenShellSandboxBinary(): string | null { + const configured = process.env.NEMOCLAW_OPENSHELL_SANDBOX_BIN; + if (configured && configured.trim()) return path.resolve(configured.trim()); + const sibling = resolveSiblingBinary("openshell-sandbox"); + if (sibling) return sibling; + for (const candidate of [ + path.join(os.homedir(), ".local", "bin", "openshell-sandbox"), + "/usr/local/bin/openshell-sandbox", + "/usr/bin/openshell-sandbox", + ]) { + if (fs.existsSync(candidate)) return candidate; + } + return null; +} + +function getOpenShellDockerSupervisorImage(versionOutput: string | null = null): string { + if (process.env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE) { + return process.env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE; + } + const installedVersion = getInstalledOpenshellVersion(versionOutput); + if (shouldUseOpenshellDevChannel() || isOpenshellDevVersion(versionOutput)) { + return "ghcr.io/nvidia/openshell/supervisor:dev"; + } + const supportedVersion = installedVersion ?? getBlueprintMaxOpenshellVersion() ?? "0.0.37"; + return `ghcr.io/nvidia/openshell/supervisor:${supportedVersion}`; +} + +function getDockerDriverGatewayEnv( + versionOutput: string | null = null, +): Record { + const stateDir = getDockerDriverGatewayStateDir(); + const env: Record = { + OPENSHELL_DRIVERS: "docker", + OPENSHELL_BIND_ADDRESS: "127.0.0.1", + OPENSHELL_SERVER_PORT: String(GATEWAY_PORT), + OPENSHELL_DISABLE_TLS: "true", + OPENSHELL_DISABLE_GATEWAY_AUTH: "true", + OPENSHELL_DB_URL: `sqlite:${path.join(stateDir, "openshell.db")}`, + OPENSHELL_GRPC_ENDPOINT: getDockerDriverGatewayEndpoint(), + OPENSHELL_SSH_GATEWAY_HOST: "127.0.0.1", + OPENSHELL_SSH_GATEWAY_PORT: String(GATEWAY_PORT), + OPENSHELL_DOCKER_NETWORK_NAME: + process.env.OPENSHELL_DOCKER_NETWORK_NAME || "openshell-docker", + OPENSHELL_DOCKER_SUPERVISOR_IMAGE: getOpenShellDockerSupervisorImage(versionOutput), + }; + const sandboxBin = resolveOpenShellSandboxBinary(); + if (sandboxBin) { + env.OPENSHELL_DOCKER_SUPERVISOR_BIN = sandboxBin; + } + return env; +} + +function isPidAlive(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return isErrnoException(error) && error.code === "EPERM"; + } +} + +function getDockerDriverGatewayPid(): number | null { + try { + const raw = fs.readFileSync(getDockerDriverGatewayPidFile(), "utf-8").trim(); + const pid = Number.parseInt(raw, 10); + return Number.isInteger(pid) && pid > 0 ? pid : null; + } catch { + return null; + } +} + +function readProcessEnv(pid: number): Record | null { + const procEnvPath = `/proc/${pid}/environ`; + const env: Record = {}; + try { + if (!fs.existsSync(procEnvPath)) return null; + for (const entry of fs.readFileSync(procEnvPath, "utf-8").split("\0")) { + if (!entry) continue; + const idx = entry.indexOf("="); + if (idx <= 0) continue; + env[entry.slice(0, idx)] = entry.slice(idx + 1); + } + } catch { + return null; + } + return env; +} + +function hasDockerDriverGatewayEnv(pid: number): boolean { + const env = readProcessEnv(pid); + if (!env) return false; + return ( + env.OPENSHELL_DRIVERS === "docker" || + Boolean(env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE) || + env.OPENSHELL_GRPC_ENDPOINT === getDockerDriverGatewayEndpoint() + ); +} + +function readProcessExe(pid: number): string | null { + try { + const procExePath = `/proc/${pid}/exe`; + if (!fs.existsSync(procExePath)) return null; + return fs.readlinkSync(procExePath); + } catch { + return null; + } +} + +function normalizeGatewayExecutablePath(value: string | null | undefined): string | null { + if (!value) return null; + const withoutDeletedSuffix = value.replace(/ \(deleted\)$/, ""); + try { + return fs.realpathSync.native(withoutDeletedSuffix); + } catch { + return path.resolve(withoutDeletedSuffix); + } +} + +type DockerDriverGatewayRuntimeDrift = { reason: string }; + +const DOCKER_DRIVER_GATEWAY_RUNTIME_ENV_KEYS = [ + "OPENSHELL_DRIVERS", + "OPENSHELL_BIND_ADDRESS", + "OPENSHELL_SERVER_PORT", + "OPENSHELL_DISABLE_TLS", + "OPENSHELL_DISABLE_GATEWAY_AUTH", + "OPENSHELL_DB_URL", + "OPENSHELL_GRPC_ENDPOINT", + "OPENSHELL_SSH_GATEWAY_HOST", + "OPENSHELL_SSH_GATEWAY_PORT", + "OPENSHELL_DOCKER_NETWORK_NAME", + "OPENSHELL_DOCKER_SUPERVISOR_IMAGE", + "OPENSHELL_DOCKER_SUPERVISOR_BIN", +] as const; + +function getDockerDriverGatewayRuntimeDriftFromSnapshot({ + processEnv, + processExe, + desiredEnv, + gatewayBin, +}: { + processEnv: Record | null; + processExe: string | null; + desiredEnv: Record; + gatewayBin?: string | null; +}): DockerDriverGatewayRuntimeDrift | null { + if (!processEnv) { + return { reason: "could not verify process environment" }; + } + for (const key of DOCKER_DRIVER_GATEWAY_RUNTIME_ENV_KEYS) { + const desired = desiredEnv[key]; + if (typeof desired !== "string") continue; + const actual = processEnv[key]; + if (actual !== desired) { + return { reason: `${key}=${actual || ""} (expected ${desired})` }; + } + } + + if (processExe === null) { + return { reason: "could not verify process executable" }; + } + if (processExe.endsWith(" (deleted)")) { + return { reason: "gateway executable was replaced on disk" }; + } + const expectedExe = normalizeGatewayExecutablePath(gatewayBin); + const actualExe = normalizeGatewayExecutablePath(processExe); + if (expectedExe && actualExe && actualExe !== expectedExe) { + return { reason: `executable=${actualExe} (expected ${expectedExe})` }; + } + return null; +} + +function getDockerDriverGatewayRuntimeDrift( + pid: number, + desiredEnv: Record, + gatewayBin?: string | null, +): DockerDriverGatewayRuntimeDrift | null { + return getDockerDriverGatewayRuntimeDriftFromSnapshot({ + processEnv: readProcessEnv(pid), + processExe: readProcessExe(pid), + desiredEnv, + gatewayBin, + }); +} + +function isDockerDriverGatewayProcess( + pid: number, + gatewayBin?: string | null, + opts: { requireDockerDriverEnv?: boolean } = {}, +): boolean { + const procCmdlinePath = `/proc/${pid}/cmdline`; + let identity = ""; + try { + if (fs.existsSync(procCmdlinePath)) { + identity = fs.readFileSync(procCmdlinePath, "utf-8").replace(/\0/g, " ").trim(); + } + } catch { + identity = ""; + } + if (!identity) { + identity = captureProcessArgs(pid); + } + if (!identity) return false; + const matchesGatewayBinary = + identity.includes("openshell-gateway") || + (typeof gatewayBin === "string" && gatewayBin.length > 0 && identity.includes(gatewayBin)); + if (!matchesGatewayBinary) return false; + if (opts.requireDockerDriverEnv && !hasDockerDriverGatewayEnv(pid)) return false; + return true; +} + +function isDockerDriverGatewayProcessAlive(): boolean { + const pid = getDockerDriverGatewayPid(); + if (pid === null || !isPidAlive(pid)) return false; + if (!isDockerDriverGatewayProcess(pid, resolveOpenShellGatewayBinary(), { + requireDockerDriverEnv: true, + })) { + fs.rmSync(getDockerDriverGatewayPidFile(), { force: true }); + return false; + } + return true; +} + +function rememberDockerDriverGatewayPid(pid: number): void { + if (!Number.isInteger(pid) || pid <= 0) return; + const stateDir = getDockerDriverGatewayStateDir(); + fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync(getDockerDriverGatewayPidFile(), `${pid}\n`, { + encoding: "utf-8", + mode: 0o600, + }); +} + +function getDockerDriverGatewayPortListenerPid( + portCheck: import("./onboard/preflight").PortProbeResult, + opts: { + platform?: NodeJS.Platform; + gatewayBin?: string | null; + isPidAliveFn?: (pid: number) => boolean; + isDockerDriverGatewayProcessFn?: (pid: number, gatewayBin?: string | null) => boolean; + } = {}, +): number | null { + if (portCheck.ok) return null; + if (!isLinuxDockerDriverGatewayEnabled(opts.platform ?? process.platform)) return null; + const pid = Number(portCheck.pid); + if (!Number.isInteger(pid) || pid <= 0) return null; + const proc = String(portCheck.process || "").toLowerCase(); + if (!proc.startsWith("openshell")) return null; + const alive = opts.isPidAliveFn ?? isPidAlive; + if (!alive(pid)) return null; + const isGateway = + opts.isDockerDriverGatewayProcessFn ?? + ((candidatePid: number, gatewayBin?: string | null) => + isDockerDriverGatewayProcess(candidatePid, gatewayBin, { requireDockerDriverEnv: true })); + if (!isGateway(pid, opts.gatewayBin)) return null; + return pid; +} + +function isDockerDriverGatewayPortListener( + portCheck: import("./onboard/preflight").PortProbeResult, + opts: Parameters[1] = {}, +): boolean { + return getDockerDriverGatewayPortListenerPid(portCheck, opts) !== null; +} + +function writeDockerGatewayDebEnvOverride(): void { + const servicePath = "/usr/lib/systemd/user/openshell-gateway.service"; + const legacyServicePath = "/lib/systemd/user/openshell-gateway.service"; + if ( + !fs.existsSync("/usr/bin/openshell-gateway") && + !fs.existsSync(servicePath) && + !fs.existsSync(legacyServicePath) + ) { + return; + } + const envFile = path.join(os.homedir(), ".config", "openshell", "gateway.env"); + fs.mkdirSync(path.dirname(envFile), { recursive: true, mode: 0o700 }); + const existing = fs.existsSync(envFile) ? fs.readFileSync(envFile, "utf-8") : ""; + const preserved = existing + .split("\n") + .filter( + (line: string) => + line.trim() && + !/^OPENSHELL_(DRIVERS|DOCKER_SUPERVISOR_IMAGE|DOCKER_SUPERVISOR_BIN)=/.test(line), + ); + const override = getDockerDriverGatewayEnv(); + const next = [ + ...preserved, + `OPENSHELL_DRIVERS=${override.OPENSHELL_DRIVERS}`, + `OPENSHELL_DOCKER_SUPERVISOR_IMAGE=${override.OPENSHELL_DOCKER_SUPERVISOR_IMAGE}`, + ...(override.OPENSHELL_DOCKER_SUPERVISOR_BIN + ? [`OPENSHELL_DOCKER_SUPERVISOR_BIN=${override.OPENSHELL_DOCKER_SUPERVISOR_BIN}`] + : []), + ].join("\n"); + fs.writeFileSync(envFile, `${next}\n`, { encoding: "utf-8", mode: 0o600 }); +} + +function registerDockerDriverGatewayEndpoint(): boolean { + const selectExisting = runQuietOpenshell(["gateway", "select", GATEWAY_NAME]); + if (selectExisting.status === 0) { + const status = runCaptureOpenshell(["status"], { ignoreError: true }); + const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { + ignoreError: true, + }); + const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); + if (isGatewayHealthy(status, namedInfo, currentInfo)) { + process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; + return true; + } + } + + let addResult = runOpenshell( + ["gateway", "add", "--local", "--name", GATEWAY_NAME, getDockerDriverGatewayEndpoint()], + { ignoreError: true, suppressOutput: true }, + ); + if (addResult.status !== 0) { + removeDockerDriverGatewayRegistration(); + addResult = runOpenshell( + ["gateway", "add", "--local", "--name", GATEWAY_NAME, getDockerDriverGatewayEndpoint()], + { ignoreError: true, suppressOutput: true }, + ); + } + const selectResult = runOpenshell(["gateway", "select", GATEWAY_NAME], { + ignoreError: true, + suppressOutput: true, + }); + const ok = + (addResult.status === 0 && selectResult.status === 0) || + (selectResult.status === 0 && + isGatewayHealthy( + runCaptureOpenshell(["status"], { ignoreError: true }), + runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { ignoreError: true }), + runCaptureOpenshell(["gateway", "info"], { ignoreError: true }), + )); + if (ok) { + process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; + } else if (process.env.OPENSHELL_GATEWAY === GATEWAY_NAME) { + delete process.env.OPENSHELL_GATEWAY; + } + return ok; +} + function getGatewayBootstrapRepairPlan(missingSecrets: string[] = []) { const allowed = new Set(GATEWAY_BOOTSTRAP_SECRET_NAMES); const normalized = [ @@ -3648,6 +4511,10 @@ function attachGatewayMetadataIfNeeded({ // flow explicitly forces a refresh after recreating bootstrap secrets. if (!forceRefresh && hasStaleGateway(gwInfo)) return true; + if (isLinuxDockerDriverGatewayEnabled()) { + return registerDockerDriverGatewayEndpoint(); + } + const addResult = runOpenshell( ["gateway", "add", "--local", "--name", GATEWAY_NAME, getGatewayLocalEndpoint()], { ignoreError: true, suppressOutput: true }, @@ -3678,6 +4545,12 @@ async function ensureNamedCredential( function waitForSandboxReady(sandboxName: string, attempts = 10, delaySeconds = 2): boolean { for (let i = 0; i < attempts; i += 1) { + if (isLinuxDockerDriverGatewayEnabled()) { + const list = runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); + if (isSandboxReady(list, sandboxName)) return true; + if (i < attempts - 1) sleep(delaySeconds); + continue; + } const podPhase = runCaptureOpenshell( [ "doctor", @@ -3734,8 +4607,15 @@ function assertCdiNvidiaGpuSpecPresent( process.exit(1); } +type PreflightOptions = Pick< + OnboardOptions, + "sandboxGpu" | "sandboxGpuDevice" | "gpu" | "noGpu" +> & { + optedOutGpuPassthrough?: boolean; +}; + async function preflight( - preflightOpts: { optedOutGpuPassthrough?: boolean } = {}, + preflightOpts: PreflightOptions = {}, ): Promise> { step(1, 8, "Preflight checks"); @@ -3749,7 +4629,9 @@ async function preflight( } console.log(" ✓ Docker is running"); - assertCdiNvidiaGpuSpecPresent(host, preflightOpts.optedOutGpuPassthrough === true); + const optedOutGpuPassthrough = + preflightOpts.optedOutGpuPassthrough === true || preflightOpts.noGpu === true; + assertCdiNvidiaGpuSpecPresent(host, optedOutGpuPassthrough); // DNS resolution from inside containers (#2101). A corp firewall that // blocks outbound UDP:53 to public resolvers leaves the sandbox build @@ -3922,7 +4804,12 @@ async function preflight( if (host.runtime !== "unknown") { console.log(` ✓ Container runtime: ${host.runtime}`); } - // Podman is now supported — no unsupported runtime warning needed. + if (isLinuxDockerDriverGatewayEnabled() && host.runtime === "podman") { + console.error(" ✗ NemoClaw Linux onboarding now uses OpenShell's Docker driver."); + console.error(" Podman is not supported for this NemoClaw integration path."); + console.error(" Switch to Docker Engine and rerun onboarding."); + process.exit(1); + } if (host.notes.includes("Running under WSL")) { console.log(" ⓘ Running under WSL"); } @@ -4007,14 +4894,33 @@ async function preflight( } } else { // Source of truth: min_openshell_version in nemoclaw-blueprint/blueprint.yaml. - // Fall back to the Landlock-enforcement floor (also MIN_VERSION in + // Fall back to the released Docker-driver floor (also MIN_VERSION in // scripts/install-openshell.sh) if the blueprint cannot be read. - const minOpenshellVersion = getBlueprintMinOpenshellVersion() ?? "0.0.32"; - const needsUpgrade = !versionGte(currentVersion, minOpenshellVersion); + const minOpenshellVersion = getBlueprintMinOpenshellVersion() ?? "0.0.37"; + const currentVersionOutput = runCaptureOpenshell(["--version"], { ignoreError: true }); + const needsDevChannel = + isLinuxDockerDriverGatewayEnabled() && + shouldUseOpenshellDevChannel() && + !isOpenshellDevVersion(currentVersionOutput); + const needsDockerDriverBinaries = + isLinuxDockerDriverGatewayEnabled() && + (!resolveOpenShellGatewayBinary() || !resolveOpenShellSandboxBinary()); + const needsUpgrade = + !versionGte(currentVersion, minOpenshellVersion) || + needsDevChannel || + needsDockerDriverBinaries; if (needsUpgrade) { - console.log( - ` openshell ${currentVersion} is below minimum required version. Upgrading...`, - ); + if (needsDevChannel) { + console.log(" OpenShell Docker-driver onboarding requires the dev channel. Upgrading..."); + } else if (needsDockerDriverBinaries) { + console.log( + " OpenShell Docker-driver onboarding requires the gateway and sandbox binaries. Reinstalling...", + ); + } else { + console.log( + ` openshell ${currentVersion} is below minimum required version. Upgrading...`, + ); + } openshellInstall = installOpenshell(); if (!openshellInstall.installed) { console.error(" Failed to upgrade openshell CLI."); @@ -4061,7 +4967,8 @@ async function preflight( if ( installedOpenshellVersion && maxOpenshellVersion && - !versionGte(maxOpenshellVersion, installedOpenshellVersion) + !versionGte(maxOpenshellVersion, installedOpenshellVersion) && + !shouldAllowOpenshellAboveBlueprintMax(openshellVersionOutput) ) { console.error(""); console.error( @@ -4094,10 +5001,11 @@ async function preflight( // reuses the user's NemoClaw gateway instead of reporting a false conflict. const gatewaySnapshot = selectNamedGatewayForReuseIfNeeded(getGatewayReuseSnapshot()); let gatewayReuseState = gatewaySnapshot.gatewayReuseState; + gatewayReuseState = await refreshDockerDriverGatewayReuseState(gatewayReuseState); // Verify the gateway container is actually running — openshell CLI metadata // can be stale after a manual `docker rm`. See #2020. - if (gatewayReuseState === "healthy") { + if (gatewayReuseState === "healthy" && !isLinuxDockerDriverGatewayEnabled()) { const containerState = verifyGatewayContainerRunning(); if (containerState === "missing") { console.log(" Gateway metadata is stale (container not running). Cleaning up..."); @@ -4128,21 +5036,14 @@ async function preflight( if (gatewayReuseState === "stale" || gatewayReuseState === "active-unnamed") { console.log(` Cleaning up previous ${cliDisplayName()} session...`); runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true }); - const destroyResult = runOpenshell(["gateway", "destroy", "-g", GATEWAY_NAME], { - ignoreError: true, - }); - // Sandboxes under the destroyed gateway no longer exist in OpenShell — - // clear the local registry so `nemoclaw list` stays consistent. (#532) - if (destroyResult.status === 0) { - registry.clearAll(); - } + destroyGateway(); console.log(" ✓ Previous session cleaned up"); } // Clean up orphaned Docker containers from interrupted onboard (e.g. Ctrl+C // during gateway start). The container may still be running even though // OpenShell has no metadata for it (gatewayReuseState === "missing"). - if (gatewayReuseState === "missing") { + if (gatewayReuseState === "missing" && !isLinuxDockerDriverGatewayEnabled()) { const containerName = `openshell-cluster-${GATEWAY_NAME}`; const inspectResult = dockerInspect( ["--type", "container", "--format", "{{.State.Status}}", containerName], @@ -4205,6 +5106,16 @@ async function preflight( ); continue; } + if (port === GATEWAY_PORT) { + const dockerGatewayPid = getDockerDriverGatewayPortListenerPid(portCheck); + if (dockerGatewayPid !== null) { + rememberDockerDriverGatewayPid(dockerGatewayPid); + console.log( + ` ✓ Port ${port} already owned by NemoClaw OpenShell Docker gateway (${label})`, + ); + continue; + } + } // Auto-cleanup orphaned SSH port-forward from a previous NemoClaw session // (e.g. dashboard forward left behind after destroy). Only kill the process // if its command line contains "openshell" to avoid killing unrelated SSH @@ -4284,6 +5195,21 @@ async function preflight( console.log(" ⓘ Local NIM unavailable — no GPU detected"); } + const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { + flag: resolveSandboxGpuFlagFromOptions(preflightOpts), + device: preflightOpts.sandboxGpuDevice ?? null, + }); + validateSandboxGpuPreflight(sandboxGpuConfig); + if (sandboxGpuConfig.sandboxGpuEnabled) { + console.log( + ` ✓ Sandbox GPU: enabled (${sandboxGpuConfig.mode}${sandboxGpuConfig.sandboxGpuDevice ? `, device ${sandboxGpuConfig.sandboxGpuDevice}` : ""})`, + ); + } else if (sandboxGpuConfig.mode === "0") { + console.log(" ✓ Sandbox GPU: disabled by configuration"); + } else { + console.log(" ⓘ Sandbox GPU: disabled (no NVIDIA GPU detected)"); + } + // Memory / swap check (Linux only) if (process.platform === "linux") { const mem = getMemoryInfo(); @@ -4339,6 +5265,10 @@ async function startGatewayWithOptions( ) { step(2, 8, "Starting OpenShell gateway"); + if (isLinuxDockerDriverGatewayEnabled()) { + return startDockerDriverGateway({ exitOnFailure }); + } + const gatewaySnapshot = selectNamedGatewayForReuseIfNeeded(getGatewayReuseSnapshot()); if ( isGatewayHealthy( @@ -4491,6 +5421,148 @@ async function startGatewayWithOptions( process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; } +async function startDockerDriverGateway({ + exitOnFailure = true, +}: { exitOnFailure?: boolean } = {}): Promise { + writeDockerGatewayDebEnvOverride(); + const gatewayBin = resolveOpenShellGatewayBinary(); + const openshellVersionOutput = runCaptureOpenshell(["--version"], { + ignoreError: true, + }); + const gatewayEnv = getDockerDriverGatewayEnv(openshellVersionOutput); + + const gatewayStatus = runCaptureOpenshell(["status"], { ignoreError: true }); + const gwInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { + ignoreError: true, + }); + const activeGatewayInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); + const pidFileGatewayPid = getDockerDriverGatewayPid(); + if ( + pidFileGatewayPid !== null && + isDockerDriverGatewayProcessAlive() && + isGatewayHealthy(gatewayStatus, gwInfo, activeGatewayInfo) + ) { + const drift = getDockerDriverGatewayRuntimeDrift(pidFileGatewayPid, gatewayEnv, gatewayBin); + if (drift) { + restartDockerDriverGatewayProcessForDrift(pidFileGatewayPid, drift.reason); + } else if (registerDockerDriverGatewayEndpoint()) { + console.log(" ✓ Reusing existing Docker-driver gateway"); + return; + } + } + + const portCheck = await checkPortAvailable(GATEWAY_PORT); + const portListenerPid = getDockerDriverGatewayPortListenerPid(portCheck, { gatewayBin }); + if (portListenerPid !== null) { + const drift = getDockerDriverGatewayRuntimeDrift(portListenerPid, gatewayEnv, gatewayBin); + if (drift) { + rememberDockerDriverGatewayPid(portListenerPid); + restartDockerDriverGatewayProcessForDrift(portListenerPid, drift.reason); + } else { + rememberDockerDriverGatewayPid(portListenerPid); + } + if (!drift && registerDockerDriverGatewayEndpoint()) { + const adoptedStatus = runCaptureOpenshell(["status"], { ignoreError: true }); + const adoptedGwInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { + ignoreError: true, + }); + const adoptedActiveGatewayInfo = runCaptureOpenshell(["gateway", "info"], { + ignoreError: true, + }); + if (isGatewayHealthy(adoptedStatus, adoptedGwInfo, adoptedActiveGatewayInfo)) { + console.log(` ✓ Reusing existing Docker-driver gateway process (PID ${portListenerPid})`); + return; + } + } + } + if (!gatewayBin) { + console.error(" OpenShell Docker-driver gateway binary not found."); + console.error(" Install OpenShell v0.0.37, or set NEMOCLAW_OPENSHELL_GATEWAY_BIN."); + if (exitOnFailure) process.exit(1); + throw new Error("OpenShell gateway binary not found"); + } + + const existingPid = getDockerDriverGatewayPid() ?? portListenerPid; + if (existingPid !== null && isPidAlive(existingPid)) { + if (!isDockerDriverGatewayProcess(existingPid, gatewayBin)) { + fs.rmSync(getDockerDriverGatewayPidFile(), { force: true }); + } else { + console.log(` Restarting unhealthy Docker-driver gateway process (PID ${existingPid})...`); + try { + process.kill(existingPid, "SIGTERM"); + sleep(1); + } catch { + /* best effort; the new process will surface any remaining port conflict */ + } + } + } + + const stateDir = getDockerDriverGatewayStateDir(); + fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + const logPath = path.join(stateDir, "openshell-gateway.log"); + const outFd = fs.openSync(logPath, "a", 0o600); + const errFd = fs.openSync(logPath, "a", 0o600); + console.log(" Starting OpenShell Docker-driver gateway..."); + console.log(` Gateway log: ${logPath}`); + const child = spawn(gatewayBin, [], { + detached: true, + stdio: ["ignore", outFd, errFd], + env: { + ...process.env, + ...gatewayEnv, + }, + }); + child.unref(); + const childPid = child.pid ?? 0; + if (childPid <= 0) { + throw new Error("OpenShell gateway process did not return a pid"); + } + rememberDockerDriverGatewayPid(childPid); + + const pollCount = envInt("NEMOCLAW_HEALTH_POLL_COUNT", 30); + const pollInterval = envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2); + for (let i = 0; i < pollCount; i += 1) { + if (!isPidAlive(childPid)) { + break; + } + if (!registerDockerDriverGatewayEndpoint()) { + if (i < pollCount - 1) sleep(pollInterval); + continue; + } + const status = runCaptureOpenshell(["status"], { ignoreError: true }); + const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { + ignoreError: true, + }); + const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); + if (isGatewayHealthy(status, namedInfo, currentInfo)) { + console.log(" ✓ Docker-driver gateway is healthy"); + return; + } + if (i < pollCount - 1) sleep(pollInterval); + } + + const tail = fs.existsSync(logPath) + ? fs + .readFileSync(logPath, "utf-8") + .split("\n") + .filter(Boolean) + .slice(-20) + .join("\n") + : ""; + if (exitOnFailure) { + console.error(" Docker-driver gateway failed to start."); + if (tail) { + console.error(" Gateway log tail:"); + for (const line of tail.split("\n")) console.error(` ${redact(line)}`); + } + console.error(" Troubleshooting:"); + console.error(` tail -100 ${logPath}`); + console.error(" docker info --format '{{json .CDISpecDirs}}'"); + process.exit(1); + } + throw new Error("Docker-driver gateway failed to start"); +} + async function startGateway( _gpu: ReturnType, { gpuPassthrough = false }: { gpuPassthrough?: boolean } = {}, @@ -4608,6 +5680,15 @@ function applyOverlayfsAutoFix(upstreamImage: string): string | null { } async function recoverGatewayRuntime() { + if (isLinuxDockerDriverGatewayEnabled()) { + try { + await startDockerDriverGateway({ exitOnFailure: false }); + return true; + } catch { + return false; + } + } + runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true }); let status = runCaptureOpenshell(["status"], { ignoreError: true }); if (status.includes("Connected") && isSelectedGateway(status)) { @@ -4771,6 +5852,41 @@ function getSandboxAgentDrift( }; } +function getSandboxRuntimeRegistryFields( + config: SandboxGpuConfig, +): Pick< + SandboxEntry, + | "gpuEnabled" + | "hostGpuDetected" + | "sandboxGpuEnabled" + | "sandboxGpuMode" + | "sandboxGpuDevice" + | "openshellDriver" + | "openshellVersion" +> { + return { + gpuEnabled: config.sandboxGpuEnabled, + hostGpuDetected: config.hostGpuDetected, + sandboxGpuEnabled: config.sandboxGpuEnabled, + sandboxGpuMode: config.mode, + sandboxGpuDevice: config.sandboxGpuDevice, + openshellDriver: isLinuxDockerDriverGatewayEnabled() ? "docker" : "kubernetes", + openshellVersion: getInstalledOpenshellVersion( + runCaptureOpenshell(["--version"], { ignoreError: true }), + ), + }; +} + +function hasSandboxGpuDrift(sandboxName: string, config: SandboxGpuConfig): boolean { + const existingEntry: SandboxEntry | null = registry.getSandbox(sandboxName); + if (!existingEntry) return false; + return ( + (existingEntry.sandboxGpuEnabled === true) !== config.sandboxGpuEnabled || + (existingEntry.sandboxGpuMode || "auto") !== config.mode || + (existingEntry.sandboxGpuDevice || null) !== config.sandboxGpuDevice + ); +} + function updateReusedSandboxMetadata( sandboxName: string, agent: AgentDefinition | null | undefined, @@ -4778,6 +5894,7 @@ function updateReusedSandboxMetadata( provider: string, dashboardPort: number, selectionVerified = true, + sandboxGpuConfig: SandboxGpuConfig | null = null, ): void { const existingEntry = registry.getSandbox(sandboxName); const agentVersionKnown = existingEntry?.agentVersion !== null; @@ -4786,6 +5903,7 @@ function updateReusedSandboxMetadata( ...selectionUpdates, dashboardPort, ...getSandboxAgentRegistryFields(agent, agentVersionKnown), + ...(sandboxGpuConfig ? getSandboxRuntimeRegistryFields(sandboxGpuConfig) : {}), }); registry.setDefault(sandboxName); } @@ -4947,7 +6065,7 @@ async function createSandbox( fromDockerfile: string | null = null, agent: AgentDefinition | null = null, controlUiPort: number | null = null, - gpuPassthrough: boolean = false, + sandboxGpuConfig: SandboxGpuConfig | null = null, ) { step(6, 8, "Creating sandbox"); @@ -4955,6 +6073,8 @@ async function createSandbox( sandboxNameOverride ?? (await promptValidatedSandboxName(agent)), "sandbox name", ); + const effectiveSandboxGpuConfig = + sandboxGpuConfig ?? resolveSandboxGpuConfig(gpu, { flag: null, device: null }); // Port priority: --control-ui-port > CHAT_UI_URL env > registry (resume) > agent.forwardPort > default // Pre-resolve port availability so CHAT_UI_URL baked into the Dockerfile, @@ -5183,6 +6303,7 @@ async function createSandbox( messagingTokenDefs.some(({ name, token }) => token && !providerExistsInGateway(name)); const selectionDrift = getSelectionDrift(sandboxName, provider, model); const confirmedSelectionDrift = selectionDrift.changed && !selectionDrift.unknown; + const sandboxGpuDrift = hasSandboxGpuDrift(sandboxName, effectiveSandboxGpuConfig); // Detect whether any messaging credential has been rotated since the // sandbox was created. Provider credentials are resolved once at sandbox @@ -5195,6 +6316,7 @@ async function createSandbox( !isRecreateSandbox() && !recreateForAgentDrift && !needsProviderMigration && + !sandboxGpuDrift && !credentialRotation.changed ) { // Guard against reusing a CPU-only sandbox when GPU passthrough is enabled. @@ -5205,7 +6327,7 @@ async function createSandbox( // The gateway Docker-inspect check (above) catches legacy CPU-only gateways // before we reach this point, so a legacy sandbox behind a verified GPU // gateway is safe to reuse — the sandbox will be recreated if needed. - if (gpuPassthrough) { + if (effectiveSandboxGpuConfig.sandboxGpuEnabled) { const entry = registry.getSandbox(sandboxName); if (entry && !entry.gpuEnabled) { console.error(` Sandbox '${sandboxName}' exists but was created without GPU passthrough.`); @@ -5247,6 +6369,7 @@ async function createSandbox( provider, reusedPort, !selectionDrift.unknown, + effectiveSandboxGpuConfig, ); return sandboxName; } @@ -5283,6 +6406,7 @@ async function createSandbox( provider, reusedPort2, !selectionDrift.unknown, + effectiveSandboxGpuConfig, ); return sandboxName; } @@ -5329,11 +6453,12 @@ async function createSandbox( sandboxName, agent, model, - provider, - reusedPort3, - !selectionDrift.unknown, - ); - return sandboxName; + provider, + reusedPort3, + !selectionDrift.unknown, + effectiveSandboxGpuConfig, + ); + return sandboxName; } } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err); @@ -5357,6 +6482,7 @@ async function createSandbox( provider, reusedPort4, !selectionDrift.unknown, + effectiveSandboxGpuConfig, ); return sandboxName; } @@ -5371,6 +6497,8 @@ async function createSandbox( console.log(" Recreating to ensure credentials flow through the provider pipeline."); } else if (confirmedSelectionDrift) { note(` Sandbox '${sandboxName}' exists — recreating to apply model/provider change.`); + } else if (sandboxGpuDrift) { + note(` Sandbox '${sandboxName}' exists — recreating to apply sandbox GPU settings.`); } else if (credentialRotation.changed) { // Message already printed above during backup. } else if (existingSandboxState === "ready") { @@ -5536,6 +6664,7 @@ async function createSandbox( const initialSandboxPolicy = prepareInitialSandboxCreatePolicy( basePolicyPath, activeMessagingChannels, + { directGpu: effectiveSandboxGpuConfig.sandboxGpuEnabled }, ); if (initialSandboxPolicy.cleanup) { process.on("exit", initialSandboxPolicy.cleanup); @@ -5545,6 +6674,9 @@ async function createSandbox( ` Including policy preset(s) at sandbox boot: ${initialSandboxPolicy.appliedPresets.join(", ")}`, ); } + if (effectiveSandboxGpuConfig.sandboxGpuEnabled) { + console.log(" Direct sandbox GPU enabled; allowing only /proc task comm writes."); + } const createArgs = [ "--from", `${buildCtx}/Dockerfile`, @@ -5552,10 +6684,8 @@ async function createSandbox( sandboxName, "--policy", initialSandboxPolicy.policyPath, + ...buildSandboxGpuCreateArgs(effectiveSandboxGpuConfig), ]; - if (gpuPassthrough) { - createArgs.push("--gpu"); - } // Create OpenShell providers for messaging credentials so they flow through // the provider/placeholder system instead of raw env vars. The L7 proxy @@ -5641,9 +6771,13 @@ async function createSandbox( // Pull the base image and resolve its digest so the Dockerfile is pinned to // exactly what we just fetched. This prevents stale :latest tags from // silently reusing a cached old image after NemoClaw upgrades (#1904). - const resolved = pullAndResolveBaseImageDigest(); - if (resolved) { + const resolved = pullAndResolveBaseImageDigest({ + requireOpenshellSandboxAbi: isLinuxDockerDriverGatewayEnabled(), + }); + if (resolved?.digest) { console.log(` Pinning base image to ${resolved.digest.slice(0, 19)}...`); + } else if (resolved) { + console.log(` Using sandbox base image ${resolved.ref}`); } else { // Check if the image exists locally before falling back to unpinned :latest. // On a first-time install behind a firewall with no cached image, warn early @@ -5860,6 +6994,21 @@ async function createSandbox( } } + if (effectiveSandboxGpuConfig.sandboxGpuEnabled) { + try { + verifyDirectSandboxGpu(sandboxName); + } catch (error) { + const delResult = runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); + if (delResult.status === 0) { + console.error(" The sandbox with failed GPU access has been removed — you can retry safely."); + } else { + console.error(" Could not remove the sandbox with failed GPU access. Manual cleanup:"); + console.error(` openshell sandbox delete "${sandboxName}"`); + } + throw error; + } + } + // Verify web search config was actually accepted by the agent runtime. // Hermes silently ignores unknown web.backend values (e.g. "brave" before // upstream support lands), so we exec into the sandbox and check for a @@ -5914,7 +7063,7 @@ async function createSandbox( name: sandboxName, model: model || null, provider: provider || null, - gpuEnabled: gpuPassthrough, + ...getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig), ...getSandboxAgentRegistryFields(agent, !fromDockerfile), imageTag: resolvedImageTag, providerCredentialHashes: @@ -5947,10 +7096,12 @@ async function createSandbox( // DNS proxy — run a forwarder in the sandbox pod so the isolated // sandbox namespace can resolve hostnames (fixes #626). - console.log(" Setting up sandbox DNS proxy..."); - runFile("bash", [path.join(SCRIPTS, "setup-dns-proxy.sh"), GATEWAY_NAME, sandboxName], { - ignoreError: true, - }); + if (!isLinuxDockerDriverGatewayEnabled()) { + console.log(" Setting up sandbox DNS proxy..."); + runFile("bash", [path.join(SCRIPTS, "setup-dns-proxy.sh"), GATEWAY_NAME, sandboxName], { + ignoreError: true, + }); + } // Check that messaging providers exist in the gateway (sandbox attachment // cannot be verified via CLI yet — only gateway-level existence is checked). @@ -10112,14 +11263,33 @@ async function onboard(opts: OnboardOptions = {}): Promise { return s; }); + const recordedSandboxName = + session?.steps?.sandbox?.status === "complete" ? session?.sandboxName || null : null; + const resumeSandboxNameForGpu = recordedSandboxName || requestedSandboxName || null; + console.log(""); console.log(` ${cliDisplayName()} Onboarding`); if (isNonInteractive()) note(" (non-interactive mode)"); if (resume) note(" (resume mode)"); console.log(" ==================="); - let gpu; + const explicitSandboxGpuFlag = resolveSandboxGpuFlagFromOptions(opts); const resumePreflight = resume && session?.steps?.preflight?.status === "complete"; + const resumeHasResolvedGpuIntent = + resumePreflight && + explicitSandboxGpuFlag === null && + opts.sandboxGpuDevice == null && + process.env.NEMOCLAW_SANDBOX_GPU === undefined && + process.env.NEMOCLAW_SANDBOX_GPU_DEVICE === undefined; + const resumedSandboxGpuOverrides = resumeHasResolvedGpuIntent + ? getResumeSandboxGpuOverrides( + resumeSandboxNameForGpu ? registry.getSandbox(resumeSandboxNameForGpu) : null, + session?.gpuPassthrough, + ) + : { flag: null, device: null }; + const effectiveSandboxGpuFlag = explicitSandboxGpuFlag ?? resumedSandboxGpuOverrides.flag; + const effectiveSandboxGpuDevice = opts.sandboxGpuDevice ?? resumedSandboxGpuOverrides.device; + let gpu; if (resumePreflight) { skippedStepMessage("preflight", "cached"); gpu = nim.detectGpu(); @@ -10134,34 +11304,29 @@ async function onboard(opts: OnboardOptions = {}): Promise { const resumeOptedOutGpuPassthrough = opts.noGpu === true || (opts.gpu !== true && session?.gpuPassthrough === false); assertCdiNvidiaGpuSpecPresent(assessHost(), resumeOptedOutGpuPassthrough); + validateSandboxGpuPreflight( + resolveSandboxGpuConfig(gpu, { + flag: effectiveSandboxGpuFlag, + device: effectiveSandboxGpuDevice, + }), + ); } else { startRecordedStep("preflight"); - gpu = await preflight({ optedOutGpuPassthrough: opts.noGpu === true }); + gpu = await preflight({ ...opts, optedOutGpuPassthrough: opts.noGpu === true }); onboardSession.markStepComplete("preflight"); } + const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { + flag: effectiveSandboxGpuFlag, + device: effectiveSandboxGpuDevice, + }); const requestedGpuPassthrough = opts.gpu === true; - const optedOutGpuPassthrough = opts.noGpu === true; - const detectedNvidiaGpu = gpu?.type === "nvidia"; - const resumeHasResolvedGpuIntent = - resume && session?.steps?.preflight?.status === "complete" && !requestedGpuPassthrough; - const gpuPassthrough = optedOutGpuPassthrough - ? false - : requestedGpuPassthrough - ? true - : resumeHasResolvedGpuIntent - ? session?.gpuPassthrough === true - : detectedNvidiaGpu; - if (gpuPassthrough && gpu?.type !== "nvidia") { - console.error(" GPU passthrough requires an NVIDIA GPU detected by nvidia-smi."); - console.error(" Install NVIDIA drivers and the Container Toolkit, or rerun with --no-gpu."); - process.exit(1); - } + const gpuPassthrough = sandboxGpuConfig.sandboxGpuEnabled; if (gpuPassthrough) { note( resumeHasResolvedGpuIntent && session?.gpuPassthrough === true ? " [resume] Continuing GPU passthrough from the saved onboarding session." - : requestedGpuPassthrough + : requestedGpuPassthrough || sandboxGpuConfig.mode === "1" ? " GPU passthrough requested; passing --gpu to OpenShell gateway and sandbox creation." : " NVIDIA GPU detected; enabling OpenShell GPU passthrough. Use --no-gpu to opt out.", ); @@ -10187,10 +11352,11 @@ async function onboard(opts: OnboardOptions = {}): Promise { const gatewaySnapshot = selectNamedGatewayForReuseIfNeeded(getGatewayReuseSnapshot()); let gatewayReuseState = gatewaySnapshot.gatewayReuseState; + gatewayReuseState = await refreshDockerDriverGatewayReuseState(gatewayReuseState); // Verify the gateway container is actually running — openshell CLI metadata // can be stale after a manual `docker rm`. See #2020. - if (gatewayReuseState === "healthy") { + if (gatewayReuseState === "healthy" && !isLinuxDockerDriverGatewayEnabled()) { const containerState = verifyGatewayContainerRunning(); if (containerState === "missing") { console.log(" Gateway metadata is stale (container not running). Cleaning up..."); @@ -10243,9 +11409,11 @@ async function onboard(opts: OnboardOptions = {}): Promise { resume && session?.steps?.gateway?.status === "complete" && canReuseHealthyGateway; if (resumeGateway) { skippedStepMessage("gateway", "running"); + onboardSession.markStepComplete("gateway"); } else if (!resume && canReuseHealthyGateway) { skippedStepMessage("gateway", "running", "reuse"); note(" Reusing healthy NemoClaw gateway."); + onboardSession.markStepComplete("gateway"); } else { if (resume && session?.steps?.gateway?.status === "complete") { if (gatewayReuseState === "active-unnamed") { @@ -10258,6 +11426,12 @@ async function onboard(opts: OnboardOptions = {}): Promise { note(" [resume] Recorded gateway state is unavailable; recreating it."); } } + if (isLinuxDockerDriverGatewayEnabled() && gatewayReuseState !== "missing") { + note(" Replacing legacy OpenShell gateway metadata with Docker-driver gateway."); + runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true }); + destroyGateway(); + registry.clearAll(); + } startRecordedStep("gateway"); await startGateway(gpu, { gpuPassthrough }); onboardSession.markStepComplete("gateway"); @@ -10268,8 +11442,6 @@ async function onboard(opts: OnboardOptions = {}): Promise { // never completed; users supplying `--name` / NEMOCLAW_SANDBOX_NAME on // the resume run must win, otherwise the stale name silently overrides // their explicit recovery input. - const recordedSandboxName = - session?.steps?.sandbox?.status === "complete" ? session?.sandboxName || null : null; let sandboxName = recordedSandboxName || requestedSandboxName || null; if (sandboxName && RESERVED_SANDBOX_NAMES.has(sandboxName)) { console.error( @@ -10488,10 +11660,14 @@ async function onboard(opts: OnboardOptions = {}): Promise { const effectiveCurrent = currentTelegramRequireMention ?? false; const effectiveRecorded = recordedTelegramRequireMention ?? false; const telegramConfigChanged = effectiveCurrent !== effectiveRecorded; + const sandboxGpuConfigChanged = sandboxName + ? hasSandboxGpuDrift(sandboxName, sandboxGpuConfig) + : false; const resumeSandbox = resume && !webSearchConfigChanged && !telegramConfigChanged && + !sandboxGpuConfigChanged && !messagingChannelConfigChanged && session?.steps?.sandbox?.status === "complete" && sandboxReuseState === "ready"; @@ -10513,6 +11689,11 @@ async function onboard(opts: OnboardOptions = {}): Promise { if (sandboxName) { registry.removeSandbox(sandboxName); } + } else if (sandboxGpuConfigChanged) { + note(" [resume] Sandbox GPU settings changed; recreating sandbox."); + if (sandboxName) { + registry.removeSandbox(sandboxName); + } } else if (messagingChannelConfigChanged) { note(" [resume] Messaging channel configuration changed; recreating sandbox."); if (sandboxName) { @@ -10574,7 +11755,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { fromDockerfile, agent, opts.controlUiPort || null, - gpuPassthrough, + sandboxGpuConfig, ); webSearchConfig = nextWebSearchConfig; // Persist model and provider after the sandbox entry exists in the registry. @@ -10799,6 +11980,9 @@ module.exports = { buildCompatibleEndpointSandboxSmokeCommand, buildCompatibleEndpointSandboxSmokeScript, buildSandboxConfigSyncScript, + buildSandboxGpuCreateArgs, + buildDirectGpuPolicyYaml, + buildDirectSandboxGpuProofCommands, compactText, copyBuildContextDir, classifySandboxCreateFailure, @@ -10810,15 +11994,24 @@ module.exports = { getGatewayBootstrapRepairPlan, getGatewayLocalEndpoint, getGatewayStartEnv, + getDockerDriverGatewayEnv, + getDockerDriverGatewayRuntimeDriftFromSnapshot, getGatewayClusterContainerState, getGatewayHealthWaitConfig, getGatewayReuseState, + isDockerDriverGatewayPortListener, handleFinalGatewayStartFailure, getNavigationChoice, getSandboxInferenceConfig, getInstalledOpenshellVersion, getBlueprintMinOpenshellVersion, getBlueprintMaxOpenshellVersion, + isLinuxDockerDriverGatewayEnabled, + findReadableNvidiaCdiSpecFiles, + parseDockerCdiSpecDirs, + getResumeSandboxGpuOverrides, + resolveSandboxGpuConfig, + shouldAllowOpenshellAboveBlueprintMax, pullAndResolveBaseImageDigest, SANDBOX_BASE_IMAGE, SANDBOX_BASE_TAG, diff --git a/src/lib/onboard/legacy-command.test.ts b/src/lib/onboard/legacy-command.test.ts index 5ff0e066bba..d8d37d741fa 100644 --- a/src/lib/onboard/legacy-command.test.ts +++ b/src/lib/onboard/legacy-command.test.ts @@ -41,6 +41,8 @@ describe("onboard command", () => { recreateSandbox: false, fromDockerfile: null, sandboxName: null, + sandboxGpu: null, + sandboxGpuDevice: null, acceptThirdPartySoftware: true, agent: null, controlUiPort: null, @@ -87,6 +89,8 @@ describe("onboard command", () => { recreateSandbox: false, fromDockerfile: null, sandboxName: null, + sandboxGpu: null, + sandboxGpuDevice: null, acceptThirdPartySoftware: true, agent: null, controlUiPort: null, @@ -114,6 +118,8 @@ describe("onboard command", () => { recreateSandbox: false, fromDockerfile: null, sandboxName: null, + sandboxGpu: null, + sandboxGpuDevice: null, acceptThirdPartySoftware: false, agent: null, controlUiPort: null, @@ -170,6 +176,8 @@ describe("onboard command", () => { recreateSandbox: false, fromDockerfile: dockerfilePath, sandboxName: null, + sandboxGpu: null, + sandboxGpuDevice: null, acceptThirdPartySoftware: false, agent: null, controlUiPort: null, @@ -198,6 +206,8 @@ describe("onboard command", () => { recreateSandbox: false, fromDockerfile: null, sandboxName: null, + sandboxGpu: null, + sandboxGpuDevice: null, acceptThirdPartySoftware: false, agent: null, controlUiPort: null, @@ -247,6 +257,8 @@ describe("onboard command", () => { recreateSandbox: false, fromDockerfile: dockerfilePath, sandboxName: "second-assistant", + sandboxGpu: null, + sandboxGpuDevice: null, acceptThirdPartySoftware: false, agent: null, controlUiPort: null, @@ -385,6 +397,8 @@ describe("onboard command", () => { recreateSandbox: false, fromDockerfile: null, sandboxName: null, + sandboxGpu: null, + sandboxGpuDevice: null, acceptThirdPartySoftware: false, agent: "openclaw", controlUiPort: null, @@ -481,6 +495,38 @@ describe("onboard command", () => { expect(result.controlUiPort).toBe(19000); }); + it("parses direct sandbox GPU flags", () => { + const result = parseOnboardArgs( + ["--sandbox-gpu", "--sandbox-gpu-device", "0"], + "--yes-i-accept-third-party-software", + "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE", + { + env: {}, + error: () => {}, + exit: exitWithCode, + }, + ); + expect(result.sandboxGpu).toBe("enable"); + expect(result.sandboxGpuDevice).toBe("0"); + }); + + it("rejects conflicting sandbox GPU flags", () => { + const errors: string[] = []; + expect(() => + parseOnboardArgs( + ["--sandbox-gpu", "--no-sandbox-gpu"], + "--yes-i-accept-third-party-software", + "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE", + { + env: {}, + error: (message = "") => errors.push(message), + exit: exitWithPrefixedCode, + }, + ), + ).toThrow("exit:1"); + expect(errors.join("\n")).toContain("mutually exclusive"); + }); + it("--help includes --control-ui-port in usage", async () => { const lines: string[] = []; await runOnboardCommand({ @@ -496,6 +542,7 @@ describe("onboard command", () => { }) as never, }); expect(lines.join("\n")).toContain("--control-ui-port"); + expect(lines.join("\n")).toContain("--sandbox-gpu"); }); it("prints the setup-spark deprecation text before delegating", async () => { @@ -522,6 +569,8 @@ describe("onboard command", () => { recreateSandbox: false, fromDockerfile: null, sandboxName: null, + sandboxGpu: null, + sandboxGpuDevice: null, acceptThirdPartySoftware: false, agent: null, controlUiPort: null, diff --git a/src/lib/onboard/legacy-command.ts b/src/lib/onboard/legacy-command.ts index 7f21047343a..196fca8cdf8 100644 --- a/src/lib/onboard/legacy-command.ts +++ b/src/lib/onboard/legacy-command.ts @@ -13,6 +13,8 @@ export interface OnboardCommandOptions { recreateSandbox: boolean; fromDockerfile: string | null; sandboxName: string | null; + sandboxGpu: "enable" | "disable" | null; + sandboxGpuDevice: string | null; acceptThirdPartySoftware: boolean; agent: string | null; controlUiPort: number | null; @@ -51,9 +53,12 @@ const ONBOARD_BASE_ARGS = [ function onboardUsageLines(noticeAcceptFlag: string): string[] { const name = CLI_NAME; return [ - ` Usage: ${name} onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--agent ] [--control-ui-port ] [--yes | -y] [${noticeAcceptFlag}]`, + ` Usage: ${name} onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--sandbox-gpu | --no-sandbox-gpu] [--sandbox-gpu-device ] [--agent ] [--control-ui-port ] [--yes | -y] [${noticeAcceptFlag}]`, "", " --from uses the Dockerfile's parent directory as the Docker build context.", + " --gpu enables direct NVIDIA GPU access inside the sandbox; --no-gpu forces CPU sandbox behavior.", + " --sandbox-gpu enables direct NVIDIA GPU access inside the sandbox; --no-sandbox-gpu forces CPU sandbox behavior.", + " --sandbox-gpu-device passes a specific OpenShell GPU device selector to sandbox create.", " Put files referenced by COPY/ADD next to that Dockerfile, or move the Dockerfile into", " a dedicated build directory to avoid sending unrelated files to Docker.", " Common large directories are skipped: node_modules, .git, .venv, __pycache__.", @@ -152,6 +157,41 @@ export function parseOnboardArgs( parsedArgs.splice(portIdx, 2); } + const sandboxGpuFlag = parsedArgs.includes("--sandbox-gpu"); + const noSandboxGpuFlag = parsedArgs.includes("--no-sandbox-gpu"); + if (sandboxGpuFlag && noSandboxGpuFlag) { + error(" --sandbox-gpu and --no-sandbox-gpu are mutually exclusive."); + printOnboardUsage(error, noticeAcceptFlag); + exit(1); + } + let sandboxGpu: "enable" | "disable" | null = null; + if (sandboxGpuFlag) { + sandboxGpu = "enable"; + parsedArgs.splice(parsedArgs.indexOf("--sandbox-gpu"), 1); + } + if (noSandboxGpuFlag) { + sandboxGpu = "disable"; + parsedArgs.splice(parsedArgs.indexOf("--no-sandbox-gpu"), 1); + } + + let sandboxGpuDevice: string | null = null; + const sandboxGpuDeviceIdx = parsedArgs.indexOf("--sandbox-gpu-device"); + if (sandboxGpuDeviceIdx !== -1) { + const deviceValue = parsedArgs[sandboxGpuDeviceIdx + 1]; + if (typeof deviceValue !== "string" || deviceValue.length === 0 || deviceValue.startsWith("--")) { + error(" --sandbox-gpu-device requires a device selector"); + printOnboardUsage(error, noticeAcceptFlag); + exit(1); + } + sandboxGpuDevice = deviceValue; + parsedArgs.splice(sandboxGpuDeviceIdx, 2); + } + if (sandboxGpu === "disable" && sandboxGpuDevice) { + error(" --sandbox-gpu-device cannot be used with --no-sandbox-gpu."); + printOnboardUsage(error, noticeAcceptFlag); + exit(1); + } + const allowedArgs = new Set([...ONBOARD_BASE_ARGS, noticeAcceptFlag]); const unknownArgs = parsedArgs.filter((arg) => !allowedArgs.has(arg)); if (unknownArgs.length > 0) { @@ -174,6 +214,16 @@ export function parseOnboardArgs( printOnboardUsage(error, noticeAcceptFlag); exit(1); } + if ((gpu && sandboxGpu === "disable") || (noGpu && sandboxGpu === "enable")) { + error(" --gpu/--no-gpu conflict with the sandbox GPU flags."); + printOnboardUsage(error, noticeAcceptFlag); + exit(1); + } + if (noGpu && sandboxGpuDevice) { + error(" --sandbox-gpu-device cannot be used with --no-gpu."); + printOnboardUsage(error, noticeAcceptFlag); + exit(1); + } return { nonInteractive: parsedArgs.includes("--non-interactive"), @@ -182,6 +232,8 @@ export function parseOnboardArgs( recreateSandbox: parsedArgs.includes("--recreate-sandbox"), fromDockerfile, sandboxName, + sandboxGpu, + sandboxGpuDevice, acceptThirdPartySoftware: parsedArgs.includes(noticeAcceptFlag) || String(deps.env[noticeAcceptEnv] || "") === "1", agent, diff --git a/src/lib/sandbox-base-image.test.ts b/src/lib/sandbox-base-image.test.ts new file mode 100644 index 00000000000..86756174c04 --- /dev/null +++ b/src/lib/sandbox-base-image.test.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + getSourceShortShaTags, + parseGlibcVersion, + versionGte, +} from "../../dist/lib/sandbox-base-image"; + +describe("sandbox base image helpers", () => { + it("parses glibc versions from ldd output", () => { + expect(parseGlibcVersion("ldd (Debian GLIBC 2.41-12+deb13u2) 2.41")).toBe("2.41"); + expect(parseGlibcVersion("ldd (Ubuntu GLIBC 2.39-0ubuntu8.6) 2.39")).toBe("2.39"); + }); + + it("compares glibc versions numerically", () => { + expect(versionGte("2.41", "2.39")).toBe(true); + expect(versionGte("2.39", "2.39")).toBe(true); + expect(versionGte("2.36", "2.39")).toBe(false); + }); + + it("derives source-sha tags compatible with base-image workflow metadata", () => { + const tags = getSourceShortShaTags("/definitely/not/a/git/repo", { + GITHUB_SHA: "1E94F2E207C5456EBC35E2BD5BB380D4430292C6", + } as NodeJS.ProcessEnv); + expect(tags).toEqual(["1e94f2e2", "1e94f2e"]); + }); +}); diff --git a/src/lib/sandbox-base-image.ts b/src/lib/sandbox-base-image.ts new file mode 100644 index 00000000000..beb524ecc20 --- /dev/null +++ b/src/lib/sandbox-base-image.ts @@ -0,0 +1,252 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import { ROOT } from "./runner"; +import { + dockerBuild, + dockerCapture, + dockerImageInspect, + dockerImageInspectFormat, + dockerPull, +} from "./adapters/docker"; + +export const OPENCLAW_SANDBOX_BASE_IMAGE = "ghcr.io/nvidia/nemoclaw/sandbox-base"; +export const HERMES_SANDBOX_BASE_IMAGE = "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base"; +export const SANDBOX_BASE_TAG = "latest"; +export const OPENSHELL_SANDBOX_MIN_GLIBC = "2.39"; + +type ResolveBaseImageOptions = { + imageName: string; + dockerfilePath: string; + localTag: string; + envVar?: string; + label?: string; + requireOpenshellSandboxAbi?: boolean; + minGlibcVersion?: string; + rootDir?: string; + env?: NodeJS.ProcessEnv; +}; + +export type SandboxBaseImageResolution = { + ref: string; + digest: string | null; + source: "override" | "source-sha" | "latest" | "local"; + glibcVersion: string | null; +}; + +export function parseGlibcVersion(output: string | null | undefined): string | null { + const text = String(output || ""); + const match = text.match(/GLIBC\s+([0-9]+(?:\.[0-9]+)+)/i) || text.match(/\s([0-9]+\.[0-9]+)\s*$/); + return match ? match[1] : null; +} + +export function versionGte(left = "0.0.0", right = "0.0.0"): boolean { + const lhs = String(left) + .split(".") + .map((part) => Number.parseInt(part, 10) || 0); + const rhs = String(right) + .split(".") + .map((part) => Number.parseInt(part, 10) || 0); + const length = Math.max(lhs.length, rhs.length); + for (let index = 0; index < length; index += 1) { + const a = lhs[index] || 0; + const b = rhs[index] || 0; + if (a > b) return true; + if (a < b) return false; + } + return true; +} + +export function getImageGlibcVersion(imageRef: string): string | null { + const output = dockerCapture( + ["run", "--rm", "--entrypoint", "/usr/bin/ldd", imageRef, "--version"], + { ignoreError: true, timeout: 20_000 }, + ); + return parseGlibcVersion(output); +} + +export function imageMeetsMinimumGlibc(imageRef: string, minVersion = OPENSHELL_SANDBOX_MIN_GLIBC): { + ok: boolean; + version: string | null; +} { + const version = getImageGlibcVersion(imageRef); + return { ok: !!version && versionGte(version, minVersion), version }; +} + +export function getSourceShortShaTags(rootDir = ROOT, env: NodeJS.ProcessEnv = process.env): string[] { + const values: string[] = []; + const push = (value: string | null | undefined) => { + const normalized = String(value || "").trim().toLowerCase(); + if (!/^[0-9a-f]{7,40}$/.test(normalized)) return; + values.push(normalized.slice(0, 8), normalized.slice(0, 7)); + }; + + push(env.GITHUB_SHA); + const git = spawnSync("git", ["-C", rootDir, "rev-parse", "HEAD"], { + encoding: "utf-8", + stdio: ["ignore", "pipe", "ignore"], + timeout: 5_000, + }); + if (git.status === 0) push(git.stdout); + + return Array.from(new Set(values)); +} + +function localBuildAllowed(env: NodeJS.ProcessEnv = process.env): boolean { + const raw = String(env.NEMOCLAW_SANDBOX_BASE_LOCAL_BUILD || "auto") + .trim() + .toLowerCase(); + if (["0", "false", "no", "off"].includes(raw)) return false; + if (["1", "true", "yes", "on"].includes(raw)) return true; + return env.NODE_ENV !== "test" && env.VITEST !== "true"; +} + +function getRepoDigest(imageName: string, imageRef: string): { digest: string; ref: string } | null { + const atIndex = imageRef.indexOf("@sha256:"); + if (atIndex !== -1) { + const digest = imageRef.slice(atIndex + 1); + return { digest, ref: imageRef }; + } + + const inspectOutput = dockerImageInspectFormat("{{json .RepoDigests}}", imageRef, { + ignoreError: true, + }); + if (!inspectOutput) return null; + + let repoDigests: unknown; + try { + repoDigests = JSON.parse(inspectOutput || "[]"); + } catch { + return null; + } + const repoDigest = Array.isArray(repoDigests) + ? repoDigests.find((entry) => String(entry).startsWith(`${imageName}@sha256:`)) + : null; + if (!repoDigest) return null; + const digest = String(repoDigest).slice(String(repoDigest).indexOf("@") + 1); + return { digest, ref: `${imageName}@${digest}` }; +} + +function resolvePulledCandidate( + imageName: string, + imageRef: string, + source: SandboxBaseImageResolution["source"], + options: ResolveBaseImageOptions, +): SandboxBaseImageResolution | null { + const inspectResult = dockerImageInspect(imageRef, { + ignoreError: true, + suppressOutput: true, + }); + if (inspectResult.status !== 0) { + const pullResult = dockerPull(imageRef, { ignoreError: true, suppressOutput: true }); + if (pullResult.status !== 0) return null; + } + + let glibcVersion: string | null = null; + if (options.requireOpenshellSandboxAbi) { + const check = imageMeetsMinimumGlibc( + imageRef, + options.minGlibcVersion || OPENSHELL_SANDBOX_MIN_GLIBC, + ); + glibcVersion = check.version; + if (!check.ok) { + console.warn( + ` Warning: ${options.label || "sandbox base image"} ${imageRef} has glibc ` + + `${glibcVersion || "unknown"}; OpenShell sandbox supervisor requires ` + + `glibc >= ${options.minGlibcVersion || OPENSHELL_SANDBOX_MIN_GLIBC}.`, + ); + return null; + } + } + + const repoDigest = getRepoDigest(imageName, imageRef); + return { + ref: repoDigest?.ref || imageRef, + digest: repoDigest?.digest || null, + source, + glibcVersion, + }; +} + +function resolveLocalCandidate( + options: ResolveBaseImageOptions, +): SandboxBaseImageResolution | null { + const imageRef = options.localTag; + const inspectResult = dockerImageInspect(imageRef, { ignoreError: true, suppressOutput: true }); + if (inspectResult.status === 0) { + const check = options.requireOpenshellSandboxAbi + ? imageMeetsMinimumGlibc(imageRef, options.minGlibcVersion || OPENSHELL_SANDBOX_MIN_GLIBC) + : { ok: true, version: null }; + if (check.ok) { + return { ref: imageRef, digest: null, source: "local", glibcVersion: check.version }; + } + } + + if (!localBuildAllowed(options.env)) return null; + + console.warn( + ` Building ${options.label || "sandbox base image"} locally because no compatible ` + + `published base image was found.`, + ); + dockerBuild(options.dockerfilePath, imageRef, options.rootDir || ROOT, { + stdio: ["ignore", "inherit", "inherit"], + }); + + const check = options.requireOpenshellSandboxAbi + ? imageMeetsMinimumGlibc(imageRef, options.minGlibcVersion || OPENSHELL_SANDBOX_MIN_GLIBC) + : { ok: true, version: null }; + if (!check.ok) { + console.error( + ` Local ${options.label || "sandbox base image"} ${imageRef} has glibc ` + + `${check.version || "unknown"}; expected >= ` + + `${options.minGlibcVersion || OPENSHELL_SANDBOX_MIN_GLIBC}.`, + ); + return null; + } + + return { ref: imageRef, digest: null, source: "local", glibcVersion: check.version }; +} + +export function resolveSandboxBaseImage( + options: ResolveBaseImageOptions, +): SandboxBaseImageResolution | null { + const env = options.env || process.env; + const override = options.envVar ? String(env[options.envVar] || "").trim() : ""; + + if (override) { + const resolved = resolvePulledCandidate(options.imageName, override, "override", options); + if (resolved) return resolved; + if (!options.requireOpenshellSandboxAbi) return null; + } else { + for (const tag of getSourceShortShaTags(options.rootDir || ROOT, env)) { + const imageRef = `${options.imageName}:${tag}`; + const resolved = resolvePulledCandidate(options.imageName, imageRef, "source-sha", options); + if (resolved) return resolved; + } + + const latestRef = `${options.imageName}:${SANDBOX_BASE_TAG}`; + const resolved = resolvePulledCandidate(options.imageName, latestRef, "latest", options); + if (resolved) return resolved; + } + + if (options.requireOpenshellSandboxAbi) { + return resolveLocalCandidate(options); + } + return null; +} + +export function buildLocalBaseTag(prefix: string, rootDir = ROOT, env = process.env): string { + const tag = getSourceShortShaTags(rootDir, env)[0] || "local"; + return `${prefix}:${tag}`; +} + +export function defaultOpenclawBaseDockerfile(rootDir = ROOT): string { + return path.join(rootDir, "Dockerfile.base"); +} + +export function defaultHermesBaseDockerfile(rootDir = ROOT): string { + return path.join(rootDir, "agents", "hermes", "Dockerfile.base"); +} diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index cc95b5d18e1..1fd11e7b9f9 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -15,6 +15,10 @@ const path = require("path"); const { fork } = require("child_process"); const { run, runCapture, validateName, shellQuote } = require("../runner"); const { dockerExecFileSync } = require("../adapters/docker/exec"); +const { dockerCapture } = require("../adapters/docker/run"); +const registry = require("../state/registry") as { + getSandbox?: (name: string) => { openshellDriver?: string | null } | null; +}; const { buildPolicyGetCommand, buildPolicySetCommand, @@ -28,16 +32,37 @@ const { resolveAgentConfig } = require("../sandbox-config"); const STATE_DIR = path.join(process.env.HOME ?? "/tmp", ".nemoclaw", "state"); // --------------------------------------------------------------------------- -// kubectl exec — bypasses the sandbox's Landlock context +// privileged sandbox exec — bypasses the sandbox's Landlock context // // openshell sandbox exec runs commands INSIDE the Landlock domain, so it // can't modify read_only paths or change chattr flags. kubectl exec starts // a new process in the pod that does NOT inherit the Landlock ruleset. -// We reach kubectl via the K3s container: docker exec kubectl exec ... +// On the legacy gateway we reach kubectl via the K3s container. On the +// Docker-driver gateway there is no K3s container, so we exec into the +// sandbox Docker container directly as root. // --------------------------------------------------------------------------- const K3S_CONTAINER = "openshell-cluster-nemoclaw"; +function resolveDockerDriverSandboxContainer(sandboxName: string): string | null { + try { + if (registry.getSandbox?.(sandboxName)?.openshellDriver !== "docker") { + return null; + } + } catch { + return null; + } + const prefix = `openshell-${sandboxName}-`; + const exact = `openshell-${sandboxName}`; + const output = dockerCapture(["ps", "--format", "{{.Names}}"], { ignoreError: true }); + return ( + output + .split("\n") + .map((line: string) => line.trim()) + .find((name: string) => name === exact || name.startsWith(prefix)) || null + ); +} + function kubectlExecArgv(sandboxName: string, cmd: string[]): string[] { return [ "exec", @@ -54,15 +79,23 @@ function kubectlExecArgv(sandboxName: string, cmd: string[]): string[] { ]; } -function kubectlExec(sandboxName: string, cmd: string[]): void { - dockerExecFileSync(kubectlExecArgv(sandboxName, cmd), { +function privilegedSandboxExecArgv(sandboxName: string, cmd: string[]): string[] { + const dockerDriverContainer = resolveDockerDriverSandboxContainer(sandboxName); + if (dockerDriverContainer) { + return ["exec", "--user", "root", dockerDriverContainer, ...cmd]; + } + return kubectlExecArgv(sandboxName, cmd); +} + +function privilegedSandboxExec(sandboxName: string, cmd: string[]): void { + dockerExecFileSync(privilegedSandboxExecArgv(sandboxName, cmd), { stdio: ["ignore", "pipe", "pipe"], timeout: 15000, }); } -function kubectlExecCapture(sandboxName: string, cmd: string[]): string { - return dockerExecFileSync(kubectlExecArgv(sandboxName, cmd), { +function privilegedSandboxExecCapture(sandboxName: string, cmd: string[]): string { + return dockerExecFileSync(privilegedSandboxExecArgv(sandboxName, cmd), { stdio: ["ignore", "pipe", "pipe"], timeout: 15000, }).trim(); @@ -272,24 +305,24 @@ function applyStateDirLockMode(sandboxName: string, configDir: string, owner: st for (const dirName of HIGH_RISK_STATE_DIRS) { const dirPath = `${configDir}/${dirName}`; try { - kubectlExec(sandboxName, ["chown", "-R", owner, dirPath]); + privilegedSandboxExec(sandboxName, ["chown", "-R", owner, dirPath]); } catch { // Directory may not exist for this agent — silently skip } try { - kubectlExec(sandboxName, ["chmod", dirMode, dirPath]); + privilegedSandboxExec(sandboxName, ["chmod", dirMode, dirPath]); } catch { // Silently skip } if (isLocking) { try { - kubectlExec(sandboxName, ["chmod", "g-s", dirPath]); + privilegedSandboxExec(sandboxName, ["chmod", "g-s", dirPath]); } catch { // Best effort; do not skip recursive write stripping. } } try { - kubectlExec(sandboxName, ["chmod", "-R", recursiveMode, dirPath]); + privilegedSandboxExec(sandboxName, ["chmod", "-R", recursiveMode, dirPath]); } catch { // Silently skip } @@ -299,7 +332,7 @@ function applyStateDirLockMode(sandboxName: string, configDir: string, owner: st // discovered dynamically because they are configured by openclaw.json. const clearSetgid = isLocking ? "1" : "0"; try { - kubectlExec(sandboxName, [ + privilegedSandboxExec(sandboxName, [ "sh", "-c", ` @@ -338,7 +371,7 @@ function assertNoLegacyStateLayout(sandboxName: string, configDir: string): void const script = 'set -u; config_dir="$1"; data_dir="$2"; data_real="$(readlink -f "$data_dir" 2>/dev/null || printf "%s" "$data_dir")"; if [ -e "$data_dir" ] || [ -L "$data_dir" ]; then echo "legacy data dir exists: $data_dir"; exit 1; fi; for entry in "$config_dir"/*; do [ -L "$entry" ] || continue; target="$(readlink -f "$entry" 2>/dev/null || readlink "$entry" 2>/dev/null || true)"; case "$target" in "$data_real"/*|"$data_dir"/*) echo "legacy symlink remains: $entry -> $target"; exit 1;; esac; done'; try { - kubectlExecCapture(sandboxName, ["sh", "-c", script, "sh", configDir, dataDir]); + privilegedSandboxExecCapture(sandboxName, ["sh", "-c", script, "sh", configDir, dataDir]); } catch (err) { const execErr = err as { stdout?: Buffer | string; stderr?: Buffer | string; message?: string }; const captured = [execErr.stdout, execErr.stderr] @@ -381,28 +414,28 @@ function unlockAgentConfig( const dirMode = target.agentName === "hermes" ? "750" : "2770"; for (const f of filesToUnlock) { try { - kubectlExec(sandboxName, ["chattr", "-i", f]); + privilegedSandboxExec(sandboxName, ["chattr", "-i", f]); } catch { errors.push(`chattr -i ${f}`); } try { - kubectlExec(sandboxName, ["chown", "sandbox:sandbox", f]); + privilegedSandboxExec(sandboxName, ["chown", "sandbox:sandbox", f]); } catch { errors.push(`chown ${f}`); } try { - kubectlExec(sandboxName, ["chmod", fileMode, f]); + privilegedSandboxExec(sandboxName, ["chmod", fileMode, f]); } catch { errors.push(`chmod ${fileMode} ${f}`); } } try { - kubectlExec(sandboxName, ["chown", "sandbox:sandbox", target.configDir]); + privilegedSandboxExec(sandboxName, ["chown", "sandbox:sandbox", target.configDir]); } catch { errors.push("chown config dir"); } try { - kubectlExec(sandboxName, ["chmod", dirMode, target.configDir]); + privilegedSandboxExec(sandboxName, ["chmod", dirMode, target.configDir]); } catch { errors.push(`chmod ${dirMode} config dir`); } @@ -421,7 +454,7 @@ function unlockAgentConfig( const issues: string[] = []; for (const f of filesToUnlock) { try { - const perms = kubectlExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", f]); + const perms = privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", f]); const [mode, owner] = perms.split(" "); if (mode !== fileMode) issues.push(`${f} mode=${mode} (expected ${fileMode})`); if (owner !== "sandbox:sandbox") issues.push(`${f} owner=${owner} (expected sandbox:sandbox)`); @@ -430,7 +463,7 @@ function unlockAgentConfig( issues.push(`${f} stat failed: ${msg}`); } try { - const attrs = kubectlExecCapture(sandboxName, ["lsattr", "-d", f]); + const attrs = privilegedSandboxExecCapture(sandboxName, ["lsattr", "-d", f]); const [flags] = attrs.trim().split(/\s+/, 1); if (flags.includes("i")) issues.push(`${f} immutable bit still set`); } catch { @@ -439,7 +472,7 @@ function unlockAgentConfig( } try { - const dirPerms = kubectlExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", target.configDir]); + const dirPerms = privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", target.configDir]); const [mode, owner] = dirPerms.split(" "); if (mode !== dirMode) issues.push(`config dir mode=${mode} (expected ${dirMode})`); if (owner !== "sandbox:sandbox") { @@ -482,25 +515,25 @@ function lockAgentConfig( for (const f of filesToLock) { try { - kubectlExec(sandboxName, ["chmod", "444", f]); + privilegedSandboxExec(sandboxName, ["chmod", "444", f]); } catch { errors.push(`chmod 444 ${f}`); } try { - kubectlExec(sandboxName, ["chown", "root:root", f]); + privilegedSandboxExec(sandboxName, ["chown", "root:root", f]); } catch { errors.push(`chown root:root ${f}`); } } try { - kubectlExec(sandboxName, ["chmod", "755", target.configDir]); + privilegedSandboxExec(sandboxName, ["chmod", "755", target.configDir]); } catch { errors.push("chmod 755 config dir"); } try { - kubectlExec(sandboxName, ["chown", "root:root", target.configDir]); + privilegedSandboxExec(sandboxName, ["chown", "root:root", target.configDir]); } catch { errors.push("chown root:root config dir"); } @@ -510,7 +543,7 @@ function lockAgentConfig( let chattrSucceeded = true; for (const f of filesToLock) { try { - kubectlExec(sandboxName, ["chattr", "+i", f]); + privilegedSandboxExec(sandboxName, ["chattr", "+i", f]); } catch { chattrSucceeded = false; } @@ -525,12 +558,12 @@ function lockAgentConfig( // after descendant locking so shields-up verifies the root config dir as // plain 755, not 2755. try { - kubectlExec(sandboxName, ["chmod", "g-s", target.configDir]); + privilegedSandboxExec(sandboxName, ["chmod", "g-s", target.configDir]); } catch { errors.push("chmod g-s config dir"); } try { - kubectlExec(sandboxName, ["chmod", "755", target.configDir]); + privilegedSandboxExec(sandboxName, ["chmod", "755", target.configDir]); } catch { errors.push("chmod 755 config dir"); } @@ -545,7 +578,7 @@ function lockAgentConfig( const issues: string[] = []; for (const f of filesToLock) { try { - const perms = kubectlExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", f]); + const perms = privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", f]); const [mode, owner] = perms.split(" "); if (!/^4[0-4][0-4]$/.test(mode)) issues.push(`${f} mode=${mode} (expected 444)`); if (owner !== "root:root") issues.push(`${f} owner=${owner} (expected root:root)`); @@ -556,7 +589,7 @@ function lockAgentConfig( } try { - const dirPerms = kubectlExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", target.configDir]); + const dirPerms = privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", target.configDir]); const [dirMode, dirOwner] = dirPerms.split(" "); if (dirMode !== "755") issues.push(`dir mode=${dirMode} (expected 755)`); if (dirOwner !== "root:root") issues.push(`dir owner=${dirOwner} (expected root:root)`); @@ -568,7 +601,7 @@ function lockAgentConfig( if (chattrSucceeded) { for (const f of filesToLock) { try { - const attrs = kubectlExecCapture(sandboxName, ["lsattr", "-d", f]); + const attrs = privilegedSandboxExecCapture(sandboxName, ["lsattr", "-d", f]); // lsattr format: "----i---------e----- /path/to/file" // First whitespace-delimited token is the flags field. const [flags] = attrs.trim().split(/\s+/, 1); diff --git a/src/lib/state/gateway.ts b/src/lib/state/gateway.ts index 80810fc9453..908f9458fad 100644 --- a/src/lib/state/gateway.ts +++ b/src/lib/state/gateway.ts @@ -170,5 +170,8 @@ export function getSandboxStateFromOutputs( ): SandboxState { if (!sandboxName) return "missing"; if (!getOutput) return "missing"; + if (/\bNotFound\b|\bNot Found\b|sandbox not found/i.test(stripAnsi(getOutput))) { + return "missing"; + } return isSandboxReady(listOutput, sandboxName) ? "ready" : "not_ready"; } diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index b29e0b9acd8..5e816e96c9b 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -22,6 +22,12 @@ export interface SandboxEntry { nimContainer?: string | null; provider?: string | null; gpuEnabled?: boolean; + hostGpuDetected?: boolean; + sandboxGpuEnabled?: boolean; + sandboxGpuMode?: "auto" | "1" | "0" | string | null; + sandboxGpuDevice?: string | null; + openshellDriver?: string | null; + openshellVersion?: string | null; policies?: string[]; customPolicies?: CustomPolicyEntry[]; policyTier?: string | null; @@ -191,6 +197,12 @@ export function registerSandbox(entry: SandboxEntry): void { nimContainer: entry.nimContainer || null, provider: entry.provider || null, gpuEnabled: entry.gpuEnabled || false, + hostGpuDetected: entry.hostGpuDetected === true, + sandboxGpuEnabled: entry.sandboxGpuEnabled === true, + sandboxGpuMode: entry.sandboxGpuMode || null, + sandboxGpuDevice: entry.sandboxGpuDevice || null, + openshellDriver: entry.openshellDriver || null, + openshellVersion: entry.openshellVersion || null, policies: entry.policies || [], policyTier: entry.policyTier || null, agent: entry.agent || null, diff --git a/test/cli.test.ts b/test/cli.test.ts index a18f2a428f5..94fac49bd27 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -420,13 +420,18 @@ describe("CLI dispatch", () => { { mode: 0o755 }, ); - const r = runWithEnv("liost", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - expect(r.code).toBe(1); - expect(r.out).toContain("Unknown command: liost"); - expect(r.out).toContain("Did you mean: nemoclaw list?"); + try { + const r = runWithEnv("liost", { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + NEMOCLAW_HEALTH_POLL_COUNT: "0", + }); + expect(r.code).toBe(1); + expect(r.out).toContain("Unknown command: liost"); + expect(r.out).toContain("Did you mean: nemoclaw list?"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } }); it("recovers a live sandbox before suggesting a bare command typo", () => { @@ -593,6 +598,12 @@ describe("CLI dispatch", () => { isDefault: true, activeSessionCount: 1, connected: true, + hostGpuDetected: false, + sandboxGpuEnabled: true, + sandboxGpuMode: null, + sandboxGpuDevice: null, + openshellDriver: null, + openshellVersion: null, }, ], }); @@ -932,6 +943,8 @@ describe("CLI dispatch", () => { expect(r.out).toContain("USAGE"); expect(r.out).toContain("nemoclaw onboard"); expect(r.out).toContain("--from "); + expect(r.out).toContain("--yes"); + expect(r.out).toContain("--sandbox-gpu-device="); }); it("unknown onboard option exits 1", () => { @@ -952,6 +965,32 @@ describe("CLI dispatch", () => { expect(r.out).toContain("Nonexistent flag: --non-interactiv"); }); + it("accepts install automation --yes in onboard CLI parsing", () => { + const r = run("onboard --resume --non-interactive --yes-i-accept-third-party-software --yes"); + expect(r.code).toBe(1); + expect(r.out.includes("No resumable onboarding session was found")).toBeTruthy(); + expect(r.out).not.toContain("Nonexistent flag: --yes"); + }); + + it("passes onboard sandbox GPU flags to legacy validation", () => { + const r = run( + "onboard --sandbox-gpu --no-sandbox-gpu --non-interactive --yes-i-accept-third-party-software --yes", + ); + expect(r.code).toBe(1); + expect(r.out).toContain("--sandbox-gpu and --no-sandbox-gpu are mutually exclusive"); + expect(r.out).not.toContain("Nonexistent flag: --sandbox-gpu"); + expect(r.out).not.toContain("Nonexistent flag: --no-sandbox-gpu"); + }); + + it("passes onboard sandbox GPU device flags to legacy validation", () => { + const r = run( + "onboard --sandbox-gpu-device nvidia.com/gpu=0 --no-sandbox-gpu --non-interactive --yes-i-accept-third-party-software --yes", + ); + expect(r.code).toBe(1); + expect(r.out).toContain("--sandbox-gpu-device cannot be used with --no-sandbox-gpu"); + expect(r.out).not.toContain("Nonexistent flag: --sandbox-gpu-device"); + }); + it("setup --help exits 0 and shows onboard usage", () => { const r = run("setup --help"); expect(r.code).toBe(0); @@ -967,14 +1006,14 @@ describe("CLI dispatch", () => { }); it("setup forwards --resume into onboard parsing", () => { - const r = run("setup --resume --non-interactive --yes-i-accept-third-party-software"); + const r = run("setup --resume --non-interactive --yes-i-accept-third-party-software --yes"); expect(r.code).toBe(1); expect(r.out.includes("deprecated")).toBeTruthy(); expect(r.out.includes("No resumable onboarding session was found")).toBeTruthy(); }); it("resume rejection clarifies --resume semantics and points to onboard (#2281)", () => { - const r = run("onboard --resume --non-interactive --yes-i-accept-third-party-software"); + const r = run("onboard --resume --non-interactive --yes-i-accept-third-party-software --yes"); expect(r.code).toBe(1); expect(r.out.includes("No resumable onboarding session was found")).toBeTruthy(); expect(r.out.includes("--resume only continues an interrupted onboarding run")).toBeTruthy(); @@ -996,7 +1035,7 @@ describe("CLI dispatch", () => { path.join(localBin, "openshell"), [ "#!/usr/bin/env bash", - 'if [ "$1" = "--version" ]; then echo "openshell 0.0.36"; exit 0; fi', + 'if [ "$1" = "--version" ]; then echo "openshell 0.0.37"; exit 0; fi', "exit 0", ].join("\n"), { mode: 0o755 }, @@ -1072,7 +1111,7 @@ describe("CLI dispatch", () => { path.join(localBin, "openshell"), [ "#!/usr/bin/env bash", - 'if [ "$1" = "--version" ]; then echo "openshell 0.0.36"; exit 0; fi', + 'if [ "$1" = "--version" ]; then echo "openshell 0.0.37"; exit 0; fi', "exit 0", ].join("\n"), { mode: 0o755 }, @@ -1142,7 +1181,9 @@ describe("CLI dispatch", () => { }); it("setup-spark is a deprecated compatibility alias for onboard", () => { - const r = run("setup-spark --resume --non-interactive --yes-i-accept-third-party-software"); + const r = run( + "setup-spark --resume --non-interactive --yes-i-accept-third-party-software --yes", + ); expect(r.code).toBe(1); expect(r.out.includes("setup-spark` is deprecated")).toBeTruthy(); expect(r.out.includes("Use `nemoclaw onboard` instead")).toBeTruthy(); @@ -2088,6 +2129,7 @@ describe("CLI dispatch", () => { // `nemoclaw onboard` reuses it. expect(openshellOutput).not.toContain("forward stop 18789"); expect(openshellOutput).not.toContain("gateway destroy -g nemoclaw"); + expect(openshellOutput).not.toContain("gateway remove nemoclaw"); expect(fs.readFileSync(bashLog, "utf8")).not.toContain("volume ls -q --filter"); }); @@ -2149,7 +2191,9 @@ describe("CLI dispatch", () => { const openshellOutput = fs.readFileSync(openshellLog, "utf8"); expect(openshellOutput).toContain("sandbox delete alpha"); expect(openshellOutput).toContain("forward stop 18789"); - expect(openshellOutput).toContain("gateway destroy -g nemoclaw"); + expect(openshellOutput).toContain( + process.platform === "linux" ? "gateway remove nemoclaw" : "gateway destroy -g nemoclaw", + ); expect(fs.readFileSync(bashLog, "utf8")).toContain("volume ls -q --filter"); }); @@ -2211,7 +2255,9 @@ describe("CLI dispatch", () => { expect(r.code).toBe(0); const openshellOutput = fs.readFileSync(openshellLog, "utf8"); expect(openshellOutput).toContain("forward stop 18789"); - expect(openshellOutput).toContain("gateway destroy -g nemoclaw"); + expect(openshellOutput).toContain( + process.platform === "linux" ? "gateway remove nemoclaw" : "gateway destroy -g nemoclaw", + ); }); it("keeps the gateway runtime when other sandboxes still exist", () => { @@ -2280,6 +2326,7 @@ describe("CLI dispatch", () => { expect(fs.readFileSync(openshellLog, "utf8")).toContain("sandbox delete alpha"); expect(fs.readFileSync(openshellLog, "utf8")).not.toContain("forward stop 18789"); expect(fs.readFileSync(openshellLog, "utf8")).not.toContain("gateway destroy -g nemoclaw"); + expect(fs.readFileSync(openshellLog, "utf8")).not.toContain("gateway remove nemoclaw"); if (fs.existsSync(bashLog)) { expect(fs.readFileSync(bashLog, "utf8")).not.toContain("volume ls -q --filter"); } @@ -2345,6 +2392,7 @@ describe("CLI dispatch", () => { expect(fs.readFileSync(openshellLog, "utf8")).toContain("beta Ready"); expect(fs.readFileSync(openshellLog, "utf8")).not.toContain("forward stop 18789"); expect(fs.readFileSync(openshellLog, "utf8")).not.toContain("gateway destroy -g nemoclaw"); + expect(fs.readFileSync(openshellLog, "utf8")).not.toContain("gateway remove nemoclaw"); if (fs.existsSync(bashLog)) { expect(fs.readFileSync(bashLog, "utf8")).not.toContain("volume ls -q --filter"); } @@ -2404,6 +2452,7 @@ describe("CLI dispatch", () => { expect(registryAfter.sandboxes.alpha).toBeTruthy(); expect(fs.readFileSync(openshellLog, "utf8")).toContain("sandbox delete alpha"); expect(fs.readFileSync(openshellLog, "utf8")).not.toContain("gateway destroy -g nemoclaw"); + expect(fs.readFileSync(openshellLog, "utf8")).not.toContain("gateway remove nemoclaw"); }); it("treats an already-missing sandbox as destroyed and clears the stale registry entry", () => { @@ -4905,11 +4954,11 @@ describe("list shows live gateway inference", () => { expect(r.code).toBe(0); // Live gateway values render on the default sandbox's main row. expect(r.out).toContain( - "agent: openclaw model: nvidia/nemotron-3-super-120b-a12b provider: nvidia-prod GPU policies: pypi, npm", + "agent: openclaw model: nvidia/nemotron-3-super-120b-a12b provider: nvidia-prod sandbox GPU policies: pypi, npm", ); // The stale (stored) row must not appear. expect(r.out).not.toContain( - "agent: openclaw model: configured-model provider: configured-provider GPU policies: pypi, npm", + "agent: openclaw model: configured-model provider: configured-provider sandbox GPU policies: pypi, npm", ); // Onboarded values appear in the drift annotation. expect(r.out).toContain("(onboarded: model=configured-model, provider=configured-provider)"); diff --git a/test/e2e/test-double-onboard.sh b/test/e2e/test-double-onboard.sh index ef8dfb40b82..7608b8d3bf4 100755 --- a/test/e2e/test-double-onboard.sh +++ b/test/e2e/test-double-onboard.sh @@ -75,8 +75,85 @@ registry_has() { [ -f "$REGISTRY" ] && grep -q "$sandbox_name" "$REGISTRY" } +wait_openshell_sandbox_absent() { + local sandbox_name="$1" + local timeout="${2:-60}" + local deadline=$((SECONDS + timeout)) + local output status + + while [ "$SECONDS" -le "$deadline" ]; do + output="$(openshell sandbox get "$sandbox_name" 2>&1)" + status=$? + if [ "$status" -ne 0 ] && grep -qiE 'NotFound|Not Found|sandbox not found' <<<"$output"; then + return 0 + fi + sleep 1 + done + + info "OpenShell still reports sandbox '$sandbox_name' after ${timeout}s:" + printf '%s\n' "$output" | sed 's/^/ /' + return 1 +} + +docker_driver_gateway_pid_file() { + printf '%s/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.pid\n' "$HOME" +} + +gateway_runtime_id() { + local pid_file pid cid + pid_file="$(docker_driver_gateway_pid_file)" + if [ -f "$pid_file" ]; then + pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + printf 'pid:%s\n' "$pid" + return 0 + fi + fi + + cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)" + if [ -n "$cid" ]; then + printf 'container:%s\n' "$cid" + return 0 + fi + + return 1 +} + +gateway_alias_endpoint() { + local scheme="https" + if [ "$(uname -s)" = "Linux" ]; then + scheme="http" + fi + printf '%s://127.0.0.1:%s\n' "$scheme" "${NEMOCLAW_GATEWAY_PORT:-8080}" +} + +stop_gateway_runtime() { + local pid_file pid cid + openshell forward stop 18789 2>/dev/null || true + openshell gateway stop -g nemoclaw 2>/dev/null || true + + pid_file="$(docker_driver_gateway_pid_file)" + if [ -f "$pid_file" ]; then + pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + for _ in $(seq 1 10); do + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + kill -0 "$pid" 2>/dev/null && kill -9 "$pid" 2>/dev/null || true + fi + fi + + cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)" + if [ -n "$cid" ]; then + docker stop "$cid" >/dev/null 2>&1 || true + fi +} + SANDBOX_A="e2e-double-a" SANDBOX_B="e2e-double-b" +INSTALL_SANDBOX_NAME="${NEMOCLAW_E2E_INSTALL_SANDBOX_NAME:-}" ALT_GATEWAY_NAME="e2e-double-alt" REGISTRY="$HOME/.nemoclaw/sandboxes.json" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" @@ -286,12 +363,18 @@ PY section "Phase 0: Pre-cleanup" info "Destroying any leftover test sandboxes/gateway from previous runs..." if [ -x "$REPO_ROOT/bin/nemoclaw.js" ] || command -v nemoclaw >/dev/null 2>&1; then + if [ -n "$INSTALL_SANDBOX_NAME" ]; then + run_nemoclaw "$INSTALL_SANDBOX_NAME" destroy --yes 2>/dev/null || true + fi run_nemoclaw "$SANDBOX_A" destroy --yes 2>/dev/null || true run_nemoclaw "$SANDBOX_B" destroy --yes 2>/dev/null || true fi +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + openshell sandbox delete "$INSTALL_SANDBOX_NAME" 2>/dev/null || true +fi openshell sandbox delete "$SANDBOX_A" 2>/dev/null || true openshell sandbox delete "$SANDBOX_B" 2>/dev/null || true -openshell forward stop 18789 2>/dev/null || true +stop_gateway_runtime openshell gateway destroy -g nemoclaw 2>/dev/null || true openshell gateway destroy -g "$ALT_GATEWAY_NAME" 2>/dev/null || true pass "Pre-cleanup complete" @@ -390,7 +473,7 @@ fi section "Phase 3: Second onboard ($SANDBOX_A — same name, recreate)" info "Running nemoclaw onboard with NEMOCLAW_RECREATE_SANDBOX=1..." -GATEWAY_ID_BEFORE=$(docker ps -qf "name=openshell-cluster-nemoclaw" | head -1) +GATEWAY_ID_BEFORE=$(gateway_runtime_id || true) PHASE3_START="$(phase_start_time)" run_onboard "$SANDBOX_A" "1" output2="$RUN_ONBOARD_OUTPUT" @@ -407,11 +490,11 @@ else dump_diagnostics "Phase 3" fi -GATEWAY_ID_AFTER=$(docker ps -qf "name=openshell-cluster-nemoclaw" | head -1) +GATEWAY_ID_AFTER=$(gateway_runtime_id || true) if [ -n "$GATEWAY_ID_BEFORE" ] && [ "$GATEWAY_ID_BEFORE" = "$GATEWAY_ID_AFTER" ]; then - pass "Healthy gateway reused on second onboard (container $GATEWAY_ID_BEFORE)" + pass "Healthy gateway runtime reused on second onboard ($GATEWAY_ID_BEFORE)" else - fail "Gateway container changed on second onboard (before=$GATEWAY_ID_BEFORE after=$GATEWAY_ID_AFTER)" + fail "Gateway runtime changed on second onboard (before=$GATEWAY_ID_BEFORE after=$GATEWAY_ID_AFTER)" fi if grep -q "Port 8080 is not available" <<<"$output2"; then @@ -438,8 +521,8 @@ fi section "Phase 4: Third onboard ($SANDBOX_B — different name)" info "Running nemoclaw onboard with new sandbox name..." -ALT_GATEWAY_ENDPOINT="https://127.0.0.1:${NEMOCLAW_GATEWAY_PORT:-8080}" -openshell gateway add --local --name "$ALT_GATEWAY_NAME" "$ALT_GATEWAY_ENDPOINT" >/dev/null 2>&1 || true +ALT_GATEWAY_ENDPOINT="$(gateway_alias_endpoint)" +alt_gateway_add_output="$(openshell gateway add --local --name "$ALT_GATEWAY_NAME" "$ALT_GATEWAY_ENDPOINT" 2>&1 || true)" if openshell gateway select "$ALT_GATEWAY_NAME" >/dev/null 2>&1; then selected_gateway_output="$( openshell status 2>&1 || true @@ -452,10 +535,10 @@ if openshell gateway select "$ALT_GATEWAY_NAME" >/dev/null 2>&1; then fail "Alternate gateway alias was not selected before third onboard (selected=${selected_gateway:-unknown})" fi else - fail "Could not select alternate gateway alias before third onboard" + fail "Could not select alternate gateway alias before third onboard (add output=${alt_gateway_add_output:-empty})" fi -GATEWAY_ID_BEFORE3=$(docker ps -qf "name=openshell-cluster-nemoclaw" | head -1) +GATEWAY_ID_BEFORE3=$(gateway_runtime_id || true) PHASE4_START="$(phase_start_time)" run_onboard "$SANDBOX_B" output3="$RUN_ONBOARD_OUTPUT" @@ -472,11 +555,11 @@ else dump_diagnostics "Phase 4" fi -GATEWAY_ID_AFTER3=$(docker ps -qf "name=openshell-cluster-nemoclaw" | head -1) +GATEWAY_ID_AFTER3=$(gateway_runtime_id || true) if [ -n "$GATEWAY_ID_BEFORE3" ] && [ "$GATEWAY_ID_BEFORE3" = "$GATEWAY_ID_AFTER3" ]; then - pass "Healthy gateway reused on third onboard (container $GATEWAY_ID_BEFORE3)" + pass "Healthy gateway runtime reused on third onboard ($GATEWAY_ID_BEFORE3)" else - fail "Gateway container changed on third onboard (before=$GATEWAY_ID_BEFORE3 after=$GATEWAY_ID_AFTER3)" + fail "Gateway runtime changed on third onboard (before=$GATEWAY_ID_BEFORE3 after=$GATEWAY_ID_AFTER3)" fi if grep -q "Port 8080 is not available" <<<"$output3"; then @@ -590,6 +673,11 @@ section "Phase 5: Stale registry reconciliation" info "Deleting '$SANDBOX_A' directly in OpenShell to leave a stale NemoClaw registry entry..." openshell sandbox delete "$SANDBOX_A" 2>/dev/null || true +if wait_openshell_sandbox_absent "$SANDBOX_A" 60; then + pass "OpenShell reports '$SANDBOX_A' absent after direct deletion" +else + fail "OpenShell still reports '$SANDBOX_A' after direct deletion" +fi if registry_has "$SANDBOX_A"; then pass "Registry still contains stale '$SANDBOX_A' entry" @@ -628,7 +716,7 @@ section "Phase 6: Gateway lifecycle response" info "Stopping the NemoClaw gateway runtime to verify current lifecycle behavior..." openshell forward stop 18789 2>/dev/null || true -openshell gateway stop -g nemoclaw 2>/dev/null || true +stop_gateway_runtime GATEWAY_LOG="$(mktemp)" run_nemoclaw "$SANDBOX_B" status >"$GATEWAY_LOG" 2>&1 @@ -636,10 +724,10 @@ gateway_status_exit=$? gateway_status_output="$(cat "$GATEWAY_LOG")" rm -f "$GATEWAY_LOG" -if [ "$gateway_status_exit" -eq 1 ]; then - pass "Post-stop status exited 1" +if [ "$gateway_status_exit" -eq 0 ] || [ "$gateway_status_exit" -eq 1 ]; then + pass "Post-stop status exited $gateway_status_exit" else - fail "Post-stop status exited $gateway_status_exit (expected 1)" + fail "Post-stop status exited $gateway_status_exit (expected 0 or 1)" fi if grep -qE \ @@ -665,11 +753,18 @@ section "Phase 7: Final cleanup" run_nemoclaw "$SANDBOX_A" destroy --yes 2>/dev/null || true run_nemoclaw "$SANDBOX_B" destroy --yes 2>/dev/null || true +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + run_nemoclaw "$INSTALL_SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi openshell sandbox delete "$SANDBOX_A" 2>/dev/null || true openshell sandbox delete "$SANDBOX_B" 2>/dev/null || true +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + openshell sandbox delete "$INSTALL_SANDBOX_NAME" 2>/dev/null || true +fi stop_forward_if_set "${port_a:-}" stop_forward_if_set "${port_b:-}" openshell forward stop 18789 2>/dev/null || true +stop_gateway_runtime openshell gateway destroy -g nemoclaw 2>/dev/null || true openshell gateway destroy -g "$ALT_GATEWAY_NAME" 2>/dev/null || true diff --git a/test/e2e/test-gpu-e2e.sh b/test/e2e/test-gpu-e2e.sh index 69faa8195fb..5f00b1798be 100755 --- a/test/e2e/test-gpu-e2e.sh +++ b/test/e2e/test-gpu-e2e.sh @@ -276,7 +276,39 @@ else fail "nemoclaw ${SANDBOX_NAME} status failed" fi -# 4c: Inference provider is ollama-local +# 4c: Direct sandbox GPU is enabled by default on NVIDIA hosts +if status_output=$(nemoclaw "$SANDBOX_NAME" status 2>&1); then + if echo "$status_output" | grep -Fq "Sandbox GPU: enabled"; then + pass "Sandbox GPU is enabled by default" + else + fail "Sandbox GPU is not enabled in status output" + fi +else + fail "Could not read sandbox GPU status" +fi + +# 4d: Direct sandbox GPU proofs +if openshell sandbox exec -n "$SANDBOX_NAME" -- nvidia-smi >/dev/null 2>&1; then + pass "Sandbox nvidia-smi works" +else + fail "Sandbox nvidia-smi failed" +fi + +# shellcheck disable=SC2016 # expanded inside the sandbox by sh -lc +if openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc \ + 'tid="$(ls /proc/self/task | head -n 1)"; old="$(cat "/proc/self/task/${tid}/comm" 2>/dev/null || true)"; printf nemoclaw-gpu >"/proc/self/task/${tid}/comm"; [ -z "$old" ] || printf "%s" "$old" >"/proc/self/task/${tid}/comm" || true' >/dev/null 2>&1; then + pass "Sandbox /proc/self/task//comm write works" +else + fail "Sandbox /proc comm write failed" +fi + +if openshell sandbox exec -n "$SANDBOX_NAME" -- python3 -c 'import ctypes; lib=ctypes.CDLL("libcuda.so.1"); rc=lib.cuInit(0); print(f"cuInit(0)={rc}"); raise SystemExit(0 if rc == 0 else 1)' >/dev/null 2>&1; then + pass "Sandbox cuInit(0) succeeds" +else + fail "Sandbox cuInit(0) failed" +fi + +# 4e: Inference provider is ollama-local if inf_check=$(openshell inference get 2>&1); then if echo "$inf_check" | grep -qi "ollama"; then pass "Inference provider is Ollama-based" @@ -287,7 +319,7 @@ else fail "openshell inference get failed: ${inf_check:0:200}" fi -# 4d: Ollama is running and reachable +# 4f: Ollama is running and reachable if curl -sf http://127.0.0.1:11434/api/tags >/dev/null 2>&1; then pass "Ollama running on 127.0.0.1:11434 (started by onboard)" else diff --git a/test/e2e/test-hermes-discord-e2e.sh b/test/e2e/test-hermes-discord-e2e.sh index c4eb5f67307..3a16e5fc8b6 100755 --- a/test/e2e/test-hermes-discord-e2e.sh +++ b/test/e2e/test-hermes-discord-e2e.sh @@ -372,20 +372,24 @@ fi env_probe=$( sandbox_exec_stdin "EXPECTED_ALLOWED_USERS=$expected_allowed_users EXPECTED_GUILD_IDS=$expected_guild_ids python3 -" <<'PY' import os +import re from pathlib import Path text = Path("/sandbox/.hermes/.env").read_text(encoding="utf-8") +lines = set(text.splitlines()) errors = [] +token_pattern = re.compile(r"^DISCORD_BOT_TOKEN=openshell:resolve:env:(?:v[0-9]+_)?DISCORD_BOT_TOKEN$") +if not any(token_pattern.match(line) for line in lines): + errors.append("missing DISCORD_BOT_TOKEN=openshell:resolve:env:DISCORD_BOT_TOKEN") required = [ - "DISCORD_BOT_TOKEN=openshell:resolve:env:DISCORD_BOT_TOKEN", "DISCORD_PROXY=http://127.0.0.1:3129", "NEMOCLAW_DISCORD_FACADE_URL=http://127.0.0.1:3130", f"NEMOCLAW_DISCORD_GUILD_IDS={os.environ['EXPECTED_GUILD_IDS']}", f"DISCORD_ALLOWED_USERS={os.environ['EXPECTED_ALLOWED_USERS']}", ] for line in required: - if line not in text.splitlines(): + if line not in lines: errors.append(f"missing {line}") -if "API_SERVER_PORT=18642" not in text.splitlines(): +if "API_SERVER_PORT=18642" not in lines: errors.append("missing API_SERVER_PORT") if errors: print("FAIL " + "; ".join(errors)) @@ -552,29 +556,49 @@ section "Phase 6: Discord REST placeholder egress" dc_api=$(sandbox_exec 'NODE_NO_WARNINGS=1 node -e " const fs = require(\"fs\"); const https = require(\"https\"); -const env = fs.readFileSync(\"/sandbox/.hermes/.env\", \"utf8\"); -const line = env.split(/\\n/).find((entry) => entry.startsWith(\"DISCORD_BOT_TOKEN=\")); -const token = line ? line.slice(\"DISCORD_BOT_TOKEN=\".length) : \"\"; +let token = \"\"; +const runtimeToken = process.env.DISCORD_BOT_TOKEN || \"\"; +if (runtimeToken.startsWith(\"openshell:resolve:env:\")) { + token = runtimeToken; +} +if (!token) { + const env = fs.readFileSync(\"/sandbox/.hermes/.env\", \"utf8\"); + const line = env.split(/\\n/).find((entry) => entry.startsWith(\"DISCORD_BOT_TOKEN=\")); + token = line ? line.slice(\"DISCORD_BOT_TOKEN=\".length) : runtimeToken; +} if (!token) { console.log(JSON.stringify({ error: \"missing_token\" })); process.exit(0); } -const req = https.request({ - hostname: \"discord.com\", - path: \"/api/v10/users/@me\", - method: \"GET\", - headers: { \"Authorization\": \"Bot \" + token }, -}, (res) => { - let body = \"\"; - res.on(\"data\", (d) => body += d); - res.on(\"end\", () => console.log(JSON.stringify({ - statusCode: res.statusCode, - body: body.slice(0, 200), - }))); -}); -req.on(\"error\", (e) => console.log(JSON.stringify({ error: e.message }))); -req.setTimeout(20000, () => { req.destroy(); console.log(JSON.stringify({ error: \"timeout\" })); }); -req.end(); +const maxAttempts = 5; +let attempt = 0; +function probe() { + attempt += 1; + const req = https.request({ + hostname: \"discord.com\", + path: \"/api/v10/users/@me\", + method: \"GET\", + headers: { \"Authorization\": \"Bot \" + token }, + }, (res) => { + let body = \"\"; + res.on(\"data\", (d) => body += d); + res.on(\"end\", () => { + if (res.statusCode === 503 && attempt < maxAttempts) { + setTimeout(probe, 2000); + return; + } + console.log(JSON.stringify({ + statusCode: res.statusCode, + body: body.slice(0, 200), + attempt, + })); + }); + }); + req.on(\"error\", (e) => console.log(JSON.stringify({ error: e.message, attempt }))); + req.setTimeout(20000, () => { req.destroy(); console.log(JSON.stringify({ error: \"timeout\", attempt })); }); + req.end(); +} +probe(); "' 2>/dev/null || true) info "Discord users/@me response: ${dc_api:0:300}" diff --git a/test/e2e/test-hermes-slack-e2e.sh b/test/e2e/test-hermes-slack-e2e.sh index c3dc49a9930..89a0e79190a 100755 --- a/test/e2e/test-hermes-slack-e2e.sh +++ b/test/e2e/test-hermes-slack-e2e.sh @@ -400,7 +400,7 @@ if policy_output=$(openshell policy get --full "$SANDBOX_NAME" 2>&1); then fi if echo "$slack_block" | grep -Fq "/usr/local/bin/hermes" \ - && echo "$slack_block" | grep -Fq "/usr/bin/python3.11" \ + && echo "$slack_block" | grep -Fq "/usr/bin/python3*" \ && echo "$slack_block" | grep -Fq "/opt/hermes/.venv/bin/python"; then pass "Slack policy is scoped to Hermes and Python binaries" else @@ -427,13 +427,17 @@ fi section "Phase 6: Slack placeholder egress from Python" slack_probe=$( - sandbox_exec_stdin 'sh -lc ". /tmp/nemoclaw-proxy-env.sh 2>/dev/null || true; /usr/bin/python3.11 -"' <<'PY' + sandbox_exec_stdin 'sh -lc ". /tmp/nemoclaw-proxy-env.sh 2>/dev/null || true; if [ -x /opt/hermes/.venv/bin/python ]; then exec /opt/hermes/.venv/bin/python -; fi; exec python3 -" 2>&1' <<'PY' import json +import http.client import socket +import ssl import sys import urllib.error import urllib.request +TLS_CONTEXT = ssl._create_unverified_context() + def call(label, path, env_key, allowed_errors): prefix = { "SLACK_BOT_TOKEN": "xoxb", @@ -450,7 +454,11 @@ def call(label, path, env_key, allowed_errors): }, ) try: - with urllib.request.urlopen(req, timeout=30) as resp: + # The assertion here is placeholder substitution + Slack egress. CA + # wiring is covered separately by proxy-env tests and can vary by + # OpenShell proxy runner, so this probe does not make TLS trust the + # signal. + with urllib.request.urlopen(req, timeout=30, context=TLS_CONTEXT) as resp: status = resp.status body = resp.read().decode("utf-8", errors="replace") except socket.timeout: @@ -463,6 +471,13 @@ def call(label, path, env_key, allowed_errors): return False print(f"ERROR {label}: {reason}") return False + except Exception as exc: + reason = f"{type(exc).__name__}: {exc}" + if isinstance(exc, http.client.RemoteDisconnected) or "timed out" in reason.lower(): + print(f"TIMEOUT {label}: {reason}") + return False + print(f"ERROR {label}: {reason}") + return False print(json.dumps({"label": label, "status": status, "body": body[:300]})) try: diff --git a/test/e2e/test-messaging-compatible-endpoint.sh b/test/e2e/test-messaging-compatible-endpoint.sh index 815baa36576..a58069c0fab 100755 --- a/test/e2e/test-messaging-compatible-endpoint.sh +++ b/test/e2e/test-messaging-compatible-endpoint.sh @@ -425,13 +425,32 @@ check_gateway_ready() { local result script script=$( cat <<'SH' -node <<'NODE' +last="" +for _attempt in $(seq 1 30); do + result=$(node <<'NODE' 2>&1 || true const net = require("net"); +let done = false; const sock = net.connect(18789, "127.0.0.1"); -sock.on("connect", () => { console.log("OPEN"); sock.end(); }); -sock.on("error", (err) => console.log("ERROR " + err.message)); -setTimeout(() => { console.log("TIMEOUT"); sock.destroy(); }, 5000); +function finish(line) { + if (done) return; + done = true; + console.log(line); + sock.destroy(); +} +sock.on("connect", () => finish("OPEN")); +sock.on("error", (err) => finish("ERROR " + err.message)); +sock.setTimeout(1000, () => finish("TIMEOUT")); NODE + ) + if echo "$result" | grep -q "OPEN"; then + echo "$result" + exit 0 + fi + last="$result" + sleep 1 +done +echo "$last" +exit 1 SH ) result=$(sandbox_exec_sh_script "$script" 2>&1 || true) diff --git a/test/e2e/test-messaging-providers.sh b/test/e2e/test-messaging-providers.sh index c7838ace9d1..1ca44b1639f 100755 --- a/test/e2e/test-messaging-providers.sh +++ b/test/e2e/test-messaging-providers.sh @@ -573,10 +573,10 @@ fi # token body to a 127.0.0.1 listener (loopback bypasses the L7 proxy), have # the listener echo what it actually received, then assert the placeholder is # gone. If the rewriter is loaded and wrapping http.request/write/end, the -# listener sees the canonical openshell:resolve:env:VAR form. If the rewriter -# is a no-op, the listener sees the raw Bolt-shape placeholder. +# listener sees the active openshell:resolve:env:VAR placeholder form. If the +# rewriter is a no-op, the listener sees the raw Bolt-shape placeholder. info "Probing rewriter via loopback listener (proves http.request is wrapped)..." -sl_loopback=$(sandbox_exec 'node -e " +sl_loopback=$(sandbox_exec 'NODE_NO_WARNINGS=1 node -e " const http = require(\"http\"); const server = http.createServer((req, res) => { let body = \"\"; @@ -615,8 +615,8 @@ server.listen(0, \"127.0.0.1\", () => { info "Loopback echoed request: ${sl_loopback:0:300}" if echo "$sl_loopback" | grep -qF 'OPENSHELL-RESOLVE-ENV-'; then fail "M-S5h: rewriter did NOT translate Bolt-shape on http.request/write/end — the preload is loaded but incomplete or a no-op" -elif echo "$sl_loopback" | grep -qE '"authorization"\s*:\s*"Bearer openshell:resolve:env:SLACK_BOT_TOKEN' \ - && echo "$sl_loopback" | grep -qE '"body"\s*:\s*"token=openshell:resolve:env:SLACK_BOT_TOKEN'; then +elif echo "$sl_loopback" | grep -qE '"authorization"\s*:\s*"Bearer openshell:resolve:env:(v[0-9]+_)?SLACK_BOT_TOKEN' \ + && echo "$sl_loopback" | grep -qE '"body"\s*:\s*"token=openshell:resolve:env:(v[0-9]+_)?SLACK_BOT_TOKEN'; then pass "M-S5h: rewriter wraps http.request/write/end — Bolt-shape header and body were translated before egress" elif echo "$sl_loopback" | grep -q "ERROR"; then fail "M-S5h: loopback probe errored: ${sl_loopback:0:200}" @@ -1198,21 +1198,33 @@ fi # M17: Discord users/@me with placeholder token info "Calling discord.com/api/v10/users/@me from inside sandbox..." -dc_api=$(sandbox_exec 'node -e " +dc_api=$(sandbox_exec 'NODE_NO_WARNINGS=1 node -e " const https = require(\"https\"); const token = process.env.DISCORD_BOT_TOKEN || \"missing\"; -const options = { - hostname: \"discord.com\", - path: \"/api/v10/users/@me\", - headers: { \"Authorization\": \"Bot \" + token }, -}; -const req = https.get(options, (res) => { - let body = \"\"; - res.on(\"data\", (d) => body += d); - res.on(\"end\", () => console.log(res.statusCode + \" \" + body.slice(0, 300))); -}); -req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); -req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); +const maxAttempts = 5; +let attempt = 0; +function probe() { + attempt += 1; + const options = { + hostname: \"discord.com\", + path: \"/api/v10/users/@me\", + headers: { \"Authorization\": \"Bot \" + token }, + }; + const req = https.get(options, (res) => { + let body = \"\"; + res.on(\"data\", (d) => body += d); + res.on(\"end\", () => { + if (res.statusCode === 503 && attempt < maxAttempts) { + setTimeout(probe, 2000); + return; + } + console.log(res.statusCode + \" \" + body.slice(0, 300)); + }); + }); + req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); + req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); +} +probe(); "' 2>/dev/null || true) info "Discord API response: ${dc_api:0:300}" @@ -1288,29 +1300,26 @@ else fi # M-S15b: L7 proxy substitution for SLACK_BOT_TOKEN, isolated from the -# rewriter. Sends the canonical openshell:resolve:env:SLACK_BOT_TOKEN -# placeholder directly (no Bolt-shape, so the rewriter is a no-op for -# this request). If the L7 proxy substitutes correctly, the fake xoxb- -# token reaches slack.com which returns invalid_auth. If the proxy -# doesn't substitute, slack.com sees the literal placeholder and STILL -# returns invalid_auth — same response shape as M-S15. To distinguish, -# we additionally call with an env var that does NOT exist in the -# sandbox (DEFINITELY_NOT_SET_XYZ); the L7 proxy's behavior on an -# unset var differs from a successful substitution. +# rewriter. Sends the active OpenShell placeholder from the sandbox env +# directly (no Bolt-shape, so the rewriter is a no-op for this request). +# Newer OpenShell builds scope provider placeholders by revision +# (openshell:resolve:env:vNN_KEY), while older builds use the canonical +# openshell:resolve:env:KEY form. # # Mirrors the proof technique already used by Telegram M15 and Discord # M17 (they get 401/404 from the real APIs because the L7 proxy -# substituted the canonical form into a real fake-token-shape value). -info "Probing L7 proxy substitution for SLACK_BOT_TOKEN (canonical placeholder, bypasses rewriter)..." +# substituted the active placeholder into a real fake-token-shape value). +info "Probing L7 proxy substitution for SLACK_BOT_TOKEN (active provider placeholder, bypasses rewriter)..." sl_canonical=$(sandbox_exec 'node -e " const https = require(\"https\"); const data = \"\"; +const token = process.env.SLACK_BOT_TOKEN || \"openshell:resolve:env:SLACK_BOT_TOKEN\"; const options = { hostname: \"slack.com\", path: \"/api/auth.test\", method: \"POST\", headers: { - \"Authorization\": \"Bearer openshell:resolve:env:SLACK_BOT_TOKEN\", + \"Authorization\": \"Bearer \" + token, \"Content-Type\": \"application/x-www-form-urlencoded\", \"Content-Length\": data.length, }, @@ -1330,11 +1339,11 @@ info "Slack auth.test (canonical) response: ${sl_canonical:0:300}" sl_canon_status=$(echo "$sl_canonical" | grep -E '^[0-9]' | head -1 | awk '{print $1}') if [ "$sl_canon_status" = "200" ] && echo "$sl_canonical" | grep -qE 'invalid_auth|not_authed'; then - pass "M-S15b: L7 proxy substitutes openshell:resolve:env:SLACK_BOT_TOKEN at egress (parallels Telegram M15 / Discord M17)" + pass "M-S15b: L7 proxy substitutes the active SLACK_BOT_TOKEN placeholder at egress (parallels Telegram M15 / Discord M17)" elif echo "$sl_canonical" | grep -q "TIMEOUT"; then - skip "M-S15b: canonical-placeholder probe timed out" + skip "M-S15b: active-placeholder probe timed out" elif echo "$sl_canonical" | grep -qF 'openshell:resolve:env:' || echo "$sl_canonical" | grep -qiF 'invalid token'; then - fail "M-S15b: L7 proxy passed canonical placeholder through unchanged — substitution not happening for SLACK_BOT_TOKEN" + fail "M-S15b: L7 proxy passed provider placeholder through unchanged — substitution not happening for SLACK_BOT_TOKEN" else fail "M-S15b: Unexpected response (status=$sl_canon_status): ${sl_canonical:0:200}" fi @@ -1435,18 +1444,19 @@ else fi # M-S16b: L7 proxy substitution for SLACK_APP_TOKEN, isolated. Same -# rationale as M-S15b — sends the canonical placeholder directly so the -# rewriter is a no-op and only the L7 proxy substitution is exercised. -info "Probing L7 proxy substitution for SLACK_APP_TOKEN (canonical placeholder)..." +# rationale as M-S15b — sends the active provider placeholder directly so +# the rewriter is a no-op and only the L7 proxy substitution is exercised. +info "Probing L7 proxy substitution for SLACK_APP_TOKEN (active provider placeholder)..." sl_app_canonical=$(sandbox_exec 'node -e " const https = require(\"https\"); const data = \"\"; +const token = process.env.SLACK_APP_TOKEN || \"openshell:resolve:env:SLACK_APP_TOKEN\"; const options = { hostname: \"slack.com\", path: \"/api/apps.connections.open\", method: \"POST\", headers: { - \"Authorization\": \"Bearer openshell:resolve:env:SLACK_APP_TOKEN\", + \"Authorization\": \"Bearer \" + token, \"Content-Type\": \"application/x-www-form-urlencoded\", \"Content-Length\": data.length, }, @@ -1462,7 +1472,7 @@ req.write(data); req.end(); "' 2>/dev/null || true) -info "Slack apps.connections.open (canonical) response: ${sl_app_canonical:0:300}" +info "Slack apps.connections.open (active placeholder) response: ${sl_app_canonical:0:300}" sl_app_canon_status=$(echo "$sl_app_canonical" | grep -E '^[0-9]' | head -1 | awk '{print $1}') info "Probing L7 proxy substitution for an unset app-token env var (negative control)..." @@ -1493,7 +1503,7 @@ req.end(); info "Slack apps.connections.open (unset env) response: ${sl_app_unset:0:300}" if [ "$sl_app_canon_status" = "200" ] && echo "$sl_app_canonical" | grep -qE 'invalid_auth|not_authed|not_allowed_token_type'; then if echo "$sl_app_unset" | grep -qE 'ERROR:.*(socket hang up|ECONNRESET|EPIPE|hang up|reset)'; then - pass "M-S16b: L7 proxy substitutes openshell:resolve:env:SLACK_APP_TOKEN at egress (unset-var control diverged)" + pass "M-S16b: L7 proxy substitutes the active SLACK_APP_TOKEN placeholder at egress (unset-var control diverged)" elif echo "$sl_app_unset" | grep -qE '^200\b'; then fail "M-S16b: unset app-token env returned HTTP 200 — proxy may be passing canonical placeholders through unchanged" elif [ -z "$sl_app_unset" ] || echo "$sl_app_unset" | grep -q "TIMEOUT"; then @@ -1502,9 +1512,9 @@ if [ "$sl_app_canon_status" = "200" ] && echo "$sl_app_canonical" | grep -qE 'in skip "M-S16b: unset app-token control produced an unclassified result: ${sl_app_unset:0:200}" fi elif echo "$sl_app_canonical" | grep -q "TIMEOUT"; then - skip "M-S16b: canonical-placeholder probe timed out" + skip "M-S16b: active-placeholder probe timed out" elif echo "$sl_app_canonical" | grep -qF 'openshell:resolve:env:'; then - fail "M-S16b: L7 proxy passed canonical placeholder through unchanged for SLACK_APP_TOKEN" + fail "M-S16b: L7 proxy passed provider placeholder through unchanged for SLACK_APP_TOKEN" else fail "M-S16b: Unexpected response (status=$sl_app_canon_status): ${sl_app_canonical:0:200}" fi diff --git a/test/e2e/test-onboard-repair.sh b/test/e2e/test-onboard-repair.sh index c7fb5255b07..052dafa4c56 100755 --- a/test/e2e/test-onboard-repair.sh +++ b/test/e2e/test-onboard-repair.sh @@ -62,6 +62,7 @@ run_nemoclaw() { SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-repair}" OTHER_SANDBOX_NAME="${NEMOCLAW_OTHER_SANDBOX_NAME:-e2e-other}" +INSTALL_SANDBOX_NAME="${NEMOCLAW_E2E_INSTALL_SANDBOX_NAME:-}" # Shim so the teardown helper's trap can call `nemoclaw destroy` even when # this repo-local test run has no globally-installed `nemoclaw` on PATH (it @@ -74,17 +75,46 @@ fi . "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" register_sandbox_for_teardown "$SANDBOX_NAME" register_sandbox_for_teardown "$OTHER_SANDBOX_NAME" +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + register_sandbox_for_teardown "$INSTALL_SANDBOX_NAME" +fi SESSION_FILE="$HOME/.nemoclaw/onboard-session.json" RESTORE_API_KEY="${NVIDIA_API_KEY:-}" +wait_openshell_sandbox_absent() { + local sandbox_name="$1" + local timeout="${2:-60}" + local deadline=$((SECONDS + timeout)) + local output status + + while [ "$SECONDS" -le "$deadline" ]; do + output="$(openshell sandbox get "$sandbox_name" 2>&1)" + status=$? + if [ "$status" -ne 0 ] && grep -qiE 'NotFound|Not Found|sandbox not found' <<<"$output"; then + return 0 + fi + sleep 1 + done + + info "OpenShell still reports sandbox '$sandbox_name' after ${timeout}s:" + printf '%s\n' "$output" | sed 's/^/ /' + return 1 +} + # ══════════════════════════════════════════════════════════════════ # Phase 0: Pre-cleanup # ══════════════════════════════════════════════════════════════════ section "Phase 0: Pre-cleanup" info "Destroying any leftover sandbox/gateway from previous runs..." +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + run_nemoclaw "$INSTALL_SANDBOX_NAME" destroy 2>/dev/null || true +fi run_nemoclaw "$SANDBOX_NAME" destroy 2>/dev/null || true run_nemoclaw "$OTHER_SANDBOX_NAME" destroy 2>/dev/null || true +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + openshell sandbox delete "$INSTALL_SANDBOX_NAME" 2>/dev/null || true +fi openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true openshell sandbox delete "$OTHER_SANDBOX_NAME" 2>/dev/null || true openshell forward stop 18789 2>/dev/null || true @@ -188,10 +218,10 @@ info "Deleting the recorded sandbox under the session, then resuming..." openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true openshell forward stop 18789 >/dev/null 2>&1 || true -if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then - fail "Sandbox '$SANDBOX_NAME' still exists after forced deletion" -else +if wait_openshell_sandbox_absent "$SANDBOX_NAME" 60; then pass "Sandbox '$SANDBOX_NAME' removed to simulate stale recorded state" +else + fail "Sandbox '$SANDBOX_NAME' still exists after forced deletion" fi REPAIR_LOG="$(mktemp)" diff --git a/test/e2e/test-openshell-gateway-upgrade.sh b/test/e2e/test-openshell-gateway-upgrade.sh new file mode 100755 index 00000000000..9cdea3ac0ba --- /dev/null +++ b/test/e2e/test-openshell-gateway-upgrade.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Regression coverage for PR #3001 upgrade installs: if a user already has a +# healthy Linux Docker-driver OpenShell gateway from an older runtime, NemoClaw +# must not reuse it after installing the current OpenShell release. The gateway +# process must be restarted with the supported supervisor image and current +# openshell-sandbox binary before onboarding continues. + +set -euo pipefail + +LOG_FILE="/tmp/nemoclaw-e2e-openshell-gateway-upgrade.log" +START_LOG="/tmp/nemoclaw-e2e-openshell-gateway-start.log" +GATEWAY_LOG="/tmp/nemoclaw-e2e-openshell-gateway-process.log" +exec > >(tee "$LOG_FILE") 2>&1 + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +diag() { echo -e "${YELLOW}[DIAG]${NC} $1"; } +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + diag "openshell status: $(openshell status 2>&1 || true)" + diag "gateway info: $(openshell gateway info -g nemoclaw 2>&1 || true)" + diag "pid file: $(cat "$PID_FILE" 2>/dev/null || echo missing)" + diag "gateway log tail:" + tail -100 "$GATEWAY_LOG" 2>/dev/null || true + exit 1 +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +STATE_DIR="${NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR:-$HOME/.local/state/nemoclaw/openshell-docker-gateway}" +PID_FILE="${STATE_DIR}/openshell-gateway.pid" +STALE_IMAGE="ghcr.io/nvidia/openshell/supervisor:0.0.36" +EXPECTED_IMAGE="" + +OLD_PID="" +NEW_PID="" + +load_shell_path() { + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi +} + +process_env_value() { + local pid="$1" key="$2" + tr '\0' '\n' <"/proc/${pid}/environ" 2>/dev/null \ + | awk -F= -v key="$key" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' +} + +cleanup_pid() { + local pid="$1" + [ -n "$pid" ] || return 0 + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +cleanup() { + set +e + cleanup_pid "$OLD_PID" + cleanup_pid "$NEW_PID" + openshell gateway remove nemoclaw >/dev/null 2>&1 || true + rm -f "$PID_FILE" +} +trap cleanup EXIT + +cd "$REPO_ROOT" +load_shell_path + +info "Preparing CLI build and OpenShell binaries" +if [ ! -d node_modules ]; then + npm ci --ignore-scripts +fi +npm run build:cli +bash scripts/install-openshell.sh +load_shell_path + +command -v openshell >/dev/null 2>&1 || fail "openshell not found after install" +command -v openshell-gateway >/dev/null 2>&1 || fail "openshell-gateway not found after install" +command -v openshell-sandbox >/dev/null 2>&1 || fail "openshell-sandbox not found after install" +unset OPENSHELL_DOCKER_SUPERVISOR_IMAGE +unset OPENSHELL_DOCKER_SUPERVISOR_BIN +EXPECTED_IMAGE="$( + node -e "const { execFileSync } = require('child_process'); const { getDockerDriverGatewayEnv } = require('./dist/lib/onboard'); const version = execFileSync('openshell', ['--version'], { encoding: 'utf8' }).trim(); console.log(getDockerDriverGatewayEnv(version).OPENSHELL_DOCKER_SUPERVISOR_IMAGE);" +)" + +mkdir -p "$STATE_DIR" +chmod 700 "$STATE_DIR" +rm -f "$PID_FILE" "$START_LOG" "$GATEWAY_LOG" +openshell gateway remove nemoclaw >/dev/null 2>&1 || true + +GATEWAY_BIN="$(command -v openshell-gateway)" +SANDBOX_BIN="$(command -v openshell-sandbox)" +STALE_GATEWAY_BIN="${STATE_DIR}/openshell-gateway-stale" +cp "$GATEWAY_BIN" "$STALE_GATEWAY_BIN" +chmod 700 "$STALE_GATEWAY_BIN" + +info "Starting a stale but healthy Docker-driver gateway" +( + export OPENSHELL_DRIVERS=docker + export OPENSHELL_BIND_ADDRESS=127.0.0.1 + export OPENSHELL_SERVER_PORT=8080 + export OPENSHELL_DISABLE_TLS=true + export OPENSHELL_DISABLE_GATEWAY_AUTH=true + export OPENSHELL_DB_URL="sqlite:${STATE_DIR}/openshell.db" + export OPENSHELL_GRPC_ENDPOINT=http://127.0.0.1:8080 + export OPENSHELL_SSH_GATEWAY_HOST=127.0.0.1 + export OPENSHELL_SSH_GATEWAY_PORT=8080 + export OPENSHELL_DOCKER_NETWORK_NAME="${OPENSHELL_DOCKER_NETWORK_NAME:-openshell-docker}" + export OPENSHELL_DOCKER_SUPERVISOR_IMAGE="$STALE_IMAGE" + export OPENSHELL_DOCKER_SUPERVISOR_BIN="$SANDBOX_BIN" + exec "$STALE_GATEWAY_BIN" +) >>"$GATEWAY_LOG" 2>&1 & +OLD_PID="$!" +echo "$OLD_PID" >"$PID_FILE" + +for _i in $(seq 1 60); do + kill -0 "$OLD_PID" 2>/dev/null || fail "stale gateway process exited early" + openshell gateway add --local --name nemoclaw http://127.0.0.1:8080 >/dev/null 2>&1 || true + openshell gateway select nemoclaw >/dev/null 2>&1 || true + if openshell status >/dev/null 2>&1; then + break + fi + sleep 2 +done +openshell status >/dev/null 2>&1 || fail "stale gateway never became healthy" + +OLD_IMAGE="$(process_env_value "$OLD_PID" OPENSHELL_DOCKER_SUPERVISOR_IMAGE)" +[ "$OLD_IMAGE" = "$STALE_IMAGE" ] || fail "stale gateway did not start with expected image" +pass "Stale gateway is healthy with ${OLD_IMAGE}" + +info "Invoking NemoClaw gateway start path; it must restart the stale process" +unset OPENSHELL_DOCKER_SUPERVISOR_IMAGE +unset OPENSHELL_DOCKER_SUPERVISOR_BIN +node <<'NODE' 2>&1 | tee "$START_LOG" +const { startGateway } = require("./dist/lib/onboard"); + +startGateway(null) + .then(() => undefined) + .catch((error) => { + console.error(error && error.stack ? error.stack : error); + process.exit(1); + }); +NODE + +[ -f "$PID_FILE" ] || fail "NemoClaw did not write a replacement gateway pid file" +NEW_PID="$(tr -d '[:space:]' <"$PID_FILE")" +[ -n "$NEW_PID" ] || fail "replacement gateway pid file is empty" +[ "$NEW_PID" != "$OLD_PID" ] || fail "NemoClaw reused the stale gateway pid" + +wait "$OLD_PID" 2>/dev/null || true +if kill -0 "$OLD_PID" 2>/dev/null; then + fail "stale gateway process is still alive after restart" +fi + +NEW_IMAGE="$(process_env_value "$NEW_PID" OPENSHELL_DOCKER_SUPERVISOR_IMAGE)" +[ "$NEW_IMAGE" = "$EXPECTED_IMAGE" ] || fail "replacement gateway image was ${NEW_IMAGE:-unset}, expected ${EXPECTED_IMAGE}" + +if ! grep -qi "Docker-driver gateway is stale" "$START_LOG"; then + fail "NemoClaw start log did not report stale gateway restart" +fi + +openshell status >/dev/null 2>&1 || fail "replacement gateway is not healthy" +pass "NemoClaw restarted stale gateway with ${NEW_IMAGE}" diff --git a/test/e2e/test-overlayfs-autofix.sh b/test/e2e/test-overlayfs-autofix.sh index b08a37458e0..a65537edf06 100755 --- a/test/e2e/test-overlayfs-autofix.sh +++ b/test/e2e/test-overlayfs-autofix.sh @@ -94,6 +94,15 @@ section() { printf '\033[1;36m=== %s ===\033[0m\n' "$1" } info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +print_summary() { + echo "" + printf '\033[1;33m=== Test summary ===\033[0m\n' + echo " PASS: $PASS" + echo " FAIL: $FAIL" + echo " SKIP: $SKIP" + echo " TOTAL: $TOTAL" + echo "" +} SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-overlayfs}" NEGATIVE_TIMEOUT="${NEMOCLAW_OVERLAYFS_E2E_NEGATIVE_TIMEOUT:-300}" @@ -120,6 +129,13 @@ register_sandbox_for_teardown "$SANDBOX_NAME" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +if [ "$(uname -s)" = "Linux" ] && grep -q 'return platform === "linux";' "$REPO_ROOT/src/lib/onboard.ts"; then + section "Applicability" + skip "OpenShell Docker-driver onboarding is active on Linux; k3s overlayfs auto-fix is not in the runtime path" + print_summary + exit 0 +fi + # ── Daemon revert ─────────────────────────────────────────────────── # Always restore the original daemon.json on exit so we don't leave the # runner in a degraded state if the test crashes mid-flight. @@ -523,13 +539,7 @@ fi # ══════════════════════════════════════════════════════════════════ # Test summary # ══════════════════════════════════════════════════════════════════ -echo "" -printf '\033[1;33m=== Test summary ===\033[0m\n' -echo " PASS: $PASS" -echo " FAIL: $FAIL" -echo " SKIP: $SKIP" -echo " TOTAL: $TOTAL" -echo "" +print_summary if [ $FAIL -gt 0 ]; then exit 1 diff --git a/test/e2e/test-sandbox-survival.sh b/test/e2e/test-sandbox-survival.sh index 41fd46ee6be..e57121ab416 100755 --- a/test/e2e/test-sandbox-survival.sh +++ b/test/e2e/test-sandbox-survival.sh @@ -123,6 +123,76 @@ cleanup_ssh() { ssh_config="" } +docker_driver_gateway_pid_file() { + printf '%s/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.pid\n' "$HOME" +} + +gateway_runtime_id() { + local pid_file pid cid + pid_file="$(docker_driver_gateway_pid_file)" + if [ -f "$pid_file" ]; then + pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + printf 'pid:%s\n' "$pid" + return 0 + fi + fi + + cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)" + if [ -n "$cid" ]; then + printf 'container:%s\n' "$cid" + return 0 + fi + + return 1 +} + +stop_gateway_runtime() { + local pid_file pid cid + openshell forward stop 18789 2>/dev/null || true + openshell gateway stop -g nemoclaw 2>/dev/null || true + + pid_file="$(docker_driver_gateway_pid_file)" + if [ -f "$pid_file" ]; then + pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + for _ in $(seq 1 10); do + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + kill -0 "$pid" 2>/dev/null && kill -9 "$pid" 2>/dev/null || true + fi + fi + + cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)" + if [ -n "$cid" ]; then + docker stop "$cid" >/dev/null 2>&1 || true + fi +} + +start_gateway_runtime() { + local previous_runtime="$1" + if [[ "$previous_runtime" == pid:* ]]; then + local recovery_log + recovery_log="$(mktemp)" + if nemoclaw "$SANDBOX_NAME" status >"$recovery_log" 2>&1; then + pass "Gateway recovered through NemoClaw status" + else + info "NemoClaw status recovery returned non-zero; polling gateway health" + sed 's/^/ /' "$recovery_log" | tail -40 || true + fi + rm -f "$recovery_log" + return 0 + fi + + if openshell gateway start --name nemoclaw 2>&1; then + pass "Gateway start command succeeded" + else + info "Gateway start returned non-zero — checking health..." + fi +} + # ══════════════════════════════════════════════════════════════════ # Phase 0: Prerequisites # ══════════════════════════════════════════════════════════════════ @@ -176,6 +246,7 @@ if command -v nemoclaw >/dev/null 2>&1; then fi if command -v openshell >/dev/null 2>&1; then openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + stop_gateway_runtime openshell gateway destroy -g nemoclaw 2>/dev/null || true fi rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true @@ -422,38 +493,40 @@ cleanup_ssh section "Phase 6: Gateway stop/start cycle (simulates host reboot)" # Stop any port forwards first +GATEWAY_RUNTIME_BEFORE="$(gateway_runtime_id || true)" openshell forward stop 18789 2>/dev/null || true info "Stopping gateway (simulates laptop close / VM shutdown)..." -if openshell gateway stop -g nemoclaw 2>/dev/null; then - pass "Gateway stopped" +stop_gateway_runtime +if [ -z "$(gateway_runtime_id || true)" ]; then + pass "Gateway runtime stopped" else - fail "Gateway stop failed" + fail "Gateway runtime still appears to be running after stop" # Non-fatal — continue to see what happens fi -# Verify the Docker container is actually stopped -CONTAINER_NAME="openshell-cluster-nemoclaw" -container_state=$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || echo "missing") -if [ "$container_state" = "false" ]; then - pass "Docker container confirmed stopped" -elif [ "$container_state" = "missing" ]; then - info "Container not found (may have been removed) — resume should handle this" - pass "Docker container not running" +# Verify the legacy Docker container is stopped when this run uses the +# legacy k3s gateway; Docker-driver runs use a host openshell-gateway PID. +if [[ "$GATEWAY_RUNTIME_BEFORE" == container:* ]]; then + CONTAINER_NAME="openshell-cluster-nemoclaw" + container_state=$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || echo "missing") + if [ "$container_state" = "false" ]; then + pass "Docker container confirmed stopped" + elif [ "$container_state" = "missing" ]; then + info "Container not found (may have been removed) — resume should handle this" + pass "Docker container not running" + else + fail "Docker container still running: state=$container_state" + fi else - fail "Docker container still running: state=$container_state" + pass "Docker-driver gateway process is not running" fi info "Waiting 5 seconds to simulate delay (laptop lid close / VM hibernate)..." sleep 5 info "Starting gateway (simulates laptop open / VM boot)..." -if openshell gateway start --name nemoclaw 2>&1; then - pass "Gateway start command succeeded" -else - # gateway start may exit non-zero but still recover - info "Gateway start returned non-zero — checking health..." -fi +start_gateway_runtime "$GATEWAY_RUNTIME_BEFORE" # Wait for gateway to become healthy info "Waiting for gateway to become healthy..." diff --git a/test/gateway-final-failure-cleanup.test.ts b/test/gateway-final-failure-cleanup.test.ts index cc7dfc6acb7..8f9df5a2387 100644 --- a/test/gateway-final-failure-cleanup.test.ts +++ b/test/gateway-final-failure-cleanup.test.ts @@ -64,6 +64,7 @@ describe("final gateway startup failure cleanup", () => { expect(errors).toContain(" Gateway logs:"); expect(errors).toContain(" gateway log line"); expect(errors).toContain(" Cleanup attempted."); + expect(errors).toContain(" openshell gateway remove nemoclaw"); expect(errors).toContain(" openshell gateway destroy -g nemoclaw"); expect(errors).toContain( ' docker volume ls -q --filter "name=openshell-cluster-nemoclaw" | xargs -r docker volume rm', diff --git a/test/hermes-share-mount-deps.test.ts b/test/hermes-share-mount-deps.test.ts index 5968b143ca5..3262fedc081 100644 --- a/test/hermes-share-mount-deps.test.ts +++ b/test/hermes-share-mount-deps.test.ts @@ -35,7 +35,7 @@ function runLoggedShell(command: string, tmp: string) { } describe("Hermes share mount package parity (#2947)", () => { - it("requests gnupg, procps, and openssh-sftp-server from the Hermes base apt layer", () => { + it("requests gnupg, procps, e2fsprogs, and openssh-sftp-server from the Hermes base apt layer", () => { const dockerfile = fs.readFileSync(HERMES_DOCKERFILE_BASE, "utf-8"); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-share-apt-")); const lists = path.join(tmp, "apt-lists"); @@ -50,9 +50,10 @@ describe("Hermes share mount package parity (#2947)", () => { expect(result.status).toBe(0); expect(calls).toContain("apt-get update"); - expect(calls).toContain("gnupg=2.2.40-1.1+deb12u2"); - expect(calls).toContain("procps=2:4.0.2-3"); - expect(calls).toContain("openssh-sftp-server=1:9.2p1-2+deb12u9"); + expect(calls).toContain("gnupg=2.4.7-21+deb13u1"); + expect(calls).toContain("procps=2:4.0.4-9"); + expect(calls).toContain("e2fsprogs=1.47.2-3+b10"); + expect(calls).toContain("openssh-sftp-server=1:10.0p1-7+deb13u2"); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } diff --git a/test/install-openshell-version-check.test.ts b/test/install-openshell-version-check.test.ts index 2c71c96df28..3b24487e571 100644 --- a/test/install-openshell-version-check.test.ts +++ b/test/install-openshell-version-check.test.ts @@ -19,12 +19,24 @@ function writeExecutable(target: string, contents: string) { * either exit early (version ok / too high) or hit the upgrade warn and then * the script tries to download — so we stub curl and gh to fail fast. */ -function runWithInstalledVersion(version: string) { +function runWithInstalledVersion( + version: string, + extraEnv: NodeJS.ProcessEnv = {}, + options: { driverBins?: boolean; os?: string; arch?: string } = {}, +) { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-ver-")); try { const fakeBin = path.join(tmp, "bin"); fs.mkdirSync(fakeBin); + if (options.os || options.arch) { + writeExecutable( + path.join(fakeBin, "uname"), + `#!/usr/bin/env bash +if [ "\${1:-}" = "-m" ]; then echo "${options.arch ?? "x86_64"}"; else echo "${options.os ?? "Linux"}"; fi`, + ); + } + // Fake openshell that reports the given version writeExecutable( path.join(fakeBin, "openshell"), @@ -33,6 +45,19 @@ if [ "\${1:-}" = "--version" ]; then echo "openshell ${version}"; exit 0; fi exit 99`, ); + if (options.driverBins !== false) { + writeExecutable( + path.join(fakeBin, "openshell-gateway"), + `#!/usr/bin/env bash +exit 0`, + ); + writeExecutable( + path.join(fakeBin, "openshell-sandbox"), + `#!/usr/bin/env bash +exit 0`, + ); + } + // Stub curl to fail so the install path exits without doing real network I/O writeExecutable( path.join(fakeBin, "curl"), @@ -49,7 +74,12 @@ exit 1`, ); return spawnSync("bash", [SCRIPT], { - env: { ...process.env, PATH: `${fakeBin}:/usr/bin:/bin` }, + env: { + ...process.env, + NEMOCLAW_OPENSHELL_CHANNEL: "stable", + ...extraEnv, + PATH: `${fakeBin}:/usr/bin:/bin`, + }, encoding: "utf8", }); } finally { @@ -58,10 +88,23 @@ exit 1`, } describe("install-openshell.sh version check", { timeout: 15_000 }, () => { - it("exits cleanly when openshell 0.0.32 is already installed", () => { - const result = runWithInstalledVersion("0.0.32"); + it("exits cleanly when openshell 0.0.37 and driver binaries are already installed", () => { + const result = runWithInstalledVersion("0.0.37"); expect(result.status).toBe(0); - expect(result.stdout).toMatch(/already installed.*0\.0\.32/); + expect(result.stdout).toMatch(/already installed.*0\.0\.37/); + }); + + it("triggers reinstall when openshell 0.0.37 is missing Docker-driver binaries", () => { + const result = runWithInstalledVersion("0.0.37", {}, { driverBins: false, os: "Linux" }); + expect(result.status).not.toBe(0); + expect(result.stdout).toMatch(/missing Docker-driver binaries/); + expect(result.stdout).toMatch(/Installing OpenShell from release 'v0\.0\.37'/); + }); + + it("triggers upgrade when openshell 0.0.36 is installed (below current floor)", () => { + const result = runWithInstalledVersion("0.0.36"); + expect(result.status).not.toBe(0); + expect(result.stdout).toMatch(/below minimum.*upgrading/); }); it("triggers upgrade when openshell 0.0.28 is installed (below MIN_VERSION)", () => { @@ -84,7 +127,7 @@ describe("install-openshell.sh version check", { timeout: 15_000 }, () => { }); it("fails with a clear error when openshell is above MAX_VERSION", () => { - const result = runWithInstalledVersion("0.0.37"); + const result = runWithInstalledVersion("0.0.38"); expect(result.status).toBe(1); expect(result.stdout).toMatch(/above the maximum/); }); @@ -95,6 +138,22 @@ describe("install-openshell.sh version check", { timeout: 15_000 }, () => { expect(result.stdout).toMatch(/above the maximum/); }); + it("accepts an installed OpenShell dev-channel Docker-driver build", () => { + const result = runWithInstalledVersion("0.0.37.dev84+g6b2180425", { + NEMOCLAW_OPENSHELL_CHANNEL: "dev", + }); + expect(result.status).toBe(0); + expect(result.stdout).toMatch(/dev channel/); + }); + + it("upgrades stable OpenShell when the dev channel is requested", () => { + const result = runWithInstalledVersion("0.0.36", { + NEMOCLAW_OPENSHELL_CHANNEL: "dev", + }); + expect(result.status).not.toBe(0); + expect(result.stdout).toMatch(/required dev-channel Docker-driver build/); + }); + it("proceeds to install when openshell is not present", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-noop-")); try { @@ -115,12 +174,16 @@ exit 1`, ); const result = spawnSync("bash", [SCRIPT], { - env: { ...process.env, PATH: `${fakeBin}:/usr/bin:/bin` }, + env: { + ...process.env, + NEMOCLAW_OPENSHELL_CHANNEL: "stable", + PATH: `${fakeBin}:/usr/bin:/bin`, + }, encoding: "utf8", }); // Should attempt install (not exit 0 early) and fail at the download step - expect(result.stdout).toMatch(/Installing openshell CLI/); + expect(result.stdout).toMatch(/Installing OpenShell from release/); expect(result.status).not.toBe(0); } finally { fs.rmSync(tmp, { recursive: true, force: true }); diff --git a/test/install-preflight.test.ts b/test/install-preflight.test.ts index f4bd56dfbe9..cbff2fccb25 100644 --- a/test/install-preflight.test.ts +++ b/test/install-preflight.test.ts @@ -3111,6 +3111,8 @@ exit 0`, { cwd: tmp, encoding: "utf-8", + // input: "" makes spawnSync attach a non-TTY stdin pipe. setsid above + // additionally removes /dev/tty on Linux/WSL. input: options.stdinIsTty ? undefined : "", env: { HOME: tmp, @@ -3159,7 +3161,7 @@ exit 0`, path.join(fakeBin, "openshell"), `#!/usr/bin/env bash echo "openshell $*" >> ${JSON.stringify(phaseLog)} -if [ "$1" = "--version" ] || [ "$1" = "version" ]; then echo "openshell 0.0.36"; fi +if [ "$1" = "--version" ] || [ "$1" = "version" ]; then echo "openshell 0.0.37"; fi exit 0`, ); writeExecutable( diff --git a/test/nemoclaw-start.test.ts b/test/nemoclaw-start.test.ts index 0ec308bb332..1505adfda32 100644 --- a/test/nemoclaw-start.test.ts +++ b/test/nemoclaw-start.test.ts @@ -264,6 +264,8 @@ describe("nemoclaw-start non-root fallback", () => { 'apply_model_override() { :; }', 'reconcile_agent_model_with_provider() { :; }', 'apply_cors_override() { :; }', + 'refresh_openclaw_provider_placeholders() { :; }', + 'ensure_mutable_openclaw_config_hash() { :; }', 'export_gateway_token() { :; }', 'write_runtime_shell_env() { :; }', 'ensure_runtime_shell_env_shim() { :; }', @@ -814,6 +816,62 @@ describe("runtime model override (#759)", () => { }); }); +describe("mutable OpenClaw config hash", () => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + + function runEnsureHash(owner: "sandbox" | "root") { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-config-hash-")); + const openclawDir = path.join(root, ".openclaw"); + fs.mkdirSync(openclawDir, { recursive: true }); + fs.writeFileSync(path.join(openclawDir, "openclaw.json"), '{"ok":true}\n'); + + const scriptPath = path.join(root, "run.sh"); + fs.writeFileSync( + scriptPath, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `openclaw_config_dir_owner() { printf '%s\\n' ${JSON.stringify(owner)}; }`, + extractShellFunctionFromSource(src, "ensure_mutable_openclaw_config_hash").replaceAll( + "/sandbox/.openclaw", + openclawDir, + ), + "ensure_mutable_openclaw_config_hash", + ].join("\n"), + { mode: 0o700 }, + ); + + const result = spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 5000 }); + const hashPath = path.join(openclawDir, ".config-hash"); + const hashExists = fs.existsSync(hashPath); + const hashCheck = hashExists + ? spawnSync("bash", ["-c", `cd ${JSON.stringify(openclawDir)} && sha256sum -c .config-hash --status`], { + encoding: "utf-8", + timeout: 5000, + }) + : undefined; + const hashMode = hashExists ? fs.statSync(hashPath).mode & 0o777 : undefined; + fs.rmSync(root, { recursive: true, force: true }); + return { result, hashExists, hashCheck, hashMode }; + } + + it("creates a missing hash for mutable-default OpenClaw config", () => { + const { result, hashExists, hashCheck, hashMode } = runEnsureHash("sandbox"); + + expect(result.status).toBe(0); + expect(hashExists).toBe(true); + expect(hashCheck?.status).toBe(0); + expect(hashMode).toBe(0o660); + }); + + it("does not synthesize a missing locked config trust anchor", () => { + const { result, hashExists } = runEnsureHash("root"); + + expect(result.status).toBe(0); + expect(hashExists).toBe(false); + }); +}); + describe("runtime CORS origin override (#719)", () => { const src = fs.readFileSync(START_SCRIPT, "utf-8"); @@ -1700,6 +1758,8 @@ describe("Telegram diagnostics (#2766)", () => { 'apply_model_override() { :; }', 'reconcile_agent_model_with_provider() { :; }', 'apply_cors_override() { :; }', + 'refresh_openclaw_provider_placeholders() { :; }', + 'ensure_mutable_openclaw_config_hash() { :; }', 'export_gateway_token() { :; }', 'write_runtime_shell_env() { :; }', 'ensure_runtime_shell_env_shim() { :; }', diff --git a/test/onboard-preset-diff.test.ts b/test/onboard-preset-diff.test.ts index d46c112060d..f2d701c8f2c 100644 --- a/test/onboard-preset-diff.test.ts +++ b/test/onboard-preset-diff.test.ts @@ -59,6 +59,8 @@ function buildPreamble({ return String.raw` // All stubs MUST be installed before requiring onboard so its module-level // destructuring picks up the patched functions. +Object.defineProperty(process, "platform", { value: "darwin" }); + const resolver = require(${resolveOpenshellPath}); resolver.resolveOpenshell = () => "/fake/openshell"; diff --git a/test/onboard.test.ts b/test/onboard.test.ts index f4c32b95f2c..6dc43b33253 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -8,6 +8,7 @@ import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { describe, expect, it } from "vitest"; +import YAML from "yaml"; import type { AgentDefinition } from "../dist/lib/agent/defs.js"; import { loadAgent } from "../dist/lib/agent/defs.js"; @@ -49,6 +50,12 @@ type OnboardTestInternals = { buildCompatibleEndpointSandboxSmokeCommand: (model: string) => string; buildCompatibleEndpointSandboxSmokeScript: (model: string) => string; buildSandboxConfigSyncScript: ShimFn; + buildSandboxGpuCreateArgs: (config: { + sandboxGpuEnabled: boolean; + sandboxGpuDevice?: string | null; + }) => string[]; + buildDirectGpuPolicyYaml: (basePolicy: string) => string; + buildDirectSandboxGpuProofCommands: (sandboxName: string) => { label: string; args: string[] }[]; classifySandboxCreateFailure: (output?: string) => { kind: string; uploadedToGateway: boolean }; compactText: (value?: string) => string; computeSetupPresetSuggestions: ShimFn; @@ -77,6 +84,51 @@ type OnboardTestInternals = { getInstalledOpenshellVersion: (versionOutput?: string | null) => string | null; getBlueprintMinOpenshellVersion: (rootDir?: string) => string | null; getBlueprintMaxOpenshellVersion: (rootDir?: string) => string | null; + getDockerDriverGatewayEnv: (versionOutput?: string | null) => Record; + getDockerDriverGatewayRuntimeDriftFromSnapshot: (snapshot: { + processEnv: Record | null; + processExe: string | null; + desiredEnv: Record; + gatewayBin?: string | null; + }) => { reason: string } | null; + isLinuxDockerDriverGatewayEnabled: (platform?: NodeJS.Platform) => boolean; + isDockerDriverGatewayPortListener: ( + portCheck: { + ok: boolean; + process?: string; + pid?: number | null; + }, + opts?: { + platform?: NodeJS.Platform; + gatewayBin?: string | null; + isPidAliveFn?: (pid: number) => boolean; + isDockerDriverGatewayProcessFn?: (pid: number, gatewayBin?: string | null) => boolean; + }, + ) => boolean; + findReadableNvidiaCdiSpecFiles: (dirs: string[]) => string[]; + parseDockerCdiSpecDirs: (value?: string | null) => string[]; + resolveSandboxGpuConfig: ( + gpu: { type: string } | null, + options?: { flag?: "enable" | "disable" | null; device?: string | null; env?: NodeJS.ProcessEnv }, + ) => { + mode: "auto" | "1" | "0"; + hostGpuDetected: boolean; + sandboxGpuEnabled: boolean; + sandboxGpuDevice: string | null; + errors: string[]; + }; + getResumeSandboxGpuOverrides: ( + entry: + | { sandboxGpuMode?: "auto" | "1" | "0" | string | null; sandboxGpuDevice?: string | null } + | null + | undefined, + sessionGpuPassthrough?: boolean, + ) => { flag: "enable" | "disable" | null; device: string | null }; + shouldAllowOpenshellAboveBlueprintMax: ( + versionOutput?: string | null, + platform?: NodeJS.Platform, + env?: NodeJS.ProcessEnv, + ) => boolean; versionGte: (left?: string | null, right?: string | null) => boolean; getRequestedModelHint: ShimFn; getRequestedProviderHint: ShimFn; @@ -115,7 +167,7 @@ type OnboardTestInternals = { providerNameToOptionKey: (name?: string | null) => string | null; parsePolicyPresetEnv: (value: string | null) => string[]; patchStagedDockerfile: ShimFn; - pullAndResolveBaseImageDigest: () => { digest: string; ref: string } | null; + pullAndResolveBaseImageDigest: () => { digest: string | null; ref: string } | null; SANDBOX_BASE_IMAGE: string; printSandboxCreateRecoveryHints: ShimFn; resolveDashboardForwardTarget: (chatUiUrl?: string) => string; @@ -150,7 +202,19 @@ function isOnboardTestInternals( typeof value.buildCompatibleEndpointSandboxSmokeCommand === "function" && typeof value.buildCompatibleEndpointSandboxSmokeScript === "function" && typeof value.buildSandboxConfigSyncScript === "function" && + typeof value.buildSandboxGpuCreateArgs === "function" && + typeof value.buildDirectGpuPolicyYaml === "function" && + typeof value.buildDirectSandboxGpuProofCommands === "function" && typeof value.classifySandboxCreateFailure === "function" && + typeof value.getDockerDriverGatewayEnv === "function" && + typeof value.getDockerDriverGatewayRuntimeDriftFromSnapshot === "function" && + typeof value.isLinuxDockerDriverGatewayEnabled === "function" && + typeof value.isDockerDriverGatewayPortListener === "function" && + typeof value.findReadableNvidiaCdiSpecFiles === "function" && + typeof value.parseDockerCdiSpecDirs === "function" && + typeof value.resolveSandboxGpuConfig === "function" && + typeof value.getResumeSandboxGpuOverrides === "function" && + typeof value.shouldAllowOpenshellAboveBlueprintMax === "function" && typeof value.hasChatCompletionsToolCall === "function" && typeof value.hasChatCompletionsToolCallLeak === "function" && typeof value.filterSetupPolicyPresets === "function" && @@ -182,6 +246,9 @@ const { buildCompatibleEndpointSandboxSmokeCommand, buildCompatibleEndpointSandboxSmokeScript, buildSandboxConfigSyncScript, + buildSandboxGpuCreateArgs, + buildDirectGpuPolicyYaml, + buildDirectSandboxGpuProofCommands, classifySandboxCreateFailure, compactText, computeSetupPresetSuggestions, @@ -195,6 +262,15 @@ const { getInstalledOpenshellVersion, getBlueprintMinOpenshellVersion, getBlueprintMaxOpenshellVersion, + getDockerDriverGatewayEnv, + getDockerDriverGatewayRuntimeDriftFromSnapshot, + isLinuxDockerDriverGatewayEnabled, + isDockerDriverGatewayPortListener, + findReadableNvidiaCdiSpecFiles, + parseDockerCdiSpecDirs, + resolveSandboxGpuConfig, + getResumeSandboxGpuOverrides, + shouldAllowOpenshellAboveBlueprintMax, versionGte, getRequestedModelHint, getRequestedProviderHint, @@ -233,7 +309,241 @@ const { formatSandboxBuildEstimateNote, } = onboardTestInternals; +const repoRoot = path.join(import.meta.dirname, ".."); + describe("onboard helpers", () => { + it("resolves sandbox GPU auto/force/disable modes", () => { + const gpu = { type: "nvidia" }; + expect(resolveSandboxGpuConfig(gpu, { env: {} }).sandboxGpuEnabled).toBe(true); + expect( + resolveSandboxGpuConfig(gpu, { + env: { NEMOCLAW_SANDBOX_GPU: "0" }, + }).sandboxGpuEnabled, + ).toBe(false); + const forced = resolveSandboxGpuConfig(null, { + flag: "enable", + env: {}, + }); + expect(forced.mode).toBe("1"); + expect(forced.errors.join("\n")).toContain("no NVIDIA GPU"); + }); + + it("resumes sandbox GPU auto mode without turning CPU fallback into explicit opt-out", () => { + const resumedAuto = getResumeSandboxGpuOverrides( + { sandboxGpuMode: "auto", sandboxGpuDevice: null }, + false, + ); + expect(resumedAuto).toEqual({ flag: null, device: null }); + expect( + resolveSandboxGpuConfig({ type: "nvidia" }, { ...resumedAuto, env: {} }).sandboxGpuEnabled, + ).toBe(true); + + const resumedDisabled = getResumeSandboxGpuOverrides( + { sandboxGpuMode: "0", sandboxGpuDevice: null }, + false, + ); + expect( + resolveSandboxGpuConfig({ type: "nvidia" }, { ...resumedDisabled, env: {} }) + .sandboxGpuEnabled, + ).toBe(false); + + const legacyGpuSession = getResumeSandboxGpuOverrides(null, true); + expect(legacyGpuSession.flag).toBe("enable"); + }); + + it("builds OpenShell sandbox GPU create args", () => { + expect(buildSandboxGpuCreateArgs({ sandboxGpuEnabled: false })).toEqual([]); + expect(buildSandboxGpuCreateArgs({ sandboxGpuEnabled: true })).toEqual(["--gpu"]); + expect( + buildSandboxGpuCreateArgs({ sandboxGpuEnabled: true, sandboxGpuDevice: "nvidia.com/gpu=0" }), + ).toEqual(["--gpu", "--gpu-device", "nvidia.com/gpu=0"]); + }); + + it("keeps /proc read-only and narrows GPU proc writes to comm", () => { + const basePolicy = fs.readFileSync( + path.join(repoRoot, "nemoclaw-blueprint", "policies", "openclaw-sandbox.yaml"), + "utf-8", + ); + const gpuPolicy = buildDirectGpuPolicyYaml(basePolicy); + const baseDoc = YAML.parse(basePolicy); + const gpuDoc = YAML.parse(gpuPolicy); + + expect(baseDoc.filesystem_policy.read_only).toContain("/proc"); + expect(gpuDoc.filesystem_policy.read_only).toContain("/proc"); + expect(gpuDoc.filesystem_policy.read_write).not.toContain("/proc"); + expect(gpuDoc.filesystem_policy.read_write).toContain("/proc/self/task/*/comm"); + }); + + it("removes stale broad /proc write entries from GPU policy input", () => { + const gpuPolicy = buildDirectGpuPolicyYaml(` +version: 1 +filesystem_policy: + include_workdir: true + read_only: + - /usr + read_write: + - /tmp + - /proc +network_policies: + nvidia: + name: nvidia + endpoints: + - host: integrate.api.nvidia.com + port: 443 +`); + const gpuDoc = YAML.parse(gpuPolicy); + + expect(gpuDoc.filesystem_policy.read_only).toContain("/proc"); + expect(gpuDoc.filesystem_policy.read_write).toEqual([ + "/tmp", + "/proc/self/task/*/comm", + ]); + }); + + it("models the Linux OpenShell Docker-driver gateway environment", () => { + expect(isLinuxDockerDriverGatewayEnabled("linux")).toBe(true); + expect(isLinuxDockerDriverGatewayEnabled("darwin")).toBe(false); + const env = getDockerDriverGatewayEnv("openshell 0.0.37"); + expect(env.OPENSHELL_DRIVERS).toBe("docker"); + expect(env.OPENSHELL_GRPC_ENDPOINT).toBe("http://127.0.0.1:8080"); + expect(env.OPENSHELL_CLUSTER_IMAGE).toBeUndefined(); + expect(env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE).toContain(":0.0.37"); + }); + + it("detects stale Docker-driver gateway runtime state before reuse", () => { + const desiredEnv = getDockerDriverGatewayEnv("openshell 0.0.37"); + const gatewayBin = process.execPath; + + expect( + getDockerDriverGatewayRuntimeDriftFromSnapshot({ + processEnv: desiredEnv, + processExe: gatewayBin, + desiredEnv, + gatewayBin, + }), + ).toBeNull(); + + expect( + getDockerDriverGatewayRuntimeDriftFromSnapshot({ + processEnv: { + ...desiredEnv, + OPENSHELL_DOCKER_SUPERVISOR_IMAGE: + "ghcr.io/nvidia/openshell/supervisor:0.0.36", + }, + processExe: gatewayBin, + desiredEnv, + gatewayBin, + })?.reason, + ).toContain("OPENSHELL_DOCKER_SUPERVISOR_IMAGE="); + + expect( + getDockerDriverGatewayRuntimeDriftFromSnapshot({ + processEnv: desiredEnv, + processExe: `${gatewayBin} (deleted)`, + desiredEnv, + gatewayBin, + })?.reason, + ).toContain("replaced on disk"); + + expect( + getDockerDriverGatewayRuntimeDriftFromSnapshot({ + processEnv: null, + processExe: gatewayBin, + desiredEnv, + gatewayBin, + })?.reason, + ).toContain("process environment"); + }); + + it("recognizes an existing Docker-driver gateway listener on Linux", () => { + const opts = { + platform: "linux" as NodeJS.Platform, + isPidAliveFn: (pid: number) => pid === 1234, + isDockerDriverGatewayProcessFn: (pid: number, gatewayBin?: string | null) => + pid === 1234 && gatewayBin === "/opt/openshell/openshell-gateway", + gatewayBin: "/opt/openshell/openshell-gateway", + }; + expect( + isDockerDriverGatewayPortListener({ ok: false, process: "openshell", pid: 1234 }, opts), + ).toBe(true); + expect( + isDockerDriverGatewayPortListener({ ok: false, process: "openshell-", pid: 1234 }, opts), + ).toBe(true); + expect( + isDockerDriverGatewayPortListener({ ok: false, process: "node", pid: 1234 }, opts), + ).toBe(false); + expect( + isDockerDriverGatewayPortListener( + { ok: false, process: "openshell", pid: 1234 }, + { ...opts, platform: "darwin" }, + ), + ).toBe(false); + expect( + isDockerDriverGatewayPortListener( + { ok: false, process: "openshell", pid: 4321 }, + { ...opts, isPidAliveFn: () => false }, + ), + ).toBe(false); + }); + + it("recognizes Docker CDI and explicit dev-channel version gates", () => { + expect(parseDockerCdiSpecDirs('["/etc/cdi","/var/run/cdi"]')).toEqual([ + "/etc/cdi", + "/var/run/cdi", + ]); + expect(parseDockerCdiSpecDirs("")).toEqual([]); + expect( + shouldAllowOpenshellAboveBlueprintMax("openshell 0.0.38.dev1+gabcdef", "linux", { + NEMOCLAW_OPENSHELL_CHANNEL: "dev", + }), + ).toBe(true); + expect( + shouldAllowOpenshellAboveBlueprintMax("openshell 0.0.38.dev1+gabcdef", "linux", { + NEMOCLAW_OPENSHELL_CHANNEL: "auto", + }), + ).toBe(false); + expect( + shouldAllowOpenshellAboveBlueprintMax("openshell 0.0.38", "linux", { + NEMOCLAW_OPENSHELL_CHANNEL: "dev", + }), + ).toBe(false); + }); + + it("requires readable NVIDIA CDI spec files, not just CDI directories", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cdi-specs-")); + try { + const emptyDir = path.join(tmpDir, "empty"); + const cdiDir = path.join(tmpDir, "cdi"); + fs.mkdirSync(emptyDir); + fs.mkdirSync(cdiDir); + fs.writeFileSync(path.join(cdiDir, "unrelated.yaml"), "kind: example.com/device\n"); + expect(findReadableNvidiaCdiSpecFiles([emptyDir, cdiDir])).toEqual([]); + + const specPath = path.join(cdiDir, "gpu-devices.yaml"); + fs.writeFileSync( + specPath, + ["cdiVersion: 0.6.0", "kind: nvidia.com/gpu", "devices:", " - name: all", ""].join( + "\n", + ), + ); + expect(findReadableNvidiaCdiSpecFiles([emptyDir, cdiDir])).toEqual([specPath]); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("builds direct sandbox GPU proof commands", () => { + const commands = buildDirectSandboxGpuProofCommands("alpha"); + expect(commands.map((entry) => entry.label)).toEqual([ + "nvidia-smi", + "/proc/self/task//comm write", + "cuInit(0) via libcuda.so.1", + ]); + expect(commands[0].args).toEqual(["sandbox", "exec", "-n", "alpha", "--", "nvidia-smi"]); + expect(commands[1].args.join(" ")).toContain("/proc/self/task/${tid}/comm"); + expect(commands[2].args.join(" ")).toContain("cuInit(0)"); + }); + it("uses Hermes-oriented sandbox defaults when NemoHermes selects Hermes", () => { const previousSandboxName = process.env.NEMOCLAW_SANDBOX_NAME; try { @@ -2167,6 +2477,7 @@ mod._load = function(req, parent, isMain) { } return origLoad.call(this, req, parent, isMain); }; +Object.defineProperty(process, "platform", { value: "darwin" }); const { startGateway } = require(${onboardPath}); startGateway(null).catch(() => {}); `; @@ -2247,6 +2558,13 @@ startGateway(null).catch(() => {}); "my-assistant NotReady init failed", ), ).toBe("not_ready"); + expect( + getSandboxStateFromOutputs( + "my-assistant", + "Error: NotFound: sandbox not found", + "other-sandbox Ready 2m ago", + ), + ).toBe("missing"); expect(getSandboxStateFromOutputs("my-assistant", "", "")).toBe("missing"); }); @@ -4410,6 +4728,24 @@ const { setupInference } = require(${onboardPath}); ); }); + it("records gateway completion when a fresh onboard reuses an existing gateway", () => { + const source = fs.readFileSync( + path.join(import.meta.dirname, "..", "src", "lib", "onboard.ts"), + "utf-8", + ); + const reusePos = source.indexOf('skippedStepMessage("gateway", "running", "reuse")'); + const nextBranchPos = source.indexOf("} else {", reusePos); + const reuseBlock = source.slice(reusePos, nextBranchPos); + + assert.ok(reusePos !== -1, "gateway reuse branch not found"); + assert.ok(nextBranchPos !== -1, "gateway reuse branch end not found"); + assert.match( + reuseBlock, + /onboardSession\.markStepComplete\("gateway"\)/, + "reused gateway must be persisted so a later resume can skip it", + ); + }); + it("starts the sandbox step before prompting for the sandbox name", () => { const source = fs.readFileSync( path.join(import.meta.dirname, "..", "src", "lib", "onboard.ts"), @@ -4420,7 +4756,7 @@ const { setupInference } = require(${onboardPath}); source, // #2753: sandboxName is intentionally absent from the options here so // the session does not record a name before createSandbox completes. - /startRecordedStep\("sandbox", \{ provider, model \}\);\s*const recordedMessagingChannels = getRecordedMessagingChannelsForResume\(resume, session\);[\s\S]*?selectedMessagingChannels = recordedMessagingChannels;[\s\S]*?selectedMessagingChannels = await setupMessagingChannels\(\);[\s\S]*?const messagingChannelConfig = readMessagingChannelConfigFromEnv\(\);[\s\S]*?onboardSession\.updateSession\(\(current[^)]*\) => \{\s*current\.messagingChannels = selectedMessagingChannels;\s*current\.messagingChannelConfig = messagingChannelConfig;\s*return current;\s*\}\);[\s\S]*?sandboxName = await createSandbox\(\s*gpu,\s*model,\s*provider,\s*preferredInferenceApi,\s*sandboxName,\s*nextWebSearchConfig,\s*selectedMessagingChannels,\s*fromDockerfile,\s*agent,\s*opts\.controlUiPort \|\| null,\s*gpuPassthrough,\s*\);/, + /startRecordedStep\("sandbox", \{ provider, model \}\);\s*const recordedMessagingChannels = getRecordedMessagingChannelsForResume\(resume, session\);[\s\S]*?selectedMessagingChannels = recordedMessagingChannels;[\s\S]*?selectedMessagingChannels = await setupMessagingChannels\(\);[\s\S]*?const messagingChannelConfig = readMessagingChannelConfigFromEnv\(\);[\s\S]*?onboardSession\.updateSession\(\(current[^)]*\) => \{\s*current\.messagingChannels = selectedMessagingChannels;\s*current\.messagingChannelConfig = messagingChannelConfig;\s*return current;\s*\}\);[\s\S]*?sandboxName = await createSandbox\(\s*gpu,\s*model,\s*provider,\s*preferredInferenceApi,\s*sandboxName,\s*nextWebSearchConfig,\s*selectedMessagingChannels,\s*fromDockerfile,\s*agent,\s*opts\.controlUiPort \|\| null,\s*sandboxGpuConfig,\s*\);/, ); }); @@ -4430,10 +4766,10 @@ const { setupInference } = require(${onboardPath}); "utf-8", ); - assert.match(source, /const detectedNvidiaGpu = gpu\?\.type === "nvidia";/); + assert.match(source, /const explicitSandboxGpuFlag = resolveSandboxGpuFlagFromOptions\(opts\);/); assert.match( source, - /const gpuPassthrough = optedOutGpuPassthrough[\s\S]*\? false[\s\S]*\? true[\s\S]*: detectedNvidiaGpu;/, + /const gpuPassthrough = sandboxGpuConfig\.sandboxGpuEnabled;/, ); assert.match(source, /Use --no-gpu to opt out/); }); @@ -4499,7 +4835,7 @@ const { setupInference } = require(${onboardPath}); // #2753: a stale `session.sandboxName` from an interrupted onboard // must not override a fresh `--name` / NEMOCLAW_SANDBOX_NAME, so the // session value participates only when its sandbox step completed. - /const recordedSandboxName =\s*session\?\.steps\?\.sandbox\?\.status === "complete" \? session\?\.sandboxName \|\| null : null;\s*let sandboxName = recordedSandboxName \|\| requestedSandboxName \|\| null;\s*if \(sandboxName && RESERVED_SANDBOX_NAMES\.has\(sandboxName\)\) \{[\s\S]*?process\.exit\(1\);\s*\}/, + /const recordedSandboxName =\s*session\?\.steps\?\.sandbox\?\.status === "complete" \? session\?\.sandboxName \|\| null : null;[\s\S]*?let sandboxName = recordedSandboxName \|\| requestedSandboxName \|\| null;\s*if \(sandboxName && RESERVED_SANDBOX_NAMES\.has\(sandboxName\)\) \{[\s\S]*?process\.exit\(1\);\s*\}/, ); }); it("reserves update as a sandbox name because it is a global command", () => { @@ -5466,7 +5802,7 @@ const { createSandbox } = require(${onboardPath}); assert.deepEqual(payload.registeredPolicies, ["slack"]); assert.deepEqual(payload.slackBinaryPaths, [ "/usr/local/bin/hermes", - "/usr/bin/python3.11", + "/usr/bin/python3*", "/opt/hermes/.venv/bin/python", ]); assert.ok( @@ -8777,8 +9113,8 @@ const { createSandbox } = require(${onboardPath}); path.join(import.meta.dirname, "..", "src", "lib", "onboard.ts"), "utf-8", ); - const pullPos = source.indexOf("pullAndResolveBaseImageDigest()"); - assert.ok(pullPos !== -1, "pullAndResolveBaseImageDigest() call not found in onboard.ts"); + const pullPos = source.search(/const resolved = pullAndResolveBaseImageDigest\s*\(/); + assert.ok(pullPos !== -1, "pullAndResolveBaseImageDigest call not found in onboard.ts"); const patchPos = source.indexOf("patchStagedDockerfile(", pullPos); assert.ok( patchPos > pullPos, diff --git a/test/policy-tiers-onboard.test.ts b/test/policy-tiers-onboard.test.ts index eff3946856c..ab361cfc367 100644 --- a/test/policy-tiers-onboard.test.ts +++ b/test/policy-tiers-onboard.test.ts @@ -69,6 +69,8 @@ const credentials = require(${credPath}); const runner = require(${runnerPath}); const registry = require(${registryPath}); +Object.defineProperty(process, "platform", { value: "darwin" }); + // Stub heavy I/O credentials.prompt = async (msg) => { throw new Error("unexpected prompt: " + msg); }; credentials.ensureApiKey = async () => {}; diff --git a/test/runner.test.ts b/test/runner.test.ts index a426f8a0543..eb1267f3a6a 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -661,15 +661,28 @@ describe("regression guards", () => { shift || true done if [ -n "$out" ]; then - if [ "$(basename "$out")" = "openshell-checksums-sha256.txt" ]; then + case "$(basename "$out")" in + openshell-checksums-sha256.txt) printf '%s\n' \ 'ignored openshell-x86_64-unknown-linux-musl.tar.gz' \ 'ignored openshell-aarch64-unknown-linux-musl.tar.gz' \ 'ignored openshell-x86_64-apple-darwin.tar.gz' \ 'ignored openshell-aarch64-apple-darwin.tar.gz' > "$out" - else + ;; + openshell-gateway-checksums-sha256.txt) + printf '%s\n' \ + 'ignored openshell-gateway-x86_64-unknown-linux-gnu.tar.gz' \ + 'ignored openshell-gateway-aarch64-unknown-linux-gnu.tar.gz' > "$out" + ;; + openshell-sandbox-checksums-sha256.txt) + printf '%s\n' \ + 'ignored openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz' \ + 'ignored openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz' > "$out" + ;; + *) : > "$out" - fi + ;; + esac fi return 0 } diff --git a/test/sandbox-init.test.ts b/test/sandbox-init.test.ts index 9b40d4dc440..46c6b777bbf 100644 --- a/test/sandbox-init.test.ts +++ b/test/sandbox-init.test.ts @@ -577,6 +577,13 @@ EOF expect(src).not.toContain("_PROXY_MARKER_BEGIN"); }); + it("hermes start.sh persists OpenShell proxy CA env for connect sessions", () => { + const src = readFileSync(join(import.meta.dirname, "../agents/hermes/start.sh"), "utf-8"); + expect(src).toContain("SSL_CERT_FILE CURL_CA_BUNDLE REQUESTS_CA_BUNDLE GIT_SSL_CAINFO"); + expect(src).toContain("export REQUESTS_CA_BUNDLE="); + expect(src).toContain("export GIT_SSL_CAINFO="); + }); + it("hermes start.sh routes Discord through the local decode proxy", () => { const src = readFileSync(join(import.meta.dirname, "../agents/hermes/start.sh"), "utf-8"); expect(src).toContain('export DISCORD_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}"'); diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index e8d8e26aa11..296e6dfebb7 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -180,8 +180,8 @@ describe("sandbox provisioning: unified .openclaw layout (#2227)", () => { }); }); -describe("sandbox provisioning: procps debug tools (#2343)", () => { - it("base apt layer requests procps and the SFTP server", () => { +describe("sandbox provisioning: base runtime tools", () => { + it("base apt layer requests procps, e2fsprogs, and the SFTP server", () => { const dockerfile = fs.readFileSync(DOCKERFILE_BASE, "utf-8"); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-apt-")); const lists = path.join(tmp, "apt-lists"); @@ -198,18 +198,20 @@ describe("sandbox provisioning: procps debug tools (#2343)", () => { ]); expect(result.status).toBe(0); expect(calls).toContain("apt-get update"); - expect(calls).toContain("procps=2:4.0.2-3"); - expect(calls).toContain("openssh-sftp-server=1:9.2p1-2+deb12u9"); + expect(calls).toContain("procps=2:4.0.4-9"); + expect(calls).toContain("e2fsprogs=1.47.2-3+b10"); + expect(calls).toContain("openssh-sftp-server=1:10.0p1-7+deb13u2"); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } }); - it("runtime hardening installs procps when a stale base lacks ps", () => { + it("runtime hardening installs procps and e2fsprogs when a stale base lacks ps and chattr", () => { const dockerfile = fs.readFileSync(DOCKERFILE, "utf-8"); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-procps-")); const log = path.join(tmp, "calls.log"); const marker = path.join(tmp, "ps-installed"); + const chattrMarker = path.join(tmp, "chattr-installed"); const lists = path.join(tmp, "apt-lists"); fs.mkdirSync(lists); const command = dockerRunCommandBetween( @@ -222,9 +224,10 @@ describe("sandbox provisioning: procps debug tools (#2343)", () => { "set -euo pipefail", `call_log=${JSON.stringify(log)}`, `ps_marker=${JSON.stringify(marker)}`, + `chattr_marker=${JSON.stringify(chattrMarker)}`, 'apt-mark() { printf "apt-mark %s\\n" "$*" >> "$call_log"; }', - 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; if [[ "$*" == *"install"* && "$*" == *"procps=2:4.0.2-3"* ]]; then touch "$ps_marker"; fi; }', - 'command() { if [ "${1:-}" = "-v" ] && [ "${2:-}" = "ps" ]; then [ -f "$ps_marker" ]; else builtin command "$@"; fi; }', + 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; if [[ "$*" == *"install"* && "$*" == *"procps=2:4.0.4-9"* ]]; then touch "$ps_marker"; fi; if [[ "$*" == *"install"* && "$*" == *"e2fsprogs=1.47.2-3+b10"* ]]; then touch "$chattr_marker"; fi; }', + 'command() { if [ "${1:-}" = "-v" ] && [ "${2:-}" = "ps" ]; then [ -f "$ps_marker" ]; elif [ "${1:-}" = "-v" ] && [ "${2:-}" = "chattr" ]; then [ -f "$chattr_marker" ]; else builtin command "$@"; fi; }', 'ps() { [ -f "$ps_marker" ] || return 127; printf "procps test version\\n"; }', command, ].join("\n"); @@ -234,10 +237,13 @@ describe("sandbox provisioning: procps debug tools (#2343)", () => { const result = spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 5000 }); expect(result.status).toBe(0); const calls = fs.readFileSync(log, "utf-8"); - expect(calls).toContain("apt-mark manual procps"); + expect(calls).toContain("apt-mark manual procps e2fsprogs"); expect(calls).toContain("apt-get autoremove --purge -y"); expect(calls).toContain("apt-get update"); - expect(calls).toContain("apt-get install -y --no-install-recommends procps=2:4.0.2-3"); + expect(calls).toContain( + "apt-get install -y --no-install-recommends procps=2:4.0.4-9", + ); + expect(calls).toContain("apt-get install -y --no-install-recommends e2fsprogs=1.47.2-3+b10"); expect(result.stdout).toContain("procps test version"); } finally { fs.rmSync(tmp, { recursive: true, force: true }); diff --git a/test/shellquote-sandbox.test.ts b/test/shellquote-sandbox.test.ts index 8605d8f1865..2be5c2038e5 100644 --- a/test/shellquote-sandbox.test.ts +++ b/test/shellquote-sandbox.test.ts @@ -104,6 +104,7 @@ try { process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_NON_INTERACTIVE = "1"; process.env.NEMOCLAW_HEALTH_POLL_COUNT = "1"; + Object.defineProperty(process, "platform", { value: "darwin" }); const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); console.log(JSON.stringify({ sandboxName, commands })); } catch (error) { diff --git a/test/slack-token-rewriter.test.ts b/test/slack-token-rewriter.test.ts index 9b20f54ac16..145615a9da8 100644 --- a/test/slack-token-rewriter.test.ts +++ b/test/slack-token-rewriter.test.ts @@ -12,7 +12,7 @@ import fs from "node:fs"; import path from "node:path"; -import { describe, it, expect, beforeEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; const CANONICAL_REWRITER = path.join( import.meta.dirname, @@ -21,6 +21,20 @@ const CANONICAL_REWRITER = path.join( "scripts", "slack-token-rewriter.js", ); +const TOKEN_ENV_KEYS = ["SLACK_BOT_TOKEN", "SLACK_APP_TOKEN"]; +const ORIGINAL_TOKEN_ENV = new Map(TOKEN_ENV_KEYS.map((key) => [key, process.env[key]])); + +function clearTokenEnv() { + for (const key of TOKEN_ENV_KEYS) delete process.env[key]; +} + +function restoreTokenEnv() { + for (const key of TOKEN_ENV_KEYS) { + const original = ORIGINAL_TOKEN_ENV.get(key); + if (original === undefined) delete process.env[key]; + else process.env[key] = original; + } +} // Build fresh stub modules and load the rewriter on top of them. Returns the // stub modules whose request/get methods have been monkey-patched by the @@ -95,8 +109,12 @@ function loadRewriter() { describe("slack-token-rewriter: string rewriting", () => { let mod: ReturnType; beforeEach(() => { + clearTokenEnv(); mod = loadRewriter(); }); + afterEach(() => { + restoreTokenEnv(); + }); it("rewrites Bolt-shape placeholder in a string URL argument", () => { mod.https.request( @@ -138,6 +156,30 @@ describe("slack-token-rewriter: string rewriting", () => { expect(opts.headers.Authorization).toBe("Bearer openshell:resolve:env:SLACK_BOT_TOKEN"); }); + it("rewrites to the revision-scoped OpenShell placeholder when present in env", () => { + process.env.SLACK_BOT_TOKEN = "openshell:resolve:env:v12_SLACK_BOT_TOKEN"; + const opts = { + hostname: "api.slack.com", + path: "/api/auth.test", + headers: { Authorization: "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" }, + }; + mod.https.request(opts); + expect(opts.headers.Authorization).toBe( + "Bearer openshell:resolve:env:v12_SLACK_BOT_TOKEN", + ); + }); + + it("does not copy raw env token values into rewritten requests", () => { + process.env.SLACK_BOT_TOKEN = "xoxb-real-token-must-not-leak"; + const opts = { + hostname: "api.slack.com", + path: "/api/auth.test", + headers: { Authorization: "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" }, + }; + mod.https.request(opts); + expect(opts.headers.Authorization).toBe("Bearer openshell:resolve:env:SLACK_BOT_TOKEN"); + }); + it("rewrites lowercase header name", () => { const opts = { headers: { authorization: "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" },