diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index c8b78bacd4..6dfc815e1e 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -384,6 +384,73 @@ jobs: - name: Checkout uses: actions/checkout@v6 + - name: Start gateway log streamer (background) + run: | + # Diagnostic for NVIDIA/NemoClaw#2484: container log driver in + # openshell's k3s setup doesn't allow reading container stdio — + # only working path to /tmp/gateway.log is via SSH, which + # `nemoclaw logs` uses internally. + # + # Snapshot mode (not follow): every 10s, overwrite per-sandbox + # log file with the latest gateway log content. Bounded output + # (~62 lines per snapshot). When a sandbox is destroyed by the + # test, the file holds the final pre-destroy snapshot. + mkdir -p docker-logs + nohup bash -c ' + export PATH="$HOME/.local/bin:$PATH" + # Strategy: every 5s, snapshot each live sandbox via + # `docker exec openshell-cluster-nemoclaw kubectl ...`. This + # bypasses both per-pod networking (which has had connection- + # refused races for some sandboxes) and the host openshell + # client (which loses gateway metadata after TC-SBX-06s + # docker-kill). kubectl talks directly to k3s in the cluster + # container. + # + # Snapshot mode (overwrite per iteration), not live tail-F: + # the gateway-persistent.log file accumulates everything since + # boot (mirrored from /tmp/gateway.log by nemoclaw-start.sh), + # so a single full-cat at any point gives us complete history. + # Each iteration is short-lived so transient connection issues + # do not cause us to lose the entire stream. + # + # Also snapshot kubectl pod listing per iteration so we have + # the actual pod naming convention even if the cluster is + # destroyed by teardown later. + while sleep 5; do + if ! docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^openshell-cluster-nemoclaw$"; then + continue + fi + docker exec openshell-cluster-nemoclaw kubectl get pods -A --no-headers >docker-logs/_pods.txt 2>&1 + registry="$HOME/.nemoclaw/sandboxes.json" + [ -f "$registry" ] || continue + live=$(jq -r ".sandboxes // {} | keys[]?" "$registry" 2>/dev/null) + for name in $live; do + case "$name" in + *[!a-z0-9_-]*|"") continue ;; + esac + # Find pod by sandbox name. openshell uses the sandbox + # name as the namespace and "agent" as the pod name. + # Try a few common patterns. + pod_match=$(awk -v n="$name" "\$1==n || \$2==n || \$1==\"sandbox-\" n || \$2==\"sandbox-\" n {print \$1\"/\"\$2; exit}" docker-logs/_pods.txt) + if [ -z "$pod_match" ]; then + # Fallback: any pod whose name contains the sandbox name + pod_match=$(awk -v n="$name" "index(\$2,n)>0 {print \$1\"/\"\$2; exit}" docker-logs/_pods.txt) + fi + if [ -z "$pod_match" ]; then continue; fi + pod_ns="${pod_match%%/*}" + pod_name="${pod_match##*/}" + docker exec openshell-cluster-nemoclaw kubectl exec -n "$pod_ns" "$pod_name" -- bash -c " + for f in /sandbox/.openclaw-data/logs/gateway-persistent.log /tmp/gateway.log /tmp/openclaw-*/openclaw-*.log; do + [ -f \"\$f\" ] || continue + printf \"\\n----- %s (size=%s) -----\\n\" \"\$f\" \"\$(stat -c%s \"\$f\" 2>/dev/null || echo ?)\" + cat -- \"\$f\" 2>/dev/null + done + " > "docker-logs/sandbox-${name}.log" 2>&1 + done + done + ' >/dev/null 2>&1 & + echo $! > /tmp/gateway-log-streamer.pid + - name: Run sandbox operations E2E test env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} @@ -391,8 +458,154 @@ jobs: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_POLICY_TIER: "open" GITHUB_TOKEN: ${{ github.token }} + # Override the 1800s default in test/e2e/e2e-timeout.sh. Sandbox + # creation alone is ~14 min per sandbox in current CI conditions + # (build+upload to k3s gateway), and the test creates two — leaving + # the default 30-min budget completely consumed by setup with no + # room for the actual TC-SBX cases. The job-level timeout (60 min, + # set in `timeout-minutes` above) is the real upper bound. + NEMOCLAW_E2E_TIMEOUT_SECONDS: "2700" run: bash test/e2e/test-sandbox-operations.sh + - name: Stop gateway log streamer + if: always() + # Diagnostic step: never let `bash -e` kill the snapshot loop on a + # single command failure (openshell ssh-config, nemoclaw logs, etc. + # all routinely fail post-test depending on TC-SBX-06's docker-kill + # state). We log the failures inline and continue. + shell: bash --noprofile --norc -uo pipefail {0} + run: | + [ -f /tmp/gateway-log-streamer.pid ] && kill "$(cat /tmp/gateway-log-streamer.pid)" 2>/dev/null || true + # Kill any per-sandbox SSH+tail followers spawned by the streamer. + pkill -f 'tail -n \+1 -F /tmp/gateway.log' 2>/dev/null || true + pkill -f 'ssh.*openshell-' 2>/dev/null || true + sleep 2 + # Final snapshot: tail -F glob expands once at start, so log files + # for openclaw processes that ran as a different UID (creating new + # /tmp/openclaw-/ dirs mid-test) get missed. Re-glob now and + # append every openclaw log file from each live sandbox to the + # per-sandbox docker-logs file. + # + # Use `nemoclaw logs` (not raw openshell ssh-config + ssh) + # because nemoclaw handles SSH key/host setup and is robust to + # streamer race conditions. Tested working in TC-SBX-04. + export PATH="$HOME/.local/bin:$PATH" + echo "=== final-snapshot: PATH=$PATH" + echo "=== final-snapshot: nemoclaw=$(command -v nemoclaw)" + echo "=== final-snapshot: openshell=$(command -v openshell)" + # TC-SBX-06's docker kill of the gateway pod can leave openshell + # without an active gateway selected; re-select before the snapshot + # so `nemoclaw logs` and direct `openshell sandbox exec` both + # have a target. The select is best-effort — failure (e.g., gateway + # not yet recovered) just means we fall through to ssh-config-based + # capture below. + openshell gateway select nemoclaw 2>&1 | head -5 || true + openshell gateway list 2>&1 | head -10 || true + # NEW PATH: bypass the openshell client entirely. The + # openshell-cluster-nemoclaw docker container runs k3s with + # kubectl available inside. Even after TC-SBX-06's docker-kill, + # docker auto-restarts the container and k3s state survives via + # /var/lib/rancher/k3s. Use `docker exec ... kubectl` to read + # the persistent log directly from each sandbox pod, with no + # dependency on the host's openshell metadata. + echo "=== final-snapshot: docker containers:" + docker ps --format '{{.Names}}\t{{.Status}}' 2>&1 | head -10 + echo "=== final-snapshot: cluster pods:" + docker exec openshell-cluster-nemoclaw kubectl get pods -A --no-headers 2>&1 | head -20 + if [ -f "$HOME/.nemoclaw/sandboxes.json" ]; then + echo "=== final-snapshot: sandboxes.json contents:" + cat "$HOME/.nemoclaw/sandboxes.json" 2>&1 | head -30 + registry_keys=$(jq -r ".sandboxes // {} | keys[]?" "$HOME/.nemoclaw/sandboxes.json" 2>&1) + echo "=== final-snapshot: sandbox names from jq: '$registry_keys'" + for name in $registry_keys; do + case "$name" in *[!a-z0-9_-]*|"") echo "=== final-snapshot: skipping invalid name '$name'"; continue ;; esac + echo "=== final-snapshot: capturing logs for '$name'" + { + printf '\n\n===== FINAL SNAPSHOT: %s =====\n' "$name" + # FIRST attempt: docker exec into the cluster container and + # kubectl-exec into the sandbox pod. This works even when + # the host openshell client is broken post-TC-SBX-06 because + # docker (and k3s inside the cluster) survive the gateway + # docker-kill via auto-restart + persistent k3s state. + pod_ns_name=$(docker exec openshell-cluster-nemoclaw kubectl get pods -A --no-headers 2>/dev/null | awk -v n="$name" '$2==n {print $1"/"$2; exit}') + if [ -n "$pod_ns_name" ]; then + echo "(found pod $pod_ns_name for $name)" + pod_ns="${pod_ns_name%%/*}" + pod_name="${pod_ns_name##*/}" + k_out=$(mktemp) + docker exec openshell-cluster-nemoclaw kubectl exec -n "$pod_ns" "$pod_name" -- bash -c ' + for f in /sandbox/.openclaw-data/logs/gateway-persistent.log /tmp/gateway.log /tmp/openclaw-*/openclaw-*.log; do + [ -f "$f" ] || continue + printf "\n----- %s (size=%s) -----\n" "$f" "$(stat -c%s "$f" 2>/dev/null || echo ?)" + cat -- "$f" 2>/dev/null || true + done + ' >"$k_out" 2>&1 + k_rc=$? + echo "(kubectl exec rc=$k_rc size=$(wc -c <"$k_out"))" + tail -c 500000 "$k_out" + rm -f "$k_out" + else + echo "(no kubectl pod found matching '$name')" + fi + # Existing fallbacks (raw ssh + nemoclaw logs) preserved + # below in case the docker/kubectl path also fails — they + # provide complementary coverage during transient states. + ssh_cfg="/tmp/sshcfg-final-${name}.tmp" + if openshell sandbox ssh-config "$name" >"$ssh_cfg" 2>&1 && [ -s "$ssh_cfg" ]; then + ssh_out=$(mktemp) + ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o LogLevel=ERROR \ + "openshell-${name}" \ + 'for f in /sandbox/.openclaw-data/logs/gateway-persistent.log \ + /tmp/gateway.log \ + /tmp/openclaw-*/openclaw-*.log; do + [ -f "$f" ] || continue + printf "\n----- %s (size=%s) -----\n" "$f" "$(stat -c%s "$f" 2>/dev/null || echo ?)" + cat -- "$f" 2>/dev/null || true + done' >"$ssh_out" 2>&1 + ssh_rc=$? + tail -c 500000 "$ssh_out" + rm -f "$ssh_out" + [ "$ssh_rc" -eq 0 ] || echo "(direct ssh exited rc=$ssh_rc)" + else + echo "(openshell sandbox ssh-config failed for $name)" + # Fallback to nemoclaw logs (less reliable, but try anything) + if command -v nemoclaw >/dev/null 2>&1; then + nm_out=$(mktemp) + nemoclaw "$name" logs >"$nm_out" 2>&1 + echo "(nemoclaw logs rc=$? size=$(wc -c <"$nm_out"))" + tail -c 500000 "$nm_out" + rm -f "$nm_out" + fi + fi + rm -f "$ssh_cfg" + } >> "docker-logs/sandbox-${name}.log" + done + else + echo "=== final-snapshot: sandboxes.json not found at $HOME/.nemoclaw/sandboxes.json" + fi + # Cap each log file at 5MB by keeping only the last 5MB — useful + # content (real gateway events) is mixed throughout, so tail-trim + # is fine for diagnostic purposes. + for f in docker-logs/*.log; do + [ -f "$f" ] || continue + sz=$(stat -c%s "$f" 2>/dev/null || stat -f%z "$f" 2>/dev/null || echo 0) + if [ "$sz" -gt 5242880 ]; then + tail -c 5242880 "$f" > "${f}.tail" && mv "${f}.tail" "$f" + fi + done + ls -la docker-logs/ 2>&1 | head -20 || true + du -sh docker-logs/ 2>&1 || true + + - name: Upload sandbox gateway logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: sandbox-operations-docker-logs + path: docker-logs/ + if-no-files-found: ignore + - name: Upload test log on failure if: failure() uses: actions/upload-artifact@v4 diff --git a/Dockerfile b/Dockerfile index 3a446291f0..89ccde83e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,7 +79,17 @@ RUN set -eu; \ # rmdir failure inside npm's own install path. rm -rf /usr/local/lib/node_modules/openclaw /usr/local/bin/openclaw; \ npm install -g --no-audit --no-fund --no-progress "openclaw@${MIN_VER}"; \ - fi + fi; \ + # Pre-install the codex-acp package so the embedded ACPx runtime can + # call the local binary instead of `npx @zed-industries/codex-acp`. + # The sandbox's L7 proxy denies @zed-industries/* package URLs + # (403 policy_denied), and npm still refreshes registry metadata for + # versioned npx package specs even when the package is globally installed. + # Installing the binary at build time and configuring ACPx to use it + # directly keeps TC-SBX-02 off the runtime npm path. + npm install -g --no-audit --no-fund --no-progress \ + '@zed-industries/codex-acp@0.11.1'; \ + command -v codex-acp >/dev/null # Patch OpenClaw media fetch for proxy-only sandbox (NVIDIA/NemoClaw#1755). # @@ -172,16 +182,38 @@ RUN set -eu; \ sed -i 's/baseLstat\.isSymbolicLink()/false \/* nemoclaw: symlink check disabled, realpath guards containment *\//' "$ipd_file"; \ if grep -q 'fs\.lstat(params\.installBaseDir)' "$ipd_file"; then echo "ERROR: Patch 3b (install-package-dir) left lstat in assertInstallBaseStable" >&2; exit 1; fi; \ # --- Patch 4: graceful EACCES in replaceConfigFile for sandbox (#2254) --- \ - # Plugin install persists metadata to openclaw.json via replaceConfigFile. \ - # In the sandbox, openclaw.json is immutable (444 root:root in a 755 \ - # root:root directory) by design. The write fails with EACCES. This \ - # patch wraps the writeConfigFile call inside replaceConfigFile to catch \ - # EACCES when OPENSHELL_SANDBOX=1 and emit a warning instead of crashing. \ - # Plugins still load via auto-discovery from the extensions directory. \ + # Plugin install persists metadata via replaceConfigFile. In the sandbox, \ + # openclaw.json is immutable (444 root:root) by design. OpenClaw 2026.4.24 \ + # restructured config writes: replaceConfigFile now first attempts a \ + # single-key include-file mutation (tryWriteSingleTopLevelIncludeMutation), \ + # falling back to writeConfigFile for the full config. Both paths can hit \ + # EACCES in the read-only sandbox tree. This patch wraps the entire \ + # write block in a try/catch that catches EACCES when OPENSHELL_SANDBOX=1 \ + # and emits a warning instead of crashing. Plugins still load via \ + # auto-discovery from the extensions directory. \ rcf_file="$(grep -RIlE --include='*.js' 'async function replaceConfigFile\(params\)' "$OC_DIST" | head -n 1)"; \ test -n "$rcf_file" || { echo "ERROR: replaceConfigFile function not found in OpenClaw dist" >&2; exit 1; }; \ - python3 -c "import sys; p=sys.argv[1]; f=open(p); src=f.read(); f.close(); old='\tawait writeConfigFile(params.nextConfig, {\n\t\t...writeOptions,\n\t\t...params.writeOptions\n\t});'; new='\ttry { await writeConfigFile(params.nextConfig, {\n\t\t...writeOptions,\n\t\t...params.writeOptions\n\t}); } catch(_rcfErr) { if (process.env.OPENSHELL_SANDBOX === \"1\" && _rcfErr.code === \"EACCES\") { console.error(\"[nemoclaw] Config is read-only in sandbox \\u2014 plugin metadata not persisted (plugins auto-load from extensions/)\"); } else { throw _rcfErr; } }'; assert old in src, 'writeConfigFile(params.nextConfig) pattern not found'; f=open(p,'w'); f.write(src.replace(old,new,1)); f.close()" "$rcf_file"; \ - grep -REq --include='*.js' 'OPENSHELL_SANDBOX.*EACCES' "$rcf_file" || { echo "ERROR: Patch 4 (replaceConfigFile EACCES) not applied" >&2; exit 1; } + python3 -c "import sys; p=sys.argv[1]; f=open(p); src=f.read(); f.close(); old='\tif (!await tryWriteSingleTopLevelIncludeMutation({\n\t\tsnapshot,\n\t\tnextConfig: params.nextConfig\n\t})) await writeConfigFile(params.nextConfig, {\n\t\tbaseSnapshot: snapshot,\n\t\t...writeOptions,\n\t\t...params.writeOptions\n\t});'; new='\ttry { if (!await tryWriteSingleTopLevelIncludeMutation({\n\t\tsnapshot,\n\t\tnextConfig: params.nextConfig\n\t})) await writeConfigFile(params.nextConfig, {\n\t\tbaseSnapshot: snapshot,\n\t\t...writeOptions,\n\t\t...params.writeOptions\n\t}); } catch(_rcfErr) { if (process.env.OPENSHELL_SANDBOX === \"1\" && _rcfErr.code === \"EACCES\") { console.error(\"[nemoclaw] Config is read-only in sandbox \\u2014 plugin metadata not persisted (plugins auto-load from extensions/)\"); } else { throw _rcfErr; } }'; assert old in src, 'tryWriteSingleTopLevelIncludeMutation/writeConfigFile pattern not found in replaceConfigFile'; f=open(p,'w'); f.write(src.replace(old,new,1)); f.close()" "$rcf_file"; \ + grep -REq --include='*.js' 'OPENSHELL_SANDBOX.*EACCES' "$rcf_file" || { echo "ERROR: Patch 4 (replaceConfigFile EACCES) not applied" >&2; exit 1; }; \ + # --- Patch 5: bump default WS handshake timeout 10s -> 60s (#2484) --- \ + # OpenClaw's WS connect handshake has a hard-coded 10s timeout on both \ + # client and server. Server-side connect-handler processing can exceed \ + # 10s under load (multiple concurrent connects on slow CI infra), \ + # causing `openclaw agent --json` to fail with "gateway timeout after \ + # 10000ms" and TC-SBX-02 to hit its 90s SSH timeout. \ + # \ + # Both env vars (OPENCLAW_HANDSHAKE_TIMEOUT_MS, \ + # OPENCLAW_CONNECT_CHALLENGE_TIMEOUT_MS) are clamped at the same \ + # DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS constant, so we patch the \ + # constant itself. Affects both client.js (used by openclaw CLI) and \ + # server.impl.js (gateway side). \ + # \ + # Removal criteria: drop when openclaw fixes the underlying connect \ + # latency, or exposes the timeout as an unbounded env override. \ + hto_files="$(grep -RIlE --include='*.js' 'DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 1e4' "$OC_DIST")"; \ + test -n "$hto_files" || { echo "ERROR: handshake-timeout constant not found" >&2; exit 1; }; \ + printf '%s\n' "$hto_files" | xargs sed -i -E 's|DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 1e4|DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 6e4|g'; \ + if grep -REq --include='*.js' 'DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 1e4' "$OC_DIST"; then echo "ERROR: Patch 5 left a 1e4 constant" >&2; exit 1; fi # Set up blueprint for local resolution. # Blueprints are immutable at runtime; DAC protection (root ownership) is applied @@ -192,8 +224,9 @@ RUN mkdir -p /sandbox/.nemoclaw/blueprints/0.1.0 \ # Copy startup script and shared sandbox initialisation library COPY scripts/lib/sandbox-init.sh /usr/local/lib/nemoclaw/sandbox-init.sh COPY scripts/nemoclaw-start.sh /usr/local/bin/nemoclaw-start +COPY scripts/codex-acp-wrapper.sh /usr/local/bin/nemoclaw-codex-acp COPY scripts/generate-openclaw-config.py /usr/local/lib/nemoclaw/generate-openclaw-config.py -RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/lib/nemoclaw/sandbox-init.sh +RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-codex-acp /usr/local/lib/nemoclaw/sandbox-init.sh # Build args for config that varies per deployment. # nemoclaw onboard passes these at image build time. @@ -296,9 +329,21 @@ USER sandbox # list of env vars and derivation rules. RUN python3 /usr/local/lib/nemoclaw/generate-openclaw-config.py -# Install NemoClaw plugin into OpenClaw -RUN openclaw doctor --fix > /dev/null 2>&1 || true \ - && openclaw plugins install /opt/nemoclaw > /dev/null 2>&1 || true +# Install NemoClaw plugin into OpenClaw. Prune non-runtime metadata from +# staged bundled plugin dependencies before this layer is committed; deleting +# it in a later layer would not reduce the OCI image imported by k3s. +RUN (openclaw doctor --fix > /dev/null 2>&1 || true) \ + && (openclaw plugins install /opt/nemoclaw > /dev/null 2>&1 || true) \ + && if [ -d /sandbox/.openclaw-data/plugin-runtime-deps ]; then \ + find /sandbox/.openclaw-data/plugin-runtime-deps -type f \( \ + -name '*.d.ts' -o -name '*.d.mts' -o -name '*.d.cts' -o \ + -name '*.map' -o -name '*.tsbuildinfo' \ + \) -delete; \ + find /sandbox/.openclaw-data/plugin-runtime-deps -type d \( \ + -name __tests__ -o -name test -o -name tests -o -name docs -o \ + -name examples \ + \) -prune -exec rm -rf {} +; \ + fi # Inject gateway auth token into openclaw.json. # NEMOCLAW_BUILD_ID busts the Docker cache so each image gets a unique token. @@ -326,19 +371,28 @@ os.chmod(path, 0o600)" # hadolint ignore=DL3002 USER root -# Ensure .openclaw-data subdirs and symlinks exist for logs, credentials, and -# sandbox. These are defined in Dockerfile.base but the GHCR base image may -# not have been rebuilt yet. Idempotent — harmless once the base catches up. +# Ensure .openclaw-data subdirs and symlinks exist for logs, credentials, +# sandbox, and plugin-runtime-deps. These are defined in Dockerfile.base but +# the GHCR base image may not have been rebuilt yet. Idempotent — harmless +# once the base catches up. +# +# plugin-runtime-deps was added in OpenClaw 2026.4.24: the CLI lazy-installs +# bundled plugin runtime dependencies into ~/.openclaw/plugin-runtime-deps/ +# on first invocation (Jiti loader). Without a writable target every bundled +# plugin (nvidia, openai, anthropic, ollama, …) fails to load with EACCES, +# leaving the agent CLI with no providers. See PluginLoadFailureError in #2484. # Ref: https://github.com/NVIDIA/NemoClaw/issues/804 RUN mkdir -p /sandbox/.openclaw-data/logs \ /sandbox/.openclaw-data/credentials \ /sandbox/.openclaw-data/sandbox \ /sandbox/.openclaw-data/media \ + /sandbox/.openclaw-data/plugin-runtime-deps \ && chown sandbox:sandbox /sandbox/.openclaw-data/logs \ /sandbox/.openclaw-data/credentials \ /sandbox/.openclaw-data/sandbox \ /sandbox/.openclaw-data/media \ - && for dir in logs credentials sandbox media; do \ + /sandbox/.openclaw-data/plugin-runtime-deps \ + && for dir in logs credentials sandbox media plugin-runtime-deps; do \ if [ -L "/sandbox/.openclaw/$dir" ]; then true; \ elif [ -e "/sandbox/.openclaw/$dir" ]; then \ cp -a "/sandbox/.openclaw/$dir/." "/sandbox/.openclaw-data/$dir/" 2>/dev/null || true; \ diff --git a/Dockerfile.base b/Dockerfile.base index d92d03fc57..38fa9f3c60 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -109,6 +109,7 @@ RUN mkdir -p /sandbox/.openclaw-data/agents/main/agent \ /sandbox/.openclaw-data/flows \ /sandbox/.openclaw-data/sandbox \ /sandbox/.openclaw-data/telegram \ + /sandbox/.openclaw-data/plugin-runtime-deps \ && mkdir -p /sandbox/.openclaw \ && ln -s /sandbox/.openclaw-data/agents /sandbox/.openclaw/agents \ && ln -s /sandbox/.openclaw-data/extensions /sandbox/.openclaw/extensions \ @@ -129,6 +130,7 @@ RUN mkdir -p /sandbox/.openclaw-data/agents/main/agent \ && ln -s /sandbox/.openclaw-data/exec-approvals.json /sandbox/.openclaw/exec-approvals.json \ && ln -s /sandbox/.openclaw-data/telegram /sandbox/.openclaw/telegram \ && ln -s /sandbox/.openclaw-data/flows /sandbox/.openclaw/flows \ + && ln -s /sandbox/.openclaw-data/plugin-runtime-deps /sandbox/.openclaw/plugin-runtime-deps \ && chown -R sandbox:sandbox /sandbox/.openclaw /sandbox/.openclaw-data # Pre-create shell init files for the sandbox user. @@ -152,7 +154,7 @@ RUN printf '%s\n' \ # OpenClaw version: change the OPENCLAW_VERSION ARG default so CI rebuilds # the base image on push to main, or use workflow_dispatch on base-image.yaml # with the openclaw_version input for a one-off build without editing this file. -ARG OPENCLAW_VERSION=2026.4.9 +ARG OPENCLAW_VERSION=2026.4.24 SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/agents/openclaw/manifest.yaml b/agents/openclaw/manifest.yaml index c2b87b0d32..be77feeb4e 100644 --- a/agents/openclaw/manifest.yaml +++ b/agents/openclaw/manifest.yaml @@ -19,7 +19,7 @@ homepage: "https://openclaw.ai" install_method: npm # npm install -g openclaw@ binary_path: /usr/local/bin/openclaw version_command: "openclaw --version" -expected_version: "2026.4.9" +expected_version: "2026.4.24" gateway_command: "openclaw gateway run" # ── Health probe ──────────────────────────────────────────────── diff --git a/nemoclaw-blueprint/blueprint.yaml b/nemoclaw-blueprint/blueprint.yaml index 93eb4499a8..29b55e2160 100644 --- a/nemoclaw-blueprint/blueprint.yaml +++ b/nemoclaw-blueprint/blueprint.yaml @@ -4,7 +4,7 @@ version: "0.1.0" min_openshell_version: "0.0.32" max_openshell_version: "0.0.36" -min_openclaw_version: "2026.4.9" +min_openclaw_version: "2026.4.24" # Mirrors the components.sandbox.image manifest digest below. Lets a # downstream consumer (or release tooling) verify the blueprint declares # a specific sandbox image without parsing the components tree, and diff --git a/scripts/codex-acp-wrapper.sh b/scripts/codex-acp-wrapper.sh new file mode 100755 index 0000000000..e57f42bc5d --- /dev/null +++ b/scripts/codex-acp-wrapper.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# OpenClaw's embedded ACPx runtime runs under the gateway user, while the +# container's default tool redirects are owned by the sandbox user. Keep Codex +# state in a per-UID /tmp tree so codex-acp can initialize without touching +# /sandbox or another user's XDG directories. +_uid="$(id -u)" +_base="${NEMOCLAW_CODEX_ACP_HOME:-/tmp/nemoclaw-codex-acp-${_uid}}" + +export HOME="${_base}/home" +export CODEX_HOME="${_base}/codex" +export CODEX_SQLITE_HOME="${_base}/sqlite" +export XDG_CACHE_HOME="${_base}/cache" +export XDG_CONFIG_HOME="${_base}/config" +export XDG_DATA_HOME="${_base}/data" +export XDG_STATE_HOME="${_base}/state" +export XDG_RUNTIME_DIR="${_base}/runtime" +export GIT_CONFIG_GLOBAL="${_base}/gitconfig" +export GNUPGHOME="${_base}/gnupg" + +mkdir -p \ + "$HOME" \ + "$CODEX_HOME" \ + "$CODEX_SQLITE_HOME" \ + "$XDG_CACHE_HOME" \ + "$XDG_CONFIG_HOME" \ + "$XDG_DATA_HOME" \ + "$XDG_STATE_HOME" \ + "$XDG_RUNTIME_DIR" \ + "$GNUPGHOME" +chmod 700 \ + "$_base" \ + "$HOME" \ + "$CODEX_HOME" \ + "$CODEX_SQLITE_HOME" \ + "$XDG_CACHE_HOME" \ + "$XDG_CONFIG_HOME" \ + "$XDG_DATA_HOME" \ + "$XDG_STATE_HOME" \ + "$XDG_RUNTIME_DIR" \ + "$GNUPGHOME" 2>/dev/null || true +touch "$GIT_CONFIG_GLOBAL" +chmod 600 "$GIT_CONFIG_GLOBAL" 2>/dev/null || true + +exec /usr/local/bin/codex-acp "$@" diff --git a/scripts/generate-openclaw-config.py b/scripts/generate-openclaw-config.py index e0a78a8a1c..a3ba8e0771 100755 --- a/scripts/generate-openclaw-config.py +++ b/scripts/generate-openclaw-config.py @@ -193,16 +193,82 @@ def build_config(env: dict | None = None) -> dict: } } + # OpenClaw 2026.4.24 stages runtime dependencies for every bundled + # enabledByDefault provider plugin during `openclaw doctor --fix`. + # NemoClaw bakes one model provider into openclaw.json, so keeping unused + # default providers enabled bloats the sandbox image and can exhaust the + # CI k3s/containerd import volume before tests even start. + plugin_entries = { + "acpx": { + "config": { + "agents": { + "codex": {"command": "/usr/local/bin/nemoclaw-codex-acp"}, + } + } + }, + "bonjour": {"enabled": False}, + "qqbot": {"enabled": False}, + } + _bundled_provider_plugins = { + "amazon-bedrock": {"amazon-bedrock", "bedrock"}, + "amazon-bedrock-mantle": {"amazon-bedrock-mantle"}, + "anthropic": {"anthropic"}, + "anthropic-vertex": {"anthropic-vertex"}, + "google": {"google", "google-gemini-cli"}, + } + for _plugin_id, _provider_keys in _bundled_provider_plugins.items(): + if provider_key not in _provider_keys: + plugin_entries[_plugin_id] = {"enabled": False} + config = { "agents": { "defaults": { "model": {"primary": primary_model_ref}, "timeoutSeconds": agent_timeout, + # NemoClaw sandboxes are provisioned non-interactively and the + # E2E CLI contract expects the first agent turn to answer the + # caller's prompt. OpenClaw 2026.4.24 seeds BOOTSTRAP.md by + # default, which redirects a fresh workspace into an identity + # setup conversation before normal replies. + "skipBootstrap": True, + # Keep first-turn smoke checks on the lowest-latency path. + # OpenClaw can infer thinking defaults from the model catalog; + # NemoClaw's sandbox contract is a direct CLI answer, not an + # interactive reasoning session. + "thinkingDefault": "off", } }, "models": {"mode": "merge", "providers": providers}, "channels": {"defaults": {}, **_ch_cfg}, "update": {"checkOnStart": False}, + # Disable bundled plugins/channels that hit the L7 proxy at startup + # and either crash or hang the gateway: + # + # bonjour — uses @homebridge/ciao for mDNS announcement; sandbox + # netns has no multicast, ciao either fails sync via + # uv_interface_addresses or async via "CIAO PROBING CANCELLED". + # Introduced in OpenClaw 2026.4.15. See NemoClaw#2484. + # + # qqbot — has stageRuntimeDependencies=true, so its npm deps + # (@tencent-connect/qqbot-connector et al.) install on first + # load. The sandbox L7 proxy denies the registry URL, the + # install retries for ~6 minutes, and while it's stuck the + # gateway can't service openclaw-agent requests — that's the + # TC-SBX-02 hang in 2026.4.24. + # + # acpx stays enabled, but its default codex adapter command is + # `npx @zed-industries/codex-acp@^0.11.1`. npm refreshes registry + # metadata for that package spec even when codex-acp is globally + # installed, which hits the L7 proxy deny path during gateway startup. + # The sandbox image pre-installs /usr/local/bin/codex-acp. The wrapper + # below points ACPx at that binary with writable per-UID Codex/XDG + # state so the gateway user does not try to write under /sandbox or + # the sandbox user's redirected /tmp directories. + # + # Provider plugins with staged runtime dependencies are disabled above + # unless they match NEMOCLAW_PROVIDER_KEY. That keeps the baked image + # limited to the provider selected during onboard. + "plugins": {"entries": plugin_entries}, "gateway": { "mode": "local", "controlUi": { diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index 75e00fec5e..e368d1fd95 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -898,17 +898,34 @@ while time.time() < DEADLINE: time.sleep(1) continue - if has_browser: - QUIET_POLLS += 1 - if QUIET_POLLS >= 4: + QUIET_POLLS += 1 + # Exit-on-quiet conditions, checked in order of strength: + # 1. Browser device paired — original control-UI workflow + # 2. Any paired device — covers dangerouslyDisableDeviceAuth setups + # where the gateway auto-pairs CLI clients directly without the + # watcher running `openclaw devices approve` (so APPROVED stays + # 0 forever in those configurations) + # 3. We approved at least one device explicitly + # Without these, the watcher polled `openclaw devices list --json` + # every 1 second for 10 minutes whenever no browser device joined, + # saturating the gateway connect handler and starving concurrent + # `openclaw agent` connects (NemoClaw#2484: WS handshake-timeout). + if QUIET_POLLS >= 4: + if has_browser: print(f'[auto-pair] browser pairing converged approvals={APPROVED}') break - elif APPROVED > 0: - QUIET_POLLS += 1 - else: - QUIET_POLLS = 0 + if paired: + print(f'[auto-pair] devices paired ({len(paired)}); exiting approvals={APPROVED}') + break + if APPROVED > 0: + print(f'[auto-pair] non-browser pairing converged approvals={APPROVED}') + break - time.sleep(1) + # Back off polling once anything is paired or approved: 1s when + # actively processing pending requests / waiting for first pairing, + # 5s thereafter. The 5s cadence avoids connect-handler pile-up under + # high gateway connect latency. + time.sleep(5 if (APPROVED > 0 or paired) else 1) else: print(f'[auto-pair] watcher timed out approvals={APPROVED}') PYAUTOPAIR @@ -973,17 +990,21 @@ fi # src/lib/sandbox-build-context.ts. A sync test enforces that the # embedded copy is byte-identical to the canonical file. # ── Global sandbox safety net ────────────────────────────────── -# Catch-all handler for uncaught exceptions and unhandled rejections -# that would otherwise crash the gateway. In a sandbox environment, -# a crashed gateway means total loss of inference, chat, and TUI — -# worse than degraded service from a swallowed error. +# Last-resort handler for uncaught exceptions and unhandled rejections +# that would otherwise crash the gateway. The gateway is shared sandbox +# infrastructure; user-initiated actions must not be able to take it down. # -# This MUST be the first --require preload so its handlers register -# before any library code runs. Specific guards (Slack, ciao) provide -# targeted handling; this catches everything else. +# This is intentionally NOT a catch-all swallow. Known-benign error +# patterns are documented inline in the script; unknown patterns are +# logged with full stack so they can be diagnosed and either fixed +# upstream or added to the allow-list with explicit justification. +# Specific guards (Slack, ciao) pre-empt their own error patterns; +# this is the backstop for everything else. # -# Only active when OPENSHELL_SANDBOX=1 (set by OpenShell at runtime). -# Outside a sandbox, normal Node.js crash behavior is preserved. +# Only active when OPENSHELL_SANDBOX=1 (set by OpenShell at runtime), +# and only for gateway processes. Outside a sandbox or in CLI processes +# (agent, doctor, plugins, tui, etc.) normal Node.js crash behavior is +# preserved so errors surface promptly to users running short-lived tools. _SANDBOX_SAFETY_NET="/tmp/nemoclaw-sandbox-safety-net.js" emit_sandbox_sourced_file "$_SANDBOX_SAFETY_NET" <<'SAFETY_NET_EOF' // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -991,53 +1012,110 @@ emit_sandbox_sourced_file "$_SANDBOX_SAFETY_NET" <<'SAFETY_NET_EOF' // // sandbox-safety-net.js — last-resort handler that keeps the gateway alive // when any library throws an uncaught exception or unhandled rejection. -// Only active inside OpenShell sandboxes (OPENSHELL_SANDBOX=1). +// +// Contract: +// +// 1. Inside the OpenShell sandbox the gateway is shared infrastructure. +// User-initiated actions (loading a plugin, starting a sidecar, +// running an agent against the gateway) must not be able to take it +// down. Node.js 22+ defaults --unhandled-rejections=throw which +// crashes on the first stray rejection from any library — including +// libraries we don't control. +// +// 2. Specific known-benign patterns are documented inline below. They +// get a single-line summary and are absorbed silently. Each pattern +// MUST document which library produces it, why it's safe to absorb +// in the sandbox context, and what the upstream fix is. Prefer +// disabling/configuring the upstream component so the rejection +// never fires; this list is the safety net, not the policy. +// +// 3. Unknown errors do NOT crash the gateway either, but they are +// logged with full stack so they can be diagnosed and either fixed +// upstream or added to the allow-list with explicit justification. +// "Unknown means crash" is the wrong default for shared +// infrastructure; "unknown means log loudly" is the right default. +// +// 4. No process.exit interception. An earlier iteration intercepted +// process.exit during swallow windows, which masked legitimate +// shutdown signals and was itself the kind of catch-all hack we +// want to avoid. +// +// 5. Only active when OPENSHELL_SANDBOX=1 (set by OpenShell at runtime), +// and only for `openclaw gateway run …` invocations +// (process.argv[2] === "gateway"). CLI commands (agent, doctor, +// plugins, tui, etc.) get default Node behavior so errors surface +// promptly to users running short-lived tools. (function () { 'use strict'; if (process.env.OPENSHELL_SANDBOX !== '1') return; + if (process.argv[2] !== 'gateway') return; - // Track whether we're inside an unhandledRejection we chose to swallow. - // OpenClaw's own handler calls process.exit(1) for non-transient rejections. - // We intercept process.exit during swallowed rejections to prevent that. - var _swallowing = false; - var _origExit = process.exit; - process.exit = function (code) { - if (_swallowing) { - try { - process.stderr.write( - '[sandbox-safety-net] blocked process.exit(' + code + - ') during swallowed rejection — gateway continues\n' - ); - } catch (_) {} - return; + // KNOWN-BENIGN ERROR PATTERNS + // + // ciao / @homebridge/ciao — mDNS service-discovery library used by the + // OpenClaw bonjour plugin (introduced in 2026.4.15). Sandboxes have + // restricted network namespaces with no multicast. Two failure modes: + // - sync: os.networkInterfaces() throws ERR_SYSTEM_ERROR + // uv_interface_addresses. Pre-empted by ciao-network-guard.js, + // which monkey-patches os.networkInterfaces() to return {}. + // - async: the probe state machine cancels itself during gateway + // startup/reload and emits "CIAO PROBING CANCELLED" as an unhandled + // rejection. This is the path we catch here. + // Upstream fix: bonjour is disabled via plugins.entries.bonjour.enabled + // = false in the sandbox openclaw.json. This pattern is a backstop in + // case the disable is bypassed or a future release introduces another + // mDNS code path. + function classifyBenignRejection(reason) { + if (!reason) return null; + var msg = String((reason && reason.message) || reason); + var stack = (reason && reason.stack) || ''; + + if (msg.indexOf('CIAO') !== -1 || + stack.indexOf('@homebridge/ciao') !== -1 || + stack.indexOf('/ciao/') !== -1) { + return 'ciao/mDNS (sandbox lacks multicast; bonjour should be disabled in openclaw.json)'; } - return _origExit.call(process, code); - }; + if (reason && reason.code === 'ERR_SYSTEM_ERROR' && + msg.indexOf('uv_interface_addresses') !== -1) { + return 'uv_interface_addresses (restricted netns)'; + } + return null; + } process.on('uncaughtException', function (err, origin) { + // Sync error paths are pre-empted by the targeted guards + // (ciao-network-guard.js, slack-channel-guard.js when Slack is + // configured). If we get here it's an error those guards didn't + // recognize. Log full stack and stay alive — registering this + // listener is what tells Node "don't crash on uncaughtException". try { process.stderr.write( - '[sandbox-safety-net] uncaughtException: ' + - (err && err.stack ? err.stack : String(err)) + - ' (origin: ' + origin + ') — swallowed, gateway continues\n' + '[sandbox-safety-net] uncaughtException [unhandled by upstream guards \u2014 please diagnose]: ' + + ((err && err.stack) ? err.stack : String(err)) + + ' (origin: ' + origin + ') \u2014 gateway continues\n' ); } catch (_) {} }); process.on('unhandledRejection', function (reason, promise) { - _swallowing = true; + var benign = classifyBenignRejection(reason); + if (benign) { + try { + process.stderr.write( + '[sandbox-safety-net] unhandledRejection [known-benign: ' + benign + ']: ' + + ((reason && reason.message) ? reason.message : String(reason)) + '\n' + ); + } catch (_) {} + return; + } try { process.stderr.write( - '[sandbox-safety-net] unhandledRejection: ' + - (reason && reason.stack ? reason.stack : String(reason)) + - ' — swallowed, gateway continues\n' + '[sandbox-safety-net] unhandledRejection [UNKNOWN PATTERN \u2014 please diagnose]: ' + + ((reason && reason.stack) ? reason.stack : String(reason)) + + ' \u2014 gateway continues\n' ); } catch (_) {} - // Keep _swallowing=true through this tick so OpenClaw's handler - // (which runs in the same microtask delivery) hits our process.exit - // intercept. Reset on next tick. - Promise.resolve().then(function () { _swallowing = false; }); }); })(); SAFETY_NET_EOF @@ -1419,32 +1497,37 @@ emit_sandbox_sourced_file "$_CIAO_GUARD_SCRIPT" <<'CIAO_GUARD_EOF' }; // Fallback: catch uncaughtException from ciao if the monkey-patch - // doesn't cover all call sites. - process.on('uncaughtException', function (err, origin) { - if ( - err && err.code === 'ERR_SYSTEM_ERROR' && - String(err.message || '').indexOf('uv_interface_addresses') !== -1 - ) { - process.stderr.write( - '[guard] ciao/networkInterfaces crash caught: ' + (err.message || err) + - ' — gateway continues\n' - ); - return; - } - // Check stack for ciao/NetworkManager - if (err && err.stack && err.stack.indexOf('ciao') !== -1 && - String(err.message || '').indexOf('networkInterfaces') !== -1) { - process.stderr.write( - '[guard] ciao network error caught: ' + (err.message || err) + - ' — gateway continues\n' - ); - return; - } - // Not a ciao error — re-throw to preserve normal crash behavior. - process.stderr.write((err && err.stack) || String(err)); - process.stderr.write('\n'); - process.exit(1); - }); + // doesn't cover all call sites. Gateway-only — registering ANY + // uncaughtException listener tells Node "don't crash by default", and + // we want CLI processes (agent, doctor, plugins, tui) to keep default + // Node crash behavior so errors surface promptly. + // + // For gateway processes, non-ciao errors fall through (return) to the + // sandbox safety net registered later in the preload chain. The safety + // net is the single point of "keep gateway alive on unknown errors". + if (process.argv[2] === 'gateway') { + process.on('uncaughtException', function (err, origin) { + if ( + err && err.code === 'ERR_SYSTEM_ERROR' && + String(err.message || '').indexOf('uv_interface_addresses') !== -1 + ) { + process.stderr.write( + '[guard] ciao/networkInterfaces crash caught: ' + (err.message || err) + + ' \u2014 gateway continues\n' + ); + return; + } + if (err && err.stack && err.stack.indexOf('ciao') !== -1 && + String(err.message || '').indexOf('networkInterfaces') !== -1) { + process.stderr.write( + '[guard] ciao network error caught: ' + (err.message || err) + + ' \u2014 gateway continues\n' + ); + return; + } + // Not ciao — let the sandbox safety net handle it. + }); + } })(); CIAO_GUARD_EOF export NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--require $_CIAO_GUARD_SCRIPT" @@ -1645,12 +1728,22 @@ if [ "$(id -u)" -ne 0 ]; then nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >/tmp/gateway.log 2>&1 & GATEWAY_PID=$! echo "[gateway] openclaw gateway launched (pid $GATEWAY_PID)" >&2 + # Diagnostic: mirror gateway log to PID 1's stderr — see root-mode block + # below for rationale (NVIDIA/NemoClaw#2484). + { tail -n +1 -F /tmp/gateway.log 2>/dev/null | sed -u 's/^/[gateway-log:] /' >&2; } & + GATEWAY_LOG_TAIL_PID=$! + # Persistent mirror: see root-mode block for rationale. + mkdir -p /sandbox/.openclaw-data/logs 2>/dev/null || true + { tail -n +1 -F /tmp/gateway.log 2>/dev/null >>/sandbox/.openclaw-data/logs/gateway-persistent.log; } & + GATEWAY_LOG_PERSIST_PID=$! start_auto_pair # NOTE: PIDs are collected after launch; a signal arriving between trap # registration and the final append is a small race window (same as before # the shared-library refactor). Acceptable for entrypoint-level cleanup. SANDBOX_CHILD_PIDS=("$GATEWAY_PID") [ -n "${AUTO_PAIR_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$AUTO_PAIR_PID") + [ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID") + [ -n "${GATEWAY_LOG_PERSIST_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_PERSIST_PID") # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" trap cleanup_on_signal SIGTERM SIGINT @@ -1787,12 +1880,35 @@ nohup gosu gateway "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >/tmp/gat GATEWAY_PID=$! echo "[gateway] openclaw gateway launched as 'gateway' user (pid $GATEWAY_PID)" >&2 +# Diagnostic: mirror gateway log to PID 1's stderr so its content surfaces in +# docker logs. /tmp/gateway.log is otherwise only readable from inside the +# sandbox via `nemoclaw logs` and is not captured by the e2e test +# framework on failure. Streaming it to PID 1's stderr lets a workflow-level +# `docker logs` capture pick it up. Each line is prefixed with [gateway-log:] +# so it can be filtered out post-hoc when not investigating. +# Ref: NVIDIA/NemoClaw#2484 (TC-SBX-02 hang investigation) +{ tail -n +1 -F /tmp/gateway.log 2>/dev/null | sed -u 's/^/[gateway-log:] /' >&2; } & +GATEWAY_LOG_TAIL_PID=$! + +# Persistent mirror: append /tmp/gateway.log content to a file under +# /sandbox/.openclaw-data/logs which is volume-mounted by openshell and +# survives pod restarts. /tmp/gateway.log itself is wiped when the pod +# restarts (TC-SBX-06 docker-kills the gateway container), so the +# only durable record of pre-restart events lives here. The diag +# streamer in the e2e workflow snapshots this file post-test. +mkdir -p /sandbox/.openclaw-data/logs 2>/dev/null || true +chown gateway:gateway /sandbox/.openclaw-data/logs 2>/dev/null || true +{ tail -n +1 -F /tmp/gateway.log 2>/dev/null >>/sandbox/.openclaw-data/logs/gateway-persistent.log; } & +GATEWAY_LOG_PERSIST_PID=$! + start_auto_pair # NOTE: PIDs are collected after launch; a signal arriving between trap # registration and the final append is a small race window (same as before # the shared-library refactor). Acceptable for entrypoint-level cleanup. SANDBOX_CHILD_PIDS=("$GATEWAY_PID") [ -n "${AUTO_PAIR_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$AUTO_PAIR_PID") +[ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID") +[ -n "${GATEWAY_LOG_PERSIST_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_PERSIST_PID") # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" trap cleanup_on_signal SIGTERM SIGINT diff --git a/src/lib/sandbox-build-context.ts b/src/lib/sandbox-build-context.ts index c9757d15bb..c4ab8df593 100644 --- a/src/lib/sandbox-build-context.ts +++ b/src/lib/sandbox-build-context.ts @@ -79,6 +79,10 @@ function stageOptimizedSandboxBuildContext( path.join(rootDir, "scripts", "nemoclaw-start.sh"), path.join(stagedScriptsDir, "nemoclaw-start.sh"), ); + fs.copyFileSync( + path.join(rootDir, "scripts", "codex-acp-wrapper.sh"), + path.join(stagedScriptsDir, "codex-acp-wrapper.sh"), + ); // Shared sandbox initialisation library sourced by the entrypoint (#2277) fs.mkdirSync(path.join(stagedScriptsDir, "lib"), { recursive: true }); fs.copyFileSync( diff --git a/src/lib/sandbox-version.test.ts b/src/lib/sandbox-version.test.ts index 0854243b26..19d87f2bd0 100644 --- a/src/lib/sandbox-version.test.ts +++ b/src/lib/sandbox-version.test.ts @@ -36,7 +36,7 @@ vi.mock("./agent-defs.js", () => ({ name, displayName: name === "openclaw" ? "OpenClaw" : "Hermes Agent", versionCommand: name === "openclaw" ? "openclaw --version" : "hermes --version", - expectedVersion: name === "openclaw" ? "2026.4.9" : "2026.4.9", + expectedVersion: name === "openclaw" ? "2026.4.24" : "2026.4.24", stateDirs: [], configPaths: { writableDir: "/sandbox/.openclaw-data" }, })), @@ -76,12 +76,12 @@ describe("checkAgentVersion", () => { registry.registerSandbox({ name: "test-sb", agent: null, - agentVersion: "2026.4.9", + agentVersion: "2026.4.24", }); const result = checkAgentVersion("test-sb"); expect(result.detectionMethod).toBe("registry"); - expect(result.sandboxVersion).toBe("2026.4.9"); + expect(result.sandboxVersion).toBe("2026.4.24"); expect(result.isStale).toBe(false); }); @@ -102,7 +102,7 @@ describe("checkAgentVersion", () => { registry.registerSandbox({ name: "test-sb", agent: null, - agentVersion: "2026.4.9", + agentVersion: "2026.4.24", }); const result = checkAgentVersion("test-sb"); @@ -119,7 +119,7 @@ describe("checkAgentVersion", () => { vi.mocked(spawnSync).mockReturnValue({ status: 0, - stdout: "OpenClaw 2026.4.9 (abc123)\n", + stdout: "OpenClaw 2026.4.24 (abc123)\n", stderr: "", pid: 1234, output: [], @@ -128,12 +128,12 @@ describe("checkAgentVersion", () => { const result = checkAgentVersion("test-sb"); expect(result.detectionMethod).toBe("ssh-exec"); - expect(result.sandboxVersion).toBe("2026.4.9"); + expect(result.sandboxVersion).toBe("2026.4.24"); expect(result.isStale).toBe(false); // Should have cached the version in registry const updated = registry.getSandbox("test-sb"); - expect(updated?.agentVersion).toBe("2026.4.9"); + expect(updated?.agentVersion).toBe("2026.4.24"); }); it("returns unavailable when SSH config fails", () => { @@ -163,7 +163,7 @@ describe("checkAgentVersion", () => { vi.mocked(spawnSync).mockReturnValue({ status: 0, - stdout: "OpenClaw 2026.4.9 (abc123)\n", + stdout: "OpenClaw 2026.4.24 (abc123)\n", stderr: "", pid: 1234, output: [], @@ -172,7 +172,7 @@ describe("checkAgentVersion", () => { const result = checkAgentVersion("test-sb", { forceProbe: true }); expect(result.detectionMethod).toBe("ssh-exec"); - expect(result.sandboxVersion).toBe("2026.4.9"); + expect(result.sandboxVersion).toBe("2026.4.24"); }); }); @@ -199,14 +199,14 @@ describe("formatStalenessWarning", () => { it("includes sandbox name, versions, and rebuild hint", () => { const lines = formatStalenessWarning("my-sb", { sandboxVersion: "2026.3.11", - expectedVersion: "2026.4.9", + expectedVersion: "2026.4.24", isStale: true, detectionMethod: "registry", }); const joined = lines.join("\n"); expect(joined).toContain("my-sb"); expect(joined).toContain("2026.3.11"); - expect(joined).toContain("2026.4.9"); + expect(joined).toContain("2026.4.24"); expect(joined).toContain("rebuild"); }); }); diff --git a/test/Dockerfile.sandbox b/test/Dockerfile.sandbox index f26d49bc91..ae4903af73 100644 --- a/test/Dockerfile.sandbox +++ b/test/Dockerfile.sandbox @@ -41,6 +41,11 @@ RUN mkdir -p /sandbox/.openclaw/workspace /sandbox/.openclaw/extensions /sandbox ' }' \ '}' > /sandbox/.openclaw/openclaw.json \ && chmod 600 /sandbox/.openclaw/openclaw.json \ + && rm -f /sandbox/.openclaw/openclaw.json.bak* \ + /sandbox/.openclaw/openclaw.json.last-good \ + /sandbox/.openclaw/openclaw.json.clobbered.* \ + /sandbox/.openclaw-data/logs/config-health.json \ + /sandbox/.openclaw-data/logs/config-audit.jsonl \ && echo "test-skill" > /sandbox/.openclaw/skills/test.md \ && echo "test-workspace-file" > /sandbox/.openclaw/workspace/project.md \ && printf '%s\n' '---' 'name: demo-hook' 'description: Demo hook fixture' '---' > /sandbox/.openclaw/hooks/demo-hook/HOOK.md \ diff --git a/test/e2e/test-sandbox-operations.sh b/test/e2e/test-sandbox-operations.sh index 01d788f76f..568e8ac5f5 100755 --- a/test/e2e/test-sandbox-operations.sh +++ b/test/e2e/test-sandbox-operations.sh @@ -107,6 +107,60 @@ sandbox_exec() { sandbox_exec_for "$SANDBOX_A" "$1" } +is_onboard_import_stream_reset() { + local output_file="$1" + [[ -f "$output_file" ]] || return 1 + + grep -q "Connection reset by peer (os error 104)" "$output_file" \ + && grep -Eq "The image appears to have reached the gateway before the stream failed|Recovery: nemoclaw onboard --resume" "$output_file" +} + +is_transient_onboard_resume_error() { + local output_file="$1" + [[ -f "$output_file" ]] || return 1 + + grep -Eq "Connection reset by peer \(os error 104\)|transport error|gateway unavailable|No active gateway|No gateway metadata found" "$output_file" +} + +resume_onboard_after_import_stream_reset() { + local name="$1" output_file="$2" + if ! is_onboard_import_stream_reset "$output_file"; then + return 1 + fi + + log " [onboard] Image reached gateway but import stream reset; retrying with nemoclaw onboard --resume..." + + local attempt delay resume_exit resume_output + for attempt in 1 2 3; do + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + resume_exit=0 + resume_output="$(mktemp)" + log " [onboard] Resume attempt ${attempt}/3..." + NEMOCLAW_SANDBOX_NAME="$name" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + nemoclaw onboard --resume --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" "$resume_output" || resume_exit=$? + + if [[ $resume_exit -eq 0 ]]; then + rm -f "$resume_output" + return 0 + fi + + log " [onboard] nemoclaw onboard --resume attempt ${attempt}/3 exited with code $resume_exit" + if ((attempt < 3)) && is_transient_onboard_resume_error "$resume_output"; then + delay=$((attempt * 15)) + log " [onboard] Gateway transport still settling; retrying resume in ${delay}s..." + rm -f "$resume_output" + sleep "$delay" + continue + fi + rm -f "$resume_output" + return 1 + done + return 1 +} + # Onboard a sandbox by name. Removes stale locks, runs nemoclaw onboard in # non-interactive mode, and returns 0 if the sandbox appears in nemoclaw list. onboard_sandbox() { @@ -116,18 +170,25 @@ onboard_sandbox() { # Remove stale lock from previous crashed runs rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true - local onboard_exit=0 + local onboard_exit=0 onboard_output + onboard_output="$(mktemp)" NEMOCLAW_SANDBOX_NAME="$name" \ NEMOCLAW_NON_INTERACTIVE=1 \ NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ NEMOCLAW_RECREATE_SANDBOX=1 \ nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ - 2>&1 | tee -a "$LOG_FILE" || onboard_exit=$? + 2>&1 | tee -a "$LOG_FILE" "$onboard_output" || onboard_exit=$? if [[ $onboard_exit -ne 0 ]]; then log " [onboard_sandbox] nemoclaw onboard exited with code $onboard_exit" - return 1 + if resume_onboard_after_import_stream_reset "$name" "$onboard_output"; then + onboard_exit=0 + else + rm -f "$onboard_output" + return 1 + fi fi + rm -f "$onboard_output" if ! nemoclaw list 2>/dev/null | grep -q "$name"; then log " [onboard_sandbox] Sandbox '$name' not found in nemoclaw list after onboard" @@ -158,9 +219,10 @@ install_nemoclaw() { log "=== Installing NemoClaw via install.sh ===" - local install_exit=0 + local install_exit=0 install_output + install_output="$(mktemp)" bash "$REPO_ROOT/install.sh" --non-interactive --yes-i-accept-third-party-software \ - 2>&1 | tee -a "$LOG_FILE" || install_exit=$? + 2>&1 | tee -a "$LOG_FILE" "$install_output" || install_exit=$? # Source shell profile to pick up PATH changes from install.sh if [ -f "$HOME/.bashrc" ]; then @@ -176,6 +238,15 @@ install_nemoclaw() { export PATH="$HOME/.local/bin:$PATH" fi + if [[ $install_exit -ne 0 ]]; then + local install_sandbox + install_sandbox="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" + if resume_onboard_after_import_stream_reset "$install_sandbox" "$install_output"; then + install_exit=0 + fi + fi + rm -f "$install_output" + if [[ $install_exit -ne 0 ]]; then echo -e "${RED}FATAL: install.sh failed (exit $install_exit)${NC}" exit 1 @@ -284,6 +355,8 @@ test_sbx_01_list_sandboxes() { # SSRF regression from the prior `Say exactly: HELLO_E2E` assertion. # 3. Asserts on `result.payloads[].text` from the JSON envelope, not on # merged stdout/stderr. +# 4. Pins `--thinking off` so the first-turn smoke contract is not delayed +# by model-catalog inferred reasoning defaults. test_sbx_02_connect_chat() { log "=== TC-SBX-02: Connect & Chat ===" require_sandbox "$SANDBOX_A" "TC-SBX-02" || return @@ -306,7 +379,7 @@ test_sbx_02_connect_chat() { -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ -o ConnectTimeout=10 -o LogLevel=ERROR \ "openshell-${SANDBOX_A}" \ - "openclaw agent --agent main --json --session-id '${session_id}' -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.'" \ + "openclaw agent --agent main --json --thinking off --session-id '${session_id}' -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.'" \ 2>/dev/null) || true rm -f "$ssh_cfg" diff --git a/test/generate-openclaw-config.test.ts b/test/generate-openclaw-config.test.ts index 962386e8c7..52c284582c 100644 --- a/test/generate-openclaw-config.test.ts +++ b/test/generate-openclaw-config.test.ts @@ -134,11 +134,43 @@ describe("generate-openclaw-config.py: config generation", () => { expect(config.agents.defaults.timeoutSeconds).toBe(300); }); + it("disables OpenClaw first-run workspace bootstrap", () => { + const config = runConfigScript(); + expect(config.agents.defaults.skipBootstrap).toBe(true); + }); + + it("disables inferred thinking for first-turn sandbox replies", () => { + const config = runConfigScript(); + expect(config.agents.defaults.thinkingDefault).toBe("off"); + }); + it("sets gateway auth token to empty string", () => { const config = runConfigScript(); expect(config.gateway.auth.token).toBe(""); }); + it("configures acpx codex to use the preinstalled binary", () => { + const config = runConfigScript(); + expect(config.plugins.entries.acpx.config.agents.codex.command).toBe( + "/usr/local/bin/nemoclaw-codex-acp", + ); + }); + + it("disables unused bundled provider plugins with staged runtime deps", () => { + const config = runConfigScript({ NEMOCLAW_PROVIDER_KEY: "inference" }); + expect(config.plugins.entries["amazon-bedrock"].enabled).toBe(false); + expect(config.plugins.entries["amazon-bedrock-mantle"].enabled).toBe(false); + expect(config.plugins.entries.anthropic.enabled).toBe(false); + expect(config.plugins.entries["anthropic-vertex"].enabled).toBe(false); + expect(config.plugins.entries.google.enabled).toBe(false); + }); + + it("keeps the selected bundled provider plugin available", () => { + const config = runConfigScript({ NEMOCLAW_PROVIDER_KEY: "anthropic" }); + expect(config.plugins.entries.anthropic).toBeUndefined(); + expect(config.plugins.entries.google.enabled).toBe(false); + }); + it("creates file with 0600 permissions", () => { runConfigScript(); const configPath = path.join(tmpDir, ".openclaw", "openclaw.json"); @@ -202,9 +234,7 @@ describe("generate-openclaw-config.py: empty-string env vars fall back to defaul it("treats empty CHAT_UI_URL as unset and uses the loopback default", () => { const config = runConfigScript({ CHAT_UI_URL: "" }); expect(config.gateway.controlUi.dangerouslyDisableDeviceAuth).toBe(false); - expect(config.gateway.controlUi.allowedOrigins).toEqual([ - "http://127.0.0.1:18789", - ]); + expect(config.gateway.controlUi.allowedOrigins).toEqual(["http://127.0.0.1:18789"]); }); it("treats empty NEMOCLAW_PROXY_HOST as unset and uses the documented default", () => { @@ -213,9 +243,7 @@ describe("generate-openclaw-config.py: empty-string env vars fall back to defaul NEMOCLAW_PROXY_HOST: "", NEMOCLAW_MESSAGING_CHANNELS_B64: channelB64, }); - expect(cfg.channels.telegram.accounts.default.proxy).toBe( - "http://10.200.0.1:3128", - ); + expect(cfg.channels.telegram.accounts.default.proxy).toBe("http://10.200.0.1:3128"); }); it("treats empty NEMOCLAW_PROXY_PORT as unset and uses the documented default", () => { @@ -224,8 +252,6 @@ describe("generate-openclaw-config.py: empty-string env vars fall back to defaul NEMOCLAW_PROXY_PORT: "", NEMOCLAW_MESSAGING_CHANNELS_B64: channelB64, }); - expect(cfg.channels.telegram.accounts.default.proxy).toBe( - "http://10.200.0.1:3128", - ); + expect(cfg.channels.telegram.accounts.default.proxy).toBe("http://10.200.0.1:3128"); }); }); diff --git a/test/sandbox-build-context.test.ts b/test/sandbox-build-context.test.ts index 89c668878b..e1cdf058c6 100644 --- a/test/sandbox-build-context.test.ts +++ b/test/sandbox-build-context.test.ts @@ -27,7 +27,10 @@ describe("sandbox build context staging", () => { ), ).toBe(true); expect(fs.existsSync(path.join(buildCtx, "scripts", "nemoclaw-start.sh"))).toBe(true); - expect(fs.existsSync(path.join(buildCtx, "scripts", "generate-openclaw-config.py"))).toBe(true); + expect(fs.existsSync(path.join(buildCtx, "scripts", "codex-acp-wrapper.sh"))).toBe(true); + expect(fs.existsSync(path.join(buildCtx, "scripts", "generate-openclaw-config.py"))).toBe( + true, + ); expect(fs.existsSync(path.join(buildCtx, "scripts", "setup.sh"))).toBe(false); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index 3af378e7fc..bc438f3177 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -19,6 +19,7 @@ import path from "node:path"; const ROOT = path.resolve(import.meta.dirname, ".."); const DOCKERFILE = path.join(ROOT, "Dockerfile"); const DOCKERFILE_BASE = path.join(ROOT, "Dockerfile.base"); +const DOCKERFILE_SANDBOX = path.join(ROOT, "test", "Dockerfile.sandbox"); describe("sandbox provisioning: exec-approvals / update-check symlinks (#1027, #1519)", () => { const src = fs.readFileSync(DOCKERFILE_BASE, "utf-8"); @@ -89,3 +90,45 @@ describe("sandbox provisioning: root-owned read-only config (#514)", () => { expect(src).toContain("chmod 755 /sandbox/.openclaw"); }); }); + +describe("sandbox provisioning: codex-acp wrapper (#2484)", () => { + const dockerSrc = fs.readFileSync(DOCKERFILE, "utf-8"); + const wrapperSrc = fs.readFileSync(path.join(ROOT, "scripts", "codex-acp-wrapper.sh"), "utf-8"); + + it("copies the wrapper into the sandbox image", () => { + expect(dockerSrc).toContain( + "COPY scripts/codex-acp-wrapper.sh /usr/local/bin/nemoclaw-codex-acp", + ); + expect(dockerSrc).toContain("/usr/local/bin/nemoclaw-codex-acp"); + }); + + it("runs codex-acp with writable Codex and XDG state", () => { + expect(wrapperSrc).toContain("export CODEX_HOME="); + expect(wrapperSrc).toContain("export XDG_CONFIG_HOME="); + expect(wrapperSrc).toContain("export HOME="); + expect(wrapperSrc).toContain("exec /usr/local/bin/codex-acp"); + }); +}); + +describe("sandbox test image fixtures", () => { + const src = fs.readFileSync(DOCKERFILE_SANDBOX, "utf-8"); + + it("clears production config recovery artifacts after writing the legacy fixture", () => { + expect(src).toContain("/sandbox/.openclaw/openclaw.json.bak*"); + expect(src).toContain("/sandbox/.openclaw/openclaw.json.last-good"); + expect(src).toContain("/sandbox/.openclaw-data/logs/config-health.json"); + }); +}); + +describe("sandbox operations E2E harness", () => { + const src = fs.readFileSync( + path.join(ROOT, "test", "e2e", "test-sandbox-operations.sh"), + "utf-8", + ); + + it("resumes onboard when OpenShell resets after importing the image", () => { + expect(src).toContain("is_onboard_import_stream_reset"); + expect(src).toContain("Connection reset by peer (os error 104)"); + expect(src).toContain("nemoclaw onboard --resume --non-interactive"); + }); +});