Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .github/workflows/pr-self-hosted.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -160,3 +160,23 @@ jobs:

- name: Run port override E2E tests
run: NEMOCLAW_TEST_IMAGE=nemoclaw-production bash test/e2e-port-overrides.sh

test-non-root-sandbox-smoke:
runs-on: linux-amd64-cpu4
timeout-minutes: 5
needs: build-sandbox-images
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

- name: Download image
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: isolation-image
path: /tmp

- name: Load image
run: gunzip -c /tmp/isolation-image.tar.gz | docker load

- name: Run non-root smoke test
run: NEMOCLAW_TEST_IMAGE=nemoclaw-production bash test/e2e/test-non-root-smoke.sh
208 changes: 208 additions & 0 deletions test/e2e/test-non-root-smoke.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Non-root sandbox smoke test.
#
# Validates that the sandbox container starts successfully under
# --security-opt=no-new-privileges (the constraint applied by OpenShell on
# Brev Launchable and DGX Spark). This is the exact failure mode that caused
# the 5-day outage in #2472 — install_configure_guard writing to .bashrc
# crashed under Landlock + set -e, preventing the gateway from starting.
#
# What this tests:
# 1. Container starts as non-root sandbox user with no-new-privileges
# 2. Gateway process comes up and responds on /health (HTTP 200 or 401)
# 3. openclaw tui --help succeeds (gateway auth token is available)
# 4. Non-root fallback path is exercised (log confirms "Running as non-root")
#
# Does NOT require:
# - NVIDIA_API_KEY (no live inference)
# - Network access to external services
# - Full onboard flow
#
# Prerequisites:
# - Docker running
# - NEMOCLAW_TEST_IMAGE loaded (default: nemoclaw-production)
#
# Usage:
# NEMOCLAW_TEST_IMAGE=nemoclaw-production bash test/e2e/test-non-root-smoke.sh
#
# CI usage (pr-self-hosted.yaml):
# Runs after build-sandbox-images job loads the isolation-image artifact.
#
# See: https://github.com/NVIDIA/NemoClaw/issues/2571

set -euo pipefail

export NEMOCLAW_E2E_DEFAULT_TIMEOUT=300
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=test/e2e/e2e-timeout.sh
source "${SCRIPT_DIR}/e2e-timeout.sh"

# ── Configuration ────────────────────────────────────────────────

IMAGE="${NEMOCLAW_TEST_IMAGE:-nemoclaw-production}"
CONTAINER_NAME="nemoclaw-nonroot-smoke-$$"
HEALTH_TIMEOUT=60
HEALTH_INTERVAL=2
DASHBOARD_PORT=18789
HOST_PORT="${NEMOCLAW_SMOKE_HOST_PORT:-18789}"

# ── Output helpers ───────────────────────────────────────────────

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

PASSED=0
FAILED=0

pass() {
echo -e "${GREEN}PASS${NC}: $1"
PASSED=$((PASSED + 1))
}

fail() {
echo -e "${RED}FAIL${NC}: $1"
FAILED=$((FAILED + 1))
}

info() { echo -e "${YELLOW}TEST${NC}: $1"; }

# ── Cleanup trap ─────────────────────────────────────────────────

cleanup() {
if docker ps -a --format '{{.Names}}' | grep -qx "$CONTAINER_NAME" 2>/dev/null; then
echo ""
info "Collecting container logs before cleanup..."
docker logs "$CONTAINER_NAME" 2>&1 | tail -50 || true
echo ""
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT

# ── Preflight ────────────────────────────────────────────────────

info "Preflight: Docker available"
if ! docker info >/dev/null 2>&1; then
fail "Docker is not running"
exit 1
fi
pass "Docker running"

info "Preflight: Image exists"
if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
fail "Image '$IMAGE' not found — load it before running this test"
exit 1
fi
pass "Image '$IMAGE' available"

# ── Resolve sandbox user UID:GID ─────────────────────────────────

info "1. Resolve sandbox user UID:GID from image"
ID_OUTPUT=$(docker run --rm --entrypoint "" "$IMAGE" id sandbox 2>&1)
SB_UID=$(echo "$ID_OUTPUT" | sed -n 's/uid=\([0-9]*\).*/\1/p')
SB_GID=$(echo "$ID_OUTPUT" | sed -n 's/.*gid=\([0-9]*\).*/\1/p')

if [ -z "$SB_UID" ] || [ -z "$SB_GID" ]; then
fail "Could not resolve sandbox UID:GID from image (output: $ID_OUTPUT)"
exit 1
fi
pass "Sandbox user: uid=$SB_UID gid=$SB_GID"

# ── Start container with non-root constraints ────────────────────

info "2. Start container with --security-opt no-new-privileges --user $SB_UID:$SB_GID"
docker run -d \
--name "$CONTAINER_NAME" \
--security-opt no-new-privileges \
--user "${SB_UID}:${SB_GID}" \
-p "${HOST_PORT}:${DASHBOARD_PORT}" \
"$IMAGE" >/dev/null 2>&1
Comment on lines +118 to +124

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

🧩 Analysis chain

🏁 Script executed:

cat -n test/e2e/test-non-root-smoke.sh | sed -n '100,150p'

Repository: NVIDIA/NemoClaw

Length of output: 2179


🏁 Script executed:

head -30 test/e2e/test-non-root-smoke.sh

Repository: NVIDIA/NemoClaw

Length of output: 1258


🏁 Script executed:

grep -n "set -e" test/e2e/test-non-root-smoke.sh

Repository: NVIDIA/NemoClaw

Length of output: 156


Expose docker run failure messages for debugging.

With set -euo pipefail enabled and both stdout and stderr redirected to /dev/null on line 124, if the docker run command fails, the error message is hidden and the script exits immediately due to set -e—preventing the crash-diagnostic block (lines 126-134) from executing. This makes it difficult to diagnose why the container failed to start.

Wrap the command to catch failures and preserve stderr visibility:

Suggested fix
 info "2. Start container with --security-opt no-new-privileges --user $SB_UID:$SB_GID"
-docker run -d \
+if ! docker run -d \
   --name "$CONTAINER_NAME" \
   --security-opt no-new-privileges \
   --user "${SB_UID}:${SB_GID}" \
   -p "${HOST_PORT}:${DASHBOARD_PORT}" \
-  "$IMAGE" >/dev/null 2>&1
+  "$IMAGE" >/dev/null; then
+  fail "Container failed to start"
+  exit 1
+fi
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
info "2. Start container with --security-opt no-new-privileges --user $SB_UID:$SB_GID"
docker run -d \
--name "$CONTAINER_NAME" \
--security-opt no-new-privileges \
--user "${SB_UID}:${SB_GID}" \
-p "${HOST_PORT}:${DASHBOARD_PORT}" \
"$IMAGE" >/dev/null 2>&1
info "2. Start container with --security-opt no-new-privileges --user $SB_UID:$SB_GID"
if ! docker run -d \
--name "$CONTAINER_NAME" \
--security-opt no-new-privileges \
--user "${SB_UID}:${SB_GID}" \
-p "${HOST_PORT}:${DASHBOARD_PORT}" \
"$IMAGE" >/dev/null; then
fail "Container failed to start"
exit 1
fi
🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@test/e2e/test-non-root-smoke.sh` around lines 118 - 124, The docker run
invocation currently redirects both stdout and stderr to /dev/null which hides
failures and causes the script to exit before the diagnostic block can run;
change the invocation that starts the container (the docker run that uses
CONTAINER_NAME, SB_UID, SB_GID, HOST_PORT, DASHBOARD_PORT, and IMAGE) so that
you capture stderr (e.g., to a temp file or variable) instead of discarding it,
check the docker run exit status, and on failure print the captured stderr (and
any contextual info) before exiting so the existing crash-diagnostic block can
run; ensure you still respect set -e by handling the failure explicitly
(checking $? or using ||) rather than relying on silent redirection.


# Verify container is running (not immediately crashed)
sleep 2
if ! docker ps --format '{{.Names}}' | grep -qx "$CONTAINER_NAME"; then
fail "Container exited immediately after start"
echo "Exit code: $(docker inspect --format='{{.State.ExitCode}}' "$CONTAINER_NAME" 2>/dev/null || echo 'unknown')"
echo "Last logs:"
docker logs "$CONTAINER_NAME" 2>&1 | tail -30 || true
exit 1
fi
pass "Container running as non-root"

# ── Verify non-root path is exercised ────────────────────────────

info "3. Verify non-root fallback path in logs"
# Give a moment for the entrypoint to emit its startup messages
sleep 3
LOGS=$(docker logs "$CONTAINER_NAME" 2>&1)
if echo "$LOGS" | grep -q "Running as non-root"; then
pass "Non-root fallback path confirmed in logs"
else
fail "Expected 'Running as non-root' in container logs"
echo "First 20 lines of logs:"
echo "$LOGS" | head -20
fi

# ── Poll /health endpoint ────────────────────────────────────────

info "4. Poll gateway /health (timeout: ${HEALTH_TIMEOUT}s)"
ELAPSED=0
HEALTH_OK=false

while [ "$ELAPSED" -lt "$HEALTH_TIMEOUT" ]; do
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${HOST_PORT}/health" 2>/dev/null || echo "000")
Comment on lines +153 to +158

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

🧩 Analysis chain

🏁 Script executed:

# Check file existence and read the relevant section
wc -l test/e2e/test-non-root-smoke.sh

Repository: NVIDIA/NemoClaw

Length of output: 95


🏁 Script executed:

# Read the file around lines 153-158 with context
sed -n '140,170p' test/e2e/test-non-root-smoke.sh

Repository: NVIDIA/NemoClaw

Length of output: 1008


🏁 Script executed:

# Search for HEALTH_INTERVAL and HEALTH_TIMEOUT definitions
rg 'HEALTH_(INTERVAL|TIMEOUT)' test/e2e/test-non-root-smoke.sh -B 2 -A 2

Repository: NVIDIA/NemoClaw

Length of output: 981


🏁 Script executed:

# Check if SPDX license header is present
head -5 test/e2e/test-non-root-smoke.sh

Repository: NVIDIA/NemoClaw

Length of output: 249


Add per-request timeouts to the curl health check.

Line 158 calls curl without timeout flags. If the socket accepts but never responds, a single probe can block indefinitely, defeating the 60s timeout guarantee. Add --connect-timeout 1 --max-time "$HEALTH_INTERVAL" to ensure each request times out within the polling interval:

Suggested fix
-  HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${HOST_PORT}/health" 2>/dev/null || echo "000")
+  HTTP_CODE=$(curl -s --connect-timeout 1 --max-time "$HEALTH_INTERVAL" -o /dev/null -w "%{http_code}" "http://localhost:${HOST_PORT}/health" 2>/dev/null || echo "000")
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
info "4. Poll gateway /health (timeout: ${HEALTH_TIMEOUT}s)"
ELAPSED=0
HEALTH_OK=false
while [ "$ELAPSED" -lt "$HEALTH_TIMEOUT" ]; do
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${HOST_PORT}/health" 2>/dev/null || echo "000")
info "4. Poll gateway /health (timeout: ${HEALTH_TIMEOUT}s)"
ELAPSED=0
HEALTH_OK=false
while [ "$ELAPSED" -lt "$HEALTH_TIMEOUT" ]; do
HTTP_CODE=$(curl -s --connect-timeout 1 --max-time "$HEALTH_INTERVAL" -o /dev/null -w "%{http_code}" "http://localhost:${HOST_PORT}/health" 2>/dev/null || echo "000")
🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@test/e2e/test-non-root-smoke.sh` around lines 153 - 158, The curl in the
health probe loop (where HTTP_CODE is set) lacks per-request timeouts so a
single stalled connection can block the whole polling loop; update the curl
invocation that sets HTTP_CODE to include --connect-timeout 1 and --max-time
"$HEALTH_INTERVAL" (using the existing HEALTH_INTERVAL variable) so each probe
respects the polling interval and the overall HEALTH_TIMEOUT behavior remains
effective.


# 200 = healthy, 401 = gateway alive but auth required — both are valid
if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "401" ]; then
HEALTH_OK=true
break
fi

sleep "$HEALTH_INTERVAL"
ELAPSED=$((ELAPSED + HEALTH_INTERVAL))
done

if [ "$HEALTH_OK" = true ]; then
pass "Gateway /health responded with HTTP $HTTP_CODE in ${ELAPSED}s"
else
fail "Gateway /health did not respond within ${HEALTH_TIMEOUT}s (last HTTP code: $HTTP_CODE)"
echo "Container status:"
docker inspect --format='{{.State.Status}} (exit={{.State.ExitCode}})' "$CONTAINER_NAME" 2>/dev/null || true
echo "Recent logs:"
docker logs "$CONTAINER_NAME" 2>&1 | tail -30 || true
fi

# ── Verify openclaw tui --help ────────────────────────────────────

info "5. Verify 'openclaw tui --help' succeeds inside container"
TUI_OUTPUT=$(docker exec "$CONTAINER_NAME" openclaw tui --help 2>&1) || TUI_RC=$?
TUI_RC=${TUI_RC:-0}

if [ "$TUI_RC" -eq 0 ]; then
# Double-check: no "Missing gateway auth token" error
if echo "$TUI_OUTPUT" | grep -qi "Missing gateway auth token"; then
fail "'openclaw tui --help' succeeded but output contains 'Missing gateway auth token'"
else
pass "'openclaw tui --help' exited 0 without auth token errors"
fi
else
fail "'openclaw tui --help' failed (exit $TUI_RC)"
echo "Output: $TUI_OUTPUT" | head -10
fi

# ── Summary ──────────────────────────────────────────────────────

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo -e "Results: ${GREEN}${PASSED} passed${NC}, ${RED}${FAILED} failed${NC}"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

if [ "$FAILED" -gt 0 ]; then
exit 1
fi
exit 0
Loading