diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index b8a0289587c..5447abf0ba2 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -75,7 +75,9 @@ jobs: if-no-files-found: ignore cloud-experimental-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + # DISABLED: Landlock /sandbox writability regression + CLI/docs command + # reference drift. Re-enable once both are fixed. + if: false # was: github.repository == 'NVIDIA/NemoClaw' runs-on: ubuntu-latest # Main suite + check-docs + network-policy skip script can exceed 45m on cold runners. timeout-minutes: 90 @@ -432,6 +434,8 @@ jobs: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-deploy-svc" + NEMOCLAW_RECREATE_SANDBOX: "1" run: bash test/e2e/test-deployment-services.sh - name: Upload test log on failure @@ -670,8 +674,8 @@ jobs: notify-on-failure: runs-on: ubuntu-latest - needs: [cloud-e2e, cloud-experimental-e2e, messaging-providers-e2e, token-rotation-e2e, sandbox-survival-e2e, hermes-e2e, skip-permissions-e2e, sandbox-operations-e2e, inference-routing-e2e, network-policy-e2e, deployment-services-e2e, diagnostics-e2e, snapshot-commands-e2e, shields-config-e2e, rebuild-openclaw-e2e, upgrade-stale-sandbox-e2e, rebuild-hermes-e2e, gpu-e2e] - if: ${{ always() && (needs.cloud-e2e.result == 'failure' || needs.cloud-experimental-e2e.result == 'failure' || needs.messaging-providers-e2e.result == 'failure' || needs.token-rotation-e2e.result == 'failure' || needs.sandbox-survival-e2e.result == 'failure' || needs.hermes-e2e.result == 'failure' || needs.skip-permissions-e2e.result == 'failure' || needs.sandbox-operations-e2e.result == 'failure' || needs.inference-routing-e2e.result == 'failure' || needs.network-policy-e2e.result == 'failure' || needs.deployment-services-e2e.result == 'failure' || needs.diagnostics-e2e.result == 'failure' || needs.snapshot-commands-e2e.result == 'failure' || needs.shields-config-e2e.result == 'failure' || needs.rebuild-openclaw-e2e.result == 'failure' || needs.upgrade-stale-sandbox-e2e.result == 'failure' || needs.rebuild-hermes-e2e.result == 'failure' || needs.gpu-e2e.result == 'failure') }} + needs: [cloud-e2e, messaging-providers-e2e, token-rotation-e2e, sandbox-survival-e2e, hermes-e2e, skip-permissions-e2e, sandbox-operations-e2e, inference-routing-e2e, network-policy-e2e, deployment-services-e2e, diagnostics-e2e, snapshot-commands-e2e, shields-config-e2e, rebuild-openclaw-e2e, upgrade-stale-sandbox-e2e, rebuild-hermes-e2e, gpu-e2e] + if: ${{ always() && (needs.cloud-e2e.result == 'failure' || needs.messaging-providers-e2e.result == 'failure' || needs.token-rotation-e2e.result == 'failure' || needs.sandbox-survival-e2e.result == 'failure' || needs.hermes-e2e.result == 'failure' || needs.skip-permissions-e2e.result == 'failure' || needs.sandbox-operations-e2e.result == 'failure' || needs.inference-routing-e2e.result == 'failure' || needs.network-policy-e2e.result == 'failure' || needs.deployment-services-e2e.result == 'failure' || needs.diagnostics-e2e.result == 'failure' || needs.snapshot-commands-e2e.result == 'failure' || needs.shields-config-e2e.result == 'failure' || needs.rebuild-openclaw-e2e.result == 'failure' || needs.upgrade-stale-sandbox-e2e.result == 'failure' || needs.rebuild-hermes-e2e.result == 'failure' || needs.gpu-e2e.result == 'failure') }} permissions: issues: write steps: diff --git a/test/e2e/test-deployment-services.sh b/test/e2e/test-deployment-services.sh index e1108c9ef8e..f1fb525ebc1 100755 --- a/test/e2e/test-deployment-services.sh +++ b/test/e2e/test-deployment-services.sh @@ -87,6 +87,10 @@ skip() { echo -e "${YELLOW} SKIP${NC} $1 — $2" | tee -a "$LOG_FILE" } +# ── Config ─────────────────────────────────────────────────────────────────── +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-deploy-svc}" +LOG_FILE="test-deployment-services-$(date +%Y%m%d-%H%M%S).log" + # ── Resolve repo root ──────────────────────────────────────────────────────── REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -144,16 +148,16 @@ preflight() { local arch arch=$(uname -m) case "$arch" in - x86_64) arch="amd64" ;; - aarch64 | arm64) arch="arm64" ;; - *) - log "WARNING: Unsupported arch $arch for cloudflared — skipping install" - return 0 - ;; + x86_64) arch="amd64" ;; + aarch64 | arm64) arch="arm64" ;; + *) + log "WARNING: Unsupported arch $arch for cloudflared — skipping install" + return 0 + ;; esac - if curl -fsSL "https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-${arch}" -o /tmp/cloudflared \ - && chmod +x /tmp/cloudflared \ - && sudo mv /tmp/cloudflared /usr/local/bin/cloudflared 2>/dev/null; then + if curl -fsSL "https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-${arch}" -o /tmp/cloudflared && + chmod +x /tmp/cloudflared && + sudo mv /tmp/cloudflared /usr/local/bin/cloudflared 2>/dev/null; then log "cloudflared installed" else log "WARNING: Could not install cloudflared" @@ -194,6 +198,7 @@ onboard_sandbox() { NEMOCLAW_NON_INTERACTIVE=1 \ NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ NEMOCLAW_POLICY_TIER="open" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ run_with_timeout 600 nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ 2>&1 | tee -a "$LOG_FILE" || { log "FATAL: Onboard failed for '$name'" diff --git a/test/e2e/test-network-policy.sh b/test/e2e/test-network-policy.sh index 5c1e1c9046c..775e63d9d8d 100755 --- a/test/e2e/test-network-policy.sh +++ b/test/e2e/test-network-policy.sh @@ -239,6 +239,7 @@ setup_sandbox() { NEMOCLAW_NON_INTERACTIVE=1 \ NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ NEMOCLAW_POLICY_TIER="restricted" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ run_with_timeout 600 nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ 2>&1 | tee -a "$LOG_FILE" || { log "FATAL: Onboard failed" diff --git a/test/e2e/test-snapshot-commands.sh b/test/e2e/test-snapshot-commands.sh index 4e46cbe4001..0d82948ee36 100755 --- a/test/e2e/test-snapshot-commands.sh +++ b/test/e2e/test-snapshot-commands.sh @@ -41,11 +41,29 @@ fail() { echo -e "${YELLOW}[DIAG]${NC} --- Failure diagnostics ---" >&2 echo -e "${YELLOW}[DIAG]${NC} Sandboxes: $(openshell sandbox list 2>&1 || echo 'unavailable')" >&2 echo -e "${YELLOW}[DIAG]${NC} Backup dir: $(ls -la "$HOME/.nemoclaw/rebuild-backups/${SANDBOX_NAME}/" 2>&1 || echo 'not found')" >&2 + echo -e "${YELLOW}[DIAG]${NC} Registry: $(cat "$HOME/.nemoclaw/sandboxes.json" 2>&1 || echo 'not found')" >&2 + echo -e "${YELLOW}[DIAG]${NC} Registry lock: $(ls -la "$HOME/.nemoclaw/sandboxes.json.lock" 2>&1 || echo 'no lock')" >&2 + echo -e "${YELLOW}[DIAG]${NC} Docker ps: $(docker ps --format '{{.Names}} {{.Status}}' 2>&1 || echo 'unavailable')" >&2 + echo -e "${YELLOW}[DIAG]${NC} nemoclaw path: $(command -v nemoclaw 2>&1 || echo 'not found')" >&2 + echo -e "${YELLOW}[DIAG]${NC} node version: $(node --version 2>&1 || echo 'not found')" >&2 echo -e "${YELLOW}[DIAG]${NC} --- End diagnostics ---" >&2 exit 1 } info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +# Run a command, capture its output and exit code without set -e killing us. +# Usage: run_capture VAR_NAME command [args...] +# Sets $VAR_NAME to the combined stdout+stderr and $_CAPTURE_RC to the exit code. +_CAPTURE_RC=0 +run_capture() { + local _var_name="$1" + shift + _CAPTURE_RC=0 + local _output + _output=$("$@" 2>&1) || _CAPTURE_RC=$? + eval "${_var_name}=\${_output}" +} + # ── Preflight ─────────────────────────────────────────────────────── [ -n "${NVIDIA_API_KEY:-}" ] || fail "NVIDIA_API_KEY is required" [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] || fail "NEMOCLAW_NON_INTERACTIVE=1 is required" @@ -90,21 +108,44 @@ pass "NemoClaw installed" info "Phase 2: Writing marker files into sandbox..." openshell sandbox exec --name "${SANDBOX_NAME}" -- \ - sh -c "mkdir -p /sandbox/.openclaw-data/workspace && echo '${MARKER_CONTENT}' > ${MARKER_FILE}" \ - || fail "Failed to write marker file" + sh -c "mkdir -p /sandbox/.openclaw-data/workspace && echo '${MARKER_CONTENT}' > ${MARKER_FILE}" || + fail "Failed to write marker file" VERIFY=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || true) [ "$VERIFY" = "${MARKER_CONTENT}" ] || fail "Marker verification failed: got '${VERIFY}'" pass "Marker file written" +# ── Phase 2b: Pre-snapshot diagnostics ───────────────────────────── +# Collect state that helps diagnose Phase 3 failures (see #2350). +info "Phase 2b: Pre-snapshot diagnostics..." +echo -e "${YELLOW}[DIAG]${NC} nemoclaw binary: $(command -v nemoclaw)" +echo -e "${YELLOW}[DIAG]${NC} nemoclaw version: $(nemoclaw --version 2>&1 || echo 'failed')" +echo -e "${YELLOW}[DIAG]${NC} openshell sandbox list:" +openshell sandbox list 2>&1 || echo "(openshell sandbox list failed)" +echo -e "${YELLOW}[DIAG]${NC} Registry file:" +cat "$HOME/.nemoclaw/sandboxes.json" 2>&1 || echo "(registry not found)" +echo -e "${YELLOW}[DIAG]${NC} Registry lock:" +ls -la "$HOME/.nemoclaw/sandboxes.json.lock" 2>&1 || echo "(no stale lock)" +echo -e "${YELLOW}[DIAG]${NC} Config dir:" +ls -la "$HOME/.nemoclaw/" 2>&1 || echo "(config dir not found)" +echo -e "${YELLOW}[DIAG]${NC} Docker containers:" +docker ps --format '{{.Names}} {{.Status}}' 2>&1 || echo "(docker ps failed)" + # ── Phase 3: snapshot create ──────────────────────────────────────── info "Phase 3: Creating snapshot..." -SNAPSHOT_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" snapshot create 2>&1) +# Use run_capture to prevent set -e from swallowing error output. +# Previously, $(nemoclaw ... 2>&1) would exit the script immediately on +# failure, hiding the actual error message. See #2350. +run_capture SNAPSHOT_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot create echo "$SNAPSHOT_OUTPUT" -if echo "$SNAPSHOT_OUTPUT" | grep -q "Snapshot created"; then +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot create exited with code $_CAPTURE_RC: ${SNAPSHOT_OUTPUT}" +fi + +if echo "$SNAPSHOT_OUTPUT" | grep -q "Snapshot.*created"; then pass "snapshot create succeeded" else fail "snapshot create did not report success: ${SNAPSHOT_OUTPUT}" @@ -117,9 +158,13 @@ info "Snapshot path: ${SNAPSHOT_PATH:-unknown}" # ── Phase 4: snapshot list ────────────────────────────────────────── info "Phase 4: Listing snapshots..." -LIST_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" snapshot list 2>&1) +run_capture LIST_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot list echo "$LIST_OUTPUT" +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot list exited with code $_CAPTURE_RC: ${LIST_OUTPUT}" +fi + if echo "$LIST_OUTPUT" | grep -q "snapshot(s)"; then pass "snapshot list shows snapshots" else @@ -135,29 +180,36 @@ info "Snapshot timestamp: ${SNAPSHOT_TIMESTAMP}" info "Phase 5: Modifying sandbox state and creating second snapshot..." openshell sandbox exec --name "${SANDBOX_NAME}" -- \ - sh -c "rm -f ${MARKER_FILE} && echo '${SECOND_CONTENT}' > ${SECOND_MARKER}" \ - || fail "Failed to modify sandbox state" + sh -c "rm -f ${MARKER_FILE} && echo '${SECOND_CONTENT}' > ${SECOND_MARKER}" || + fail "Failed to modify sandbox state" # Verify first marker is gone GONE=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || echo "GONE") [ "$GONE" = "GONE" ] || fail "First marker should be deleted but got: ${GONE}" -nemoclaw "${SANDBOX_NAME}" snapshot create >/dev/null 2>&1 || fail "Second snapshot create failed" +run_capture _SECOND_SNAP nemoclaw "${SANDBOX_NAME}" snapshot create +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "Second snapshot create failed (code $_CAPTURE_RC): ${_SECOND_SNAP}" +fi pass "State modified, second snapshot created" # Perturb workspace so restore has to do real work openshell sandbox exec --name "${SANDBOX_NAME}" -- \ - sh -c "rm -f ${SECOND_MARKER} && echo 'BROKEN' > ${MARKER_FILE}" \ - || fail "Failed to perturb sandbox before latest restore" + sh -c "rm -f ${SECOND_MARKER} && echo 'BROKEN' > ${MARKER_FILE}" || + fail "Failed to perturb sandbox before latest restore" # ── Phase 6: snapshot restore (latest) ────────────────────────────── info "Phase 6: Restoring latest snapshot..." -RESTORE_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" snapshot restore 2>&1) +run_capture RESTORE_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot restore echo "$RESTORE_OUTPUT" +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot restore exited with code $_CAPTURE_RC: ${RESTORE_OUTPUT}" +fi + if ! echo "$RESTORE_OUTPUT" | grep -q "Restored"; then - fail "snapshot restore failed: ${RESTORE_OUTPUT}" + fail "snapshot restore did not report success: ${RESTORE_OUTPUT}" fi SECOND_CHECK=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${SECOND_MARKER}" 2>/dev/null || echo "MISSING") @@ -167,11 +219,15 @@ pass "Latest snapshot restored expected state" # ── Phase 7: snapshot restore with timestamp (first snapshot) ─────── info "Phase 7: Restoring first snapshot by timestamp..." -TARGETED_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" snapshot restore "${SNAPSHOT_TIMESTAMP}" 2>&1) +run_capture TARGETED_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot restore "${SNAPSHOT_TIMESTAMP}" echo "$TARGETED_OUTPUT" +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "targeted snapshot restore exited with code $_CAPTURE_RC: ${TARGETED_OUTPUT}" +fi + if ! echo "$TARGETED_OUTPUT" | grep -q "Restored"; then - fail "Targeted snapshot restore failed: ${TARGETED_OUTPUT}" + fail "targeted snapshot restore did not report success: ${TARGETED_OUTPUT}" fi FIRST_CHECK=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || echo "MISSING") @@ -198,10 +254,13 @@ fi # ── Phase 9: snapshot help ────────────────────────────────────────── info "Phase 9: Verifying snapshot help output..." -HELP_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" snapshot 2>&1) -if echo "$HELP_OUTPUT" | grep -q "snapshot create" \ - && echo "$HELP_OUTPUT" | grep -q "snapshot list" \ - && echo "$HELP_OUTPUT" | grep -q "snapshot restore"; then +run_capture HELP_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot help exited with code $_CAPTURE_RC: ${HELP_OUTPUT}" +fi +if echo "$HELP_OUTPUT" | grep -q "snapshot create" && + echo "$HELP_OUTPUT" | grep -q "snapshot list" && + echo "$HELP_OUTPUT" | grep -q "snapshot restore"; then pass "snapshot help shows create/list/restore" else fail "snapshot help incomplete: ${HELP_OUTPUT}"