From 38b2b175f8d065fc23d144f8bb40721234d0b7a6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 26 Apr 2026 15:53:11 -0700 Subject: [PATCH] =?UTF-8?q?test(e2e):=20cover=20openclaw=20=E2=86=92=20inf?= =?UTF-8?q?erence.local=20and=20stop=20matching=20prompt=20tokens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "live inference" assertions in cloud-e2e (test-full-e2e.sh) and the Hermes e2e were curl-from-sandbox checks. They prove OpenShell's DNS forwarder + proxy can route inference.local; they never invoke openclaw's HTTP client and never reach openclaw's SSRF guard. That is why every openclaw 4.9 nightly-e2e run on PR #2464 reported [LIVE] Sandbox inference: PASS while real users were getting SsrFBlockedError on the same release. Changes: * test-full-e2e.sh: relabel Phase 4b from [LIVE] to [ROUTING] with a comment pointing at #2490; add Phase 4c, an actual openclaw-mediated turn that runs `openclaw agent --json` over SSH, parses result.payloads[].text, and asserts the model produced "42" for "What is 6 multiplied by 7?". The expected token is not a substring of the prompt, --json routes logs to stderr, stderr is dropped — so prompt-echo on an error path cannot satisfy the grep. * test-hermes-e2e.sh: same relabel for the equivalent curl assertion. * test-sandbox-operations.sh TC-SBX-02: replace `Say exactly: HELLO_E2E` prompt + grep on merged stdout/stderr with the same arithmetic-via-JSON pattern. The previous assertion would match the prompt itself in any error path that quoted it back, including the openclaw 4.9 SSRF rejection — false positive that hid the regression for the entire 4.2 → 4.7 → 4.8 → 4.9 bump series. * verify-sandbox-skill-via-agent.sh: stop embedding ${VERIFY_TOKEN} in the prompt (the agent must read it from SKILL.md — that is the test). Add a guard that refuses SKILL_VERIFY_PROMPT overrides which smuggle the token back in, and a negative assertion on SsrFBlockedError, transport errors, and gateway-unavailable markers before the positive grep. Signed-off-by: Aaron Erickson --- .../skill/verify-sandbox-skill-via-agent.sh | 22 +++++- test/e2e/test-full-e2e.sh | 69 +++++++++++++++++-- test/e2e/test-hermes-e2e.sh | 14 ++-- test/e2e/test-sandbox-operations.sh | 41 +++++++++-- 4 files changed, 129 insertions(+), 17 deletions(-) diff --git a/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh b/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh index 825d36f9e31..be72434bb1a 100755 --- a/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh +++ b/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh @@ -39,9 +39,21 @@ info() { printf '%s\n' "verify-sandbox-skill-via-agent: INFO: $*"; } [ -n "$SANDBOX_NAME" ] || die "set SANDBOX_NAME (or NEMOCLAW_SANDBOX_NAME)" [ -n "${NVIDIA_API_KEY:-}" ] || die "set NVIDIA_API_KEY (needed for inference inside sandbox)" -DEFAULT_PROMPT="Use the OpenClaw managed skill named '${SKILL_ID}'. Read its SKILL.md. Reply with ONLY this exact verification token string and nothing else: ${VERIFY_TOKEN}" +# Do NOT include ${VERIFY_TOKEN} in the prompt itself. The token must come +# from the agent reading the skill's SKILL.md — that is the entire point of +# this test. Embedding it in the prompt makes the downstream grep match any +# error path that echoes the prompt back (e.g. the openclaw 4.9 SSRF +# regression in NemoClaw #2490 was masked by exactly this antipattern in +# TC-SBX-02). Override SKILL_VERIFY_PROMPT only if you know what you're +# doing — overrides that re-introduce the literal token defeat the test. +DEFAULT_PROMPT="Use the OpenClaw managed skill named '${SKILL_ID}'. Read its SKILL.md and reply with ONLY the agent verification token defined in that file. No quotes, no extra words." PROMPT="${SKILL_VERIFY_PROMPT:-$DEFAULT_PROMPT}" +# Guard against an override that accidentally smuggles the token back in. +if printf '%s' "$PROMPT" | grep -Fq "$VERIFY_TOKEN"; then + die "SKILL_VERIFY_PROMPT must not contain VERIFY_TOKEN ('${VERIFY_TOKEN}'); the agent must read it from SKILL.md so a prompt-echo error path cannot satisfy the assertion" +fi + command -v openshell >/dev/null 2>&1 || die "openshell not on PATH" command -v base64 >/dev/null 2>&1 || die "base64 not on PATH" @@ -84,6 +96,14 @@ printf '\n%s\n' "--- agent stdout/stderr (trimmed for display) ---" printf '%s' "$raw_out" | tail -c 12000 printf '\n%s\n' "--- end ---" +# Fail closed on provider/transport errors so a coincidental token match +# (e.g. someone overrode SKILL_VERIFY_PROMPT to embed the token, or the +# token leaked into a stack trace via the skill manifest path) cannot mask +# an SSRF block, transport reset, or gateway error. See NemoClaw #2490. +if printf '%s' "$raw_out" | grep -qiE "SsrFBlockedError|Blocked hostname|Blocked: resolves to|transport error|provider error|ECONNREFUSED|EAI_AGAIN|gateway unavailable"; then + die "agent failed before completing turn — provider/transport error in output (exit ${agent_rc}). Session: ${SESSION_ID}" +fi + # Collapse newlines so a model-wrapped token (e.g. "SKILL_SMOKE_VER\nIFY_K9X2") still matches. collapsed_out=$(printf '%s' "$raw_out" | tr -d '\n\r') if printf '%s' "$collapsed_out" | grep -Fq "$VERIFY_TOKEN"; then diff --git a/test/e2e/test-full-e2e.sh b/test/e2e/test-full-e2e.sh index 759a7a7033f..cb902662f92 100755 --- a/test/e2e/test-full-e2e.sh +++ b/test/e2e/test-full-e2e.sh @@ -288,8 +288,13 @@ else fail "[LIVE] Direct API: empty response from curl" fi -# ── Test 4b: Inference through the sandbox (THE definitive test) ── -info "[LIVE] Sandbox inference test → user → sandbox → gateway → NVIDIA API..." +# ── Test 4b: OpenShell DNS+proxy can route inference.local from the sandbox ── +# This is a routing-layer check, not an openclaw check. The HTTP request is +# made by `curl` from inside the sandbox; nothing in this path exercises +# openclaw's HTTP client or its SSRF guard. See Phase 4c for the openclaw- +# mediated assertion. (NemoClaw #2490 / openclaw 2026.4.9 SSRF regression +# was invisible to this step because curl bypasses openclaw entirely.) +info "[ROUTING] inference.local DNS + OpenShell proxy reachable from sandbox..." ssh_config="$(mktemp)" sandbox_response="" @@ -349,10 +354,64 @@ for pong_attempt in 1 2 3; do rm -f "$ssh_config" done if $pong_ok; then - pass "[LIVE] Sandbox inference: model responded with PONG through sandbox" - info "Full path proven: user → sandbox → openshell gateway → NVIDIA Endpoints → response" + pass "[ROUTING] inference.local: OpenShell routed curl to NVIDIA Endpoints and returned PONG" + info "Routing path proven: sandbox curl → DNS forwarder → gateway proxy → NVIDIA Endpoints (does not exercise openclaw HTTP client; see Phase 4c)" else - fail "[LIVE] Sandbox inference: expected PONG after 3 attempts, got: ${sandbox_content:0:200}" + fail "[ROUTING] inference.local: expected PONG after 3 attempts, got: ${sandbox_content:0:200}" +fi + +# ── Test 4c: openclaw-mediated turn against inference.local ── +# This is the only assertion in this file that proves openclaw can complete +# a turn against inference.local. Prior to this step, every "[LIVE] inference" +# label in the suite was actually a [ROUTING] check via curl (see 4b above). +# +# Properties of this assertion that prevent the false-positive class that +# masked the openclaw 2026.4.9 SSRF regression: +# * Uses `openclaw agent --json`. With --json the CLI calls +# routeLogsToStderr() (openclaw/src/commands/agent-via-gateway.ts:57), +# so stdout is a clean JSON envelope; prompt-echo on stderr cannot +# pollute the assertion. +# * Asserts on the model's reply text inside `result.payloads[].text`, +# not on the merged stdout/stderr. +# * The expected token (the integer 42) is not a literal substring of the +# prompt, so an error path that quoted the prompt back cannot satisfy +# the grep. +info "[LIVE] openclaw agent → openclaw HTTP client → inference.local..." +ssh_config="$(mktemp)" +agent_response="" + +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + agent_session_id="e2e-live-$(date +%s)-$$" + # 2>/dev/null discards stderr (progress + log lines) so stdout is JSON-only. + agent_response=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "openclaw agent --agent main --json --session-id '${agent_session_id}' -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.'" \ + 2>/dev/null) || true +fi +rm -f "$ssh_config" + +agent_reply=$(echo "$agent_response" | python3 -c " +import json, sys +try: + doc = json.load(sys.stdin) +except Exception: + sys.exit(0) +result = doc.get('result') or {} +parts = [] +for p in result.get('payloads') or []: + if isinstance(p, dict) and isinstance(p.get('text'), str): + parts.append(p['text']) +print('\n'.join(parts)) +" 2>/dev/null) || true + +if grep -qE "(^|[^0-9])42([^0-9]|$)" <<<"$agent_reply"; then + pass "[LIVE] openclaw agent: model answered 6×7=42 through openclaw → inference.local" +else + fail "[LIVE] openclaw agent: expected '42' in agent reply, got: ${agent_reply:0:200}" fi # ══════════════════════════════════════════════════════════════════ diff --git a/test/e2e/test-hermes-e2e.sh b/test/e2e/test-hermes-e2e.sh index e9959586729..71e54ea06e0 100755 --- a/test/e2e/test-hermes-e2e.sh +++ b/test/e2e/test-hermes-e2e.sh @@ -423,7 +423,11 @@ else fi # ── Test 5b: Inference through the sandbox (THE definitive test) ── -info "[LIVE] Sandbox inference test → user → sandbox → gateway → NVIDIA API..." +# Routing-layer check, not a Hermes/openclaw check. The HTTP request is made +# by curl from inside the sandbox; nothing in this path exercises the Hermes +# agent runtime or openclaw's HTTP client. See NemoClaw #2490 for the +# openclaw 4.9 SSRF regression that was invisible to assertions of this shape. +info "[ROUTING] inference.local DNS + OpenShell proxy reachable from Hermes sandbox..." ssh_config="$(mktemp)" sandbox_response="" @@ -448,13 +452,13 @@ rm -f "$ssh_config" if [ -n "$sandbox_response" ]; then sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true if grep -qi "PONG" <<<"$sandbox_content"; then - pass "[LIVE] Sandbox inference: model responded with PONG through Hermes sandbox" - info "Full path proven: user → Hermes sandbox → openshell gateway → NVIDIA Endpoints → response" + pass "[ROUTING] inference.local: OpenShell routed curl to NVIDIA Endpoints and returned PONG" + info "Routing path proven: sandbox curl → DNS forwarder → gateway proxy → NVIDIA Endpoints (does not exercise the Hermes agent runtime or openclaw HTTP client)" else - fail "[LIVE] Sandbox inference: expected PONG, got: ${sandbox_content:0:200}" + fail "[ROUTING] inference.local: expected PONG, got: ${sandbox_content:0:200}" fi else - fail "[LIVE] Sandbox inference: no response from inference.local inside Hermes sandbox" + fail "[ROUTING] inference.local: no response from inference.local inside Hermes sandbox" fi # ══════════════════════════════════════════════════════════════════ diff --git a/test/e2e/test-sandbox-operations.sh b/test/e2e/test-sandbox-operations.sh index d384341101f..3e592e03612 100755 --- a/test/e2e/test-sandbox-operations.sh +++ b/test/e2e/test-sandbox-operations.sh @@ -271,18 +271,47 @@ test_sbx_01_list_sandboxes() { } # ── TC-SBX-02: Connect & Chat ─────────────────────────────────────────────── +# Drives one openclaw-mediated turn through the sandbox and asserts the +# model produced a real answer. Three properties keep this honest: +# +# 1. Uses `openclaw agent --json`, which calls routeLogsToStderr() in +# openclaw/src/commands/agent-via-gateway.ts:57 so stdout is a clean +# JSON envelope. Stderr is dropped (2>/dev/null) so any prompt-echo +# or wrapped error there cannot satisfy the assertion. +# 2. The expected token (the integer 42) is not a literal substring of +# the prompt, so an error path that quoted the prompt back cannot +# false-positive the grep — which is what masked the openclaw 4.9 +# SSRF regression from the prior `Say exactly: HELLO_E2E` assertion. +# 3. Asserts on `result.payloads[].text` from the JSON envelope, not on +# merged stdout/stderr. test_sbx_02_connect_chat() { log "=== TC-SBX-02: Connect & Chat ===" require_sandbox "$SANDBOX_A" "TC-SBX-02" || return - log " Sending one-shot message to agent via SSH..." - local reply - reply=$(sandbox_exec "openclaw agent --agent main -m 'Say exactly: HELLO_E2E' --session-id e2e-test" 2>&1) || true + log " Sending one-shot message to agent via SSH (openclaw agent --json)..." + local session_id raw + session_id="e2e-sbx-02-$(date +%s)-$$" + raw=$(sandbox_exec "openclaw agent --agent main --json --session-id '${session_id}' -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.'" 2>/dev/null) || true - if echo "$reply" | grep -qi "HELLO_E2E"; then - pass "TC-SBX-02: Agent replied with expected token" + local reply + reply=$(echo "$raw" | python3 -c " +import json, sys +try: + doc = json.load(sys.stdin) +except Exception: + sys.exit(0) +result = doc.get('result') or {} +parts = [] +for p in result.get('payloads') or []: + if isinstance(p, dict) and isinstance(p.get('text'), str): + parts.append(p['text']) +print('\n'.join(parts)) +" 2>/dev/null) || true + + if [[ -n "$reply" ]] && echo "$reply" | grep -qE "(^|[^0-9])42([^0-9]|$)"; then + pass "TC-SBX-02: Agent computed 6×7=42 through openclaw → inference.local" else - fail "TC-SBX-02: Connect & Chat" "Got: $(echo "$reply" | head -3)" + fail "TC-SBX-02: Connect & Chat" "Expected '42' in agent reply; reply='${reply:0:200}'; raw stdout='${raw:0:200}'" fi }