test(e2e): cover openclaw → inference.local and stop matching prompt tokens

test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh

@@ -39,9 +39,21 @@ info() { printf '%s\n' "verify-sandbox-skill-via-agent: INFO: $*"; }
 [ -n "$SANDBOX_NAME" ] || die "set SANDBOX_NAME (or NEMOCLAW_SANDBOX_NAME)"
 [ -n "${NVIDIA_API_KEY:-}" ] || die "set NVIDIA_API_KEY (needed for inference inside sandbox)"
 
-DEFAULT_PROMPT="Use the OpenClaw managed skill named '${SKILL_ID}'. Read its SKILL.md. Reply with ONLY this exact verification token string and nothing else: ${VERIFY_TOKEN}"
+# Do NOT include ${VERIFY_TOKEN} in the prompt itself. The token must come
+# from the agent reading the skill's SKILL.md — that is the entire point of
+# this test. Embedding it in the prompt makes the downstream grep match any
+# error path that echoes the prompt back (e.g. the openclaw 4.9 SSRF
+# regression in NemoClaw #2490 was masked by exactly this antipattern in
+# TC-SBX-02). Override SKILL_VERIFY_PROMPT only if you know what you're
+# doing — overrides that re-introduce the literal token defeat the test.
+DEFAULT_PROMPT="Use the OpenClaw managed skill named '${SKILL_ID}'. Read its SKILL.md and reply with ONLY the agent verification token defined in that file. No quotes, no extra words."
 PROMPT="${SKILL_VERIFY_PROMPT:-$DEFAULT_PROMPT}"
 
+# Guard against an override that accidentally smuggles the token back in.
+if printf '%s' "$PROMPT" | grep -Fq "$VERIFY_TOKEN"; then
+  die "SKILL_VERIFY_PROMPT must not contain VERIFY_TOKEN ('${VERIFY_TOKEN}'); the agent must read it from SKILL.md so a prompt-echo error path cannot satisfy the assertion"
+fi
+
 command -v openshell >/dev/null 2>&1 || die "openshell not on PATH"
 command -v base64 >/dev/null 2>&1 || die "base64 not on PATH"
 
@@ -84,6 +96,14 @@ printf '\n%s\n' "--- agent stdout/stderr (trimmed for display) ---"
 printf '%s' "$raw_out" | tail -c 12000
 printf '\n%s\n' "--- end ---"
 
+# Fail closed on provider/transport errors so a coincidental token match
+# (e.g. someone overrode SKILL_VERIFY_PROMPT to embed the token, or the
+# token leaked into a stack trace via the skill manifest path) cannot mask
+# an SSRF block, transport reset, or gateway error. See NemoClaw #2490.
+if printf '%s' "$raw_out" | grep -qiE "SsrFBlockedError|Blocked hostname|Blocked: resolves to|transport error|provider error|ECONNREFUSED|EAI_AGAIN|gateway unavailable"; then
+  die "agent failed before completing turn — provider/transport error in output (exit ${agent_rc}). Session: ${SESSION_ID}"
+fi
+
 # Collapse newlines so a model-wrapped token (e.g. "SKILL_SMOKE_VER\nIFY_K9X2") still matches.
 collapsed_out=$(printf '%s' "$raw_out" | tr -d '\n\r')
 if printf '%s' "$collapsed_out" | grep -Fq "$VERIFY_TOKEN"; then

test/e2e/test-full-e2e.sh

@@ -288,8 +288,13 @@ else
   fail "[LIVE] Direct API: empty response from curl"
 fi
 
-# ── Test 4b: Inference through the sandbox (THE definitive test) ──
-info "[LIVE] Sandbox inference test → user → sandbox → gateway → NVIDIA API..."
+# ── Test 4b: OpenShell DNS+proxy can route inference.local from the sandbox ──
+# This is a routing-layer check, not an openclaw check. The HTTP request is
+# made by `curl` from inside the sandbox; nothing in this path exercises
+# openclaw's HTTP client or its SSRF guard. See Phase 4c for the openclaw-
+# mediated assertion. (NemoClaw #2490 / openclaw 2026.4.9 SSRF regression
+# was invisible to this step because curl bypasses openclaw entirely.)
+info "[ROUTING] inference.local DNS + OpenShell proxy reachable from sandbox..."
 ssh_config="$(mktemp)"
 sandbox_response=""
 
@@ -349,10 +354,64 @@ for pong_attempt in 1 2 3; do
   rm -f "$ssh_config"
 done
 if $pong_ok; then
-  pass "[LIVE] Sandbox inference: model responded with PONG through sandbox"
-  info "Full path proven: user → sandbox → openshell gateway → NVIDIA Endpoints → response"
+  pass "[ROUTING] inference.local: OpenShell routed curl to NVIDIA Endpoints and returned PONG"
+  info "Routing path proven: sandbox curl → DNS forwarder → gateway proxy → NVIDIA Endpoints (does not exercise openclaw HTTP client; see Phase 4c)"
 else
-  fail "[LIVE] Sandbox inference: expected PONG after 3 attempts, got: ${sandbox_content:0:200}"
+  fail "[ROUTING] inference.local: expected PONG after 3 attempts, got: ${sandbox_content:0:200}"
+fi
+
+# ── Test 4c: openclaw-mediated turn against inference.local ──
+# This is the only assertion in this file that proves openclaw can complete
+# a turn against inference.local. Prior to this step, every "[LIVE] inference"
+# label in the suite was actually a [ROUTING] check via curl (see 4b above).
+#
+# Properties of this assertion that prevent the false-positive class that
+# masked the openclaw 2026.4.9 SSRF regression:
+#   * Uses `openclaw agent --json`. With --json the CLI calls
+#     routeLogsToStderr() (openclaw/src/commands/agent-via-gateway.ts:57),
+#     so stdout is a clean JSON envelope; prompt-echo on stderr cannot
+#     pollute the assertion.
+#   * Asserts on the model's reply text inside `result.payloads[].text`,
+#     not on the merged stdout/stderr.
+#   * The expected token (the integer 42) is not a literal substring of the
+#     prompt, so an error path that quoted the prompt back cannot satisfy
+#     the grep.
+info "[LIVE] openclaw agent → openclaw HTTP client → inference.local..."
+ssh_config="$(mktemp)"
+agent_response=""
+
+if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then
+  agent_session_id="e2e-live-$(date +%s)-$$"
+  # 2>/dev/null discards stderr (progress + log lines) so stdout is JSON-only.
+  agent_response=$($TIMEOUT_CMD ssh -F "$ssh_config" \
+    -o StrictHostKeyChecking=no \
+    -o UserKnownHostsFile=/dev/null \
+    -o ConnectTimeout=10 \
+    -o LogLevel=ERROR \
+    "openshell-${SANDBOX_NAME}" \
+    "openclaw agent --agent main --json --session-id '${agent_session_id}' -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.'" \
+    2>/dev/null) || true
+fi
+rm -f "$ssh_config"
+
+agent_reply=$(echo "$agent_response" | python3 -c "
+import json, sys
+try:
+    doc = json.load(sys.stdin)
+except Exception:
+    sys.exit(0)
+result = doc.get('result') or {}
+parts = []
+for p in result.get('payloads') or []:
+    if isinstance(p, dict) and isinstance(p.get('text'), str):
+        parts.append(p['text'])
+print('\n'.join(parts))
+" 2>/dev/null) || true
+
+if grep -qE "(^|[^0-9])42([^0-9]|$)" <<<"$agent_reply"; then
+  pass "[LIVE] openclaw agent: model answered 6×7=42 through openclaw → inference.local"
+else
+  fail "[LIVE] openclaw agent: expected '42' in agent reply, got: ${agent_reply:0:200}"
 fi
 
 # ══════════════════════════════════════════════════════════════════

test/e2e/test-hermes-e2e.sh

@@ -423,7 +423,11 @@ else
 fi
 
 # ── Test 5b: Inference through the sandbox (THE definitive test) ──
-info "[LIVE] Sandbox inference test → user → sandbox → gateway → NVIDIA API..."
+# Routing-layer check, not a Hermes/openclaw check. The HTTP request is made
+# by curl from inside the sandbox; nothing in this path exercises the Hermes
+# agent runtime or openclaw's HTTP client. See NemoClaw #2490 for the
+# openclaw 4.9 SSRF regression that was invisible to assertions of this shape.
+info "[ROUTING] inference.local DNS + OpenShell proxy reachable from Hermes sandbox..."
 ssh_config="$(mktemp)"
 sandbox_response=""
 
@@ -448,13 +452,13 @@ rm -f "$ssh_config"
 if [ -n "$sandbox_response" ]; then
   sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true
   if grep -qi "PONG" <<<"$sandbox_content"; then
-    pass "[LIVE] Sandbox inference: model responded with PONG through Hermes sandbox"
-    info "Full path proven: user → Hermes sandbox → openshell gateway → NVIDIA Endpoints → response"
+    pass "[ROUTING] inference.local: OpenShell routed curl to NVIDIA Endpoints and returned PONG"
+    info "Routing path proven: sandbox curl → DNS forwarder → gateway proxy → NVIDIA Endpoints (does not exercise the Hermes agent runtime or openclaw HTTP client)"
   else
-    fail "[LIVE] Sandbox inference: expected PONG, got: ${sandbox_content:0:200}"
+    fail "[ROUTING] inference.local: expected PONG, got: ${sandbox_content:0:200}"
   fi
 else
-  fail "[LIVE] Sandbox inference: no response from inference.local inside Hermes sandbox"
+  fail "[ROUTING] inference.local: no response from inference.local inside Hermes sandbox"
 fi
 
 # ══════════════════════════════════════════════════════════════════

test/e2e/test-sandbox-operations.sh

@@ -271,18 +271,47 @@ test_sbx_01_list_sandboxes() {
 }
 
 # ── TC-SBX-02: Connect & Chat ───────────────────────────────────────────────
+# Drives one openclaw-mediated turn through the sandbox and asserts the
+# model produced a real answer. Three properties keep this honest:
+#
+#   1. Uses `openclaw agent --json`, which calls routeLogsToStderr() in
+#      openclaw/src/commands/agent-via-gateway.ts:57 so stdout is a clean
+#      JSON envelope. Stderr is dropped (2>/dev/null) so any prompt-echo
+#      or wrapped error there cannot satisfy the assertion.
+#   2. The expected token (the integer 42) is not a literal substring of
+#      the prompt, so an error path that quoted the prompt back cannot
+#      false-positive the grep — which is what masked the openclaw 4.9
+#      SSRF regression from the prior `Say exactly: HELLO_E2E` assertion.
+#   3. Asserts on `result.payloads[].text` from the JSON envelope, not on
+#      merged stdout/stderr.
 test_sbx_02_connect_chat() {
   log "=== TC-SBX-02: Connect & Chat ==="
   require_sandbox "$SANDBOX_A" "TC-SBX-02" || return
 
-  log "  Sending one-shot message to agent via SSH..."
-  local reply
-  reply=$(sandbox_exec "openclaw agent --agent main -m 'Say exactly: HELLO_E2E' --session-id e2e-test" 2>&1) || true
+  log "  Sending one-shot message to agent via SSH (openclaw agent --json)..."
+  local session_id raw
+  session_id="e2e-sbx-02-$(date +%s)-$$"
+  raw=$(sandbox_exec "openclaw agent --agent main --json --session-id '${session_id}' -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.'" 2>/dev/null) || true
 
-  if echo "$reply" | grep -qi "HELLO_E2E"; then
-    pass "TC-SBX-02: Agent replied with expected token"
+  local reply
+  reply=$(echo "$raw" | python3 -c "
+import json, sys
+try:
+    doc = json.load(sys.stdin)
+except Exception:
+    sys.exit(0)
+result = doc.get('result') or {}
+parts = []
+for p in result.get('payloads') or []:
+    if isinstance(p, dict) and isinstance(p.get('text'), str):
+        parts.append(p['text'])
+print('\n'.join(parts))
+" 2>/dev/null) || true
+
+  if [[ -n "$reply" ]] && echo "$reply" | grep -qE "(^|[^0-9])42([^0-9]|$)"; then
+    pass "TC-SBX-02: Agent computed 6×7=42 through openclaw → inference.local"
   else
-    fail "TC-SBX-02: Connect & Chat" "Got: $(echo "$reply" | head -3)"
+    fail "TC-SBX-02: Connect & Chat" "Expected '42' in agent reply; reply='${reply:0:200}'; raw stdout='${raw:0:200}'"
   fi
 }