From f177ded73d458dc3f28120b1a94b34d9b3b26696 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 21 May 2026 21:30:34 -0400 Subject: [PATCH 1/2] test: tolerate openclaw json envelope changes --- test/e2e/lib/openclaw-json.sh | 90 +++++++++++++++++++ ...st-bedrock-runtime-compatible-anthropic.sh | 20 +---- test/e2e/test-brave-search-e2e.sh | 16 +--- test/e2e/test-full-e2e.sh | 43 +-------- test/e2e/test-launchable-smoke.sh | 16 +--- .../e2e/test-messaging-compatible-endpoint.sh | 16 +--- test/e2e/test-openclaw-inference-switch.sh | 17 +--- test/e2e/test-sandbox-operations.sh | 20 ++--- 8 files changed, 114 insertions(+), 124 deletions(-) create mode 100644 test/e2e/lib/openclaw-json.sh diff --git a/test/e2e/lib/openclaw-json.sh b/test/e2e/lib/openclaw-json.sh new file mode 100644 index 00000000000..8f17bab69fa --- /dev/null +++ b/test/e2e/lib/openclaw-json.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Extract human-readable assistant text from `openclaw agent --json` output. +# OpenClaw's JSON envelope has moved between result.payloads[] and top-level +# payloads[]; keep E2E assertions focused on visible reply text instead of one +# exact envelope shape. This also tolerates wrapper output before the JSON blob +# but intentionally ignores metadata fields so IDs, durations, session names, +# and model/provider details cannot satisfy reply assertions. +parse_openclaw_agent_text() { + python3 -c ' +import json +import sys + +raw = sys.stdin.read() +if not raw.strip(): + sys.exit(0) + +parts = [] +visited = set() + +TEXT_KEYS = {"text", "content", "reasoning_content"} +CONTAINER_KEYS = { + "result", "payloads", "payload", "messages", "choices", "response", + "data", "output", "outputs", "items", "segments", "delta", +} + + +def add(value): + if isinstance(value, str) and value.strip(): + parts.append(value.strip()) + + +def collect(value): + value_id = id(value) + if value_id in visited: + return + visited.add(value_id) + + if isinstance(value, str): + add(value) + return + if isinstance(value, list): + for item in value: + collect(item) + return + if not isinstance(value, dict): + return + + for key in TEXT_KEYS: + add(value.get(key)) + + # OpenAI-style choices can nest assistant text under message/delta objects. + for choice in value.get("choices") or []: + if isinstance(choice, dict): + collect(choice.get("message")) + collect(choice.get("delta")) + add(choice.get("text")) + + for key in CONTAINER_KEYS: + if key in value: + collect(value[key]) + + +def collect_from_doc(doc): + if isinstance(doc, dict) and isinstance(doc.get("result"), dict): + collect(doc["result"]) + else: + collect(doc) + +try: + collect_from_doc(json.loads(raw)) +except Exception: + decoder = json.JSONDecoder() + for idx, char in enumerate(raw): + if char != "{": + continue + try: + doc, _end = decoder.raw_decode(raw[idx:]) + except Exception: + continue + before = len(parts) + collect_from_doc(doc) + if len(parts) > before: + break + +print("\n".join(parts)) +' +} diff --git a/test/e2e/test-bedrock-runtime-compatible-anthropic.sh b/test/e2e/test-bedrock-runtime-compatible-anthropic.sh index 76e36354ae2..e22b3e2c295 100755 --- a/test/e2e/test-bedrock-runtime-compatible-anthropic.sh +++ b/test/e2e/test-bedrock-runtime-compatible-anthropic.sh @@ -25,6 +25,8 @@ export NEMOCLAW_E2E_DEFAULT_TIMEOUT=2700 SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" # shellcheck source=test/e2e/e2e-timeout.sh . "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e/lib/openclaw-json.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/openclaw-json.sh" PASS=0 FAIL=0 @@ -698,23 +700,7 @@ check_openclaw_agent_turn() { return fi - reply=$(printf '%s' "$raw" | python3 -c ' -import json -import sys - -text = sys.stdin.read() -for idx, char in enumerate(text): - if char != "{": - continue - try: - doc = json.loads(text[idx:]) - except Exception: - continue - payloads = ((doc.get("result") or {}).get("payloads") or []) - parts = [p.get("text") for p in payloads if isinstance(p, dict) and isinstance(p.get("text"), str)] - print("\n".join(parts)) - break -' 2>/dev/null) || true + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true if [ "$rc" -eq 0 ] && grep -qi "PONG" <<<"$reply"; then pass "B8: OpenClaw agent completed a Bedrock-backed turn through inference.local" diff --git a/test/e2e/test-brave-search-e2e.sh b/test/e2e/test-brave-search-e2e.sh index e6f3f92d498..a2df9aadea4 100755 --- a/test/e2e/test-brave-search-e2e.sh +++ b/test/e2e/test-brave-search-e2e.sh @@ -33,6 +33,8 @@ export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1800 SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" # shellcheck source=test/e2e/e2e-timeout.sh . "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e/lib/openclaw-json.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/openclaw-json.sh" PASS=0 FAIL=0 @@ -306,19 +308,7 @@ check_real_brave_search_via_agent() { return fi - reply=$(printf '%s' "$raw" | python3 -c " -import json, sys -try: - doc = json.load(sys.stdin) -except Exception: - sys.exit(0) -result = doc.get('result') or {} -parts = [] -for p in result.get('payloads') or []: - if isinstance(p, dict) and isinstance(p.get('text'), str): - parts.append(p['text']) -print('\n'.join(parts)) -" 2>/dev/null) || true + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true # NVIDIA-related phrasing (nvidia, gpu, cuda, geforce) is overwhelmingly # likely in any legitimate top-1 web result for the query "NVIDIA". diff --git a/test/e2e/test-full-e2e.sh b/test/e2e/test-full-e2e.sh index ca8bf7cdca0..e19c423580b 100755 --- a/test/e2e/test-full-e2e.sh +++ b/test/e2e/test-full-e2e.sh @@ -69,43 +69,8 @@ except Exception as e: " } -parse_openclaw_agent_text() { - python3 -c ' -import json -import sys - -try: - doc = json.load(sys.stdin) -except Exception: - sys.exit(0) - -parts = [] - -def collect(value): - if isinstance(value, str): - if value.strip(): - parts.append(value) - return - if isinstance(value, list): - for item in value: - collect(item) - return - if not isinstance(value, dict): - return - - for key in ("text", "content", "reasoning_content", "message"): - found = value.get(key) - if isinstance(found, str) and found.strip(): - parts.append(found) - - for key in ("payloads", "payload", "messages", "choices", "result", "response", "data", "output"): - if key in value: - collect(value[key]) - -collect(doc.get("result", doc)) -print("\n".join(parts)) -' -} +# shellcheck source=test/e2e/lib/openclaw-json.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/openclaw-json.sh" # Determine repo root if [ -d /workspace ] && [ -f /workspace/install.sh ]; then @@ -409,8 +374,8 @@ fi # routeLogsToStderr() (openclaw/src/commands/agent-via-gateway.ts:57), # so stdout is a clean JSON envelope; prompt-echo on stderr cannot # pollute the assertion. -# * Asserts on the model's reply text inside `result.payloads[].text`, -# not on the merged stdout/stderr. +# * Asserts on parsed model reply text from the JSON envelope, not on +# the merged stdout/stderr or a single brittle envelope shape. # * The expected token (the integer 42) is not a literal substring of the # prompt, so an error path that quoted the prompt back cannot satisfy # the grep. diff --git a/test/e2e/test-launchable-smoke.sh b/test/e2e/test-launchable-smoke.sh index bbb04cf1131..803a2c789fc 100755 --- a/test/e2e/test-launchable-smoke.sh +++ b/test/e2e/test-launchable-smoke.sh @@ -52,6 +52,8 @@ export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1800 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" # shellcheck source=test/e2e/e2e-timeout.sh source "${SCRIPT_DIR}/e2e-timeout.sh" +# shellcheck source=test/e2e/lib/openclaw-json.sh +source "${SCRIPT_DIR}/lib/openclaw-json.sh" PASS=0 FAIL=0 @@ -522,19 +524,7 @@ if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then fi rm -f "$ssh_config" -agent_reply=$(echo "$agent_response" | python3 -c " -import json, sys -try: - doc = json.load(sys.stdin) -except Exception: - sys.exit(0) -result = doc.get('result') or {} -parts = [] -for p in result.get('payloads') or []: - if isinstance(p, dict) and isinstance(p.get('text'), str): - parts.append(p['text']) -print('\n'.join(parts)) -" 2>/dev/null) || true +agent_reply=$(printf '%s' "$agent_response" | parse_openclaw_agent_text 2>/dev/null) || true if grep -qE "(^|[^0-9])42([^0-9]|$)" <<<"$agent_reply"; then pass "[LIVE] openclaw agent: model answered 6×7=42 through openclaw → inference.local" diff --git a/test/e2e/test-messaging-compatible-endpoint.sh b/test/e2e/test-messaging-compatible-endpoint.sh index a58069c0fab..8510a2058c6 100755 --- a/test/e2e/test-messaging-compatible-endpoint.sh +++ b/test/e2e/test-messaging-compatible-endpoint.sh @@ -37,6 +37,8 @@ export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1800 SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" # shellcheck source=test/e2e/e2e-timeout.sh . "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e/lib/openclaw-json.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/openclaw-json.sh" PASS=0 FAIL=0 @@ -525,19 +527,7 @@ check_openclaw_agent_turn() { return fi - reply=$(printf '%s' "$raw" | python3 -c " -import json, sys -try: - doc = json.load(sys.stdin) -except Exception: - sys.exit(0) -result = doc.get('result') or {} -parts = [] -for p in result.get('payloads') or []: - if isinstance(p, dict) and isinstance(p.get('text'), str): - parts.append(p['text']) -print('\n'.join(parts)) -" 2>/dev/null) || true + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true if [ "$rc" -eq 0 ] && printf '%s' "$reply" | grep -qi "PONG"; then pass "C8: openclaw agent completed turn via compatible endpoint (http-proxy-fix.js FORWARD-mode path exercised)" diff --git a/test/e2e/test-openclaw-inference-switch.sh b/test/e2e/test-openclaw-inference-switch.sh index 05eb033c424..16556af48ea 100755 --- a/test/e2e/test-openclaw-inference-switch.sh +++ b/test/e2e/test-openclaw-inference-switch.sh @@ -274,20 +274,7 @@ check_openclaw_agent_turn() { 2>/dev/null) || rc=$? rm -f "$ssh_config" - reply=$(printf '%s' "$raw" | python3 -c ' -import json -import sys -try: - doc = json.load(sys.stdin) -except Exception: - sys.exit(0) -result = doc.get("result") or {} -parts = [] -for payload in result.get("payloads") or []: - if isinstance(payload, dict) and isinstance(payload.get("text"), str): - parts.append(payload["text"]) -print("\n".join(parts)) -' 2>/dev/null) || true + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true if [ "$rc" -eq 0 ] && grep -qE '(^|[^0-9])42([^0-9]|$)' <<<"$reply"; then pass "OpenClaw agent answered through the switched inference route" @@ -306,6 +293,8 @@ else fi E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=test/e2e/lib/openclaw-json.sh +. "${E2E_DIR}/lib/openclaw-json.sh" SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-openclaw-inference-switch}" SWITCH_PROVIDER="${NEMOCLAW_SWITCH_PROVIDER:-nvidia-prod}" SWITCH_MODEL="${NEMOCLAW_SWITCH_MODEL:-z-ai/glm-5.1}" diff --git a/test/e2e/test-sandbox-operations.sh b/test/e2e/test-sandbox-operations.sh index 568e8ac5f5d..40dae39242a 100755 --- a/test/e2e/test-sandbox-operations.sh +++ b/test/e2e/test-sandbox-operations.sh @@ -24,6 +24,8 @@ export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1800 SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" # shellcheck source=test/e2e/e2e-timeout.sh source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e/lib/openclaw-json.sh +source "${SCRIPT_DIR_TIMEOUT}/lib/openclaw-json.sh" # ── Config ─────────────────────────────────────────────────────────────────── SANDBOX_A="test-sbx-a" @@ -353,8 +355,8 @@ test_sbx_01_list_sandboxes() { # the prompt, so an error path that quoted the prompt back cannot # false-positive the grep — which is what masked the openclaw 4.9 # SSRF regression from the prior `Say exactly: HELLO_E2E` assertion. -# 3. Asserts on `result.payloads[].text` from the JSON envelope, not on -# merged stdout/stderr. +# 3. Asserts on parsed model reply text from the JSON envelope, not on +# merged stdout/stderr or a single brittle envelope shape. # 4. Pins `--thinking off` so the first-turn smoke contract is not delayed # by model-catalog inferred reasoning defaults. test_sbx_02_connect_chat() { @@ -384,19 +386,7 @@ test_sbx_02_connect_chat() { rm -f "$ssh_cfg" local reply - reply=$(echo "$raw" | python3 -c " -import json, sys -try: - doc = json.load(sys.stdin) -except Exception: - sys.exit(0) -result = doc.get('result') or {} -parts = [] -for p in result.get('payloads') or []: - if isinstance(p, dict) and isinstance(p.get('text'), str): - parts.append(p['text']) -print('\n'.join(parts)) -" 2>/dev/null) || true + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true if [[ -n "$reply" ]] && echo "$reply" | grep -qE "(^|[^0-9])42([^0-9]|$)"; then pass "TC-SBX-02: Agent computed 6×7=42 through openclaw → inference.local" From 812199178f7d7a8e57fa1f909e96806671aafd85 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 21 May 2026 21:40:43 -0400 Subject: [PATCH 2/2] fix(e2e): mark OpenClaw JSON helper executable --- test/e2e/lib/openclaw-json.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 test/e2e/lib/openclaw-json.sh diff --git a/test/e2e/lib/openclaw-json.sh b/test/e2e/lib/openclaw-json.sh old mode 100644 new mode 100755