From 924c1d529dea2f3662e3437180d364ce020618de Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 06:44:28 -0700 Subject: [PATCH 1/9] chore: prove OpenClaw 2026.5.27 fixes issue 4434 --- Dockerfile | 10 +- Dockerfile.base | 6 +- agents/openclaw/manifest.yaml | 2 +- scripts/patch-openclaw-chat-send.js | 19 +- src/lib/sandbox/version.test.ts | 22 +- src/lib/verify-deployment.test.ts | 10 +- ...st-issue-4434-tui-unreachable-inference.sh | 189 ++++++++++++++++++ .../e2e/test-openclaw-tui-chat-correlation.sh | 4 +- test/fetch-guard-patch-regression.test.ts | 37 ++-- ...sue-4434-tui-unreachable-inference.test.ts | 144 +++++++++++++ test/openclaw-chat-send-patch.test.ts | 80 ++++++++ 11 files changed, 475 insertions(+), 48 deletions(-) create mode 100755 test/e2e/test-issue-4434-tui-unreachable-inference.sh create mode 100644 test/issue-4434-tui-unreachable-inference.test.ts diff --git a/Dockerfile b/Dockerfile index 1bd9054ea15..1ef762d420e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,8 +28,8 @@ RUN npm ci && npm run build # Stage 2: Runtime image — pull cached base from GHCR # hadolint ignore=DL3006 FROM ${BASE_IMAGE} -ARG OPENCLAW_VERSION=2026.5.22 -ARG OPENCLAW_2026_5_22_INTEGRITY=sha512-m+zgBELGbCHjWB1IWF5WSWNPr480cMKOMff2OF72c8A0AMD4hC/9+qwYtzjYmGkETcffnB711JymlVsQnh2Tow== +ARG OPENCLAW_VERSION=2026.5.27 +ARG OPENCLAW_2026_5_27_INTEGRITY=sha512-2N93zhdAo88KAbHt6T7KvYXf4s7XIkYXBgv1npYpn7e1Y9FvrtgtpsA38my9rtFW+70uXEojRPX5/OqnuDqJPw== # Harden: remove unnecessary build tools and network probes from base image (#830) # Protect runtime tools before autoremove — the GHCR base may predate the @@ -102,7 +102,7 @@ RUN set -eu; \ echo "ERROR: OpenClaw build target ${OPENCLAW_VERSION} is below blueprint minimum ${MIN_VER}" >&2; exit 1; \ fi; \ EXPECTED_INTEGRITY=""; \ - if [ "$OPENCLAW_VERSION" = "2026.5.22" ]; then EXPECTED_INTEGRITY="$OPENCLAW_2026_5_22_INTEGRITY"; fi; \ + if [ "$OPENCLAW_VERSION" = "2026.5.27" ]; then EXPECTED_INTEGRITY="$OPENCLAW_2026_5_27_INTEGRITY"; fi; \ if [ -n "$EXPECTED_INTEGRITY" ]; then \ REGISTRY_INTEGRITY=$(npm view "openclaw@${OPENCLAW_VERSION}" dist.integrity); \ if [ "$REGISTRY_INTEGRITY" != "$EXPECTED_INTEGRITY" ]; then \ @@ -279,7 +279,7 @@ RUN set -eu; \ fi; \ fi; \ # --- Patch 2b: allow OpenShell host gateway only through web_fetch trusted env proxy --- \ - # Reviewed against openclaw@2026.5.22 dist: fetchWithWebToolsNetworkGuard \ + # Reviewed against openclaw@2026.5.27 dist: fetchWithWebToolsNetworkGuard \ # passes useEnvProxy into withTrustedEnvProxyGuardedFetchMode(resolved), and \ # the SSRF guard consumes policy.allowedHostnames to skip private-network \ # checks for an exact normalized hostname. hostnameAllowlist only gates \ @@ -371,7 +371,7 @@ RUN set -eu; \ RUN node /usr/local/lib/nemoclaw/patch-openclaw-chat-send.js \ /usr/local/lib/node_modules/openclaw/dist -# Patch OpenClaw's pinned 2026.5.22 compiled selection runtime to expose a +# Patch OpenClaw's pinned 2026.5.27 compiled selection runtime to expose a # compact searchable tool catalog to the model while preserving the full # effective tool set behind tool_call. NEMOCLAW_TOOL_CATALOG=0 disables this # wrapper if an emergency rollback is needed. The script fails closed if the diff --git a/Dockerfile.base b/Dockerfile.base index b24a1ff3197..5738559860d 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -178,8 +178,8 @@ RUN printf '%s\n' \ # OpenClaw version: change the OPENCLAW_VERSION ARG default so CI rebuilds # the base image on push to main, or use workflow_dispatch on base-image.yaml # with the openclaw_version input for a one-off build without editing this file. -ARG OPENCLAW_VERSION=2026.5.22 -ARG OPENCLAW_2026_5_22_INTEGRITY=sha512-m+zgBELGbCHjWB1IWF5WSWNPr480cMKOMff2OF72c8A0AMD4hC/9+qwYtzjYmGkETcffnB711JymlVsQnh2Tow== +ARG OPENCLAW_VERSION=2026.5.27 +ARG OPENCLAW_2026_5_27_INTEGRITY=sha512-2N93zhdAo88KAbHt6T7KvYXf4s7XIkYXBgv1npYpn7e1Y9FvrtgtpsA38my9rtFW+70uXEojRPX5/OqnuDqJPw== SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -201,7 +201,7 @@ RUN --mount=type=bind,source=nemoclaw-blueprint/blueprint.yaml,target=/tmp/bluep echo "Hint: Check available versions with: npm view openclaw versions"; exit 1; \ fi; \ EXPECTED_INTEGRITY=""; \ - if [ "$OPENCLAW_VERSION" = "2026.5.22" ]; then EXPECTED_INTEGRITY="$OPENCLAW_2026_5_22_INTEGRITY"; fi; \ + if [ "$OPENCLAW_VERSION" = "2026.5.27" ]; then EXPECTED_INTEGRITY="$OPENCLAW_2026_5_27_INTEGRITY"; fi; \ if [ -n "$EXPECTED_INTEGRITY" ]; then \ REGISTRY_INTEGRITY=$(npm view "openclaw@${OPENCLAW_VERSION}" dist.integrity); \ if [ "$REGISTRY_INTEGRITY" != "$EXPECTED_INTEGRITY" ]; then \ diff --git a/agents/openclaw/manifest.yaml b/agents/openclaw/manifest.yaml index 7819860cebc..e0126352165 100644 --- a/agents/openclaw/manifest.yaml +++ b/agents/openclaw/manifest.yaml @@ -19,7 +19,7 @@ homepage: "https://openclaw.ai" install_method: npm # npm install -g openclaw@ binary_path: /usr/local/bin/openclaw version_command: "openclaw --version" -expected_version: "2026.5.22" +expected_version: "2026.5.27" gateway_command: "openclaw gateway run" # ── Health probe ──────────────────────────────────────────────── diff --git a/scripts/patch-openclaw-chat-send.js b/scripts/patch-openclaw-chat-send.js index 35e5748119a..0522bae2bdf 100755 --- a/scripts/patch-openclaw-chat-send.js +++ b/scripts/patch-openclaw-chat-send.js @@ -111,14 +111,24 @@ function patchFollowupRunnerFile(file) { } // Source boundary: OpenClaw 2026.5.18 passed opts into runQueuedFollowup, - // while 2026.5.22 closes over params.opts. Both shapes must have opts in + // 2026.5.22 closes over params.opts and uses createReplyOperation, and + // 2026.5.27 closes over params.opts and admits a queued reply turn before + // creating the run id. All reviewed shapes must have opts and queued in // scope before this NemoClaw run-id preservation shim is inserted. - const next = source.replace( + let next = source.replace( /(replyOperation = createReplyOperation\(\{\n\s*sessionId: run\.sessionId,\n\s*sessionKey: replySessionKey \?\? "",\n\s*resetTriggered: false,\n\s*upstreamAbortSignal: queued\.abortSignal(?: \?\? opts\?\.abortSignal)?\n\s*\}\);\n\s*)const runId = crypto\.randomUUID\(\);/, (_match, prefix) => `${prefix}const runId = queued.runId ?? opts?.runId ?? crypto.randomUUID(); ` + `// nemoclaw: preserve chat.send run ids in followup queue (#2603, #3145)`, ); + if (next === source) { + next = source.replace( + /(const admission = await admitReplyTurn\(\{\n\s*sessionId: run\.sessionId,\n\s*sessionKey: replySessionKey \?\? "",\n\s*kind: "queued_followup",\n\s*resetTriggered: false,\n\s*upstreamAbortSignal: queued\.abortSignal\n\s*\}\);[\s\S]*?replyOperation = admission\.operation;[\s\S]*?\n\s*)const runId = crypto\.randomUUID\(\);/, + (_match, prefix) => + `${prefix}const runId = queued.runId ?? opts?.runId ?? crypto.randomUUID(); ` + + `// nemoclaw: preserve chat.send run ids in followup queue (#2603, #3145)`, + ); + } if (next === source) { fail(`OpenClaw followup runner run-id shape not recognized in ${file}`); } @@ -215,7 +225,10 @@ const followupCandidates = listJsFiles(distDir).filter((file) => { const source = fs.readFileSync(file, "utf8"); return ( source.includes("function createFollowupRunner") && - source.includes("replyOperation = createReplyOperation") && + (source.includes("replyOperation = createReplyOperation") || + (source.includes("admitReplyTurn") && + source.includes("replyOperation = admission.operation")) || + source.includes("preserve chat.send run ids in followup queue")) && (source.includes("const runId = crypto.randomUUID();") || source.includes("preserve chat.send run ids in followup queue")) ); diff --git a/src/lib/sandbox/version.test.ts b/src/lib/sandbox/version.test.ts index 8e2da099b08..a7361aa61c6 100644 --- a/src/lib/sandbox/version.test.ts +++ b/src/lib/sandbox/version.test.ts @@ -36,7 +36,7 @@ vi.mock("../agent/defs.js", () => ({ name, displayName: name === "openclaw" ? "OpenClaw" : "Hermes Agent", versionCommand: name === "openclaw" ? "openclaw --version" : "hermes --version", - expectedVersion: name === "openclaw" ? "2026.5.22" : "2026.5.16", + expectedVersion: name === "openclaw" ? "2026.5.27" : "2026.5.16", stateDirs: [], configPaths: { dir: "/sandbox/.openclaw" }, })), @@ -77,12 +77,12 @@ describe("checkAgentVersion", () => { registry.registerSandbox({ name: "test-sb", agent: null, - agentVersion: "2026.5.22", + agentVersion: "2026.5.27", }); const result = checkAgentVersion("test-sb"); expect(result.detectionMethod).toBe("registry"); - expect(result.sandboxVersion).toBe("2026.5.22"); + expect(result.sandboxVersion).toBe("2026.5.27"); expect(result.isStale).toBe(false); }); @@ -103,7 +103,7 @@ describe("checkAgentVersion", () => { registry.registerSandbox({ name: "test-sb", agent: null, - agentVersion: "2026.5.22", + agentVersion: "2026.5.27", }); const result = checkAgentVersion("test-sb"); @@ -120,7 +120,7 @@ describe("checkAgentVersion", () => { vi.mocked(spawnSync).mockReturnValue({ status: 0, - stdout: "OpenClaw 2026.5.22 (abc123)\n", + stdout: "OpenClaw 2026.5.27 (abc123)\n", stderr: "", pid: 1234, output: [], @@ -129,7 +129,7 @@ describe("checkAgentVersion", () => { const result = checkAgentVersion("test-sb"); expect(result.detectionMethod).toBe("ssh-exec"); - expect(result.sandboxVersion).toBe("2026.5.22"); + expect(result.sandboxVersion).toBe("2026.5.27"); expect(result.isStale).toBe(false); expect(captureSandboxSshConfigCommand).toHaveBeenCalledWith( "/usr/local/bin/openshell", @@ -139,7 +139,7 @@ describe("checkAgentVersion", () => { // Should have cached the version in registry const updated = registry.getSandbox("test-sb"); - expect(updated?.agentVersion).toBe("2026.5.22"); + expect(updated?.agentVersion).toBe("2026.5.27"); }); it("returns unavailable when SSH config fails", () => { @@ -183,7 +183,7 @@ describe("checkAgentVersion", () => { vi.mocked(spawnSync).mockReturnValue({ status: 0, - stdout: "OpenClaw 2026.5.22 (abc123)\n", + stdout: "OpenClaw 2026.5.27 (abc123)\n", stderr: "", pid: 1234, output: [], @@ -192,7 +192,7 @@ describe("checkAgentVersion", () => { const result = checkAgentVersion("test-sb", { forceProbe: true }); expect(result.detectionMethod).toBe("ssh-exec"); - expect(result.sandboxVersion).toBe("2026.5.22"); + expect(result.sandboxVersion).toBe("2026.5.27"); }); }); @@ -219,14 +219,14 @@ describe("formatStalenessWarning", () => { it("includes sandbox name, versions, and rebuild hint", () => { const lines = formatStalenessWarning("my-sb", { sandboxVersion: "2026.3.11", - expectedVersion: "2026.5.22", + expectedVersion: "2026.5.27", isStale: true, detectionMethod: "registry", }); const joined = lines.join("\n"); expect(joined).toContain("my-sb"); expect(joined).toContain("2026.3.11"); - expect(joined).toContain("2026.5.22"); + expect(joined).toContain("2026.5.27"); expect(joined).toContain("rebuild"); }); }); diff --git a/src/lib/verify-deployment.test.ts b/src/lib/verify-deployment.test.ts index d9abb31e93f..bc377f3b9b6 100644 --- a/src/lib/verify-deployment.test.ts +++ b/src/lib/verify-deployment.test.ts @@ -124,13 +124,13 @@ describe("verifyDeployment", () => { const deps = makeDeps({ executeSandboxCommand: (_name: string, script: string) => { if (script.includes("openclaw --version")) { - return { status: 0, stdout: "2026.5.22", stderr: "" }; + return { status: 0, stdout: "2026.5.27", stderr: "" }; } return { status: 0, stdout: "200", stderr: "" }; }, }); const result = await verifyDeployment("my-sandbox", chain, deps, NO_RETRY); - expect(result.verification.gatewayVersion).toBe("2026.5.22"); + expect(result.verification.gatewayVersion).toBe("2026.5.27"); }); it("reports null version when gateway is down (skips version probe)", async () => { @@ -179,7 +179,7 @@ describe("verifyDeployment", () => { const deps = makeDeps({ executeSandboxCommand: (_name: string, script: string) => { if (script.includes("openclaw --version")) { - return { status: 0, stdout: "2026.5.22", stderr: "" }; + return { status: 0, stdout: "2026.5.27", stderr: "" }; } if (script.includes("inference.local")) { return { status: 0, stdout: "200", stderr: "" }; @@ -240,14 +240,14 @@ describe("formatVerificationDiagnostics", () => { const result = await verifyDeployment("my-sandbox", chain, makeDeps({ executeSandboxCommand: (_name: string, script: string) => { if (script.includes("openclaw --version")) { - return { status: 0, stdout: "2026.5.22", stderr: "" }; + return { status: 0, stdout: "2026.5.27", stderr: "" }; } return { status: 0, stdout: "200", stderr: "" }; }, }), NO_RETRY); const lines = formatVerificationDiagnostics(result); expect(lines.some((l) => l.includes("verified"))).toBe(true); - expect(lines.some((l) => l.includes("2026.5.22"))).toBe(true); + expect(lines.some((l) => l.includes("2026.5.27"))).toBe(true); }); it("prints failure diagnostics with hints when unhealthy", async () => { diff --git a/test/e2e/test-issue-4434-tui-unreachable-inference.sh b/test/e2e/test-issue-4434-tui-unreachable-inference.sh new file mode 100755 index 00000000000..8353709b448 --- /dev/null +++ b/test/e2e/test-issue-4434-tui-unreachable-inference.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Opt-in live repro for #4434: +# openclaw tui must show a visible error, and stop the active spinner, when +# the NVIDIA endpoint is unreachable from the sandbox. +# +# This mutates host firewall state. Run only on a Linux Docker host you control: +# +# NEMOCLAW_ISSUE_4434_LIVE=1 NVIDIA_API_KEY=nvapi-... \ +# bash test/e2e/test-issue-4434-tui-unreachable-inference.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-issue-4434-tui-unreachable}" +INSTALL_LOG="${E2E_ISSUE_4434_INSTALL_LOG:-/tmp/nemoclaw-e2e-issue-4434-install.log}" +CAPTURE_DIR="${NEMOCLAW_ISSUE_4434_CAPTURE_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/nemoclaw-issue-4434.XXXXXX")}" +CAPTURE_FILE="${CAPTURE_DIR}/openclaw-tui-capture.log" +PLAIN_CAPTURE_FILE="${CAPTURE_DIR}/openclaw-tui-capture.plain.log" +TUI_TIMEOUT_SEC="${NEMOCLAW_ISSUE_4434_TUI_TIMEOUT_SEC:-210}" +VISIBLE_ERROR_RE="error|failed|timeout|timed out|unavailable|fetch failed|ETIMEDOUT|ECONN|upstream" +SPINNER_CONNECTED_RE="flibbertigibbeting|[0-9]+m[[:space:]][0-9]+s[[:space:]]*\\|[[:space:]]*connected" +BLOCKED_IPS=("75.2.113.119" "99.83.136.103") +INSERTED_IPS=() +CLEANUP_SANDBOX=0 + +info() { printf '[issue-4434] %s\n' "$*"; } +fail() { + printf '[issue-4434] FAIL: %s\n' "$*" >&2 + printf '[issue-4434] capture: %s\n' "$CAPTURE_FILE" >&2 + exit 1 +} + +cleanup_firewall() { + local ip + for ip in "${INSERTED_IPS[@]}"; do + sudo iptables -D DOCKER-USER -d "$ip" -j DROP >/dev/null 2>&1 || true + done +} + +cleanup_sandbox() { + if [ "$CLEANUP_SANDBOX" != "1" ]; then + return + fi + if [ "${NEMOCLAW_E2E_SKIP_CLEANUP:-0}" = "1" ]; then + return + fi + SANDBOX_NAME="$SANDBOX_NAME" bash "${SCRIPT_DIR}/e2e-cloud-experimental/cleanup.sh" --verify >/dev/null 2>&1 || true +} + +cleanup() { + cleanup_firewall + cleanup_sandbox +} +trap cleanup EXIT + +if [ "${NEMOCLAW_ISSUE_4434_LIVE:-0}" != "1" ]; then + info "skipping: set NEMOCLAW_ISSUE_4434_LIVE=1 to run the privileged live repro" + exit 0 +fi + +if [ "$(uname -s)" != "Linux" ]; then + fail "Linux host required for DOCKER-USER iptables repro" +fi +for command in docker sudo expect curl openshell timeout perl; do + command -v "$command" >/dev/null 2>&1 || fail "missing required command: $command" +done +docker info >/dev/null 2>&1 || fail "Docker is not running" +sudo -n true >/dev/null 2>&1 || fail "passwordless sudo is required for non-interactive iptables cleanup" +if [ -z "${NVIDIA_API_KEY:-}" ] || [[ "${NVIDIA_API_KEY}" != nvapi-* ]]; then + fail "NVIDIA_API_KEY must be set and start with nvapi-" +fi + +mkdir -p "$CAPTURE_DIR" +CLEANUP_SANDBOX=1 + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export E2E_CLOUD_ONBOARD_INSTALL_LOG="$INSTALL_LOG" +export NEMOCLAW_E2E_KEEP_SANDBOX=1 +export NEMOCLAW_NON_INTERACTIVE="${NEMOCLAW_NON_INTERACTIVE:-1}" +export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE="${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-1}" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" +export NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL="${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL:-nvidia/nemotron-3-super-120b-a12b}" + +info "onboarding sandbox ${SANDBOX_NAME} with ${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL}" +bash "${SCRIPT_DIR}/test-cloud-onboard-e2e.sh" + +# Pick up PATH changes from the installer in this shell. +# shellcheck source=test/e2e/lib/install-path-refresh.sh +. "${SCRIPT_DIR}/lib/install-path-refresh.sh" +nemoclaw_refresh_install_env +nemoclaw_ensure_local_bin_on_path +export PATH="/usr/local/bin:${HOME}/.local/bin:${PATH}" + +openclaw_version="$(openshell sandbox exec --name "$SANDBOX_NAME" -- openclaw --version 2>&1 || true)" +info "sandbox OpenClaw version: ${openclaw_version}" +if ! grep -q "2026.5.27" <<<"$openclaw_version"; then + fail "expected sandbox OpenClaw 2026.5.27" +fi + +status_log="${CAPTURE_DIR}/nemoclaw-status-before-block.log" +if ! nemoclaw "$SANDBOX_NAME" status >"$status_log" 2>&1; then + fail "nemoclaw ${SANDBOX_NAME} status failed before firewall block" +fi +if ! grep -Eiq "inference.*healthy|healthy.*inference" "$status_log"; then + fail "pre-block status did not report healthy inference" +fi + +info "installing DOCKER-USER DROP rules for NVIDIA endpoint IPs" +for ip in "${BLOCKED_IPS[@]}"; do + sudo iptables -I DOCKER-USER -d "$ip" -j DROP + INSERTED_IPS+=("$ip") +done + +block_probe_log="${CAPTURE_DIR}/blocked-endpoint-probe.log" +set +e +timeout 25 openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'curl -sk --connect-timeout 5 --max-time 12 https://integrate.api.nvidia.com/v1/models >/tmp/issue4434-models.out 2>&1' \ + >"$block_probe_log" 2>&1 +block_probe_rc=$? +set -e +if [ "$block_probe_rc" -eq 0 ]; then + fail "integrate.api.nvidia.com was still reachable from inside the sandbox after firewall block" +fi +info "sandbox endpoint block verified (probe exit ${block_probe_rc})" + +info "launching openclaw tui through nemoclaw ${SANDBOX_NAME} connect" +set +e +env \ + NEMOCLAW_ISSUE_4434_SANDBOX="$SANDBOX_NAME" \ + NEMOCLAW_ISSUE_4434_CAPTURE="$CAPTURE_FILE" \ + NEMOCLAW_ISSUE_4434_TUI_TIMEOUT="$TUI_TIMEOUT_SEC" \ + expect >"${CAPTURE_DIR}/expect.log" 2>&1 <<'EXPECT' +set timeout $env(NEMOCLAW_ISSUE_4434_TUI_TIMEOUT) +set sandbox $env(NEMOCLAW_ISSUE_4434_SANDBOX) +set capture $env(NEMOCLAW_ISSUE_4434_CAPTURE) +log_file -a $capture +spawn nemoclaw $sandbox connect +expect { + -re {[$#>] $} { + send "export TERM=xterm-256color\r" + expect -re {[$#>] $} + send "openclaw tui\r" + sleep 8 + send -- "hello\r" + } + timeout { puts "Timed out waiting for sandbox shell prompt."; exit 10 } + eof { exit 11 } +} +expect { + -nocase -re {(error|failed|timeout|timed out|unavailable|fetch failed|ETIMEDOUT|ECONN|upstream)} { + sleep 5 + send "\003" + sleep 1 + send "\003" + exit 0 + } + timeout { + send "\003" + sleep 1 + send "\003" + exit 20 + } + eof { exit 21 } +} +EXPECT +expect_rc=$? +set -e + +perl -pe 's/\x1b\[[0-9;?]*[ -\/]*[@-~]//g' "$CAPTURE_FILE" >"$PLAIN_CAPTURE_FILE" + +if ! grep -Eiq "$VISIBLE_ERROR_RE" "$PLAIN_CAPTURE_FILE"; then + if grep -Eiq "$SPINNER_CONNECTED_RE" "$PLAIN_CAPTURE_FILE"; then + fail "matched #4434 signature: spinner plus connected status with no visible error" + fi + fail "TUI did not surface a visible inference error before the timeout window" +fi +if [ "$expect_rc" -ne 0 ]; then + fail "expect harness exited ${expect_rc} even though an error-looking capture was found" +fi +if tail -40 "$PLAIN_CAPTURE_FILE" | grep -Eiq "$SPINNER_CONNECTED_RE"; then + fail "TUI capture still ends with active connected spinner after the visible error" +fi + +info "PASS: openclaw tui surfaced a visible unreachable-inference error and stopped the spinner" +info "capture: ${PLAIN_CAPTURE_FILE}" diff --git a/test/e2e/test-openclaw-tui-chat-correlation.sh b/test/e2e/test-openclaw-tui-chat-correlation.sh index 99d105420bc..8d1aa8a432b 100755 --- a/test/e2e/test-openclaw-tui-chat-correlation.sh +++ b/test/e2e/test-openclaw-tui-chat-correlation.sh @@ -46,8 +46,8 @@ openclaw_version="$( openshell sandbox exec --name "$SANDBOX_NAME" -- openclaw --version 2>&1 || true )" echo "Sandbox OpenClaw version: ${openclaw_version}" -if ! grep -q "2026.5.22" <<<"$openclaw_version"; then - echo "Expected fresh sandbox to run OpenClaw 2026.5.22" >&2 +if ! grep -q "2026.5.27" <<<"$openclaw_version"; then + echo "Expected fresh sandbox to run OpenClaw 2026.5.27" >&2 exit 1 fi diff --git a/test/fetch-guard-patch-regression.test.ts b/test/fetch-guard-patch-regression.test.ts index 6102253c58c..371a64371b3 100644 --- a/test/fetch-guard-patch-regression.test.ts +++ b/test/fetch-guard-patch-regression.test.ts @@ -14,11 +14,12 @@ const REVIEWED_OPENCLAW_PATCH_CLASSIFIER_VERSIONS = [ "2026.4.24", "2026.5.18", "2026.5.22", + "2026.5.27", ] as const; -const CURRENT_REVIEWED_OPENCLAW_PATCH_CLASSIFIER_VERSION = "2026.5.22"; +const CURRENT_REVIEWED_OPENCLAW_PATCH_CLASSIFIER_VERSION = "2026.5.27"; const EXPECTED_OPENCLAW_INTEGRITY = - "sha512-m+zgBELGbCHjWB1IWF5WSWNPr480cMKOMff2OF72c8A0AMD4hC/9+qwYtzjYmGkETcffnB711JymlVsQnh2Tow=="; -const REVIEWED_OPENCLAW_2026_5_22_WEB_FETCH_SHAPE = [ + "sha512-2N93zhdAo88KAbHt6T7KvYXf4s7XIkYXBgv1npYpn7e1Y9FvrtgtpsA38my9rtFW+70uXEojRPX5/OqnuDqJPw=="; +const REVIEWED_OPENCLAW_2026_5_27_WEB_FETCH_SHAPE = [ "async function fetchWithWebToolsNetworkGuard(params) {", " const { timeoutSeconds, useEnvProxy, ...rest } = params;", " const resolved = {", @@ -31,7 +32,7 @@ const REVIEWED_OPENCLAW_2026_5_22_WEB_FETCH_SHAPE = [ " return fetchWithSsrFGuard(useEnvProxy ? withTrustedEnvProxyGuardedFetchMode(resolved) : withStrictGuardedFetchMode(resolved));", "}", ].join("\n"); -const REVIEWED_OPENCLAW_2026_5_22_SSRF_POLICY_SHAPE = [ +const REVIEWED_OPENCLAW_2026_5_27_SSRF_POLICY_SHAPE = [ "function shouldSkipPrivateNetworkChecks(hostname, policy) {", " return isPrivateNetworkAllowedByPolicy(policy) || normalizeHostnameSet(policy?.allowedHostnames).has(hostname);", "}", @@ -49,7 +50,7 @@ const REVIEWED_OPENCLAW_2026_5_22_SSRF_POLICY_SHAPE = [ "}", ].join("\n"); -function loadReviewedOpenClaw20260522SsrfPolicyShape() { +function loadReviewedOpenClaw20260527SsrfPolicyShape() { return new Function(` class SsrFBlockedError extends Error {} function normalizeHostname(value) { @@ -74,7 +75,7 @@ function assertAllowedHostOrIpOrThrow(hostnameOrIp) { throw new SsrFBlockedError("blocked " + hostnameOrIp); } } -${REVIEWED_OPENCLAW_2026_5_22_SSRF_POLICY_SHAPE} +${REVIEWED_OPENCLAW_2026_5_27_SSRF_POLICY_SHAPE} return { shouldSkipPrivateNetworkChecks, resolveHostnamePolicyChecks }; `)() as { shouldSkipPrivateNetworkChecks: (hostname: string, policy?: Record) => boolean; @@ -128,7 +129,7 @@ function readDockerfileOpenClawVersion(): string { function readDockerfileBaseOpenClawIntegrity(): string { return readRequiredMatch( DOCKERFILE_BASE, - /^ARG OPENCLAW_2026_5_22_INTEGRITY=([^\s]+)/m, + /^ARG OPENCLAW_2026_5_27_INTEGRITY=([^\s]+)/m, "OpenClaw base image integrity", ); } @@ -136,7 +137,7 @@ function readDockerfileBaseOpenClawIntegrity(): string { function readDockerfileOpenClawIntegrity(): string { return readRequiredMatch( DOCKERFILE, - /^ARG OPENCLAW_2026_5_22_INTEGRITY=([^\s]+)/m, + /^ARG OPENCLAW_2026_5_27_INTEGRITY=([^\s]+)/m, "OpenClaw runtime integrity", ); } @@ -187,12 +188,12 @@ function runOpenClawUpgradeBlock(currentVersion: string) { "set -euo pipefail", `call_log=${JSON.stringify(log)}`, `OPENCLAW_VERSION=${JSON.stringify(openclawVersion)}`, - `OPENCLAW_2026_5_22_INTEGRITY=${JSON.stringify(openclawIntegrity)}`, + `OPENCLAW_2026_5_27_INTEGRITY=${JSON.stringify(openclawIntegrity)}`, `openclaw() { if [ "\${1:-}" = "--version" ]; then printf 'openclaw ${currentVersion}\\n'; else return 127; fi; }`, "npm() {", ' printf "npm %s\\n" "$*" >> "$call_log";', ' if [ "${1:-}" = "view" ] && [ "${2:-}" = "openclaw@${OPENCLAW_VERSION}" ] && [ "${3:-}" = "dist.integrity" ]; then', - ' printf "%s\\n" "$OPENCLAW_2026_5_22_INTEGRITY";', + ' printf "%s\\n" "$OPENCLAW_2026_5_27_INTEGRITY";', " fi", "}", 'command() { if [ "${1:-}" = "-v" ] && [ "${2:-}" = "codex-acp" ]; then return 0; fi; builtin command "$@"; }', @@ -247,7 +248,7 @@ function runDockerfilePatchBlock( dist: string, tmp: string, endMarker: string, - version = "2026.5.22", + version = "2026.5.27", ) { const command = dockerRunCommandBetween( "# Patch OpenClaw media fetch for proxy-only sandbox", @@ -271,7 +272,7 @@ function runDockerfilePatchBlock( }); } -function runFetchGuardPatchBlock(dist: string, tmp: string, version = "2026.5.22") { +function runFetchGuardPatchBlock(dist: string, tmp: string, version = "2026.5.27") { return runDockerfilePatchBlock( dist, tmp, @@ -314,21 +315,21 @@ function webGuardedFetchFixtureSource(): string { } describe("fetch-guard patch regression guard", () => { - it("anchors web_fetch host-gateway policy to the reviewed OpenClaw 2026.5.22 SSRF contract", () => { - expect(REVIEWED_OPENCLAW_2026_5_22_WEB_FETCH_SHAPE).toContain( + it("anchors web_fetch host-gateway policy to the reviewed OpenClaw 2026.5.27 SSRF contract", () => { + expect(REVIEWED_OPENCLAW_2026_5_27_WEB_FETCH_SHAPE).toContain( "function fetchWithWebToolsNetworkGuard(params)", ); - expect(REVIEWED_OPENCLAW_2026_5_22_WEB_FETCH_SHAPE).toContain( + expect(REVIEWED_OPENCLAW_2026_5_27_WEB_FETCH_SHAPE).toContain( "withTrustedEnvProxyGuardedFetchMode(resolved)", ); - expect(REVIEWED_OPENCLAW_2026_5_22_SSRF_POLICY_SHAPE).toContain( + expect(REVIEWED_OPENCLAW_2026_5_27_SSRF_POLICY_SHAPE).toContain( "normalizeHostnameSet(policy?.allowedHostnames).has(hostname)", ); - expect(REVIEWED_OPENCLAW_2026_5_22_SSRF_POLICY_SHAPE).toContain( + expect(REVIEWED_OPENCLAW_2026_5_27_SSRF_POLICY_SHAPE).toContain( "normalizeHostnameAllowlist(policy?.hostnameAllowlist)", ); - const reviewed = loadReviewedOpenClaw20260522SsrfPolicyShape(); + const reviewed = loadReviewedOpenClaw20260527SsrfPolicyShape(); expect( reviewed.shouldSkipPrivateNetworkChecks("host.openshell.internal", { allowedHostnames: ["HOST.OPENSHELL.INTERNAL."], diff --git a/test/issue-4434-tui-unreachable-inference.test.ts b/test/issue-4434-tui-unreachable-inference.test.ts new file mode 100644 index 00000000000..6423a6c4d75 --- /dev/null +++ b/test/issue-4434-tui-unreachable-inference.test.ts @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +type ChatEvent = { + state: "delta" | "final" | "error"; + message?: { role: string; text?: string }; + errorMessage?: string; +}; + +type TuiState = { + spinnerActive: boolean; + status: "connected" | "error"; + terminalLines: string[]; +}; + +const VISIBLE_ERROR_RE = /\b(error|failed|timeout|timed out|unavailable|fetch failed|upstream|connection)\b/i; +const CONNECTED_SPINNER_RE = + /(?:flibbertigibbeting|thinking|waiting|processing).*?\|\s*connected|[0-9]+m\s+[0-9]+s\s*\|\s*connected/i; + +function stripAnsi(value: string): string { + return value.replace(/\x1b\[[0-9;?]*[ -/]*[@-~]/g, ""); +} + +function analyzeIssue4434TuiCapture(capture: string) { + const plain = stripAnsi(capture); + const visibleError = VISIBLE_ERROR_RE.test(plain); + const connectedSpinner = CONNECTED_SPINNER_RE.test(plain); + return { + visibleError, + connectedSpinner, + issue4434Signature: connectedSpinner && !visibleError, + }; +} + +function renderTui(state: TuiState): string { + const statusLine = state.spinnerActive + ? "flibbertigibbeting... | connected" + : `status: ${state.status}`; + return [...state.terminalLines, statusLine].join("\n"); +} + +function applyChatEventToTui(state: TuiState, event: ChatEvent): TuiState { + if (event.state === "error") { + const text = event.errorMessage || event.message?.text || "OpenClaw chat.send failed"; + return { + spinnerActive: false, + status: "error", + terminalLines: [...state.terminalLines, `Error: ${text}`], + }; + } + if (event.state === "final") { + return { + spinnerActive: false, + status: "connected", + terminalLines: event.message?.text + ? [...state.terminalLines, event.message.text] + : state.terminalLines, + }; + } + return state; +} + +function driveMockOpenClawGatewayChatPath(params: { + endpointReachable: boolean; + broadcastSyncErrors: boolean; +}) { + const events: ChatEvent[] = []; + const tuiInitialState: TuiState = { + spinnerActive: true, + status: "connected", + terminalLines: ["user: hello"], + }; + + try { + if (!params.endpointReachable) { + throw new Error("fetch failed: connect ETIMEDOUT 75.2.113.119:443"); + } + events.push({ state: "final", message: { role: "assistant", text: "hello" } }); + } catch (error) { + if (params.broadcastSyncErrors) { + events.push({ + state: "error", + errorMessage: error instanceof Error ? error.message : String(error), + }); + } + } + + const finalState = events.reduce(applyChatEventToTui, tuiInitialState); + return { + events, + capture: renderTui(finalState), + state: finalState, + }; +} + +describe("issue #4434 unreachable inference TUI behavior", () => { + it("classifies the captured spinner plus connected status with no error as the broken signature", () => { + const capture = [ + " flibbertigibbeting... - 3m 42s | connected", + "agent main | session main | inference/nvidia/nemotron-3-super-120b-a12b", + "", + ].join("\n"); + + expect(analyzeIssue4434TuiCapture(capture)).toEqual({ + visibleError: false, + connectedSpinner: true, + issue4434Signature: true, + }); + }); + + it("drives the gateway chat path with an unreachable endpoint and requires a visible TUI error", () => { + const result = driveMockOpenClawGatewayChatPath({ + endpointReachable: false, + broadcastSyncErrors: true, + }); + + expect(result.events).toEqual([ + { + state: "error", + errorMessage: "fetch failed: connect ETIMEDOUT 75.2.113.119:443", + }, + ]); + expect(result.state.spinnerActive).toBe(false); + expect(result.state.status).toBe("error"); + expect(analyzeIssue4434TuiCapture(result.capture)).toMatchObject({ + visibleError: true, + connectedSpinner: false, + issue4434Signature: false, + }); + }); + + it("keeps failing when the gateway drops the synchronous chat.send error event", () => { + const result = driveMockOpenClawGatewayChatPath({ + endpointReachable: false, + broadcastSyncErrors: false, + }); + + expect(result.events).toEqual([]); + expect(result.state.spinnerActive).toBe(true); + expect(analyzeIssue4434TuiCapture(result.capture).issue4434Signature).toBe(true); + }); +}); diff --git a/test/openclaw-chat-send-patch.test.ts b/test/openclaw-chat-send-patch.test.ts index 46513571994..9c4aa9e36b2 100644 --- a/test/openclaw-chat-send-patch.test.ts +++ b/test/openclaw-chat-send-patch.test.ts @@ -124,6 +124,43 @@ function writeFollowupRunner20260522Fixture(dist: string): string { return fixture; } +function writeFollowupRunner20260527Fixture(dist: string): string { + const fixture = path.join(dist, "agent-runner.fixture.js"); + fs.writeFileSync( + fixture, + [ + "function createFollowupRunner(params) {", + " const { opts, typing, sessionEntry } = params;", + " return async (queued) => {", + " let replyOperation;", + " let run = queued.run;", + " const replySessionKey = queued.run.sessionKey ?? sessionKey;", + " const admission = await admitReplyTurn({", + " sessionId: run.sessionId,", + ' sessionKey: replySessionKey ?? "",', + ' kind: "queued_followup",', + " resetTriggered: false,", + " upstreamAbortSignal: queued.abortSignal", + " });", + ' if (admission.status === "skipped") return;', + " replyOperation = admission.operation;", + " if (replyOperation.sessionId !== run.sessionId) {", + " run = { ...run, sessionId: replyOperation.sessionId };", + " }", + " const runId = crypto.randomUUID();", + " if (run.sessionKey) registerAgentRunContext(runId, {", + " sessionKey: run.sessionKey,", + " verboseLevel: run.verboseLevel", + " });", + " return runId;", + " }", + "}", + "", + ].join("\n"), + ); + return fixture; +} + function writeFollowupRunnerWithoutOptsBindingFixture(dist: string): string { const fixture = path.join(dist, "agent-runner.fixture.js"); fs.writeFileSync( @@ -211,8 +248,10 @@ async function runPatchedFollowupFixture( const context = { createReplyOperation: (value: unknown) => value, crypto: { randomUUID: () => "fallback-run-id" }, + admitReplyTurn: async () => ({ status: "admitted", operation: { sessionId: "session" } }), registerAgentRunContext: (runId: string) => registeredRuns.push(runId), replySessionKey: "reply-session", + sessionKey: "fallback-session-key", }; const createFollowupRunner = vm.runInNewContext( `${patchedSource}\ncreateFollowupRunner;`, @@ -323,6 +362,47 @@ describe("OpenClaw chat.send compatibility patch", () => { } }); + it("recognizes the 2026.5.27 followup runner admission shape", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-chat-send-527-")); + const dist = path.join(tmp, "dist"); + fs.mkdirSync(dist); + writeChatSendFixture(dist); + const followupFixture = writeFollowupRunner20260527Fixture(dist); + writeGetReplyFixture(dist); + + try { + const patch = runPatch(dist); + expect(patch.status, `${patch.stdout}${patch.stderr}`).toBe(0); + const patchedFollowup = fs.readFileSync(followupFixture, "utf-8"); + expect(patchedFollowup).toContain( + "const runId = queued.runId ?? opts?.runId ?? crypto.randomUUID(); // nemoclaw: preserve chat.send run ids in followup queue (#2603, #3145)", + ); + await expect( + runPatchedFollowupFixture( + patchedFollowup, + { opts: { runId: "opts-run-id" } }, + { runId: "queued-run-id", run: { sessionId: "session", sessionKey: "key" } }, + ), + ).resolves.toMatchObject({ runId: "queued-run-id", registeredRuns: ["queued-run-id"] }); + await expect( + runPatchedFollowupFixture( + patchedFollowup, + { opts: { runId: "opts-run-id" } }, + { run: { sessionId: "session", sessionKey: "key" } }, + ), + ).resolves.toMatchObject({ runId: "opts-run-id", registeredRuns: ["opts-run-id"] }); + await expect( + runPatchedFollowupFixture( + patchedFollowup, + {}, + { run: { sessionId: "session", sessionKey: "key" } }, + ), + ).resolves.toMatchObject({ runId: "fallback-run-id", registeredRuns: ["fallback-run-id"] }); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("fails closed when the followup runner opts binding is absent", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-chat-send-no-opts-")); const dist = path.join(tmp, "dist"); From 8b747f31a9a91527f2146dcce1d5346212105f1b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 06:58:53 -0700 Subject: [PATCH 2/9] test: wire issue 4434 live repro into nightly e2e --- .github/workflows/nightly-e2e.yaml | 56 +++++++++++++++++++ ...st-issue-4434-tui-unreachable-inference.sh | 5 +- test/validate-e2e-coverage.test.ts | 4 ++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index efca3afc5da..4b047e24ae0 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -52,6 +52,9 @@ # openclaw-inference-switch-e2e # Switches a running OpenClaw sandbox with `nemoclaw inference set` # and verifies route, openclaw.json, hashes, and live requests. +# issue-4434-tui-unreachable-inference-e2e +# Recreates #4434's NVIDIA endpoint firewall block and verifies +# OpenClaw TUI shows a visible error and stops the active spinner. # credential-migration-e2e Validates legacy ~/.nemoclaw/credentials.json migration to the # OpenShell gateway, secure zero-fill on unlink, allowlist filter # on non-credential env keys, and symlink-safe deletion. @@ -86,6 +89,7 @@ on: skill-agent-e2e, docs-validation-e2e, messaging-providers-e2e, openclaw-slack-pairing-e2e, openclaw-tui-chat-correlation-e2e, + issue-4434-tui-unreachable-inference-e2e, issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, messaging-compatible-endpoint-e2e, @@ -338,6 +342,55 @@ jobs: path: /tmp/nemoclaw-e2e-openclaw-tui-correlation-install.log if-no-files-found: ignore + issue-4434-tui-unreachable-inference-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',issue-4434-tui-unreachable-inference-e2e,')) + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.target_ref || github.ref }} + + - name: Resolve public install ref + id: public_install_ref + shell: bash + run: | + printf 'ref=%s\n' "$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: "Install issue #4434 test dependencies" + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y expect iptables + + - name: "Run issue #4434 TUI unreachable inference E2E test" + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + NEMOCLAW_ISSUE_4434_LIVE: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-issue-4434-tui-unreachable" + NEMOCLAW_PUBLIC_INSTALL_REF: ${{ steps.public_install_ref.outputs.ref }} + GITHUB_TOKEN: ${{ github.token }} + run: bash test/e2e/test-issue-4434-tui-unreachable-inference.sh + + - name: "Upload issue #4434 logs on failure" + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: issue-4434-tui-unreachable-inference-logs + path: | + /tmp/nemoclaw-e2e-issue-4434-install.log + /tmp/nemoclaw-issue-4434.* + if-no-files-found: ignore + # ── DGX Station GPU optional proof validation (#3600) ────────── # CI cannot emulate GB300, but this guards the release-blocker mitigation: # optional direct GPU proofs must not abort onboard before the fatal throw. @@ -1833,6 +1886,7 @@ jobs: messaging-providers-e2e, openclaw-slack-pairing-e2e, openclaw-tui-chat-correlation-e2e, + issue-4434-tui-unreachable-inference-e2e, issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, messaging-compatible-endpoint-e2e, @@ -1937,6 +1991,7 @@ jobs: messaging-providers-e2e, openclaw-slack-pairing-e2e, openclaw-tui-chat-correlation-e2e, + issue-4434-tui-unreachable-inference-e2e, issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, messaging-compatible-endpoint-e2e, @@ -2098,6 +2153,7 @@ jobs: messaging-providers-e2e, openclaw-slack-pairing-e2e, openclaw-tui-chat-correlation-e2e, + issue-4434-tui-unreachable-inference-e2e, issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, messaging-compatible-endpoint-e2e, diff --git a/test/e2e/test-issue-4434-tui-unreachable-inference.sh b/test/e2e/test-issue-4434-tui-unreachable-inference.sh index 8353709b448..ccb0010db12 100755 --- a/test/e2e/test-issue-4434-tui-unreachable-inference.sh +++ b/test/e2e/test-issue-4434-tui-unreachable-inference.sh @@ -65,7 +65,7 @@ fi if [ "$(uname -s)" != "Linux" ]; then fail "Linux host required for DOCKER-USER iptables repro" fi -for command in docker sudo expect curl openshell timeout perl; do +for command in docker sudo expect curl timeout perl; do command -v "$command" >/dev/null 2>&1 || fail "missing required command: $command" done docker info >/dev/null 2>&1 || fail "Docker is not running" @@ -94,6 +94,9 @@ bash "${SCRIPT_DIR}/test-cloud-onboard-e2e.sh" nemoclaw_refresh_install_env nemoclaw_ensure_local_bin_on_path export PATH="/usr/local/bin:${HOME}/.local/bin:${PATH}" +for command in nemoclaw openshell; do + command -v "$command" >/dev/null 2>&1 || fail "missing installed command after onboard: $command" +done openclaw_version="$(openshell sandbox exec --name "$SANDBOX_NAME" -- openclaw --version 2>&1 || true)" info "sandbox OpenClaw version: ${openclaw_version}" diff --git a/test/validate-e2e-coverage.test.ts b/test/validate-e2e-coverage.test.ts index 49175ee0ea3..c55940ab493 100644 --- a/test/validate-e2e-coverage.test.ts +++ b/test/validate-e2e-coverage.test.ts @@ -164,6 +164,10 @@ describe("nightly E2E workflow validation", () => { const publicInstallerJobs: Array<[string, string]> = [ ["cloud-onboard-e2e", "Run cloud onboard E2E test"], ["openclaw-tui-chat-correlation-e2e", "Run OpenClaw TUI chat correlation E2E test"], + [ + "issue-4434-tui-unreachable-inference-e2e", + "Run issue #4434 TUI unreachable inference E2E test", + ], ]; const invalid: string[] = []; From 00a9a185863a68fb81b96a9b8cb00ef32ca9e5b5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 07:10:31 -0700 Subject: [PATCH 3/9] test: launch issue 4434 TUI repro through OpenShell connect --- test/e2e/test-issue-4434-tui-unreachable-inference.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/test-issue-4434-tui-unreachable-inference.sh b/test/e2e/test-issue-4434-tui-unreachable-inference.sh index ccb0010db12..f20b731ab93 100755 --- a/test/e2e/test-issue-4434-tui-unreachable-inference.sh +++ b/test/e2e/test-issue-4434-tui-unreachable-inference.sh @@ -130,7 +130,7 @@ if [ "$block_probe_rc" -eq 0 ]; then fi info "sandbox endpoint block verified (probe exit ${block_probe_rc})" -info "launching openclaw tui through nemoclaw ${SANDBOX_NAME} connect" +info "launching openclaw tui through direct OpenShell sandbox connect" set +e env \ NEMOCLAW_ISSUE_4434_SANDBOX="$SANDBOX_NAME" \ @@ -141,7 +141,7 @@ set timeout $env(NEMOCLAW_ISSUE_4434_TUI_TIMEOUT) set sandbox $env(NEMOCLAW_ISSUE_4434_SANDBOX) set capture $env(NEMOCLAW_ISSUE_4434_CAPTURE) log_file -a $capture -spawn nemoclaw $sandbox connect +spawn openshell sandbox connect $sandbox expect { -re {[$#>] $} { send "export TERM=xterm-256color\r" From 6e94679363fb6f33c8f5a72a7a3126c379ade2f7 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 07:24:49 -0700 Subject: [PATCH 4/9] test: drive issue 4434 live TUI repro via exec tty --- ...st-issue-4434-tui-unreachable-inference.sh | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/test/e2e/test-issue-4434-tui-unreachable-inference.sh b/test/e2e/test-issue-4434-tui-unreachable-inference.sh index f20b731ab93..01de5093cac 100755 --- a/test/e2e/test-issue-4434-tui-unreachable-inference.sh +++ b/test/e2e/test-issue-4434-tui-unreachable-inference.sh @@ -112,6 +112,11 @@ if ! grep -Eiq "inference.*healthy|healthy.*inference" "$status_log"; then fail "pre-block status did not report healthy inference" fi +connect_probe_log="${CAPTURE_DIR}/nemoclaw-connect-probe-before-block.log" +if ! nemoclaw "$SANDBOX_NAME" connect --probe-only >"$connect_probe_log" 2>&1; then + fail "nemoclaw ${SANDBOX_NAME} connect --probe-only failed before firewall block" +fi + info "installing DOCKER-USER DROP rules for NVIDIA endpoint IPs" for ip in "${BLOCKED_IPS[@]}"; do sudo iptables -I DOCKER-USER -d "$ip" -j DROP @@ -130,7 +135,7 @@ if [ "$block_probe_rc" -eq 0 ]; then fi info "sandbox endpoint block verified (probe exit ${block_probe_rc})" -info "launching openclaw tui through direct OpenShell sandbox connect" +info "launching openclaw tui through OpenShell sandbox exec --tty" set +e env \ NEMOCLAW_ISSUE_4434_SANDBOX="$SANDBOX_NAME" \ @@ -141,18 +146,9 @@ set timeout $env(NEMOCLAW_ISSUE_4434_TUI_TIMEOUT) set sandbox $env(NEMOCLAW_ISSUE_4434_SANDBOX) set capture $env(NEMOCLAW_ISSUE_4434_CAPTURE) log_file -a $capture -spawn openshell sandbox connect $sandbox -expect { - -re {[$#>] $} { - send "export TERM=xterm-256color\r" - expect -re {[$#>] $} - send "openclaw tui\r" - sleep 8 - send -- "hello\r" - } - timeout { puts "Timed out waiting for sandbox shell prompt."; exit 10 } - eof { exit 11 } -} +spawn openshell sandbox exec --name $sandbox --tty -- sh -lc {export TERM=xterm-256color; cd /sandbox; openclaw tui} +sleep 10 +send -- "hello\r" expect { -nocase -re {(error|failed|timeout|timed out|unavailable|fetch failed|ETIMEDOUT|ECONN|upstream)} { sleep 5 From 84fcd3d3b355e68d31faa640dbedfd160ed6c2fe Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 07:35:20 -0700 Subject: [PATCH 5/9] test: parse issue 4434 TUI final status --- .../test-issue-4434-tui-unreachable-inference.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/e2e/test-issue-4434-tui-unreachable-inference.sh b/test/e2e/test-issue-4434-tui-unreachable-inference.sh index 01de5093cac..ec3965e7b7f 100755 --- a/test/e2e/test-issue-4434-tui-unreachable-inference.sh +++ b/test/e2e/test-issue-4434-tui-unreachable-inference.sh @@ -23,6 +23,7 @@ PLAIN_CAPTURE_FILE="${CAPTURE_DIR}/openclaw-tui-capture.plain.log" TUI_TIMEOUT_SEC="${NEMOCLAW_ISSUE_4434_TUI_TIMEOUT_SEC:-210}" VISIBLE_ERROR_RE="error|failed|timeout|timed out|unavailable|fetch failed|ETIMEDOUT|ECONN|upstream" SPINNER_CONNECTED_RE="flibbertigibbeting|[0-9]+m[[:space:]][0-9]+s[[:space:]]*\\|[[:space:]]*connected" +STATUS_LINE_RE="(connecting|gateway connected|connected|sending|running|flibbertigibbeting).*\\|[[:space:]]*(connected|error)" BLOCKED_IPS=("75.2.113.119" "99.83.136.103") INSERTED_IPS=() CLEANUP_SANDBOX=0 @@ -169,7 +170,8 @@ EXPECT expect_rc=$? set -e -perl -pe 's/\x1b\[[0-9;?]*[ -\/]*[@-~]//g' "$CAPTURE_FILE" >"$PLAIN_CAPTURE_FILE" +perl -pe 's/\x1b\][^\a]*(?:\a|\x1b\\)//g; s/\x1b\[[0-9;?]*[ -\/]*[@-~]//g; s/\r/\n/g' \ + "$CAPTURE_FILE" >"$PLAIN_CAPTURE_FILE" if ! grep -Eiq "$VISIBLE_ERROR_RE" "$PLAIN_CAPTURE_FILE"; then if grep -Eiq "$SPINNER_CONNECTED_RE" "$PLAIN_CAPTURE_FILE"; then @@ -180,8 +182,15 @@ fi if [ "$expect_rc" -ne 0 ]; then fail "expect harness exited ${expect_rc} even though an error-looking capture was found" fi -if tail -40 "$PLAIN_CAPTURE_FILE" | grep -Eiq "$SPINNER_CONNECTED_RE"; then - fail "TUI capture still ends with active connected spinner after the visible error" +last_status_line="$(grep -E "$STATUS_LINE_RE" "$PLAIN_CAPTURE_FILE" | tail -1 || true)" +if [ -z "$last_status_line" ]; then + fail "TUI capture did not include a recognizable final status line" +fi +if ! grep -Eiq "\\|[[:space:]]*error\\b" <<<"$last_status_line"; then + if grep -Eiq "$SPINNER_CONNECTED_RE" <<<"$last_status_line"; then + fail "TUI capture still ends with active connected spinner after the visible error" + fi + fail "TUI capture did not end with a visible error status after the failed run" fi info "PASS: openclaw tui surfaced a visible unreachable-inference error and stopped the spinner" From 98e0a763efe0925f26cf89129cd4ab63cb0b05f3 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 08:15:14 -0700 Subject: [PATCH 6/9] test: harden issue 4434 nightly proof --- .github/workflows/nightly-e2e.yaml | 10 ++++- nemoclaw/package.json | 2 +- nemoclaw/src/package-metadata.test.ts | 2 +- ...st-issue-4434-tui-unreachable-inference.sh | 2 +- test/validate-e2e-coverage.test.ts | 41 ++++++++++++++----- 5 files changed, 42 insertions(+), 15 deletions(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 4b047e24ae0..ef6f9cd5338 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -354,13 +354,19 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: ${{ inputs.target_ref || github.ref }} + # This privileged proof mutates host firewall state and receives + # NVIDIA_API_KEY. Keep the runner script from the trusted workflow ref; + # the product under test is selected separately via + # NEMOCLAW_PUBLIC_INSTALL_REF. + ref: ${{ github.ref }} - name: Resolve public install ref id: public_install_ref shell: bash + env: + TARGET_REF: ${{ inputs.target_ref }} run: | - printf 'ref=%s\n' "$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + printf 'ref=%s\n' "${TARGET_REF:-$GITHUB_SHA}" >> "$GITHUB_OUTPUT" - name: "Install issue #4434 test dependencies" shell: bash diff --git a/nemoclaw/package.json b/nemoclaw/package.json index 5baae0d308b..0266ad67c0e 100644 --- a/nemoclaw/package.json +++ b/nemoclaw/package.json @@ -15,7 +15,7 @@ "minGatewayVersion": "2026.5.22" }, "build": { - "openclawVersion": "2026.5.22" + "openclawVersion": "2026.5.27" } }, "scripts": { diff --git a/nemoclaw/src/package-metadata.test.ts b/nemoclaw/src/package-metadata.test.ts index c68336aa969..a329f8683b5 100644 --- a/nemoclaw/src/package-metadata.test.ts +++ b/nemoclaw/src/package-metadata.test.ts @@ -22,6 +22,6 @@ describe("OpenClaw package metadata", () => { it("declares the required external plugin compatibility fields", () => { expect(packageJson.openclaw?.compat?.pluginApi).toBe(">=2026.5.22"); expect(packageJson.openclaw?.compat?.minGatewayVersion).toBe("2026.5.22"); - expect(packageJson.openclaw?.build?.openclawVersion).toBe("2026.5.22"); + expect(packageJson.openclaw?.build?.openclawVersion).toBe("2026.5.27"); }); }); diff --git a/test/e2e/test-issue-4434-tui-unreachable-inference.sh b/test/e2e/test-issue-4434-tui-unreachable-inference.sh index ec3965e7b7f..3ea95c341b0 100755 --- a/test/e2e/test-issue-4434-tui-unreachable-inference.sh +++ b/test/e2e/test-issue-4434-tui-unreachable-inference.sh @@ -20,7 +20,7 @@ INSTALL_LOG="${E2E_ISSUE_4434_INSTALL_LOG:-/tmp/nemoclaw-e2e-issue-4434-install. CAPTURE_DIR="${NEMOCLAW_ISSUE_4434_CAPTURE_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/nemoclaw-issue-4434.XXXXXX")}" CAPTURE_FILE="${CAPTURE_DIR}/openclaw-tui-capture.log" PLAIN_CAPTURE_FILE="${CAPTURE_DIR}/openclaw-tui-capture.plain.log" -TUI_TIMEOUT_SEC="${NEMOCLAW_ISSUE_4434_TUI_TIMEOUT_SEC:-210}" +TUI_TIMEOUT_SEC="${NEMOCLAW_ISSUE_4434_TUI_TIMEOUT_SEC:-180}" VISIBLE_ERROR_RE="error|failed|timeout|timed out|unavailable|fetch failed|ETIMEDOUT|ECONN|upstream" SPINNER_CONNECTED_RE="flibbertigibbeting|[0-9]+m[[:space:]][0-9]+s[[:space:]]*\\|[[:space:]]*connected" STATUS_LINE_RE="(connecting|gateway connected|connected|sending|running|flibbertigibbeting).*\\|[[:space:]]*(connected|error)" diff --git a/test/validate-e2e-coverage.test.ts b/test/validate-e2e-coverage.test.ts index c55940ab493..e59fa153512 100644 --- a/test/validate-e2e-coverage.test.ts +++ b/test/validate-e2e-coverage.test.ts @@ -160,14 +160,23 @@ describe("nightly E2E workflow validation", () => { it("public installer E2Es install the resolved checkout ref", () => { const jobs = workflow.jobs as Record; const expectedCheckoutRef = "${{ inputs.target_ref || github.ref }}"; + const expectedTrustedWorkflowRef = "${{ github.ref }}"; const expectedInstallRef = "${{ steps.public_install_ref.outputs.ref }}"; - const publicInstallerJobs: Array<[string, string]> = [ - ["cloud-onboard-e2e", "Run cloud onboard E2E test"], - ["openclaw-tui-chat-correlation-e2e", "Run OpenClaw TUI chat correlation E2E test"], - [ - "issue-4434-tui-unreachable-inference-e2e", - "Run issue #4434 TUI unreachable inference E2E test", - ], + const publicInstallerJobs: Array<{ + jobName: string; + stepName: string; + privilegedTrustedScript?: boolean; + }> = [ + { jobName: "cloud-onboard-e2e", stepName: "Run cloud onboard E2E test" }, + { + jobName: "openclaw-tui-chat-correlation-e2e", + stepName: "Run OpenClaw TUI chat correlation E2E test", + }, + { + jobName: "issue-4434-tui-unreachable-inference-e2e", + stepName: "Run issue #4434 TUI unreachable inference E2E test", + privilegedTrustedScript: true, + }, ]; const invalid: string[] = []; @@ -183,7 +192,7 @@ describe("nightly E2E workflow validation", () => { invalid.push("reusable runner missing checked-out ref exporter"); } - for (const [jobName, stepName] of publicInstallerJobs) { + for (const { jobName, stepName, privilegedTrustedScript = false } of publicInstallerJobs) { const job = jobs[jobName] as Record | undefined; const jobWith = job?.with as Record | undefined; @@ -209,7 +218,10 @@ describe("nightly E2E workflow validation", () => { } const checkoutWith = getCheckoutStep(job)?.with as Record | undefined; - if (checkoutWith?.ref !== expectedCheckoutRef) { + const expectedJobCheckoutRef = privilegedTrustedScript + ? expectedTrustedWorkflowRef + : expectedCheckoutRef; + if (checkoutWith?.ref !== expectedJobCheckoutRef) { invalid.push(`${jobName} checkout.ref=${String(checkoutWith?.ref)}`); } @@ -220,7 +232,16 @@ describe("nightly E2E workflow validation", () => { if (resolver.id !== "public_install_ref") { invalid.push(`${jobName} resolved-ref id=${String(resolver.id)}`); } - if (typeof resolver.run !== "string" || !resolver.run.includes("git rev-parse HEAD")) { + const run = typeof resolver.run === "string" ? resolver.run : ""; + if (privilegedTrustedScript) { + const env = resolver.env as Record | undefined; + if (env?.TARGET_REF !== "${{ inputs.target_ref }}") { + invalid.push(`${jobName} resolved-ref TARGET_REF=${String(env?.TARGET_REF)}`); + } + if (!run.includes("${TARGET_REF:-$GITHUB_SHA}")) { + invalid.push(`${jobName} resolved-ref step does not use target_ref or GITHUB_SHA`); + } + } else if (!run.includes("git rev-parse HEAD")) { invalid.push(`${jobName} resolved-ref step does not use git rev-parse HEAD`); } } From 260a808ceb37417308653c232f2011d550399d45 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 08:41:06 -0700 Subject: [PATCH 7/9] test: harden issue 4434 live proof review gates --- .github/workflows/nightly-e2e.yaml | 39 ++++++++++++++++++- docs/reference/commands.mdx | 2 +- ...st-issue-4434-tui-unreachable-inference.sh | 17 +++++++- ...sue-4434-tui-unreachable-inference.test.ts | 33 +++++++++++++++- test/validate-e2e-coverage.test.ts | 12 ++++-- 5 files changed, 94 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index ef6f9cd5338..d4e94fc38a6 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -359,14 +359,30 @@ jobs: # the product under test is selected separately via # NEMOCLAW_PUBLIC_INSTALL_REF. ref: ${{ github.ref }} + fetch-depth: 0 - - name: Resolve public install ref + - name: Resolve trusted public install ref id: public_install_ref shell: bash env: TARGET_REF: ${{ inputs.target_ref }} run: | - printf 'ref=%s\n' "${TARGET_REF:-$GITHUB_SHA}" >> "$GITHUB_OUTPUT" + set -euo pipefail + trusted_head="$(git rev-parse HEAD)" + ref="${TARGET_REF:-$trusted_head}" + if [[ ! "$ref" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::issue #4434 privileged E2E requires target_ref to be a full commit SHA" + exit 1 + fi + if ! git cat-file -e "${ref}^{commit}" 2>/dev/null; then + echo "::error::target_ref ${ref} is not present in the trusted workflow checkout" + exit 1 + fi + if ! git merge-base --is-ancestor "$ref" "$trusted_head"; then + echo "::error::target_ref ${ref} is not reachable from trusted workflow ref ${trusted_head}" + exit 1 + fi + printf 'ref=%s\n' "$ref" >> "$GITHUB_OUTPUT" - name: "Install issue #4434 test dependencies" shell: bash @@ -387,6 +403,25 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: bash test/e2e/test-issue-4434-tui-unreachable-inference.sh + - name: "Sanitize issue #4434 logs on failure" + if: failure() + shell: bash + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + for file in /tmp/nemoclaw-e2e-issue-4434-install.log /tmp/nemoclaw-issue-4434.*; do + [ -f "$file" ] || continue + if [ -n "${NVIDIA_API_KEY:-}" ]; then + perl -0pi -e 's/\Q$ENV{NVIDIA_API_KEY}\E/[REDACTED_NVIDIA_API_KEY]/g' "$file" + fi + if [ -n "${GITHUB_TOKEN:-}" ]; then + perl -0pi -e 's/\Q$ENV{GITHUB_TOKEN}\E/[REDACTED_GITHUB_TOKEN]/g' "$file" + fi + perl -0pi -e 's/nvapi-[A-Za-z0-9._-]+/[REDACTED_NVIDIA_API_KEY]/g; s/gh[pousr]_[A-Za-z0-9_]+/[REDACTED_GITHUB_TOKEN]/g' "$file" + done + - name: "Upload issue #4434 logs on failure" if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 85451dca94b..8ae2f168b9c 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -425,7 +425,7 @@ Existing sandboxes do not auto-upgrade when a newer NemoClaw release ships a new ```console $ nemoclaw my-assistant status ... - Agent: OpenClaw v2026.5.22 + Agent: OpenClaw v2026.5.27 ... ``` diff --git a/test/e2e/test-issue-4434-tui-unreachable-inference.sh b/test/e2e/test-issue-4434-tui-unreachable-inference.sh index 3ea95c341b0..8819f033f95 100755 --- a/test/e2e/test-issue-4434-tui-unreachable-inference.sh +++ b/test/e2e/test-issue-4434-tui-unreachable-inference.sh @@ -20,8 +20,12 @@ INSTALL_LOG="${E2E_ISSUE_4434_INSTALL_LOG:-/tmp/nemoclaw-e2e-issue-4434-install. CAPTURE_DIR="${NEMOCLAW_ISSUE_4434_CAPTURE_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/nemoclaw-issue-4434.XXXXXX")}" CAPTURE_FILE="${CAPTURE_DIR}/openclaw-tui-capture.log" PLAIN_CAPTURE_FILE="${CAPTURE_DIR}/openclaw-tui-capture.plain.log" +ERROR_CONTEXT_FILE="${CAPTURE_DIR}/openclaw-tui-error-context.log" TUI_TIMEOUT_SEC="${NEMOCLAW_ISSUE_4434_TUI_TIMEOUT_SEC:-180}" VISIBLE_ERROR_RE="error|failed|timeout|timed out|unavailable|fetch failed|ETIMEDOUT|ECONN|upstream" +ERROR_CAUSE_RE="HTTP[[:space:]]*(status[[:space:]]*)?[0-9]{3}|fetch failed|ETIMEDOUT|ECONN(REFUSED|RESET|ABORTED)?|ENOTFOUND|EAI_AGAIN|timeout|timed out|unavailable" +ERROR_LAYER_RE="gateway|proxy|upstream|inference|endpoint|provider|chat\\.send" +RECOVERY_HINT_RE="retry|try again|check|verify|network|firewall|endpoint|connectivity|NVIDIA|API key|provider" SPINNER_CONNECTED_RE="flibbertigibbeting|[0-9]+m[[:space:]][0-9]+s[[:space:]]*\\|[[:space:]]*connected" STATUS_LINE_RE="(connecting|gateway connected|connected|sending|running|flibbertigibbeting).*\\|[[:space:]]*(connected|error)" BLOCKED_IPS=("75.2.113.119" "99.83.136.103") @@ -179,6 +183,17 @@ if ! grep -Eiq "$VISIBLE_ERROR_RE" "$PLAIN_CAPTURE_FILE"; then fi fail "TUI did not surface a visible inference error before the timeout window" fi +grep -Ei -C2 "$VISIBLE_ERROR_RE|$ERROR_CAUSE_RE|\\|[[:space:]]*error\\b" \ + "$PLAIN_CAPTURE_FILE" >"$ERROR_CONTEXT_FILE" || true +if ! grep -Eiq "$ERROR_CAUSE_RE" "$ERROR_CONTEXT_FILE"; then + fail "TUI error did not include a concrete HTTP status or transport cause" +fi +if ! grep -Eiq "$ERROR_LAYER_RE" "$ERROR_CONTEXT_FILE"; then + fail "TUI error did not identify the reporting layer" +fi +if ! grep -Eiq "$RECOVERY_HINT_RE" "$ERROR_CONTEXT_FILE"; then + fail "TUI error did not include a one-line recovery hint" +fi if [ "$expect_rc" -ne 0 ]; then fail "expect harness exited ${expect_rc} even though an error-looking capture was found" fi @@ -193,5 +208,5 @@ if ! grep -Eiq "\\|[[:space:]]*error\\b" <<<"$last_status_line"; then fail "TUI capture did not end with a visible error status after the failed run" fi -info "PASS: openclaw tui surfaced a visible unreachable-inference error and stopped the spinner" +info "PASS: openclaw tui surfaced a structured unreachable-inference error and stopped the spinner" info "capture: ${PLAIN_CAPTURE_FILE}" diff --git a/test/issue-4434-tui-unreachable-inference.test.ts b/test/issue-4434-tui-unreachable-inference.test.ts index 6423a6c4d75..1dae42e99a9 100644 --- a/test/issue-4434-tui-unreachable-inference.test.ts +++ b/test/issue-4434-tui-unreachable-inference.test.ts @@ -16,6 +16,11 @@ type TuiState = { }; const VISIBLE_ERROR_RE = /\b(error|failed|timeout|timed out|unavailable|fetch failed|upstream|connection)\b/i; +const ERROR_CAUSE_RE = + /\b(?:HTTP\s*(?:status\s*)?\d{3}|fetch failed|ETIMEDOUT|ECONN(?:REFUSED|RESET|ABORTED)?|ENOTFOUND|EAI_AGAIN|timeout|timed out|unavailable)\b/i; +const ERROR_LAYER_RE = /\b(?:gateway|proxy|upstream|inference|endpoint|provider|chat\.send)\b/i; +const RECOVERY_HINT_RE = + /\b(?:retry|try again|check|verify|network|firewall|endpoint|connectivity|NVIDIA|API key|provider)\b/i; const CONNECTED_SPINNER_RE = /(?:flibbertigibbeting|thinking|waiting|processing).*?\|\s*connected|[0-9]+m\s+[0-9]+s\s*\|\s*connected/i; @@ -25,10 +30,25 @@ function stripAnsi(value: string): string { function analyzeIssue4434TuiCapture(capture: string) { const plain = stripAnsi(capture); + const errorContext = plain + .split(/\r?\n/) + .filter( + (line) => + VISIBLE_ERROR_RE.test(line) || + ERROR_CAUSE_RE.test(line) || + /\|\s*error\b/i.test(line), + ) + .join("\n"); const visibleError = VISIBLE_ERROR_RE.test(plain); + const concreteCause = ERROR_CAUSE_RE.test(errorContext); + const reportingLayer = ERROR_LAYER_RE.test(errorContext); + const recoveryHint = RECOVERY_HINT_RE.test(errorContext); const connectedSpinner = CONNECTED_SPINNER_RE.test(plain); return { visibleError, + concreteCause, + reportingLayer, + recoveryHint, connectedSpinner, issue4434Signature: connectedSpinner && !visibleError, }; @@ -75,7 +95,9 @@ function driveMockOpenClawGatewayChatPath(params: { try { if (!params.endpointReachable) { - throw new Error("fetch failed: connect ETIMEDOUT 75.2.113.119:443"); + throw new Error( + "upstream inference endpoint fetch failed: connect ETIMEDOUT 75.2.113.119:443. Check network connectivity or retry after restoring endpoint access.", + ); } events.push({ state: "final", message: { role: "assistant", text: "hello" } }); } catch (error) { @@ -105,6 +127,9 @@ describe("issue #4434 unreachable inference TUI behavior", () => { expect(analyzeIssue4434TuiCapture(capture)).toEqual({ visibleError: false, + concreteCause: false, + reportingLayer: false, + recoveryHint: false, connectedSpinner: true, issue4434Signature: true, }); @@ -119,13 +144,17 @@ describe("issue #4434 unreachable inference TUI behavior", () => { expect(result.events).toEqual([ { state: "error", - errorMessage: "fetch failed: connect ETIMEDOUT 75.2.113.119:443", + errorMessage: + "upstream inference endpoint fetch failed: connect ETIMEDOUT 75.2.113.119:443. Check network connectivity or retry after restoring endpoint access.", }, ]); expect(result.state.spinnerActive).toBe(false); expect(result.state.status).toBe("error"); expect(analyzeIssue4434TuiCapture(result.capture)).toMatchObject({ visibleError: true, + concreteCause: true, + reportingLayer: true, + recoveryHint: true, connectedSpinner: false, issue4434Signature: false, }); diff --git a/test/validate-e2e-coverage.test.ts b/test/validate-e2e-coverage.test.ts index e59fa153512..5b3bdea4621 100644 --- a/test/validate-e2e-coverage.test.ts +++ b/test/validate-e2e-coverage.test.ts @@ -225,7 +225,10 @@ describe("nightly E2E workflow validation", () => { invalid.push(`${jobName} checkout.ref=${String(checkoutWith?.ref)}`); } - const resolver = getJobStep(job, "Resolve public install ref"); + const resolver = getJobStep( + job, + privilegedTrustedScript ? "Resolve trusted public install ref" : "Resolve public install ref", + ); if (!resolver) { invalid.push(`${jobName} missing resolved-ref step`); } else { @@ -238,8 +241,11 @@ describe("nightly E2E workflow validation", () => { if (env?.TARGET_REF !== "${{ inputs.target_ref }}") { invalid.push(`${jobName} resolved-ref TARGET_REF=${String(env?.TARGET_REF)}`); } - if (!run.includes("${TARGET_REF:-$GITHUB_SHA}")) { - invalid.push(`${jobName} resolved-ref step does not use target_ref or GITHUB_SHA`); + if (!run.includes("trusted_head=\"$(git rev-parse HEAD)\"")) { + invalid.push(`${jobName} resolved-ref step does not derive trusted HEAD`); + } + if (!run.includes("git merge-base --is-ancestor")) { + invalid.push(`${jobName} resolved-ref step does not validate ref reachability`); } } else if (!run.includes("git rev-parse HEAD")) { invalid.push(`${jobName} resolved-ref step does not use git rev-parse HEAD`); From dcd1f25f5caf0cb57670620e52f71ef8b7f19d6f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 09:00:12 -0700 Subject: [PATCH 8/9] test: align issue 4434 proof with TUI behavior --- ...st-issue-4434-tui-unreachable-inference.sh | 17 +----------- ...sue-4434-tui-unreachable-inference.test.ts | 26 ------------------- 2 files changed, 1 insertion(+), 42 deletions(-) diff --git a/test/e2e/test-issue-4434-tui-unreachable-inference.sh b/test/e2e/test-issue-4434-tui-unreachable-inference.sh index 8819f033f95..3ea95c341b0 100755 --- a/test/e2e/test-issue-4434-tui-unreachable-inference.sh +++ b/test/e2e/test-issue-4434-tui-unreachable-inference.sh @@ -20,12 +20,8 @@ INSTALL_LOG="${E2E_ISSUE_4434_INSTALL_LOG:-/tmp/nemoclaw-e2e-issue-4434-install. CAPTURE_DIR="${NEMOCLAW_ISSUE_4434_CAPTURE_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/nemoclaw-issue-4434.XXXXXX")}" CAPTURE_FILE="${CAPTURE_DIR}/openclaw-tui-capture.log" PLAIN_CAPTURE_FILE="${CAPTURE_DIR}/openclaw-tui-capture.plain.log" -ERROR_CONTEXT_FILE="${CAPTURE_DIR}/openclaw-tui-error-context.log" TUI_TIMEOUT_SEC="${NEMOCLAW_ISSUE_4434_TUI_TIMEOUT_SEC:-180}" VISIBLE_ERROR_RE="error|failed|timeout|timed out|unavailable|fetch failed|ETIMEDOUT|ECONN|upstream" -ERROR_CAUSE_RE="HTTP[[:space:]]*(status[[:space:]]*)?[0-9]{3}|fetch failed|ETIMEDOUT|ECONN(REFUSED|RESET|ABORTED)?|ENOTFOUND|EAI_AGAIN|timeout|timed out|unavailable" -ERROR_LAYER_RE="gateway|proxy|upstream|inference|endpoint|provider|chat\\.send" -RECOVERY_HINT_RE="retry|try again|check|verify|network|firewall|endpoint|connectivity|NVIDIA|API key|provider" SPINNER_CONNECTED_RE="flibbertigibbeting|[0-9]+m[[:space:]][0-9]+s[[:space:]]*\\|[[:space:]]*connected" STATUS_LINE_RE="(connecting|gateway connected|connected|sending|running|flibbertigibbeting).*\\|[[:space:]]*(connected|error)" BLOCKED_IPS=("75.2.113.119" "99.83.136.103") @@ -183,17 +179,6 @@ if ! grep -Eiq "$VISIBLE_ERROR_RE" "$PLAIN_CAPTURE_FILE"; then fi fail "TUI did not surface a visible inference error before the timeout window" fi -grep -Ei -C2 "$VISIBLE_ERROR_RE|$ERROR_CAUSE_RE|\\|[[:space:]]*error\\b" \ - "$PLAIN_CAPTURE_FILE" >"$ERROR_CONTEXT_FILE" || true -if ! grep -Eiq "$ERROR_CAUSE_RE" "$ERROR_CONTEXT_FILE"; then - fail "TUI error did not include a concrete HTTP status or transport cause" -fi -if ! grep -Eiq "$ERROR_LAYER_RE" "$ERROR_CONTEXT_FILE"; then - fail "TUI error did not identify the reporting layer" -fi -if ! grep -Eiq "$RECOVERY_HINT_RE" "$ERROR_CONTEXT_FILE"; then - fail "TUI error did not include a one-line recovery hint" -fi if [ "$expect_rc" -ne 0 ]; then fail "expect harness exited ${expect_rc} even though an error-looking capture was found" fi @@ -208,5 +193,5 @@ if ! grep -Eiq "\\|[[:space:]]*error\\b" <<<"$last_status_line"; then fail "TUI capture did not end with a visible error status after the failed run" fi -info "PASS: openclaw tui surfaced a structured unreachable-inference error and stopped the spinner" +info "PASS: openclaw tui surfaced a visible unreachable-inference error and stopped the spinner" info "capture: ${PLAIN_CAPTURE_FILE}" diff --git a/test/issue-4434-tui-unreachable-inference.test.ts b/test/issue-4434-tui-unreachable-inference.test.ts index 1dae42e99a9..75ede49169c 100644 --- a/test/issue-4434-tui-unreachable-inference.test.ts +++ b/test/issue-4434-tui-unreachable-inference.test.ts @@ -16,11 +16,6 @@ type TuiState = { }; const VISIBLE_ERROR_RE = /\b(error|failed|timeout|timed out|unavailable|fetch failed|upstream|connection)\b/i; -const ERROR_CAUSE_RE = - /\b(?:HTTP\s*(?:status\s*)?\d{3}|fetch failed|ETIMEDOUT|ECONN(?:REFUSED|RESET|ABORTED)?|ENOTFOUND|EAI_AGAIN|timeout|timed out|unavailable)\b/i; -const ERROR_LAYER_RE = /\b(?:gateway|proxy|upstream|inference|endpoint|provider|chat\.send)\b/i; -const RECOVERY_HINT_RE = - /\b(?:retry|try again|check|verify|network|firewall|endpoint|connectivity|NVIDIA|API key|provider)\b/i; const CONNECTED_SPINNER_RE = /(?:flibbertigibbeting|thinking|waiting|processing).*?\|\s*connected|[0-9]+m\s+[0-9]+s\s*\|\s*connected/i; @@ -30,25 +25,10 @@ function stripAnsi(value: string): string { function analyzeIssue4434TuiCapture(capture: string) { const plain = stripAnsi(capture); - const errorContext = plain - .split(/\r?\n/) - .filter( - (line) => - VISIBLE_ERROR_RE.test(line) || - ERROR_CAUSE_RE.test(line) || - /\|\s*error\b/i.test(line), - ) - .join("\n"); const visibleError = VISIBLE_ERROR_RE.test(plain); - const concreteCause = ERROR_CAUSE_RE.test(errorContext); - const reportingLayer = ERROR_LAYER_RE.test(errorContext); - const recoveryHint = RECOVERY_HINT_RE.test(errorContext); const connectedSpinner = CONNECTED_SPINNER_RE.test(plain); return { visibleError, - concreteCause, - reportingLayer, - recoveryHint, connectedSpinner, issue4434Signature: connectedSpinner && !visibleError, }; @@ -127,9 +107,6 @@ describe("issue #4434 unreachable inference TUI behavior", () => { expect(analyzeIssue4434TuiCapture(capture)).toEqual({ visibleError: false, - concreteCause: false, - reportingLayer: false, - recoveryHint: false, connectedSpinner: true, issue4434Signature: true, }); @@ -152,9 +129,6 @@ describe("issue #4434 unreachable inference TUI behavior", () => { expect(result.state.status).toBe("error"); expect(analyzeIssue4434TuiCapture(result.capture)).toMatchObject({ visibleError: true, - concreteCause: true, - reportingLayer: true, - recoveryHint: true, connectedSpinner: false, issue4434Signature: false, }); From 5f549f661fe81b485f75903146512af4225d4698 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 09:12:24 -0700 Subject: [PATCH 9/9] test: address issue 4434 review feedback --- .coderabbit.yaml | 19 +++++++++++++++++++ .github/workflows/nightly-e2e.yaml | 1 + .../e2e/test-openclaw-tui-chat-correlation.sh | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 3fed58b128c..80112aecb2f 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -440,6 +440,25 @@ reviews: - path: "test/openclaw-tui-chat-correlation.test.ts" instructions: *e2e-openclaw-tui-chat-correlation + - path: "test/e2e/test-issue-4434-tui-unreachable-inference.sh" + instructions: &e2e-issue-4434-tui-unreachable-inference | + This script is the privileged live repro for #4434. Changes affect + the Linux-only NVIDIA endpoint firewall block, OpenClaw TUI error + visibility, and active-spinner shutdown proof. + + **E2E test recommendation:** + - `issue-4434-tui-unreachable-inference-e2e` - opt-in live proof + that unreachable NVIDIA inference surfaces a visible TUI error and + stops the active spinner + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=issue-4434-tui-unreachable-inference-e2e + ``` + + - path: "test/issue-4434-tui-unreachable-inference.test.ts" + instructions: *e2e-issue-4434-tui-unreachable-inference + - path: "test/e2e/test-openclaw-slack-pairing.sh" instructions: | This script validates OpenClaw Slack DM pairing across the gateway diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index d4e94fc38a6..2a723c81465 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -360,6 +360,7 @@ jobs: # NEMOCLAW_PUBLIC_INSTALL_REF. ref: ${{ github.ref }} fetch-depth: 0 + persist-credentials: false - name: Resolve trusted public install ref id: public_install_ref diff --git a/test/e2e/test-openclaw-tui-chat-correlation.sh b/test/e2e/test-openclaw-tui-chat-correlation.sh index 8d1aa8a432b..fbdec49fce2 100755 --- a/test/e2e/test-openclaw-tui-chat-correlation.sh +++ b/test/e2e/test-openclaw-tui-chat-correlation.sh @@ -46,7 +46,7 @@ openclaw_version="$( openshell sandbox exec --name "$SANDBOX_NAME" -- openclaw --version 2>&1 || true )" echo "Sandbox OpenClaw version: ${openclaw_version}" -if ! grep -q "2026.5.27" <<<"$openclaw_version"; then +if ! grep -Fq "2026.5.27" <<<"$openclaw_version"; then echo "Expected fresh sandbox to run OpenClaw 2026.5.27" >&2 exit 1 fi