diff --git a/src/lib/onboard/compatible-endpoint-smoke.test.ts b/src/lib/onboard/compatible-endpoint-smoke.test.ts index b1cf1189624..57679c5dd70 100644 --- a/src/lib/onboard/compatible-endpoint-smoke.test.ts +++ b/src/lib/onboard/compatible-endpoint-smoke.test.ts @@ -17,6 +17,7 @@ import { buildCompatibleEndpointSandboxSmokeScript, shouldRunCompatibleEndpointSandboxSmoke, spawnOutputToString, + verifyCompatibleEndpointSandboxSmoke, } from "./compatible-endpoint-smoke"; describe("compatible endpoint sandbox smoke helpers", () => { @@ -100,6 +101,31 @@ ${bodyForCall} expect(spawnOutputToString(42)).toBe("42"); }); + it("budgets the host command timeout for every retry attempt", () => { + const runOpenshell = vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: "provider ready" }) + .mockReturnValueOnce({ + status: 0, + stdout: "OPENCLAW_CONFIG_OK\nINFERENCE_SMOKE_OK PONG", + }); + + verifyCompatibleEndpointSandboxSmoke({ + sandboxName: "smoke-sandbox", + provider: "compatible-endpoint", + model: "nvidia/nemotron-3-ultra", + runOpenshell, + redact: (value) => value, + messagingChannels: ["telegram"], + }); + + expect(runOpenshell).toHaveBeenNthCalledWith( + 2, + expect.any(Array), + expect.objectContaining({ timeout: 220_000 }), + ); + }); + it("builds a sandbox script that checks managed provider routing", () => { const script = buildCompatibleEndpointSandboxSmokeScript("provider/model'"); @@ -109,6 +135,9 @@ ${bodyForCall} expect(script).toContain("https://inference.local/v1/chat/completions"); expect(script).toContain("INITIAL_MAX_TOKENS=256"); expect(script).toContain("RETRY_MAX_TOKENS=1024"); + expect(script).toContain("SMOKE_ATTEMPTS=3"); + expect(script).toContain("SMOKE_REQUEST_TIMEOUT_SECONDS=60"); + expect(script).toContain("SMOKE_RETRY_DELAY_SECONDS=5"); expect(script).toContain("MODEL='provider/model'\\'''"); }); @@ -131,8 +160,10 @@ fi `, ); const script = buildCompatibleEndpointSandboxSmokeScript(model, { + attempts: 2, configPath, initialMaxTokens: 32, + retryDelaySeconds: 0, retryMaxTokens: 512, }); @@ -145,6 +176,82 @@ fi expect(fs.readFileSync(callFile, "utf-8")).toBe("2"); }); + it("retries a transient non-JSON gateway response", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-compat-smoke-transient-")); + const model = "nvidia/nemotron-3-ultra"; + const configPath = writeSmokeConfig(tmpDir, model); + const { binDir, callFile } = writeFakeCurl( + tmpDir, + String.raw` +if [ "$count" -eq 1 ]; then + printf '%s\n' '504 Gateway Time-out' +else + printf '%s\n' '{"choices":[{"message":{"content":"PONG"},"finish_reason":"stop"}]}' +fi +`, + ); + const script = buildCompatibleEndpointSandboxSmokeScript(model, { + attempts: 3, + configPath, + retryDelaySeconds: 0, + }); + + const result = runSmokeScript(script, tmpDir, binDir); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("INFERENCE_SMOKE_OK PONG"); + expect(result.stderr).toContain("inference.local returned non-JSON response"); + expect(result.stderr).toContain("smoke attempt 1/3 failed; retrying in 0s"); + expect(fs.readFileSync(callFile, "utf-8")).toBe("2"); + }); + + it("does not retry a permanent JSON response validation failure", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-compat-smoke-permanent-")); + const model = "nvidia/nemotron-3-ultra"; + const configPath = writeSmokeConfig(tmpDir, model); + const { binDir, callFile } = writeFakeCurl( + tmpDir, + `printf '%s\\n' '{"error":{"message":"invalid model"}}'`, + ); + const script = buildCompatibleEndpointSandboxSmokeScript(model, { + attempts: 3, + configPath, + retryDelaySeconds: 0, + }); + + const result = runSmokeScript(script, tmpDir, binDir); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("did not contain non-empty choices[0].message.content"); + expect(result.stderr).not.toContain("retrying in"); + expect(fs.readFileSync(callFile, "utf-8")).toBe("1"); + }); + + it("fails after the bounded transient retry budget", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-compat-smoke-exhausted-")); + const model = "nvidia/nemotron-3-ultra"; + const configPath = writeSmokeConfig(tmpDir, model); + const { binDir, callFile } = writeFakeCurl( + tmpDir, + "printf '%s\\n' '504 Gateway Time-outAuthorization: Bearer test-secret'", + ); + const script = buildCompatibleEndpointSandboxSmokeScript(model, { + attempts: 3, + configPath, + retryDelaySeconds: 0, + }); + + const result = runSmokeScript(script, tmpDir, binDir); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("http_status=504"); + expect(result.stderr).toContain("response_bytes="); + expect(result.stderr).not.toContain("test-secret"); + expect(result.stderr).toContain("smoke attempt 1/3 failed; retrying in 0s"); + expect(result.stderr).toContain("smoke attempt 2/3 failed; retrying in 0s"); + expect(fs.readFileSync(callFile, "utf-8")).toBe("3"); + }); + it("reports a model-output budget problem when the retry also has no assistant content", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-compat-smoke-no-content-")); const model = "minimaxai/minimax-m2.7"; @@ -158,8 +265,10 @@ JSON `, ); const script = buildCompatibleEndpointSandboxSmokeScript(model, { + attempts: 2, configPath, initialMaxTokens: 32, + retryDelaySeconds: 0, retryMaxTokens: 64, }); diff --git a/src/lib/onboard/compatible-endpoint-smoke.ts b/src/lib/onboard/compatible-endpoint-smoke.ts index 7b1ce9acee7..40de9bec26e 100644 --- a/src/lib/onboard/compatible-endpoint-smoke.ts +++ b/src/lib/onboard/compatible-endpoint-smoke.ts @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { StdioOptions } from "node:child_process"; import { shellQuote } from "../core/shell-quote"; import { compactText } from "../core/url-utils"; import { INFERENCE_ROUTE_URL, MANAGED_PROVIDER_ID } from "../inference/config"; -import type { StdioOptions } from "node:child_process"; type CompatibleEndpointSmokeAgent = | { @@ -14,9 +14,11 @@ type CompatibleEndpointSmokeAgent = | undefined; type CompatibleEndpointSandboxSmokeScriptOptions = { + attempts?: number; configPath?: string; inferenceUrl?: string; initialMaxTokens?: number; + retryDelaySeconds?: number; retryMaxTokens?: number; }; @@ -30,6 +32,16 @@ type CompatibleEndpointSmokeRun = ( }, ) => { status: number | null; stdout?: unknown; stderr?: unknown }; +const COMPATIBLE_ENDPOINT_SMOKE_ATTEMPTS = 3; +const COMPATIBLE_ENDPOINT_SMOKE_REQUEST_TIMEOUT_SECONDS = 60; +const COMPATIBLE_ENDPOINT_SMOKE_RETRY_DELAY_SECONDS = 5; +const COMPATIBLE_ENDPOINT_SMOKE_COMMAND_OVERHEAD_SECONDS = 30; +const COMPATIBLE_ENDPOINT_SMOKE_COMMAND_TIMEOUT_MS = + (COMPATIBLE_ENDPOINT_SMOKE_ATTEMPTS * COMPATIBLE_ENDPOINT_SMOKE_REQUEST_TIMEOUT_SECONDS + + (COMPATIBLE_ENDPOINT_SMOKE_ATTEMPTS - 1) * COMPATIBLE_ENDPOINT_SMOKE_RETRY_DELAY_SECONDS + + COMPATIBLE_ENDPOINT_SMOKE_COMMAND_OVERHEAD_SECONDS) * + 1000; + /** * Normalizes optional token-budget overrides while preserving safe defaults for * the generated sandbox smoke script. @@ -40,6 +52,12 @@ function positiveInt(value: number | undefined, fallback: number): number { return rounded > 0 ? rounded : fallback; } +function nonNegativeInt(value: number | undefined, fallback: number): number { + if (!Number.isFinite(value)) return fallback; + const rounded = Math.floor(Number(value)); + return rounded >= 0 ? rounded : fallback; +} + /** * Returns whether onboarding should validate the compatible endpoint through * the OpenClaw sandbox instead of only checking host-side configuration. @@ -146,7 +164,7 @@ export function verifyCompatibleEndpointSandboxSmoke(options: { ignoreError: true, suppressOutput: true, stdio: ["ignore", "pipe", "pipe"], - timeout: 90_000, + timeout: COMPATIBLE_ENDPOINT_SMOKE_COMMAND_TIMEOUT_MS, }, ); const smokeOutput = [ @@ -178,6 +196,11 @@ export function buildCompatibleEndpointSandboxSmokeScript( const configPath = options.configPath || "/sandbox/.openclaw/openclaw.json"; const inferenceUrl = options.inferenceUrl || `${INFERENCE_ROUTE_URL}/chat/completions`; const initialMaxTokens = positiveInt(options.initialMaxTokens, 256); + const attempts = positiveInt(options.attempts, COMPATIBLE_ENDPOINT_SMOKE_ATTEMPTS); + const retryDelaySeconds = nonNegativeInt( + options.retryDelaySeconds, + COMPATIBLE_ENDPOINT_SMOKE_RETRY_DELAY_SECONDS, + ); const retryMaxTokens = positiveInt(options.retryMaxTokens, 1024); return ` @@ -187,6 +210,9 @@ CONFIG=${shellQuote(configPath)} INFERENCE_URL=${shellQuote(inferenceUrl)} INITIAL_MAX_TOKENS=${initialMaxTokens} RETRY_MAX_TOKENS=${retryMaxTokens} +SMOKE_ATTEMPTS=${attempts} +SMOKE_REQUEST_TIMEOUT_SECONDS=${COMPATIBLE_ENDPOINT_SMOKE_REQUEST_TIMEOUT_SECONDS} +SMOKE_RETRY_DELAY_SECONDS=${retryDelaySeconds} python3 - "$CONFIG" "$MODEL" <<'PYCFG' import json @@ -250,20 +276,21 @@ PYPAYLOAD } run_smoke_request() { - curl -sS --connect-timeout 10 --max-time 60 \ + curl -sS --connect-timeout 10 --max-time "$SMOKE_REQUEST_TIMEOUT_SECONDS" \ "$INFERENCE_URL" \ -H "Content-Type: application/json" \ -d "@$payload_file" >"$response_file" 2>"$error_file" || { rc=$? printf 'curl exit %s: ' "$rc" >&2 cat "$error_file" >&2 - exit "$rc" + return "$rc" } } check_response() { python3 - "$response_file" "$1" "$2" "$3" <<'PYRESP' import json +import re import sys path = sys.argv[1] @@ -280,8 +307,17 @@ except Exception as exc: body = f.read(1000) except Exception: pass - print("inference.local returned non-JSON response: %s; body=%s" % (exc, body), file=sys.stderr) - sys.exit(1) + status_match = re.search(r"<(?:title|h1)>\\s*([1-5][0-9][0-9])\\b", body, re.IGNORECASE) + status_detail = "; http_status=%s" % status_match.group(1) if status_match else "" + print( + "inference.local returned non-JSON response: %s; response_bytes=%s%s" + % (exc, len(body.encode("utf-8", errors="replace")), status_detail), + file=sys.stderr, + ) + retryable_gateway_error = bool( + status_match and 500 <= int(status_match.group(1)) <= 599 + ) + sys.exit(3 if can_retry and retryable_gateway_error else 1) choices = data.get("choices") choice = choices[0] if isinstance(choices, list) and choices and isinstance(choices[0], dict) else {} @@ -317,20 +353,45 @@ print("INFERENCE_SMOKE_OK " + content.strip()[:200]) PYRESP } -write_payload "$INITIAL_MAX_TOKENS" -run_smoke_request -status=0 -check_response initial "$INITIAL_MAX_TOKENS" 1 || status=$? -if [ "$status" -eq 0 ]; then - exit 0 -fi -if [ "$status" -ne 2 ]; then - exit "$status" -fi - -write_payload "$RETRY_MAX_TOKENS" -run_smoke_request -check_response retry "$RETRY_MAX_TOKENS" 0 +attempt=1 +while [ "$attempt" -le "$SMOKE_ATTEMPTS" ]; do + max_tokens="$RETRY_MAX_TOKENS" + attempt_label=retry + if [ "$attempt" -eq 1 ]; then + max_tokens="$INITIAL_MAX_TOKENS" + attempt_label=initial + fi + + write_payload "$max_tokens" + status=0 + request_failed=0 + run_smoke_request || { + status=$? + request_failed=1 + } + if [ "$status" -eq 0 ]; then + can_retry=0 + if [ "$attempt" -lt "$SMOKE_ATTEMPTS" ]; then + can_retry=1 + fi + check_response "$attempt_label" "$max_tokens" "$can_retry" || status=$? + fi + if [ "$status" -eq 0 ]; then + exit 0 + fi + if [ "$request_failed" -eq 0 ] && [ "$status" -ne 2 ] && [ "$status" -ne 3 ]; then + exit "$status" + fi + if [ "$attempt" -ge "$SMOKE_ATTEMPTS" ]; then + exit "$status" + fi + if [ "$status" -ne 2 ]; then + printf 'inference.local smoke attempt %s/%s failed; retrying in %ss\n' \ + "$attempt" "$SMOKE_ATTEMPTS" "$SMOKE_RETRY_DELAY_SECONDS" >&2 + fi + sleep "$SMOKE_RETRY_DELAY_SECONDS" + attempt=$((attempt + 1)) +done `.trim(); } diff --git a/test/e2e/test-hermes-e2e.sh b/test/e2e/test-hermes-e2e.sh index e0589eea2d9..482d8c5205a 100755 --- a/test/e2e/test-hermes-e2e.sh +++ b/test/e2e/test-hermes-e2e.sh @@ -610,21 +610,40 @@ section "Phase 5: Live inference" # ── Test 5a: Direct hosted inference endpoint ── info "[LIVE] Direct API test → ${HOSTED_INFERENCE_BASE_URL}..." -api_response=$(curl -s --max-time 30 \ - -X POST "${HOSTED_INFERENCE_BASE_URL}/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $HOSTED_INFERENCE_KEY" \ - -d "$(printf '{"model":"%s","messages":[{"role":"user","content":"Reply with exactly one word: PONG"}],"max_tokens":100}' "$HOSTED_INFERENCE_MODEL")" 2>/dev/null) || true - -if [ -n "$api_response" ]; then - api_content=$(echo "$api_response" | parse_chat_content 2>/dev/null) || true +api_response="" +api_content="" +api_failure_summary="" +for attempt in 1 2 3; do + max_tokens=1024 + if [ "$attempt" -eq 1 ]; then + max_tokens=256 + fi + api_response_file="$(mktemp)" + api_http_status="$(curl -sS --max-time 90 \ + -o "$api_response_file" \ + -w "%{http_code}" \ + -X POST "${HOSTED_INFERENCE_BASE_URL}/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $HOSTED_INFERENCE_KEY" \ + -d "$(printf '{"model":"%s","messages":[{"role":"user","content":"Reply with exactly one word: PONG"}],"max_tokens":%s}' "$HOSTED_INFERENCE_MODEL" "$max_tokens")" 2>/dev/null || true)" + api_response="$(cat "$api_response_file" 2>/dev/null || true)" + rm -f "$api_response_file" + [ -n "$api_http_status" ] || api_http_status="000" + api_content="$(printf '%s' "$api_response" | parse_chat_content 2>/dev/null || true)" + api_failure_summary="response without PONG (HTTP ${api_http_status: -3}, chars=${#api_response})" if grep -qi "PONG" <<<"$api_content"; then - pass "[LIVE] Direct API: model responded with PONG" - else - fail "[LIVE] Direct API: expected PONG, got: ${api_content:0:200}" + break fi + if [ "$attempt" -lt 3 ]; then + info "[LIVE] Direct API attempt ${attempt}/3 ${api_failure_summary}; retrying..." + sleep $((5 * attempt)) + fi +done + +if grep -qi "PONG" <<<"$api_content"; then + pass "[LIVE] Direct API: model responded with PONG" else - fail "[LIVE] Direct API: empty response from curl" + fail "[LIVE] Direct API: expected PONG after 3 attempts; ${api_failure_summary}" fi # ── Test 5b: Inference through the sandbox (THE definitive test) ──