From 30ca8fc34e6c99a2c98f98fc088349a816081ac2 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 08:08:04 -0700 Subject: [PATCH 1/3] fix(e2e): retry transient hosted inference Signed-off-by: Carlos Villela --- .../onboard/compatible-endpoint-smoke.test.ts | 57 +++++++++++++++++ src/lib/onboard/compatible-endpoint-smoke.ts | 62 ++++++++++++++----- test/e2e/test-hermes-e2e.sh | 37 +++++++---- 3 files changed, 128 insertions(+), 28 deletions(-) diff --git a/src/lib/onboard/compatible-endpoint-smoke.test.ts b/src/lib/onboard/compatible-endpoint-smoke.test.ts index b1cf1189624..ca33ce7cdb0 100644 --- a/src/lib/onboard/compatible-endpoint-smoke.test.ts +++ b/src/lib/onboard/compatible-endpoint-smoke.test.ts @@ -109,6 +109,8 @@ ${bodyForCall} expect(script).toContain("https://inference.local/v1/chat/completions"); expect(script).toContain("INITIAL_MAX_TOKENS=256"); expect(script).toContain("RETRY_MAX_TOKENS=1024"); + expect(script).toContain("SMOKE_ATTEMPTS=3"); + expect(script).toContain("SMOKE_RETRY_DELAY_SECONDS=5"); expect(script).toContain("MODEL='provider/model'\\'''"); }); @@ -131,8 +133,10 @@ fi `, ); const script = buildCompatibleEndpointSandboxSmokeScript(model, { + attempts: 2, configPath, initialMaxTokens: 32, + retryDelaySeconds: 0, retryMaxTokens: 512, }); @@ -145,6 +149,57 @@ fi expect(fs.readFileSync(callFile, "utf-8")).toBe("2"); }); + it("retries a transient non-JSON gateway response", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-compat-smoke-transient-")); + const model = "nvidia/nemotron-3-ultra"; + const configPath = writeSmokeConfig(tmpDir, model); + const { binDir, callFile } = writeFakeCurl( + tmpDir, + String.raw` +if [ "$count" -eq 1 ]; then + printf '%s\n' '504 Gateway Time-out' +else + printf '%s\n' '{"choices":[{"message":{"content":"PONG"},"finish_reason":"stop"}]}' +fi +`, + ); + const script = buildCompatibleEndpointSandboxSmokeScript(model, { + attempts: 3, + configPath, + retryDelaySeconds: 0, + }); + + const result = runSmokeScript(script, tmpDir, binDir); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("INFERENCE_SMOKE_OK PONG"); + expect(result.stderr).toContain("inference.local returned non-JSON response"); + expect(result.stderr).toContain("smoke attempt 1/3 failed; retrying in 0s"); + expect(fs.readFileSync(callFile, "utf-8")).toBe("2"); + }); + + it("fails after the bounded transient retry budget", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-compat-smoke-exhausted-")); + const model = "nvidia/nemotron-3-ultra"; + const configPath = writeSmokeConfig(tmpDir, model); + const { binDir, callFile } = writeFakeCurl( + tmpDir, + "printf '%s\\n' '504 Gateway Time-out'", + ); + const script = buildCompatibleEndpointSandboxSmokeScript(model, { + attempts: 3, + configPath, + retryDelaySeconds: 0, + }); + + const result = runSmokeScript(script, tmpDir, binDir); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("smoke attempt 1/3 failed; retrying in 0s"); + expect(result.stderr).toContain("smoke attempt 2/3 failed; retrying in 0s"); + expect(fs.readFileSync(callFile, "utf-8")).toBe("3"); + }); + it("reports a model-output budget problem when the retry also has no assistant content", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-compat-smoke-no-content-")); const model = "minimaxai/minimax-m2.7"; @@ -158,8 +213,10 @@ JSON `, ); const script = buildCompatibleEndpointSandboxSmokeScript(model, { + attempts: 2, configPath, initialMaxTokens: 32, + retryDelaySeconds: 0, retryMaxTokens: 64, }); diff --git a/src/lib/onboard/compatible-endpoint-smoke.ts b/src/lib/onboard/compatible-endpoint-smoke.ts index 7b1ce9acee7..7ea11290415 100644 --- a/src/lib/onboard/compatible-endpoint-smoke.ts +++ b/src/lib/onboard/compatible-endpoint-smoke.ts @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { StdioOptions } from "node:child_process"; import { shellQuote } from "../core/shell-quote"; import { compactText } from "../core/url-utils"; import { INFERENCE_ROUTE_URL, MANAGED_PROVIDER_ID } from "../inference/config"; -import type { StdioOptions } from "node:child_process"; type CompatibleEndpointSmokeAgent = | { @@ -14,9 +14,11 @@ type CompatibleEndpointSmokeAgent = | undefined; type CompatibleEndpointSandboxSmokeScriptOptions = { + attempts?: number; configPath?: string; inferenceUrl?: string; initialMaxTokens?: number; + retryDelaySeconds?: number; retryMaxTokens?: number; }; @@ -40,6 +42,12 @@ function positiveInt(value: number | undefined, fallback: number): number { return rounded > 0 ? rounded : fallback; } +function nonNegativeInt(value: number | undefined, fallback: number): number { + if (!Number.isFinite(value)) return fallback; + const rounded = Math.floor(Number(value)); + return rounded >= 0 ? rounded : fallback; +} + /** * Returns whether onboarding should validate the compatible endpoint through * the OpenClaw sandbox instead of only checking host-side configuration. @@ -178,6 +186,8 @@ export function buildCompatibleEndpointSandboxSmokeScript( const configPath = options.configPath || "/sandbox/.openclaw/openclaw.json"; const inferenceUrl = options.inferenceUrl || `${INFERENCE_ROUTE_URL}/chat/completions`; const initialMaxTokens = positiveInt(options.initialMaxTokens, 256); + const attempts = positiveInt(options.attempts, 3); + const retryDelaySeconds = nonNegativeInt(options.retryDelaySeconds, 5); const retryMaxTokens = positiveInt(options.retryMaxTokens, 1024); return ` @@ -187,6 +197,8 @@ CONFIG=${shellQuote(configPath)} INFERENCE_URL=${shellQuote(inferenceUrl)} INITIAL_MAX_TOKENS=${initialMaxTokens} RETRY_MAX_TOKENS=${retryMaxTokens} +SMOKE_ATTEMPTS=${attempts} +SMOKE_RETRY_DELAY_SECONDS=${retryDelaySeconds} python3 - "$CONFIG" "$MODEL" <<'PYCFG' import json @@ -257,7 +269,7 @@ run_smoke_request() { rc=$? printf 'curl exit %s: ' "$rc" >&2 cat "$error_file" >&2 - exit "$rc" + return "$rc" } } @@ -317,20 +329,38 @@ print("INFERENCE_SMOKE_OK " + content.strip()[:200]) PYRESP } -write_payload "$INITIAL_MAX_TOKENS" -run_smoke_request -status=0 -check_response initial "$INITIAL_MAX_TOKENS" 1 || status=$? -if [ "$status" -eq 0 ]; then - exit 0 -fi -if [ "$status" -ne 2 ]; then - exit "$status" -fi - -write_payload "$RETRY_MAX_TOKENS" -run_smoke_request -check_response retry "$RETRY_MAX_TOKENS" 0 +attempt=1 +while [ "$attempt" -le "$SMOKE_ATTEMPTS" ]; do + max_tokens="$RETRY_MAX_TOKENS" + attempt_label=retry + if [ "$attempt" -eq 1 ]; then + max_tokens="$INITIAL_MAX_TOKENS" + attempt_label=initial + fi + + write_payload "$max_tokens" + status=0 + run_smoke_request || status=$? + if [ "$status" -eq 0 ]; then + can_retry=0 + if [ "$attempt" -lt "$SMOKE_ATTEMPTS" ]; then + can_retry=1 + fi + check_response "$attempt_label" "$max_tokens" "$can_retry" || status=$? + fi + if [ "$status" -eq 0 ]; then + exit 0 + fi + if [ "$attempt" -ge "$SMOKE_ATTEMPTS" ]; then + exit "$status" + fi + if [ "$status" -ne 2 ]; then + printf 'inference.local smoke attempt %s/%s failed; retrying in %ss\n' \ + "$attempt" "$SMOKE_ATTEMPTS" "$SMOKE_RETRY_DELAY_SECONDS" >&2 + fi + sleep "$SMOKE_RETRY_DELAY_SECONDS" + attempt=$((attempt + 1)) +done `.trim(); } diff --git a/test/e2e/test-hermes-e2e.sh b/test/e2e/test-hermes-e2e.sh index e0589eea2d9..66e89d78994 100755 --- a/test/e2e/test-hermes-e2e.sh +++ b/test/e2e/test-hermes-e2e.sh @@ -610,21 +610,34 @@ section "Phase 5: Live inference" # ── Test 5a: Direct hosted inference endpoint ── info "[LIVE] Direct API test → ${HOSTED_INFERENCE_BASE_URL}..." -api_response=$(curl -s --max-time 30 \ - -X POST "${HOSTED_INFERENCE_BASE_URL}/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $HOSTED_INFERENCE_KEY" \ - -d "$(printf '{"model":"%s","messages":[{"role":"user","content":"Reply with exactly one word: PONG"}],"max_tokens":100}' "$HOSTED_INFERENCE_MODEL")" 2>/dev/null) || true - -if [ -n "$api_response" ]; then - api_content=$(echo "$api_response" | parse_chat_content 2>/dev/null) || true +api_response="" +api_content="" +for attempt in 1 2 3; do + max_tokens=1024 + if [ "$attempt" -eq 1 ]; then + max_tokens=256 + fi + api_response="$(curl -sS --max-time 90 \ + -X POST "${HOSTED_INFERENCE_BASE_URL}/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $HOSTED_INFERENCE_KEY" \ + -d "$(printf '{"model":"%s","messages":[{"role":"user","content":"Reply with exactly one word: PONG"}],"max_tokens":%s}' "$HOSTED_INFERENCE_MODEL" "$max_tokens")" 2>/dev/null || true)" + api_content="$(printf '%s' "$api_response" | parse_chat_content 2>/dev/null || true)" if grep -qi "PONG" <<<"$api_content"; then - pass "[LIVE] Direct API: model responded with PONG" - else - fail "[LIVE] Direct API: expected PONG, got: ${api_content:0:200}" + break fi + if [ "$attempt" -lt 3 ]; then + info "[LIVE] Direct API attempt ${attempt}/3 did not return PONG; retrying..." + sleep $((5 * attempt)) + fi +done + +if grep -qi "PONG" <<<"$api_content"; then + pass "[LIVE] Direct API: model responded with PONG" +elif [ -n "$api_response" ]; then + fail "[LIVE] Direct API: expected PONG after 3 attempts, got: ${api_content:0:200}" else - fail "[LIVE] Direct API: empty response from curl" + fail "[LIVE] Direct API: empty response from curl after 3 attempts" fi # ── Test 5b: Inference through the sandbox (THE definitive test) ── From 4c5abf1bbfb4cb8c94f10dc6238196c68fc96b58 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 08:15:44 -0700 Subject: [PATCH 2/3] fix(onboard): bound smoke retry diagnostics Signed-off-by: Carlos Villela --- .../onboard/compatible-endpoint-smoke.test.ts | 32 ++++++++++++++++++- src/lib/onboard/compatible-endpoint-smoke.ts | 31 +++++++++++++++--- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/lib/onboard/compatible-endpoint-smoke.test.ts b/src/lib/onboard/compatible-endpoint-smoke.test.ts index ca33ce7cdb0..b21b26ad4d6 100644 --- a/src/lib/onboard/compatible-endpoint-smoke.test.ts +++ b/src/lib/onboard/compatible-endpoint-smoke.test.ts @@ -17,6 +17,7 @@ import { buildCompatibleEndpointSandboxSmokeScript, shouldRunCompatibleEndpointSandboxSmoke, spawnOutputToString, + verifyCompatibleEndpointSandboxSmoke, } from "./compatible-endpoint-smoke"; describe("compatible endpoint sandbox smoke helpers", () => { @@ -100,6 +101,31 @@ ${bodyForCall} expect(spawnOutputToString(42)).toBe("42"); }); + it("budgets the host command timeout for every retry attempt", () => { + const runOpenshell = vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: "provider ready" }) + .mockReturnValueOnce({ + status: 0, + stdout: "OPENCLAW_CONFIG_OK\nINFERENCE_SMOKE_OK PONG", + }); + + verifyCompatibleEndpointSandboxSmoke({ + sandboxName: "smoke-sandbox", + provider: "compatible-endpoint", + model: "nvidia/nemotron-3-ultra", + runOpenshell, + redact: (value) => value, + messagingChannels: ["telegram"], + }); + + expect(runOpenshell).toHaveBeenNthCalledWith( + 2, + expect.any(Array), + expect.objectContaining({ timeout: 220_000 }), + ); + }); + it("builds a sandbox script that checks managed provider routing", () => { const script = buildCompatibleEndpointSandboxSmokeScript("provider/model'"); @@ -110,6 +136,7 @@ ${bodyForCall} expect(script).toContain("INITIAL_MAX_TOKENS=256"); expect(script).toContain("RETRY_MAX_TOKENS=1024"); expect(script).toContain("SMOKE_ATTEMPTS=3"); + expect(script).toContain("SMOKE_REQUEST_TIMEOUT_SECONDS=60"); expect(script).toContain("SMOKE_RETRY_DELAY_SECONDS=5"); expect(script).toContain("MODEL='provider/model'\\'''"); }); @@ -184,7 +211,7 @@ fi const configPath = writeSmokeConfig(tmpDir, model); const { binDir, callFile } = writeFakeCurl( tmpDir, - "printf '%s\\n' '504 Gateway Time-out'", + "printf '%s\\n' '504 Gateway Time-outAuthorization: Bearer test-secret'", ); const script = buildCompatibleEndpointSandboxSmokeScript(model, { attempts: 3, @@ -195,6 +222,9 @@ fi const result = runSmokeScript(script, tmpDir, binDir); expect(result.status).toBe(1); + expect(result.stderr).toContain("http_status=504"); + expect(result.stderr).toContain("response_bytes="); + expect(result.stderr).not.toContain("test-secret"); expect(result.stderr).toContain("smoke attempt 1/3 failed; retrying in 0s"); expect(result.stderr).toContain("smoke attempt 2/3 failed; retrying in 0s"); expect(fs.readFileSync(callFile, "utf-8")).toBe("3"); diff --git a/src/lib/onboard/compatible-endpoint-smoke.ts b/src/lib/onboard/compatible-endpoint-smoke.ts index 7ea11290415..767f6590391 100644 --- a/src/lib/onboard/compatible-endpoint-smoke.ts +++ b/src/lib/onboard/compatible-endpoint-smoke.ts @@ -32,6 +32,16 @@ type CompatibleEndpointSmokeRun = ( }, ) => { status: number | null; stdout?: unknown; stderr?: unknown }; +const COMPATIBLE_ENDPOINT_SMOKE_ATTEMPTS = 3; +const COMPATIBLE_ENDPOINT_SMOKE_REQUEST_TIMEOUT_SECONDS = 60; +const COMPATIBLE_ENDPOINT_SMOKE_RETRY_DELAY_SECONDS = 5; +const COMPATIBLE_ENDPOINT_SMOKE_COMMAND_OVERHEAD_SECONDS = 30; +const COMPATIBLE_ENDPOINT_SMOKE_COMMAND_TIMEOUT_MS = + (COMPATIBLE_ENDPOINT_SMOKE_ATTEMPTS * COMPATIBLE_ENDPOINT_SMOKE_REQUEST_TIMEOUT_SECONDS + + (COMPATIBLE_ENDPOINT_SMOKE_ATTEMPTS - 1) * COMPATIBLE_ENDPOINT_SMOKE_RETRY_DELAY_SECONDS + + COMPATIBLE_ENDPOINT_SMOKE_COMMAND_OVERHEAD_SECONDS) * + 1000; + /** * Normalizes optional token-budget overrides while preserving safe defaults for * the generated sandbox smoke script. @@ -154,7 +164,7 @@ export function verifyCompatibleEndpointSandboxSmoke(options: { ignoreError: true, suppressOutput: true, stdio: ["ignore", "pipe", "pipe"], - timeout: 90_000, + timeout: COMPATIBLE_ENDPOINT_SMOKE_COMMAND_TIMEOUT_MS, }, ); const smokeOutput = [ @@ -186,8 +196,11 @@ export function buildCompatibleEndpointSandboxSmokeScript( const configPath = options.configPath || "/sandbox/.openclaw/openclaw.json"; const inferenceUrl = options.inferenceUrl || `${INFERENCE_ROUTE_URL}/chat/completions`; const initialMaxTokens = positiveInt(options.initialMaxTokens, 256); - const attempts = positiveInt(options.attempts, 3); - const retryDelaySeconds = nonNegativeInt(options.retryDelaySeconds, 5); + const attempts = positiveInt(options.attempts, COMPATIBLE_ENDPOINT_SMOKE_ATTEMPTS); + const retryDelaySeconds = nonNegativeInt( + options.retryDelaySeconds, + COMPATIBLE_ENDPOINT_SMOKE_RETRY_DELAY_SECONDS, + ); const retryMaxTokens = positiveInt(options.retryMaxTokens, 1024); return ` @@ -198,6 +211,7 @@ INFERENCE_URL=${shellQuote(inferenceUrl)} INITIAL_MAX_TOKENS=${initialMaxTokens} RETRY_MAX_TOKENS=${retryMaxTokens} SMOKE_ATTEMPTS=${attempts} +SMOKE_REQUEST_TIMEOUT_SECONDS=${COMPATIBLE_ENDPOINT_SMOKE_REQUEST_TIMEOUT_SECONDS} SMOKE_RETRY_DELAY_SECONDS=${retryDelaySeconds} python3 - "$CONFIG" "$MODEL" <<'PYCFG' @@ -262,7 +276,7 @@ PYPAYLOAD } run_smoke_request() { - curl -sS --connect-timeout 10 --max-time 60 \ + curl -sS --connect-timeout 10 --max-time "$SMOKE_REQUEST_TIMEOUT_SECONDS" \ "$INFERENCE_URL" \ -H "Content-Type: application/json" \ -d "@$payload_file" >"$response_file" 2>"$error_file" || { @@ -276,6 +290,7 @@ run_smoke_request() { check_response() { python3 - "$response_file" "$1" "$2" "$3" <<'PYRESP' import json +import re import sys path = sys.argv[1] @@ -292,7 +307,13 @@ except Exception as exc: body = f.read(1000) except Exception: pass - print("inference.local returned non-JSON response: %s; body=%s" % (exc, body), file=sys.stderr) + status_match = re.search(r"<(?:title|h1)>\\s*([1-5][0-9][0-9])\\b", body, re.IGNORECASE) + status_detail = "; http_status=%s" % status_match.group(1) if status_match else "" + print( + "inference.local returned non-JSON response: %s; response_bytes=%s%s" + % (exc, len(body.encode("utf-8", errors="replace")), status_detail), + file=sys.stderr, + ) sys.exit(1) choices = data.get("choices") From cb65f819621b0d73fe78768e57aced8714032daa Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 08:20:24 -0700 Subject: [PATCH 3/3] fix(onboard): retry only transient smoke failures Signed-off-by: Carlos Villela --- .../onboard/compatible-endpoint-smoke.test.ts | 22 +++++++++++++++++++ src/lib/onboard/compatible-endpoint-smoke.ts | 14 ++++++++++-- test/e2e/test-hermes-e2e.sh | 16 +++++++++----- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/lib/onboard/compatible-endpoint-smoke.test.ts b/src/lib/onboard/compatible-endpoint-smoke.test.ts index b21b26ad4d6..57679c5dd70 100644 --- a/src/lib/onboard/compatible-endpoint-smoke.test.ts +++ b/src/lib/onboard/compatible-endpoint-smoke.test.ts @@ -205,6 +205,28 @@ fi expect(fs.readFileSync(callFile, "utf-8")).toBe("2"); }); + it("does not retry a permanent JSON response validation failure", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-compat-smoke-permanent-")); + const model = "nvidia/nemotron-3-ultra"; + const configPath = writeSmokeConfig(tmpDir, model); + const { binDir, callFile } = writeFakeCurl( + tmpDir, + `printf '%s\\n' '{"error":{"message":"invalid model"}}'`, + ); + const script = buildCompatibleEndpointSandboxSmokeScript(model, { + attempts: 3, + configPath, + retryDelaySeconds: 0, + }); + + const result = runSmokeScript(script, tmpDir, binDir); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("did not contain non-empty choices[0].message.content"); + expect(result.stderr).not.toContain("retrying in"); + expect(fs.readFileSync(callFile, "utf-8")).toBe("1"); + }); + it("fails after the bounded transient retry budget", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-compat-smoke-exhausted-")); const model = "nvidia/nemotron-3-ultra"; diff --git a/src/lib/onboard/compatible-endpoint-smoke.ts b/src/lib/onboard/compatible-endpoint-smoke.ts index 767f6590391..40de9bec26e 100644 --- a/src/lib/onboard/compatible-endpoint-smoke.ts +++ b/src/lib/onboard/compatible-endpoint-smoke.ts @@ -314,7 +314,10 @@ except Exception as exc: % (exc, len(body.encode("utf-8", errors="replace")), status_detail), file=sys.stderr, ) - sys.exit(1) + retryable_gateway_error = bool( + status_match and 500 <= int(status_match.group(1)) <= 599 + ) + sys.exit(3 if can_retry and retryable_gateway_error else 1) choices = data.get("choices") choice = choices[0] if isinstance(choices, list) and choices and isinstance(choices[0], dict) else {} @@ -361,7 +364,11 @@ while [ "$attempt" -le "$SMOKE_ATTEMPTS" ]; do write_payload "$max_tokens" status=0 - run_smoke_request || status=$? + request_failed=0 + run_smoke_request || { + status=$? + request_failed=1 + } if [ "$status" -eq 0 ]; then can_retry=0 if [ "$attempt" -lt "$SMOKE_ATTEMPTS" ]; then @@ -372,6 +379,9 @@ while [ "$attempt" -le "$SMOKE_ATTEMPTS" ]; do if [ "$status" -eq 0 ]; then exit 0 fi + if [ "$request_failed" -eq 0 ] && [ "$status" -ne 2 ] && [ "$status" -ne 3 ]; then + exit "$status" + fi if [ "$attempt" -ge "$SMOKE_ATTEMPTS" ]; then exit "$status" fi diff --git a/test/e2e/test-hermes-e2e.sh b/test/e2e/test-hermes-e2e.sh index 66e89d78994..482d8c5205a 100755 --- a/test/e2e/test-hermes-e2e.sh +++ b/test/e2e/test-hermes-e2e.sh @@ -612,32 +612,38 @@ section "Phase 5: Live inference" info "[LIVE] Direct API test → ${HOSTED_INFERENCE_BASE_URL}..." api_response="" api_content="" +api_failure_summary="" for attempt in 1 2 3; do max_tokens=1024 if [ "$attempt" -eq 1 ]; then max_tokens=256 fi - api_response="$(curl -sS --max-time 90 \ + api_response_file="$(mktemp)" + api_http_status="$(curl -sS --max-time 90 \ + -o "$api_response_file" \ + -w "%{http_code}" \ -X POST "${HOSTED_INFERENCE_BASE_URL}/chat/completions" \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $HOSTED_INFERENCE_KEY" \ -d "$(printf '{"model":"%s","messages":[{"role":"user","content":"Reply with exactly one word: PONG"}],"max_tokens":%s}' "$HOSTED_INFERENCE_MODEL" "$max_tokens")" 2>/dev/null || true)" + api_response="$(cat "$api_response_file" 2>/dev/null || true)" + rm -f "$api_response_file" + [ -n "$api_http_status" ] || api_http_status="000" api_content="$(printf '%s' "$api_response" | parse_chat_content 2>/dev/null || true)" + api_failure_summary="response without PONG (HTTP ${api_http_status: -3}, chars=${#api_response})" if grep -qi "PONG" <<<"$api_content"; then break fi if [ "$attempt" -lt 3 ]; then - info "[LIVE] Direct API attempt ${attempt}/3 did not return PONG; retrying..." + info "[LIVE] Direct API attempt ${attempt}/3 ${api_failure_summary}; retrying..." sleep $((5 * attempt)) fi done if grep -qi "PONG" <<<"$api_content"; then pass "[LIVE] Direct API: model responded with PONG" -elif [ -n "$api_response" ]; then - fail "[LIVE] Direct API: expected PONG after 3 attempts, got: ${api_content:0:200}" else - fail "[LIVE] Direct API: empty response from curl after 3 attempts" + fail "[LIVE] Direct API: expected PONG after 3 attempts; ${api_failure_summary}" fi # ── Test 5b: Inference through the sandbox (THE definitive test) ──