From 6d659f0a0d47dad5ab4566fc6372376c84d29324 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 14 May 2026 07:54:07 -0500 Subject: [PATCH 1/2] fix(onboard): retry compatible smoke after reasoning-only output --- .../onboard/compatible-endpoint-smoke.test.ts | 119 ++++++++++++++++++ src/lib/onboard/compatible-endpoint-smoke.ts | 106 +++++++++++++--- test/onboard.test.ts | 3 +- 3 files changed, 209 insertions(+), 19 deletions(-) diff --git a/src/lib/onboard/compatible-endpoint-smoke.test.ts b/src/lib/onboard/compatible-endpoint-smoke.test.ts index dd57b579717..cce1ab0cf67 100644 --- a/src/lib/onboard/compatible-endpoint-smoke.test.ts +++ b/src/lib/onboard/compatible-endpoint-smoke.test.ts @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it, vi } from "vitest"; vi.mock("../inference/config", () => ({ @@ -16,6 +20,60 @@ import { } from "./compatible-endpoint-smoke"; describe("compatible endpoint sandbox smoke helpers", () => { + function writeSmokeConfig(tmpDir: string, model: string): string { + const configDir = path.join(tmpDir, ".openclaw"); + fs.mkdirSync(configDir, { recursive: true }); + const configPath = path.join(configDir, "openclaw.json"); + fs.writeFileSync( + configPath, + JSON.stringify({ + agents: { defaults: { model: { primary: `inference/${model}` } } }, + models: { + providers: { + inference: { + baseUrl: "https://inference.local/v1", + apiKey: "unused", + }, + }, + }, + }), + ); + return configPath; + } + + function writeFakeCurl(tmpDir: string, bodyForCall: string): { binDir: string; callFile: string } { + const binDir = path.join(tmpDir, "bin"); + const callFile = path.join(tmpDir, "curl-calls"); + fs.mkdirSync(binDir, { recursive: true }); + fs.writeFileSync( + path.join(binDir, "curl"), + `#!/usr/bin/env bash +set -eu +call_file="${callFile}" +count=0 +if [ -f "$call_file" ]; then + count="$(cat "$call_file")" +fi +count=$((count + 1)) +printf '%s' "$count" >"$call_file" +${bodyForCall} +`, + { mode: 0o755 }, + ); + return { binDir, callFile }; + } + + function runSmokeScript(script: string, tmpDir: string, binDir: string) { + return spawnSync("sh", ["-c", script], { + cwd: tmpDir, + encoding: "utf-8", + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH || ""}`, + }, + }); + } + it("runs only for OpenClaw compatible-endpoint sandboxes with messaging", () => { expect(shouldRunCompatibleEndpointSandboxSmoke("compatible-endpoint", ["telegram"])).toBe( true, @@ -48,9 +106,70 @@ describe("compatible endpoint sandbox smoke helpers", () => { expect(script).toContain("INFERENCE_SMOKE_OK"); expect(script).toContain("models.providers.inference"); expect(script).toContain("https://inference.local/v1/chat/completions"); + expect(script).toContain("INITIAL_MAX_TOKENS=256"); + expect(script).toContain("RETRY_MAX_TOKENS=1024"); expect(script).toContain("MODEL='provider/model'\\'''"); }); + it("retries a reasoning-only length response before failing the sandbox smoke", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-compat-smoke-reasoning-")); + const model = "minimaxai/minimax-m2.7"; + const configPath = writeSmokeConfig(tmpDir, model); + const { binDir, callFile } = writeFakeCurl( + tmpDir, + String.raw` +if [ "$count" -eq 1 ]; then + cat <<'JSON' +{"id":"82f5ff","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":null,"reasoning_content":"The user asked for PONG."},"finish_reason":"length"}],"usage":{"completion_tokens":32,"reasoning_tokens":32}} +JSON +else + cat <<'JSON' +{"id":"82f5ff","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"PONG"},"finish_reason":"stop"}]} +JSON +fi +`, + ); + const script = buildCompatibleEndpointSandboxSmokeScript(model, { + configPath, + initialMaxTokens: 32, + retryMaxTokens: 512, + }); + + const result = runSmokeScript(script, tmpDir, binDir); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("OPENCLAW_CONFIG_OK"); + expect(result.stdout).toContain("INFERENCE_SMOKE_OK PONG"); + expect(result.stderr).toContain("exhausted max_tokens=32 in reasoning_content"); + expect(fs.readFileSync(callFile, "utf-8")).toBe("2"); + }); + + it("reports a model-output budget problem when the retry also has no assistant content", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-compat-smoke-no-content-")); + const model = "minimaxai/minimax-m2.7"; + const configPath = writeSmokeConfig(tmpDir, model); + const { binDir, callFile } = writeFakeCurl( + tmpDir, + String.raw` +cat <<'JSON' +{"id":"82f5ff","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":null,"reasoning_content":"Still reasoning."},"finish_reason":"length"}],"usage":{"completion_tokens":32,"reasoning_tokens":32}} +JSON +`, + ); + const script = buildCompatibleEndpointSandboxSmokeScript(model, { + configPath, + initialMaxTokens: 32, + retryMaxTokens: 64, + }); + + const result = runSmokeScript(script, tmpDir, binDir); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("initial smoke attempt exhausted max_tokens=32"); + expect(result.stderr).toContain("retry smoke attempt still exhausted max_tokens=64"); + expect(fs.readFileSync(callFile, "utf-8")).toBe("2"); + }); + it("wraps the script as a base64 decoded temporary shell command", () => { const command = buildCompatibleEndpointSandboxSmokeCommand("nvidia/model"); diff --git a/src/lib/onboard/compatible-endpoint-smoke.ts b/src/lib/onboard/compatible-endpoint-smoke.ts index 2740cc04586..cb4e747367e 100644 --- a/src/lib/onboard/compatible-endpoint-smoke.ts +++ b/src/lib/onboard/compatible-endpoint-smoke.ts @@ -8,6 +8,19 @@ type CompatibleEndpointSmokeAgent = { name?: string | null; } | null | undefined; +type CompatibleEndpointSandboxSmokeScriptOptions = { + configPath?: string; + inferenceUrl?: string; + initialMaxTokens?: number; + retryMaxTokens?: number; +}; + +function positiveInt(value: number | undefined, fallback: number): number { + if (!Number.isFinite(value)) return fallback; + const rounded = Math.floor(Number(value)); + return rounded > 0 ? rounded : fallback; +} + export function shouldRunCompatibleEndpointSandboxSmoke( provider: string | null | undefined, messagingChannels: string[] | null | undefined, @@ -29,11 +42,22 @@ export function spawnOutputToString(value: unknown): string { return String(value); } -export function buildCompatibleEndpointSandboxSmokeScript(model: string): string { +export function buildCompatibleEndpointSandboxSmokeScript( + model: string, + options: CompatibleEndpointSandboxSmokeScriptOptions = {}, +): string { + const configPath = options.configPath || "/sandbox/.openclaw/openclaw.json"; + const inferenceUrl = options.inferenceUrl || `${INFERENCE_ROUTE_URL}/chat/completions`; + const initialMaxTokens = positiveInt(options.initialMaxTokens, 256); + const retryMaxTokens = positiveInt(options.retryMaxTokens, 1024); + return ` set -eu MODEL=${shellQuote(model)} -CONFIG=/sandbox/.openclaw/openclaw.json +CONFIG=${shellQuote(configPath)} +INFERENCE_URL=${shellQuote(inferenceUrl)} +INITIAL_MAX_TOKENS=${initialMaxTokens} +RETRY_MAX_TOKENS=${retryMaxTokens} python3 - "$CONFIG" "$MODEL" <<'PYCFG' import json @@ -79,35 +103,44 @@ response_file="$(mktemp)" error_file="$(mktemp)" trap 'rm -f "$payload_file" "$response_file" "$error_file"' EXIT -python3 - "$MODEL" >"$payload_file" <<'PYPAYLOAD' +write_payload() { + python3 - "$MODEL" "$1" >"$payload_file" <<'PYPAYLOAD' import json import sys model = sys.argv[1] +max_tokens = int(sys.argv[2]) print(json.dumps({ "model": model, "messages": [ {"role": "user", "content": "Reply with exactly: PONG"} ], - "max_tokens": 32, + "max_tokens": max_tokens, })) PYPAYLOAD +} -curl -sS --connect-timeout 10 --max-time 60 \ - "${INFERENCE_ROUTE_URL}/chat/completions" \ +run_smoke_request() { + curl -sS --connect-timeout 10 --max-time 60 \ + "$INFERENCE_URL" \ -H "Content-Type: application/json" \ -d "@$payload_file" >"$response_file" 2>"$error_file" || { - rc=$? - printf 'curl exit %s: ' "$rc" >&2 - cat "$error_file" >&2 - exit "$rc" + rc=$? + printf 'curl exit %s: ' "$rc" >&2 + cat "$error_file" >&2 + exit "$rc" + } } -python3 - "$response_file" <<'PYRESP' +check_response() { + python3 - "$response_file" "$1" "$2" "$3" <<'PYRESP' import json import sys path = sys.argv[1] +attempt = sys.argv[2] +max_tokens = sys.argv[3] +can_retry = sys.argv[4] == "1" try: with open(path, "r", encoding="utf-8") as f: data = json.load(f) @@ -121,18 +154,55 @@ except Exception as exc: print("inference.local returned non-JSON response: %s; body=%s" % (exc, body), file=sys.stderr) sys.exit(1) -content = ( - data.get("choices", [{}])[0] - .get("message", {}) - .get("content") -) +choices = data.get("choices") +choice = choices[0] if isinstance(choices, list) and choices and isinstance(choices[0], dict) else {} +message = choice.get("message") if isinstance(choice.get("message"), dict) else {} +content = message.get("content") if not isinstance(content, str) or not content.strip(): - print("inference.local response did not contain choices[0].message.content: %s" % json.dumps(data)[:1000], file=sys.stderr) + finish_reason = choice.get("finish_reason") + reasoning_content = message.get("reasoning_content") + if not isinstance(reasoning_content, str) or not reasoning_content.strip(): + reasoning_content = message.get("reasoning") + if finish_reason == "length" and isinstance(reasoning_content, str) and reasoning_content.strip(): + if can_retry: + print( + "inference.local reached the model, but the %s smoke attempt exhausted max_tokens=%s in reasoning_content before emitting choices[0].message.content; retrying with a larger smoke budget" + % (attempt, max_tokens), + file=sys.stderr, + ) + sys.exit(2) + print( + "inference.local reached the model, but the %s smoke attempt still exhausted max_tokens=%s in reasoning_content before emitting choices[0].message.content: %s" + % (attempt, max_tokens, json.dumps(data)[:1000]), + file=sys.stderr, + ) + sys.exit(1) + print( + "inference.local response did not contain non-empty choices[0].message.content (finish_reason=%r): %s" + % (finish_reason, json.dumps(data)[:1000]), + file=sys.stderr, + ) sys.exit(1) print("INFERENCE_SMOKE_OK " + content.strip()[:200]) PYRESP -`.trim(); +} + +write_payload "$INITIAL_MAX_TOKENS" +run_smoke_request +status=0 +check_response initial "$INITIAL_MAX_TOKENS" 1 || status=$? +if [ "$status" -eq 0 ]; then + exit 0 +fi +if [ "$status" -ne 2 ]; then + exit "$status" +fi + +write_payload "$RETRY_MAX_TOKENS" +run_smoke_request +check_response retry "$RETRY_MAX_TOKENS" 0 + `.trim(); } export function buildCompatibleEndpointSandboxSmokeCommand(model: string): string { diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 770ee6d374e..103bf3a7bea 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -885,7 +885,8 @@ network_policies: assert.match(script, /https:\/\/inference\.local\/v1/); assert.match(script, /apiKey.*unused/); assert.match(script, /agents\.defaults\.model\.primary/); - assert.match(script, /curl[\s\S]*\/chat\/completions/); + assert.match(script, /INFERENCE_URL=.*\/chat\/completions/); + assert.match(script, /curl[\s\S]*"\$INFERENCE_URL"/); assert.doesNotMatch(script, /COMPATIBLE_API_KEY/); assert.doesNotMatch(script, /api\.deepinfra\.com/); }); From b6e1c0e7c9d9315bbb363432b0c1d3bb26ca5bd9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 14 May 2026 12:14:51 -0500 Subject: [PATCH 2/2] docs(onboard): document compatible smoke helpers --- src/lib/onboard/compatible-endpoint-smoke.ts | 21 ++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/lib/onboard/compatible-endpoint-smoke.ts b/src/lib/onboard/compatible-endpoint-smoke.ts index cb4e747367e..880c665dcef 100644 --- a/src/lib/onboard/compatible-endpoint-smoke.ts +++ b/src/lib/onboard/compatible-endpoint-smoke.ts @@ -15,12 +15,20 @@ type CompatibleEndpointSandboxSmokeScriptOptions = { retryMaxTokens?: number; }; +/** + * Normalizes optional token-budget overrides while preserving safe defaults for + * the generated sandbox smoke script. + */ function positiveInt(value: number | undefined, fallback: number): number { if (!Number.isFinite(value)) return fallback; const rounded = Math.floor(Number(value)); return rounded > 0 ? rounded : fallback; } +/** + * Returns whether onboarding should validate the compatible endpoint through + * the OpenClaw sandbox instead of only checking host-side configuration. + */ export function shouldRunCompatibleEndpointSandboxSmoke( provider: string | null | undefined, messagingChannels: string[] | null | undefined, @@ -35,6 +43,10 @@ export function shouldRunCompatibleEndpointSandboxSmoke( ); } +/** + * Converts child-process output into text for diagnostics without assuming + * whether Node returned strings, buffers, nulls, or primitive values. + */ export function spawnOutputToString(value: unknown): string { if (typeof value === "string") return value; if (Buffer.isBuffer(value)) return value.toString("utf-8"); @@ -42,6 +54,11 @@ export function spawnOutputToString(value: unknown): string { return String(value); } +/** + * Builds the shell script that runs inside the sandbox to confirm OpenClaw is + * routed through NemoClaw's managed inference provider and can receive assistant + * content from the compatible endpoint. + */ export function buildCompatibleEndpointSandboxSmokeScript( model: string, options: CompatibleEndpointSandboxSmokeScriptOptions = {}, @@ -205,6 +222,10 @@ check_response retry "$RETRY_MAX_TOKENS" 0 `.trim(); } +/** + * Wraps the sandbox smoke script as a one-line command suitable for execution + * through the existing OpenShell command path. + */ export function buildCompatibleEndpointSandboxSmokeCommand(model: string): string { const script = buildCompatibleEndpointSandboxSmokeScript(model); const encoded = Buffer.from(script, "utf8").toString("base64");