diff --git a/package.json b/package.json index 2f40073d94c..a88235a6ef1 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "test:imports:check": "tsx scripts/checks/no-test-dist-imports.ts", "test:projects:check": "tsx scripts/checks/vitest-project-overlap.ts", "test:titles:check": "tsx scripts/checks/test-title-style.ts", + "bench": "tsx scripts/bench/run.ts", "check": "npx prek run --all-files", "checks": "tsx scripts/checks/run.ts", "lint": "npx @biomejs/biome lint . && npm run checks", diff --git a/scripts/bench/README.md b/scripts/bench/README.md new file mode 100644 index 00000000000..0977d9577d9 --- /dev/null +++ b/scripts/bench/README.md @@ -0,0 +1,120 @@ + + +# NemoClaw value benchmark + +A small, developer- and agent-runnable benchmark that answers "is NemoClaw fast +enough on this machine?". It measures core first-use and inference-path timings +and emits both machine-readable JSON and a concise Markdown value report. + +It addresses [#5604](https://github.com/NVIDIA/NemoClaw/issues/5604). v1 is +deliberately **advisory**: it does not ship owner-approved pass/warn/fail +thresholds (those are tracked by #3776), so the numbers are for comparing runs, +not for gating. + +> The harness only sends requests to the inference endpoint you configure. It +> never uploads results or sends telemetry to any external service. + +The configured endpoint must use HTTPS, except that HTTP is allowed for loopback +hosts (`localhost`, `127.0.0.0/8`, and `::1`) so local inference stays easy to +benchmark. URL userinfo is rejected. Redirects are refused, query values are +redacted from shareable reports, remote error bodies are never copied into +reports, and a successful sample must contain a valid OpenAI-compatible chat +completion rather than an arbitrary HTTP 2xx body. + +## Metrics + +| Metric | Source | Notes | +|--------|--------|-------| +| `inference-round-trip` | live request | Times N OpenAI-compatible `/v1/chat/completions` calls (warm-up + samples), reports min/median/p95/mean/max. | +| `sandbox-cold-start` | onboard trace | Total duration of the emitted `nemoclaw.onboard.phase.sandbox` span, which encloses sandbox creation and readiness. The nested `nemoclaw.sandbox.readiness_wait` span is reported as an optional breakdown without being added twice. | +| `policy-shield-overhead` | onboard trace | Marked `unsupported` in v1: the available `nemoclaw.policy.application` span measures setup, not request-path shield overhead. Interactive traces can also include human think time. | + +Trace metrics require a completed NemoClaw onboard trace with successful root +and metric spans. A valid trace without a selected metric reports that metric as +`unsupported`; a malformed trace or failed metric span reports `error` and exits +non-zero. + +## Prerequisites + +- Node `>=22.16` (`tsx` is a dev dependency; run via `npm`/`npx`). +- An OpenAI-compatible inference endpoint and model you can reach from the host + (e.g. an NVIDIA endpoint, a local vLLM/Ollama server, or — from inside a + sandbox — `https://inference.local/v1`). +- The API key in `OPENAI_API_KEY` or `NVIDIA_INFERENCE_API_KEY` (the value is + never passed as a flag). Put a compatible provider's key in one of these + benchmark-specific names rather than selecting an unrelated process secret. +- Optional: an onboard trace artifact for the sandbox/policy metrics. Produce one + by running `NEMOCLAW_TRACE=1 nemoclaw onboard --non-interactive ...`; the trace + file path is printed and also controlled by `NEMOCLAW_TRACE_FILE` / + `NEMOCLAW_TRACE_DIR`. Non-interactive collection provides more comparable + context; request-path policy overhead remains unsupported until dedicated + instrumentation exists. + +## Usage + +One documented command produces both outputs: + +```bash +export OPENAI_API_KEY=... # or NVIDIA_INFERENCE_API_KEY +npm run bench -- \ + --base-url https://integrate.api.nvidia.com/v1 \ + --model nvidia/nemotron-3-super-120b-a12b \ + --samples 10 \ + --json bench-result.json +``` + +This prints the Markdown report to stdout and writes structured JSON to +`bench-result.json`. Add the sandbox/policy metrics by pointing at an onboard +trace: + +```bash +npm run bench -- \ + --base-url https://inference.local/v1 --model \ + --trace .e2e/traces/onboard.json \ + --report bench-report.md --json bench-result.json +``` + +Trace-only run (no live inference): + +```bash +npm run bench -- --no-inference --trace .e2e/traces/onboard.json +``` + +Run `npm run bench -- --help` for all flags. + +## How an agent should use this + +1. Confirm a provider is configured (`nemoclaw status`) and export the key. +2. Run `npm run bench -- --base-url --model --json bench.json`. +3. Read `bench.json` (`schema_version: nemoclaw.bench.v1`). Summarize each + metric's `status` and `stats` (median + p95) and surface any `error`/ + `unsupported` `reason`. Do not present the timings as pass/fail — they are + advisory until thresholds land (#3776). +4. On `error` exit status, report the `reason` and the troubleshooting pointers + from the Markdown report. + +## Output schema (`nemoclaw.bench.v1`) + +```jsonc +{ + "schema_version": "nemoclaw.bench.v1", + "generated_at": "", + "environment": { "os", "arch", "node", "cpus", "cpu_model", "total_mem_gib" }, + "target": { "base_url": "", "model": "...", "api_key_present": true }, + "metrics": [ + { "id": "inference-round-trip", "status": "ok", "unit": "ms", + "source": "live-request", "interpretation": "advisory-non-normative", + "samples": 10, "stats": { "min_ms", "median_ms", "p95_ms", "mean_ms", "max_ms" } } + ] +} +``` + +Trace-backed metrics also include a sanitized `context` object when available +(`provider`, `model`, `agent`, `non_interactive`, and `fresh`) so runs can be +compared without exposing sandbox names or credentials. + +The harness exits non-zero when a selected metric errors, a supplied trace is +invalid, or required prerequisites (endpoint, model, API key) are missing. diff --git a/scripts/bench/lib.ts b/scripts/bench/lib.ts new file mode 100644 index 00000000000..332f7920356 --- /dev/null +++ b/scripts/bench/lib.ts @@ -0,0 +1,448 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Core, side-effect-free building blocks for the NemoClaw value benchmark harness +// (issue #5604). The CLI entry point lives in run.ts; everything here is pure or +// dependency-injected so it can be unit tested without a live sandbox or network. + +import { isIP } from "node:net"; +import os from "node:os"; + +import { redactFull } from "../../src/lib/security/redact"; + +export { + ingestPolicyOverhead, + ingestSandboxColdStart, + POLICY_APPLICATION_SPAN, + SANDBOX_PHASE_SPAN, + SANDBOX_READINESS_SPAN, +} from "./trace-ingest"; + +export const BENCH_SCHEMA_VERSION = "nemoclaw.bench.v1" as const; + +export type MetricId = "inference-round-trip" | "sandbox-cold-start" | "policy-shield-overhead"; +export type MetricStatus = "ok" | "unsupported" | "error"; +export type MetricSource = "live-request" | "trace-artifact" | "none"; + +export interface LatencyStats { + min_ms: number; + median_ms: number; + p95_ms: number; + mean_ms: number; + max_ms: number; +} + +export interface BenchMetric { + id: MetricId; + status: MetricStatus; + unit: "ms"; + source: MetricSource; + // Pass/warn/fail interpretation is deliberately advisory until owners approve + // normative thresholds (issue #5604 / #3776 non-goal). + interpretation: "advisory-non-normative"; + samples?: number; + stats?: LatencyStats; + breakdown?: Record; + context?: BenchMetricContext; + reason?: string; +} + +export interface BenchMetricContext { + provider?: string; + model?: string; + agent?: string; + non_interactive?: boolean; + fresh?: boolean; +} + +export interface BenchEnvironment { + os: string; + arch: string; + node: string; + cpus: number; + cpu_model: string; + total_mem_gib: number; +} + +export interface BenchTarget { + base_url: string; + model: string; + api_key_present: boolean; +} + +export interface BenchReport { + schema_version: typeof BENCH_SCHEMA_VERSION; + generated_at: string; + environment: BenchEnvironment; + target: BenchTarget; + metrics: BenchMetric[]; +} + +export function buildBenchTarget( + baseUrl: string | undefined, + model: string | undefined, + apiKeyPresent: boolean, + knownSecrets: readonly string[] = [], +): BenchTarget { + return { + base_url: baseUrl ? redactBaseUrl(baseUrl, knownSecrets) : "(none)", + model: scrubSecrets(model ?? "(none)", knownSecrets), + api_key_present: apiKeyPresent, + }; +} + +export function computeStats(samplesMs: readonly number[]): LatencyStats { + const sorted = [...samplesMs].sort((a, b) => a - b); + const n = sorted.length; + if (n === 0) { + return { min_ms: 0, median_ms: 0, p95_ms: 0, mean_ms: 0, max_ms: 0 }; + } + const sum = sorted.reduce((acc, value) => acc + value, 0); + return { + min_ms: round3(sorted[0]), + median_ms: round3(percentile(sorted, 50)), + p95_ms: round3(percentile(sorted, 95)), + mean_ms: round3(sum / n), + max_ms: round3(sorted[n - 1]), + }; +} + +// Nearest-rank percentile over an already-sorted ascending array. +function percentile(sortedAsc: readonly number[], p: number): number { + const n = sortedAsc.length; + if (n === 0) return 0; + const rank = Math.ceil((p / 100) * n); + const index = Math.min(Math.max(rank, 1), n) - 1; + return sortedAsc[index]; +} + +function round3(value: number): number { + return Number(value.toFixed(3)); +} + +export function collectEnvironment(): BenchEnvironment { + const cpus = os.cpus(); + return { + os: `${os.type()} ${os.release()}`, + arch: os.arch(), + node: process.version, + cpus: cpus.length, + cpu_model: cpus[0]?.model?.trim() ?? "unknown", + total_mem_gib: Number((os.totalmem() / 1024 ** 3).toFixed(2)), + }; +} + +// Drop URL userinfo and scrub any secret-shaped substring so the report is safe +// to share. Never let a credential reach JSON/Markdown output. +export function redactBaseUrl(rawUrl: string, knownSecrets: readonly string[] = []): string { + try { + const url = new URL(rawUrl); + if (url.protocol !== "http:" && url.protocol !== "https:") return "(invalid URL)"; + url.username = ""; + url.password = ""; + for (const key of [...url.searchParams.keys()]) { + // Query values are not needed to identify a benchmark target and may use + // provider-specific names that a key-name allowlist cannot recognize. + url.searchParams.set(key, ""); + } + url.hash = ""; + return scrubSecrets(url.toString(), knownSecrets); + } catch { + return "(invalid URL)"; + } +} + +export function scrubSecrets(text: string, knownSecrets: readonly string[] = []): string { + let scrubbed = text; + for (const secret of knownSecrets) { + if (secret.length > 0) scrubbed = scrubbed.replaceAll(secret, ""); + } + return redactFull(scrubbed); +} + +export interface InferenceRoundTripOptions { + fetchImpl: typeof fetch; + clock: () => number; + baseUrl: string; + apiKey: string; + model: string; + samples: number; + warmup: number; + prompt: string; + maxTokens: number; + timeoutMs: number; +} + +interface ChatRequestResult { + ok: boolean; + status: number; + detail: string; +} + +class InvalidBenchmarkEndpointError extends Error {} + +function isLoopbackHost(hostname: string): boolean { + const normalized = hostname.toLowerCase(); + return ( + normalized === "localhost" || + normalized === "[::1]" || + (isIP(normalized) === 4 && normalized.startsWith("127.")) + ); +} + +export function buildChatCompletionsUrl(baseUrl: string): string { + let url: URL; + try { + url = new URL(baseUrl); + } catch { + throw new InvalidBenchmarkEndpointError("base URL must be a valid HTTP(S) URL"); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new InvalidBenchmarkEndpointError("base URL must use HTTP or HTTPS"); + } + if (url.protocol === "http:" && !isLoopbackHost(url.hostname)) { + throw new InvalidBenchmarkEndpointError("base URL must use HTTPS unless the host is loopback"); + } + if (url.username || url.password) { + throw new InvalidBenchmarkEndpointError("base URL must not include username or password"); + } + url.hash = ""; + url.pathname = `${url.pathname.replace(/\/+$/, "")}/chat/completions`; + return url.toString(); +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function isValidChatCompletion(payload: unknown): boolean { + if (!isRecord(payload) || !Array.isArray(payload.choices) || payload.choices.length === 0) { + return false; + } + const firstChoice = payload.choices[0]; + if (!isRecord(firstChoice)) return false; + const message = isRecord(firstChoice.message) ? firstChoice.message : {}; + return [message.content, message.reasoning_content, message.reasoning, firstChoice.text].some( + (value) => typeof value === "string" && value.trim().length > 0, + ); +} + +async function discardResponseBody(response: Response): Promise { + try { + await response.body?.cancel(); + } catch { + // The request has already failed; body cleanup must not replace that signal. + } +} + +async function postChatCompletion(options: InferenceRoundTripOptions): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), options.timeoutMs); + try { + const response = await options.fetchImpl(buildChatCompletionsUrl(options.baseUrl), { + method: "POST", + redirect: "error", + headers: { + "content-type": "application/json", + authorization: `Bearer ${options.apiKey}`, + }, + body: JSON.stringify({ + model: options.model, + messages: [{ role: "user", content: options.prompt }], + max_tokens: options.maxTokens, + stream: false, + temperature: 0, + }), + signal: controller.signal, + }); + if (!response.ok) { + // Never copy a remote error body into a shareable report. Providers may + // echo the prompt, model, Authorization header, or endpoint credentials. + await discardResponseBody(response); + return { ok: false, status: response.status, detail: "remote error body omitted" }; + } + + // Drain and validate the body so the timing reflects a real OpenAI-compatible + // completion rather than headers or an arbitrary HTTP 2xx response. + const bodyText = await response.text(); + let payload: unknown; + try { + payload = JSON.parse(bodyText); + } catch { + return { ok: false, status: response.status, detail: "response was not valid JSON" }; + } + if (!isValidChatCompletion(payload)) { + return { + ok: false, + status: response.status, + detail: "response was not an OpenAI-compatible chat completion", + }; + } + return { + ok: true, + status: response.status, + detail: "", + }; + } finally { + clearTimeout(timer); + } +} + +export async function runInferenceRoundTrip( + options: InferenceRoundTripOptions, +): Promise { + const base: BenchMetric = { + id: "inference-round-trip", + status: "ok", + unit: "ms", + source: "live-request", + interpretation: "advisory-non-normative", + }; + + try { + for (let i = 0; i < options.warmup; i += 1) { + const result = await postChatCompletion(options); + if (!result.ok) { + return { + ...base, + status: "error", + reason: `warm-up request ${i + 1} failed (HTTP ${result.status}): ${result.detail}`, + }; + } + } + + const samplesMs: number[] = []; + for (let i = 0; i < options.samples; i += 1) { + const startedAt = options.clock(); + const result = await postChatCompletion(options); + const elapsed = options.clock() - startedAt; + if (!result.ok) { + return { + ...base, + status: "error", + reason: `request ${i + 1} failed (HTTP ${result.status}): ${result.detail}`, + }; + } + samplesMs.push(elapsed); + } + + return { ...base, samples: samplesMs.length, stats: computeStats(samplesMs) }; + } catch (error) { + return { ...base, status: "error", reason: describeRequestError(error, options.timeoutMs) }; + } +} + +function describeRequestError(error: unknown, timeoutMs: number): string { + if (error instanceof InvalidBenchmarkEndpointError) return error.message; + if (error instanceof Error && error.name === "AbortError") { + return `request timed out after ${timeoutMs} ms`; + } + return error instanceof Error ? `${error.name}: request failed` : "request failed"; +} + +export function unsupportedTraceMetric(id: MetricId): BenchMetric { + return { + id, + status: "unsupported", + unit: "ms", + source: "none", + interpretation: "advisory-non-normative", + reason: + "no onboard trace provided; set NEMOCLAW_TRACE=1 during `nemoclaw onboard`, then pass --trace ", + }; +} + +// --- Reporting --- + +export function renderMarkdownReport(report: BenchReport): string { + const env = report.environment; + const lines: string[] = [ + "# NemoClaw value benchmark", + "", + `Generated: ${report.generated_at}`, + "", + "## Environment", + "", + `- OS: ${env.os} (${env.arch})`, + `- Node: ${env.node}`, + `- CPU: ${env.cpu_model} x${env.cpus}`, + `- Memory: ${env.total_mem_gib} GiB`, + "", + "## Inference target", + "", + `- Endpoint: ${report.target.base_url}`, + `- Model: ${report.target.model}`, + `- API key present: ${report.target.api_key_present ? "yes" : "no"}`, + "", + "## Metrics", + "", + "| Metric | Status | Source | min | median | p95 | mean | max |", + "|--------|--------|--------|-----|--------|-----|------|-----|", + ]; + + for (const metric of report.metrics) { + lines.push(renderMetricRow(metric)); + } + + lines.push(""); + for (const metric of report.metrics) { + const note = metricNote(metric); + if (note) lines.push(note); + } + + lines.push( + "", + "> Interpretation is **advisory and non-normative**: these timings describe this", + "> machine and provider only. NemoClaw does not ship owner-approved pass/warn/fail", + "> thresholds yet (see issue #3776), so use the numbers to compare runs, not to gate.", + "", + "## Troubleshooting", + "", + "- High inference latency: check `nemoclaw status` for the active provider and", + " the `Inference` line; for local Ollama/vLLM confirm the backend is reachable.", + "- Missing sandbox/policy timings: re-run onboarding with `NEMOCLAW_TRACE=1` and pass", + " the written trace file with `--trace`.", + "- See docs/inference/use-local-inference and docs/reference/troubleshooting.", + "", + ); + + return scrubSecrets(`${lines.join("\n")}`); +} + +function renderMetricRow(metric: BenchMetric): string { + const stats = metric.stats; + const cells = stats + ? [stats.min_ms, stats.median_ms, stats.p95_ms, stats.mean_ms, stats.max_ms].map(fmtMs) + : ["-", "-", "-", "-", "-"]; + return `| ${metric.id} | ${metric.status} | ${metric.source} | ${cells.join(" | ")} |`; +} + +function metricNote(metric: BenchMetric): string { + const parts: string[] = []; + if (metric.reason) parts.push(`- **${metric.id}**: ${metric.reason}`); + if (metric.breakdown) { + const detail = Object.entries(metric.breakdown) + .map(([key, value]) => `${key}=${fmtMs(value)}`) + .join(", "); + parts.push(`- **${metric.id}** breakdown: ${detail}`); + } + if (metric.context) { + const detail = Object.entries(metric.context) + .map(([key, value]) => `${key}=${inlineMarkdownValue(String(value))}`) + .join(", "); + parts.push(`- **${metric.id}** context: ${detail}`); + } + return parts.join("\n"); +} + +function inlineMarkdownValue(value: string): string { + return value.replace(/[\r\n|]+/g, " ").trim(); +} + +function fmtMs(value: number): string { + return `${value.toFixed(1)} ms`; +} + +export function hasBlockingError(report: BenchReport): boolean { + return report.metrics.some((metric) => metric.status === "error"); +} diff --git a/scripts/bench/run.ts b/scripts/bench/run.ts new file mode 100644 index 00000000000..fdfed09abb1 --- /dev/null +++ b/scripts/bench/run.ts @@ -0,0 +1,277 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// NemoClaw value benchmark harness (issue #5604). +// +// Measures core "is NemoClaw fast enough on this machine" signals and emits a +// machine-readable JSON document plus a Markdown value report. It only contacts +// the inference endpoint you configure and never posts results anywhere. +// +// tsx scripts/bench/run.ts --base-url --model [--json out.json] +// tsx scripts/bench/run.ts --trace .e2e/traces/onboard.json --base-url ... --model ... +// +// The API key is read from an environment variable (default OPENAI_API_KEY or +// NVIDIA_INFERENCE_API_KEY), never from a command-line flag. + +import fs from "node:fs"; + +import { + BENCH_SCHEMA_VERSION, + type BenchMetric, + type BenchReport, + buildBenchTarget, + collectEnvironment, + hasBlockingError, + ingestPolicyOverhead, + ingestSandboxColdStart, + renderMarkdownReport, + runInferenceRoundTrip, + unsupportedTraceMetric, +} from "./lib"; + +interface CliOptions { + baseUrl?: string; + model?: string; + apiKeyEnv?: string; + samples: number; + warmup: number; + prompt: string; + maxTokens: number; + timeoutMs: number; + tracePath?: string; + jsonPath?: string; + reportPath?: string; + runInference: boolean; +} + +const USAGE = `NemoClaw value benchmark (issue #5604) + +Usage: + tsx scripts/bench/run.ts --base-url --model [options] + +Options: + --base-url OpenAI-compatible base URL (or env OPENAI_BASE_URL / NEMOCLAW_BENCH_BASE_URL) + --model Model id to send (or env OPENAI_MODEL / NEMOCLAW_BENCH_MODEL) + --api-key-env API key env: OPENAI_API_KEY or NVIDIA_INFERENCE_API_KEY (checked in that order by default) + --samples Timed inference requests (default 5) + --warmup Untimed warm-up requests (default 1) + --prompt Prompt to send (default: a tiny deterministic prompt) + --max-tokens max_tokens per request (default 16) + --timeout-ms Per-request timeout in ms (default 60000) + --trace Onboard trace artifact for sandbox cold-start + policy overhead + --no-inference Skip the live inference round-trip metric + --json Write machine-readable JSON to ('-' for stdout) + --report Also write the Markdown report to + -h, --help Show this help + +The harness sends requests only to the configured endpoint and never uploads results.`; + +function parseArgs(argv: string[]): CliOptions { + const options: CliOptions = { + baseUrl: process.env.OPENAI_BASE_URL ?? process.env.NEMOCLAW_BENCH_BASE_URL, + model: process.env.OPENAI_MODEL ?? process.env.NEMOCLAW_BENCH_MODEL, + samples: 5, + warmup: 1, + prompt: "Reply with exactly one word: PONG", + maxTokens: 16, + timeoutMs: 60_000, + runInference: true, + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + const value = (): string => { + i += 1; + return takeValue(argv, i, arg); + }; + switch (arg) { + case "--base-url": + options.baseUrl = value(); + break; + case "--model": + options.model = value(); + break; + case "--api-key-env": + options.apiKeyEnv = value(); + break; + case "--samples": + options.samples = toPositiveInt(value(), arg); + break; + case "--warmup": + options.warmup = toNonNegativeInt(value(), arg); + break; + case "--prompt": + options.prompt = value(); + break; + case "--max-tokens": + options.maxTokens = toPositiveInt(value(), arg); + break; + case "--timeout-ms": + options.timeoutMs = toPositiveInt(value(), arg); + break; + case "--trace": + options.tracePath = value(); + break; + case "--json": + options.jsonPath = value(); + break; + case "--report": + options.reportPath = value(); + break; + case "--no-inference": + options.runInference = false; + break; + case "-h": + case "--help": + fs.writeSync(1, `${USAGE}\n`); + process.exit(0); + break; + default: + throw new Error(`Unknown argument: ${arg}\n\n${USAGE}`); + } + } + + return options; +} + +function takeValue(argv: string[], index: number, flag: string): string { + const value = argv[index]; + if (value === undefined || (value.startsWith("--") && value.length > 2)) { + throw new Error(`Missing value for ${flag}`); + } + return value; +} + +function toPositiveInt(value: string, flag: string): number { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`${flag} must be a positive integer, got "${value}"`); + } + return parsed; +} + +function toNonNegativeInt(value: string, flag: string): number { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`${flag} must be a non-negative integer, got "${value}"`); + } + return parsed; +} + +function resolveApiKey(envName?: string): { name: string; value: string | undefined } { + const allowedNames = ["OPENAI_API_KEY", "NVIDIA_INFERENCE_API_KEY"] as const; + if (envName && !allowedNames.includes(envName as (typeof allowedNames)[number])) { + throw new Error("--api-key-env must be OPENAI_API_KEY or NVIDIA_INFERENCE_API_KEY"); + } + const candidates = envName ? [envName] : [...allowedNames]; + for (const name of candidates) { + const value = process.env[name]; + if (value) return { name, value }; + } + return { name: candidates[0], value: undefined }; +} + +function readTraceArtifact(tracePath: string): unknown { + const raw = fs.readFileSync(tracePath, "utf8"); + return JSON.parse(raw); +} + +async function buildReport(options: CliOptions): Promise { + const metrics: BenchMetric[] = []; + const apiKey = resolveApiKey(options.apiKeyEnv); + + if (options.runInference) { + metrics.push( + await runInferenceRoundTrip({ + fetchImpl: fetch, + clock: () => performance.now(), + baseUrl: options.baseUrl as string, + apiKey: apiKey.value as string, + model: options.model as string, + samples: options.samples, + warmup: options.warmup, + prompt: options.prompt, + maxTokens: options.maxTokens, + timeoutMs: options.timeoutMs, + }), + ); + } + + if (options.tracePath) { + const artifact = readTraceArtifact(options.tracePath); + metrics.push(ingestSandboxColdStart(artifact)); + metrics.push(ingestPolicyOverhead(artifact)); + } else { + metrics.push(unsupportedTraceMetric("sandbox-cold-start")); + metrics.push(unsupportedTraceMetric("policy-shield-overhead")); + } + + return { + schema_version: BENCH_SCHEMA_VERSION, + generated_at: new Date().toISOString(), + environment: collectEnvironment(), + target: buildBenchTarget( + options.baseUrl, + options.model, + apiKey.value !== undefined, + apiKey.value ? [apiKey.value] : [], + ), + metrics, + }; +} + +function preflight(options: CliOptions): void { + const missing: string[] = []; + if (options.runInference) { + const apiKey = resolveApiKey(options.apiKeyEnv); + if (!options.baseUrl) missing.push("--base-url (or OPENAI_BASE_URL / NEMOCLAW_BENCH_BASE_URL)"); + if (!options.model) missing.push("--model (or OPENAI_MODEL / NEMOCLAW_BENCH_MODEL)"); + if (!apiKey.value) missing.push(`API key in env ${apiKey.name}`); + } + if (missing.length > 0) { + throw new Error( + `Cannot run the inference benchmark, missing:\n - ${missing.join("\n - ")}\n\n` + + `Provide them, or pass --no-inference to run only trace-based metrics.\n\n${USAGE}`, + ); + } + if (!options.runInference && !options.tracePath) { + throw new Error( + `Nothing to benchmark: pass an inference target or --trace .\n\n${USAGE}`, + ); + } +} + +function writeOutputs(report: BenchReport, options: CliOptions): void { + const json = `${JSON.stringify(report, null, 2)}\n`; + const markdown = renderMarkdownReport(report); + + if (options.jsonPath === "-") { + process.stdout.write(json); + } else if (options.jsonPath) { + fs.writeFileSync(options.jsonPath, json); + process.stderr.write(`Wrote JSON to ${options.jsonPath}\n`); + } + + if (options.reportPath) { + fs.writeFileSync(options.reportPath, `${markdown}\n`); + process.stderr.write(`Wrote Markdown report to ${options.reportPath}\n`); + } + + if (options.jsonPath !== "-") { + process.stdout.write(`${markdown}\n`); + } +} + +async function main(): Promise { + const options = parseArgs(process.argv.slice(2)); + preflight(options); + const report = await buildReport(options); + writeOutputs(report, options); + process.exitCode = hasBlockingError(report) ? 1 : 0; +} + +main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${message}\n`); + process.exitCode = 1; +}); diff --git a/scripts/bench/trace-ingest.ts b/scripts/bench/trace-ingest.ts new file mode 100644 index 00000000000..0073e381161 --- /dev/null +++ b/scripts/bench/trace-ingest.ts @@ -0,0 +1,297 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { redactFull } from "../../src/lib/security/redact"; + +import type { BenchMetric, BenchMetricContext, LatencyStats, MetricId } from "./lib"; + +// Span names emitted by src/lib/onboard/tracing.ts into the nemoclaw.trace_timing +// artifact. The benchmark reads canonical emitted spans rather than adding +// parallel instrumentation to onboarding. +export const SANDBOX_PHASE_SPAN = "nemoclaw.onboard.phase.sandbox"; +export const SANDBOX_READINESS_SPAN = "nemoclaw.sandbox.readiness_wait"; +export const POLICY_APPLICATION_SPAN = "nemoclaw.policy.application"; + +interface TraceLikeSpan { + trace_id?: unknown; + span_id?: unknown; + parent_span_id?: unknown; + name?: unknown; + duration_ms?: unknown; + status?: unknown; + attributes?: unknown; +} + +interface ValidTrace { + rootSpanId: string; + rootDurationMs: number; + rootAttributes: Record; + spans: TraceLikeSpan[]; +} + +type TraceMetricId = Extract; +type TraceInspection = { ok: true; trace: ValidTrace } | { ok: false; reason: string }; +type MetricSpan = + | { + kind: "ok"; + durationMs: number; + spanId: string; + parentSpanId?: string; + attributes: Record; + } + | { kind: "missing" } + | { kind: "error"; reason: string }; + +const TRACE_SCOPE_NAME = "nemoclaw.onboard"; +const TRACE_ROOT_SPAN = "nemoclaw.onboard"; + +function asRecord(value: unknown): Record | null { + return value !== null && typeof value === "object" ? (value as Record) : null; +} + +function inspectTraceArtifact(artifact: unknown): TraceInspection { + const artifactRecord = asRecord(artifact); + const summary = asRecord(artifactRecord?.summary); + const traceId = summary?.trace_id; + if (typeof traceId !== "string" || traceId.length === 0) { + return { ok: false, reason: "trace summary is missing trace_id" }; + } + + const resourceSpans = artifactRecord?.resource_spans; + if (!Array.isArray(resourceSpans)) { + return { ok: false, reason: "trace artifact is missing resource_spans" }; + } + + const spans: TraceLikeSpan[] = []; + let matchedScope = false; + for (const resourceSpan of resourceSpans) { + const scopeSpans = asRecord(resourceSpan)?.scope_spans; + if (!Array.isArray(scopeSpans)) continue; + for (const scopeSpan of scopeSpans) { + const scopeSpanRecord = asRecord(scopeSpan); + const scope = asRecord(scopeSpanRecord?.scope); + if (scope?.name !== TRACE_SCOPE_NAME) continue; + matchedScope = true; + const inner = scopeSpanRecord?.spans; + if (!Array.isArray(inner) || inner.some((span) => asRecord(span) === null)) { + return { ok: false, reason: "onboard trace scope contains malformed spans" }; + } + spans.push(...(inner as TraceLikeSpan[])); + } + } + + if (!matchedScope) { + return { ok: false, reason: `trace artifact is missing the ${TRACE_SCOPE_NAME} scope` }; + } + const roots = spans.filter((span) => span.name === TRACE_ROOT_SPAN); + if (roots.length !== 1) { + return { ok: false, reason: "trace artifact must contain exactly one onboard root span" }; + } + if (spans.some((span) => span.trace_id !== traceId)) { + return { ok: false, reason: "trace spans do not match the summary trace_id" }; + } + + const root = roots[0]; + if (typeof root.span_id !== "string" || root.span_id.length === 0) { + return { ok: false, reason: "onboard root span is missing span_id" }; + } + const rootStatus = asRecord(root.status)?.code; + if (rootStatus !== "OK") { + return { ok: false, reason: "onboard root span status is missing or not OK" }; + } + if (!isValidDuration(root.duration_ms)) { + return { ok: false, reason: "onboard root span has an invalid duration" }; + } + return { + ok: true, + trace: { + rootSpanId: root.span_id, + rootDurationMs: root.duration_ms, + rootAttributes: asRecord(root.attributes) ?? {}, + spans, + }, + }; +} + +function isValidDuration(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function readMetricSpan(trace: ValidTrace, name: string): MetricSpan { + const matches = trace.spans.filter((span) => span.name === name); + if (matches.length === 0) return { kind: "missing" }; + if (matches.length > 1) { + return { kind: "error", reason: `trace contains multiple ${name} spans` }; + } + const span = matches[0]; + if (typeof span.span_id !== "string" || span.span_id.length === 0) { + return { kind: "error", reason: `${name} span is missing span_id` }; + } + const status = asRecord(span.status)?.code; + if (status !== "OK") { + return { kind: "error", reason: `${name} span status is missing or not OK` }; + } + if (!isValidDuration(span.duration_ms)) { + return { kind: "error", reason: `${name} span has an invalid duration` }; + } + return { + kind: "ok", + durationMs: round3(span.duration_ms), + spanId: span.span_id, + attributes: asRecord(span.attributes) ?? {}, + ...(typeof span.parent_span_id === "string" ? { parentSpanId: span.parent_span_id } : {}), + }; +} + +function safeContextString(value: unknown): string | undefined { + if (typeof value !== "string" || value.trim().length === 0) return undefined; + return redactFull(value) + .replace(/[\u0000-\u001f\u007f-\u009f]+/g, " ") + .trim() + .slice(0, 160); +} + +function traceMetricContext( + trace: ValidTrace, + metricAttributes: Record, +): BenchMetricContext { + const sandboxAttributes = + asRecord(trace.spans.find((span) => span.name === SANDBOX_PHASE_SPAN)?.attributes) ?? {}; + const provider = safeContextString(metricAttributes.provider ?? sandboxAttributes.provider); + const model = safeContextString(sandboxAttributes.model); + const agent = safeContextString(trace.rootAttributes.agent ?? sandboxAttributes.agent); + const nonInteractive = trace.rootAttributes.non_interactive; + const fresh = trace.rootAttributes.fresh; + return { + ...(provider ? { provider } : {}), + ...(model ? { model } : {}), + ...(agent ? { agent } : {}), + ...(typeof nonInteractive === "boolean" ? { non_interactive: nonInteractive } : {}), + ...(typeof fresh === "boolean" ? { fresh } : {}), + }; +} + +function traceMetricBase(id: TraceMetricId): BenchMetric { + return { + id, + status: "ok", + unit: "ms", + source: "trace-artifact", + interpretation: "advisory-non-normative", + }; +} + +function invalidTraceMetric(id: TraceMetricId, reason: string): BenchMetric { + return { ...traceMetricBase(id), status: "error", reason: `invalid onboard trace: ${reason}` }; +} + +export function ingestSandboxColdStart(artifact: unknown): BenchMetric { + const inspected = inspectTraceArtifact(artifact); + if (!inspected.ok) return invalidTraceMetric("sandbox-cold-start", inspected.reason); + const phase = readMetricSpan(inspected.trace, SANDBOX_PHASE_SPAN); + const base = traceMetricBase("sandbox-cold-start"); + if (phase.kind === "error") return invalidTraceMetric("sandbox-cold-start", phase.reason); + if (phase.kind === "missing") { + return { + ...base, + status: "unsupported", + source: "none", + reason: `no ${SANDBOX_PHASE_SPAN} span in the trace artifact (re-run \`nemoclaw onboard\` with NEMOCLAW_TRACE=1, then pass --trace )`, + }; + } + if (phase.parentSpanId !== inspected.trace.rootSpanId) { + return invalidTraceMetric( + "sandbox-cold-start", + `${SANDBOX_PHASE_SPAN} is not a child of the onboard root`, + ); + } + if (phase.durationMs > inspected.trace.rootDurationMs) { + return invalidTraceMetric( + "sandbox-cold-start", + `${SANDBOX_PHASE_SPAN} duration exceeds the onboard root`, + ); + } + + const breakdown: Record = { sandbox_phase_ms: phase.durationMs }; + const readiness = readMetricSpan(inspected.trace, SANDBOX_READINESS_SPAN); + if (readiness.kind === "error") { + return invalidTraceMetric("sandbox-cold-start", readiness.reason); + } + if (readiness.kind === "ok") { + if (readiness.parentSpanId !== phase.spanId) { + return invalidTraceMetric( + "sandbox-cold-start", + `${SANDBOX_READINESS_SPAN} is not nested under the sandbox phase`, + ); + } + if (readiness.durationMs > phase.durationMs) { + return invalidTraceMetric( + "sandbox-cold-start", + `${SANDBOX_READINESS_SPAN} duration exceeds its enclosing sandbox phase`, + ); + } + breakdown.readiness_wait_ms = readiness.durationMs; + } + return { + ...base, + breakdown, + context: traceMetricContext(inspected.trace, phase.attributes), + stats: singleValueStats(phase.durationMs), + }; +} + +export function ingestPolicyOverhead(artifact: unknown): BenchMetric { + const inspected = inspectTraceArtifact(artifact); + if (!inspected.ok) return invalidTraceMetric("policy-shield-overhead", inspected.reason); + const policy = readMetricSpan(inspected.trace, POLICY_APPLICATION_SPAN); + const base = traceMetricBase("policy-shield-overhead"); + if (policy.kind === "error") return invalidTraceMetric("policy-shield-overhead", policy.reason); + if (policy.kind === "missing") { + return { + ...base, + status: "unsupported", + source: "none", + reason: + "no policy.application span in the trace artifact (re-run `nemoclaw onboard` with NEMOCLAW_TRACE=1, then pass --trace )", + }; + } + if (policy.parentSpanId !== inspected.trace.rootSpanId) { + return invalidTraceMetric( + "policy-shield-overhead", + `${POLICY_APPLICATION_SPAN} is not a child of the onboard root`, + ); + } + if (policy.durationMs > inspected.trace.rootDurationMs) { + return invalidTraceMetric( + "policy-shield-overhead", + `${POLICY_APPLICATION_SPAN} duration exceeds the onboard root`, + ); + } + const context = traceMetricContext(inspected.trace, policy.attributes); + if (inspected.trace.rootAttributes.non_interactive !== true) { + return { + ...base, + status: "unsupported", + source: "none", + context, + reason: + "interactive policy selection can include human think time; collect the trace with `nemoclaw onboard --non-interactive`", + }; + } + return { + ...base, + status: "unsupported", + source: "none", + context, + reason: + "the onboard trace records policy application setup time, not request-path shield overhead; dedicated request-path timing is not available", + }; +} + +function round3(value: number): number { + return Number(value.toFixed(3)); +} + +function singleValueStats(value: number): LatencyStats { + return { min_ms: value, median_ms: value, p95_ms: value, mean_ms: value, max_ms: value }; +} diff --git a/test/bench/bench-cli.test.ts b/test/bench/bench-cli.test.ts new file mode 100644 index 00000000000..77001781779 --- /dev/null +++ b/test/bench/bench-cli.test.ts @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import fs from "node:fs"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../.."); +const RUNNER = path.join(REPO_ROOT, "scripts", "bench", "run.ts"); +const VALID_COMPLETION = JSON.stringify({ + choices: [{ message: { role: "assistant", content: "PONG" } }], +}); + +interface RunResult { + code: number | null; + stdout: string; + stderr: string; +} + +function cleanBenchEnv(overrides: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + const env = { ...process.env }; + for (const key of [ + "OPENAI_API_KEY", + "NVIDIA_INFERENCE_API_KEY", + "OPENAI_BASE_URL", + "NEMOCLAW_BENCH_BASE_URL", + "OPENAI_MODEL", + "NEMOCLAW_BENCH_MODEL", + ]) { + delete env[key]; + } + return { ...env, ...overrides }; +} + +async function runBench(args: string[], env: NodeJS.ProcessEnv = {}): Promise { + const child = spawn(process.execPath, ["--import", "tsx", RUNNER, ...args], { + cwd: REPO_ROOT, + env: cleanBenchEnv(env), + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + const [code] = (await once(child, "close")) as [number | null]; + return { code, stdout, stderr }; +} + +async function startInferenceServer( + body: string, + status = 200, +): Promise<{ + server: http.Server; + baseUrl: string; + requests: string[]; +}> { + const requests: string[] = []; + const server = http.createServer((request, response) => { + requests.push(request.url ?? ""); + request.resume(); + response.writeHead(status, { "content-type": "application/json" }); + response.end(body); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert(address && typeof address !== "string", "test server did not bind TCP"); + return { server, baseUrl: `http://127.0.0.1:${address.port}`, requests }; +} + +async function closeServer(server: http.Server): Promise { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); +} + +describe("benchmark CLI", () => { + it("writes JSON and Markdown from a valid completion without leaking target secrets", async () => { + const fixture = await startInferenceServer(VALID_COMPLETION); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bench-cli-")); + const jsonPath = path.join(tempDir, "bench.json"); + const reportPath = path.join(tempDir, "bench.md"); + const apiKey = "custom-key-that-must-not-leak"; + const querySecret = "clear-query-secret"; + try { + const result = await runBench( + [ + "--base-url", + `${fixture.baseUrl}/v1?tenant=${querySecret}#ignored`, + "--model", + "test-model", + "--samples", + "1", + "--warmup", + "0", + "--json", + jsonPath, + "--report", + reportPath, + ], + { OPENAI_API_KEY: apiKey }, + ); + const json = fs.readFileSync(jsonPath, "utf8"); + const markdown = fs.readFileSync(reportPath, "utf8"); + const report = JSON.parse(json) as { + schema_version: string; + metrics: Array<{ id: string; status: string }>; + }; + expect(result.code).toBe(0); + expect(fixture.requests).toEqual([`/v1/chat/completions?tenant=${querySecret}`]); + expect(report.schema_version).toBe("nemoclaw.bench.v1"); + expect(report.metrics[0]).toMatchObject({ id: "inference-round-trip", status: "ok" }); + expect(`${json}\n${markdown}\n${result.stdout}`).not.toContain(apiKey); + expect(`${json}\n${markdown}\n${result.stdout}`).not.toContain(querySecret); + } finally { + await closeServer(fixture.server); + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("fails when an HTTP 2xx response is not an OpenAI chat completion", async () => { + const fixture = await startInferenceServer("{}"); + try { + const result = await runBench( + [ + "--base-url", + `${fixture.baseUrl}/v1`, + "--model", + "test-model", + "--samples", + "1", + "--warmup", + "0", + ], + { OPENAI_API_KEY: "test-key" }, + ); + expect(result.code).toBe(1); + expect(result.stdout).toContain("not an OpenAI-compatible chat completion"); + } finally { + await closeServer(fixture.server); + } + }); + + it("fails clearly when required inference configuration is missing", async () => { + const result = await runBench([]); + expect(result.code).toBe(1); + expect(result.stderr).toContain("Cannot run the inference benchmark, missing:"); + expect(result.stderr).toContain("NEMOCLAW_BENCH_BASE_URL"); + expect(result.stderr).toContain("NEMOCLAW_BENCH_MODEL"); + }); + + it("rejects an unrelated API key environment before sending a request", async () => { + const fixture = await startInferenceServer(VALID_COMPLETION); + const unrelatedSecret = "github-token-that-must-not-leak"; + try { + const result = await runBench( + [ + "--base-url", + `${fixture.baseUrl}/v1`, + "--model", + "test-model", + "--api-key-env", + "GITHUB_TOKEN", + "--samples", + "1", + "--warmup", + "0", + ], + { GITHUB_TOKEN: unrelatedSecret }, + ); + expect(result.code).toBe(1); + expect(result.stderr).toContain( + "--api-key-env must be OPENAI_API_KEY or NVIDIA_INFERENCE_API_KEY", + ); + expect(`${result.stdout}\n${result.stderr}`).not.toContain(unrelatedSecret); + expect(fixture.requests).toEqual([]); + } finally { + await closeServer(fixture.server); + } + }); +}); diff --git a/test/bench/bench.test.ts b/test/bench/bench.test.ts new file mode 100644 index 00000000000..1a2ba1a9d95 --- /dev/null +++ b/test/bench/bench.test.ts @@ -0,0 +1,607 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + BENCH_SCHEMA_VERSION, + type BenchReport, + buildBenchTarget, + buildChatCompletionsUrl, + computeStats, + hasBlockingError, + ingestPolicyOverhead, + ingestSandboxColdStart, + POLICY_APPLICATION_SPAN, + redactBaseUrl, + renderMarkdownReport, + runInferenceRoundTrip, + SANDBOX_PHASE_SPAN, + SANDBOX_READINESS_SPAN, + unsupportedTraceMetric, +} from "../../scripts/bench/lib"; +import { + finishOnboardTrace, + startOnboardTrace, + withSandboxPhaseTrace, +} from "../../src/lib/onboard/tracing"; +import type { TraceArtifact, TraceSpan } from "../../src/lib/trace"; +import { resetTraceForTests } from "../../src/lib/trace"; + +function queueClock(values: readonly number[]): () => number { + let index = 0; + return () => { + const value = values[Math.min(index, values.length - 1)]; + index += 1; + return value; + }; +} + +function fakeFetch(status: number, body: string): typeof fetch { + return (async () => + ({ + ok: status >= 200 && status < 300, + status, + text: async () => body, + }) as Response) as unknown as typeof fetch; +} + +const inferenceOptionsBase = { + baseUrl: "https://inference.local/v1", + apiKey: "nvapi-test-key", + model: "test-model", + warmup: 0, + prompt: "ping", + maxTokens: 4, + timeoutMs: 1000, +}; + +const VALID_COMPLETION = JSON.stringify({ + choices: [{ message: { role: "assistant", content: "PONG" } }], +}); + +const TRACE_ID = "0123456789abcdef0123456789abcdef"; +const ROOT_SPAN_ID = "0123456789abcdef"; +let spanSequence = 1; + +function traceSpan( + name: string, + durationMs: number, + overrides: Partial = {}, +): TraceSpan { + return { + trace_id: TRACE_ID, + span_id: (spanSequence++).toString(16).padStart(16, "0"), + parent_span_id: ROOT_SPAN_ID, + name, + kind: "INTERNAL", + start_time_unix_nano: "1000000", + end_time_unix_nano: "2000000", + duration_ms: durationMs, + status: { code: "OK" }, + attributes: {}, + events: [], + ...overrides, + }; +} + +function traceArtifact( + spans: TraceSpan[], + options: { + rootStatus?: TraceSpan["status"]; + rootDurationMs?: number; + summaryTraceId?: string; + scopeName?: string; + rootAttributes?: Record; + } = {}, +): TraceArtifact { + const root = traceSpan("nemoclaw.onboard", options.rootDurationMs ?? 3000, { + span_id: ROOT_SPAN_ID, + parent_span_id: undefined, + status: options.rootStatus ?? { code: "OK" }, + attributes: { + fresh: false, + non_interactive: true, + agent: "openclaw", + ...options.rootAttributes, + }, + }); + return { + resource_spans: [ + { + resource: { attributes: { "service.name": "nemoclaw" } }, + scope_spans: [ + { + scope: { name: options.scopeName ?? "nemoclaw.onboard", version: "1.0.0" }, + spans: [root, ...spans], + }, + ], + }, + ], + summary: { + trace_id: options.summaryTraceId ?? TRACE_ID, + generated_at: "2026-07-03T00:00:00.000Z", + total_duration_ms: 3000, + slowest_spans: [], + output_path: ".e2e/traces/test.json", + }, + }; +} + +describe("computeStats", () => { + it.each([ + { input: [10], expected: { min: 10, median: 10, p95: 10, mean: 10, max: 10 } }, + { input: [10, 30], expected: { min: 10, median: 10, p95: 30, mean: 20, max: 30 } }, + { + input: [50, 10, 20, 40, 30], + expected: { min: 10, median: 30, p95: 50, mean: 30, max: 50 }, + }, + ])("summarizes $input", ({ input, expected }) => { + const stats = computeStats(input); + expect(stats.min_ms).toBe(expected.min); + expect(stats.median_ms).toBe(expected.median); + expect(stats.p95_ms).toBe(expected.p95); + expect(stats.mean_ms).toBe(expected.mean); + expect(stats.max_ms).toBe(expected.max); + }); + + it("returns zeros for an empty sample set", () => { + expect(computeStats([])).toEqual({ + min_ms: 0, + median_ms: 0, + p95_ms: 0, + mean_ms: 0, + max_ms: 0, + }); + }); +}); + +describe("buildChatCompletionsUrl", () => { + it.each([ + "https://inference.local/v1", + "https://inference.local/v1/", + "https://inference.local/v1///", + ])("normalizes trailing slashes for %s", (base) => { + expect(buildChatCompletionsUrl(base)).toBe("https://inference.local/v1/chat/completions"); + }); + + it("appends the completion path before query parameters and removes fragments", () => { + expect(buildChatCompletionsUrl("https://host.test/v1?tenant=alpha#ignored")).toBe( + "https://host.test/v1/chat/completions?tenant=alpha", + ); + }); + + it.each([ + "http://localhost:8000/v1", + "http://127.0.0.1:8000/v1", + "http://[::1]:8000/v1", + ])("allows a plaintext loopback endpoint: %s", (base) => { + expect(buildChatCompletionsUrl(base)).toContain("/v1/chat/completions"); + }); + + it("rejects non-HTTP and credential-bearing endpoints", () => { + expect(() => buildChatCompletionsUrl("file:///tmp/inference")).toThrow("HTTP or HTTPS"); + expect(() => buildChatCompletionsUrl("http://example.com/v1")).toThrow( + "must use HTTPS unless the host is loopback", + ); + expect(() => buildChatCompletionsUrl("http://127.evil/v1")).toThrow( + "must use HTTPS unless the host is loopback", + ); + expect(() => buildChatCompletionsUrl("https://user:pass@host.test/v1")).toThrow( + "must not include username or password", + ); + }); +}); + +describe("redactBaseUrl", () => { + it("strips URL userinfo so credentials never reach the report", () => { + const redacted = redactBaseUrl("https://user:s3cr3t-token@host:8000/v1"); + expect(redacted).not.toContain("s3cr3t-token"); + expect(redacted).not.toContain("user:"); + expect(redacted).toContain("host:8000"); + }); + + it("passes through a clean URL host and path", () => { + expect(redactBaseUrl("https://inference.local/v1")).toContain("inference.local/v1"); + }); + + it("redacts credential-bearing query parameters", () => { + const redacted = redactBaseUrl( + "https://inference.local/v1?api_key=clear-api-secret&password=clear-password&custom=clear-query-secret", + ); + expect(redacted).not.toContain("clear-api-secret"); + expect(redacted).not.toContain("clear-password"); + expect(redacted).not.toContain("clear-query-secret"); + }); + + it("does not echo malformed or unsupported endpoint URLs", () => { + expect(redactBaseUrl("https//user:clear-password@host")).toBe("(invalid URL)"); + expect(redactBaseUrl("file:///tmp/clear-secret")).toBe("(invalid URL)"); + }); + + it("builds a shareable target without URL or model secrets", () => { + const target = buildBenchTarget( + "https://inference.local/v1?api_key=clear-api-secret", + "model api_key=clear-model-secret", + true, + ); + const serialized = JSON.stringify(target); + expect(serialized).not.toContain("clear-api-secret"); + expect(serialized).not.toContain("clear-model-secret"); + expect(target.api_key_present).toBe(true); + }); +}); + +describe("runInferenceRoundTrip", () => { + it("produces ok stats from timed samples", async () => { + const metric = await runInferenceRoundTrip({ + ...inferenceOptionsBase, + samples: 2, + fetchImpl: fakeFetch(200, VALID_COMPLETION), + clock: queueClock([0, 10, 100, 130]), + }); + expect(metric.status).toBe("ok"); + expect(metric.samples).toBe(2); + expect(metric.stats?.min_ms).toBe(10); + expect(metric.stats?.max_ms).toBe(30); + expect(metric.source).toBe("live-request"); + }); + + it("returns an error metric on a non-2xx response", async () => { + const echoedSecret = inferenceOptionsBase.apiKey; + const metric = await runInferenceRoundTrip({ + ...inferenceOptionsBase, + samples: 1, + fetchImpl: fakeFetch(500, `echoed prompt and credential: ${echoedSecret}`), + clock: queueClock([0, 5]), + }); + expect(metric.status).toBe("error"); + expect(metric.reason).toContain("HTTP 500"); + expect(metric.reason).not.toContain(echoedSecret); + expect(metric.reason).not.toContain("echoed prompt"); + }); + + it("rejects an HTTP 2xx body that is not a chat completion", async () => { + const metric = await runInferenceRoundTrip({ + ...inferenceOptionsBase, + samples: 1, + fetchImpl: fakeFetch(200, "{}"), + clock: queueClock([0, 5]), + }); + expect(metric.status).toBe("error"); + expect(metric.reason).toContain("not an OpenAI-compatible chat completion"); + }); + + it.each([ + { message: { content: null, reasoning_content: "reasoning output" } }, + { message: { content: "", reasoning: "reasoning output" } }, + { text: "legacy completion output" }, + ])("accepts compatible reasoning or text output: $message $text", async (choice) => { + const metric = await runInferenceRoundTrip({ + ...inferenceOptionsBase, + samples: 1, + fetchImpl: fakeFetch(200, JSON.stringify({ choices: [choice] })), + clock: queueClock([0, 5]), + }); + expect(metric.status).toBe("ok"); + }); + + it("returns an error metric when the request throws", async () => { + const throwingFetch = (async () => { + throw new Error("ECONNREFUSED"); + }) as unknown as typeof fetch; + const metric = await runInferenceRoundTrip({ + ...inferenceOptionsBase, + samples: 1, + fetchImpl: throwingFetch, + clock: queueClock([0, 5]), + }); + expect(metric.status).toBe("error"); + expect(metric.reason).toBe("Error: request failed"); + }); + + it("rejects remote plaintext before sending the API key", async () => { + let requestCount = 0; + const fetchImpl = (async () => { + requestCount += 1; + return { ok: true, status: 200, text: async () => VALID_COMPLETION } as Response; + }) as typeof fetch; + const metric = await runInferenceRoundTrip({ + ...inferenceOptionsBase, + baseUrl: "http://example.com/v1", + samples: 1, + fetchImpl, + clock: queueClock([0, 5]), + }); + expect(metric.status).toBe("error"); + expect(metric.reason).toContain("must use HTTPS unless the host is loopback"); + expect(requestCount).toBe(0); + }); + + it("does not copy a credential-bearing fetch error into the report", async () => { + const throwingFetch = (async () => { + throw new TypeError( + "request to https://user:clear-password@host/v1?secret=clear-query-secret failed", + ); + }) as unknown as typeof fetch; + const metric = await runInferenceRoundTrip({ + ...inferenceOptionsBase, + samples: 1, + fetchImpl: throwingFetch, + clock: queueClock([0, 5]), + }); + expect(metric.reason).toBe("TypeError: request failed"); + expect(metric.reason).not.toContain("clear-password"); + expect(metric.reason).not.toContain("clear-query-secret"); + }); + + it("refuses redirects so prompts stay on the configured origin", async () => { + let requestInit: RequestInit | undefined; + const fetchImpl: typeof fetch = async (_input, init) => { + requestInit = init; + return { ok: true, status: 200, text: async () => VALID_COMPLETION } as Response; + }; + const metric = await runInferenceRoundTrip({ + ...inferenceOptionsBase, + samples: 1, + fetchImpl, + clock: queueClock([0, 5]), + }); + expect(metric.status).toBe("ok"); + expect(requestInit?.redirect).toBe("error"); + }); +}); + +describe("trace ingestion", () => { + it("ingests the canonical sandbox phase emitted by onboarding", () => { + const traceDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bench-trace-")); + const tracePath = path.join(traceDir, "onboard.json"); + const previousTraceFile = process.env.NEMOCLAW_TRACE_FILE; + process.env.NEMOCLAW_TRACE_FILE = tracePath; + resetTraceForTests(); + try { + const handle = startOnboardTrace({ agent: "openclaw" }, process.env); + withSandboxPhaseTrace("bench", "openai", "test-model", "openclaw", () => undefined); + finishOnboardTrace(handle, true); + const artifact = JSON.parse(fs.readFileSync(tracePath, "utf8")) as unknown; + expect(ingestSandboxColdStart(artifact)).toMatchObject({ + status: "ok", + breakdown: { sandbox_phase_ms: expect.any(Number) }, + }); + } finally { + resetTraceForTests(); + delete process.env.NEMOCLAW_TRACE_FILE; + Object.assign( + process.env, + previousTraceFile === undefined ? {} : { NEMOCLAW_TRACE_FILE: previousTraceFile }, + ); + fs.rmSync(traceDir, { recursive: true, force: true }); + } + }); + + it("uses the enclosing sandbox phase as cold-start total without double-counting readiness", () => { + const phase = traceSpan(SANDBOX_PHASE_SPAN, 2000); + const readiness = traceSpan(SANDBOX_READINESS_SPAN, 800, { + parent_span_id: phase.span_id, + }); + const metric = ingestSandboxColdStart(traceArtifact([phase, readiness])); + expect(metric.status).toBe("ok"); + expect(metric.breakdown).toEqual({ sandbox_phase_ms: 2000, readiness_wait_ms: 800 }); + expect(metric.stats?.median_ms).toBe(2000); + // This span exists only around createSandbox(); an initial cold creation can + // have fresh=false because --fresh controls forced recreation. + expect(metric.context?.fresh).toBe(false); + }); + + it("marks sandbox cold-start unsupported when spans are absent", () => { + const metric = ingestSandboxColdStart(traceArtifact([])); + expect(metric.status).toBe("unsupported"); + expect(metric.source).toBe("none"); + expect(metric.reason).toContain("trace"); + }); + + it("does not present policy application setup time as request-path overhead", () => { + const metric = ingestPolicyOverhead( + traceArtifact([ + traceSpan(POLICY_APPLICATION_SPAN, 42, { attributes: { provider: "nvidia" } }), + ]), + ); + expect(metric.status).toBe("unsupported"); + expect(metric.stats).toBeUndefined(); + expect(metric.reason).toContain("not request-path shield overhead"); + expect(metric.context).toMatchObject({ + provider: "nvidia", + agent: "openclaw", + non_interactive: true, + fresh: false, + }); + }); + + it("marks policy overhead unsupported when the span is absent", () => { + const metric = ingestPolicyOverhead(traceArtifact([])); + expect(metric.status).toBe("unsupported"); + }); + + it("reports malformed supplied traces as errors", () => { + expect(ingestSandboxColdStart(null)).toMatchObject({ status: "error" }); + expect(ingestPolicyOverhead({ resource_spans: "nope" })).toMatchObject({ + status: "error", + }); + }); + + it("rejects artifacts from a foreign trace scope", () => { + const artifact = traceArtifact([], { scopeName: "other.tool" }); + expect(ingestSandboxColdStart(artifact)).toMatchObject({ status: "error" }); + }); + + it("rejects a failed onboard root", () => { + const artifact = traceArtifact([traceSpan(SANDBOX_PHASE_SPAN, 2000)], { + rootStatus: { code: "ERROR", message: "onboard failed" }, + }); + expect(ingestSandboxColdStart(artifact).status).toBe("error"); + expect(ingestPolicyOverhead(artifact).status).toBe("error"); + }); + + it("rejects failed and invalid metric spans", () => { + const failed = traceArtifact([ + traceSpan(SANDBOX_PHASE_SPAN, 2000, { status: { code: "ERROR" } }), + ]); + const negative = traceArtifact([traceSpan(POLICY_APPLICATION_SPAN, -25)]); + const nonFinite = traceArtifact([traceSpan(POLICY_APPLICATION_SPAN, Number.POSITIVE_INFINITY)]); + expect(ingestSandboxColdStart(failed)).toMatchObject({ status: "error" }); + expect(ingestPolicyOverhead(negative)).toMatchObject({ status: "error" }); + expect(ingestPolicyOverhead(nonFinite)).toMatchObject({ status: "error" }); + }); + + it("does not echo untrusted root or metric status text into report reasons", () => { + const leakedStatus = { code: "arbitrary-trace-secret" } as unknown as TraceSpan["status"]; + const metrics = [ + ingestSandboxColdStart(traceArtifact([], { rootStatus: leakedStatus })), + ingestSandboxColdStart( + traceArtifact([ + traceSpan(SANDBOX_PHASE_SPAN, 2000, { + status: leakedStatus, + }), + ]), + ), + ]; + const serialized = JSON.stringify(metrics); + expect(serialized).not.toContain("arbitrary-trace-secret"); + expect(metrics[0].reason).toContain("status is missing or not OK"); + expect(metrics[1].reason).toContain("status is missing or not OK"); + }); + + it("rejects spans from a different trace identity", () => { + const artifact = traceArtifact([ + traceSpan(SANDBOX_PHASE_SPAN, 2000, { + trace_id: "ffffffffffffffffffffffffffffffff", + }), + ]); + expect(ingestSandboxColdStart(artifact)).toMatchObject({ status: "error" }); + }); + + it("rejects readiness durations larger than the enclosing sandbox phase", () => { + const phase = traceSpan(SANDBOX_PHASE_SPAN, 1000); + const readiness = traceSpan(SANDBOX_READINESS_SPAN, 1001, { + parent_span_id: phase.span_id, + }); + const artifact = traceArtifact([phase, readiness]); + expect(ingestSandboxColdStart(artifact)).toMatchObject({ status: "error" }); + }); + + it("rejects readiness spans outside the sandbox phase", () => { + const artifact = traceArtifact([ + traceSpan(SANDBOX_PHASE_SPAN, 1000), + traceSpan(SANDBOX_READINESS_SPAN, 500, { parent_span_id: ROOT_SPAN_ID }), + ]); + expect(ingestSandboxColdStart(artifact)).toMatchObject({ status: "error" }); + }); + + it("rejects a sandbox phase longer than the onboard root", () => { + const artifact = traceArtifact([traceSpan(SANDBOX_PHASE_SPAN, 3001)]); + expect(ingestSandboxColdStart(artifact)).toMatchObject({ status: "error" }); + }); + + it("rejects foreign and impossible policy spans", () => { + const foreign = traceArtifact([ + traceSpan(POLICY_APPLICATION_SPAN, 42, { parent_span_id: "foreign" }), + ]); + const tooLong = traceArtifact([traceSpan(POLICY_APPLICATION_SPAN, 3001)]); + expect(ingestPolicyOverhead(foreign)).toMatchObject({ status: "error" }); + expect(ingestPolicyOverhead(tooLong)).toMatchObject({ status: "error" }); + }); + + it("marks interactive policy timing unsupported because it can include human think time", () => { + const artifact = traceArtifact([traceSpan(POLICY_APPLICATION_SPAN, 42)], { + rootAttributes: { non_interactive: false }, + }); + const metric = ingestPolicyOverhead(artifact); + expect(metric).toMatchObject({ status: "unsupported", source: "none" }); + expect(metric.reason).toContain("human think time"); + }); +}); + +describe("unsupportedTraceMetric", () => { + it.each([ + "sandbox-cold-start", + "policy-shield-overhead", + ] as const)("describes %s as unsupported with guidance", (id) => { + const metric = unsupportedTraceMetric(id); + expect(metric.id).toBe(id); + expect(metric.status).toBe("unsupported"); + expect(metric.reason).toContain("NEMOCLAW_TRACE"); + }); +}); + +describe("renderMarkdownReport", () => { + const report: BenchReport = { + schema_version: BENCH_SCHEMA_VERSION, + generated_at: "2026-06-23T00:00:00.000Z", + environment: { + os: "Linux 6.0", + arch: "x64", + node: "v22.16.0", + cpus: 8, + cpu_model: "Test CPU", + total_mem_gib: 32, + }, + target: { base_url: "https://inference.local/v1", model: "test-model", api_key_present: true }, + metrics: [ + { + id: "inference-round-trip", + status: "ok", + unit: "ms", + source: "live-request", + interpretation: "advisory-non-normative", + samples: 3, + stats: { min_ms: 10, median_ms: 20, p95_ms: 30, mean_ms: 20, max_ms: 30 }, + }, + unsupportedTraceMetric("sandbox-cold-start"), + ], + }; + + it("includes environment, target, metrics, and the advisory disclaimer", () => { + const markdown = renderMarkdownReport(report); + expect(markdown).toContain("# NemoClaw value benchmark"); + expect(markdown).toContain("test-model"); + expect(markdown).toContain("inference-round-trip"); + expect(markdown).toContain("advisory and non-normative"); + expect(markdown).toContain("Troubleshooting"); + }); +}); + +describe("hasBlockingError", () => { + it.each([ + { status: "ok" as const, expected: false }, + { status: "unsupported" as const, expected: false }, + { status: "error" as const, expected: true }, + ])("returns $expected for a $status metric", ({ status, expected }) => { + const report: BenchReport = { + schema_version: BENCH_SCHEMA_VERSION, + generated_at: "2026-06-23T00:00:00.000Z", + environment: { + os: "Linux", + arch: "x64", + node: "v22.16.0", + cpus: 1, + cpu_model: "x", + total_mem_gib: 1, + }, + target: { base_url: "x", model: "x", api_key_present: false }, + metrics: [ + { + id: "inference-round-trip", + status, + unit: "ms", + source: "live-request", + interpretation: "advisory-non-normative", + }, + ], + }; + expect(hasBlockingError(report)).toBe(expected); + }); +});