diff --git a/test/e2e/live/full-e2e-inference-probe.ts b/test/e2e/live/full-e2e-inference-probe.ts new file mode 100644 index 0000000000..3d4dd9c070 --- /dev/null +++ b/test/e2e/live/full-e2e-inference-probe.ts @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { resolveMaxTokensField } from "../../../src/lib/inference/max-tokens-field.ts"; +import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts"; + +const ARITHMETIC_PROMPT = "What is 6 multiplied by 7? Reply with only the integer, no extra words."; + +export const FULL_E2E_INFERENCE_REPLY_BUDGETS = [512, 1024] as const; +export const FULL_E2E_INFERENCE_CAPTURE_LIMIT_BYTES = 256 * 1024; +export const FULL_E2E_INFERENCE_EVIDENCE_LIMIT_BYTES = 32 * 1024; + +const EVIDENCE_TEXT_LIMIT_BYTES = 4 * 1024; +const EVIDENCE_PARSE_ERROR_LIMIT_BYTES = 2 * 1024; +const EVIDENCE_MODEL_LIMIT_BYTES = 512; +const EVIDENCE_FINISH_REASON_LIMIT_BYTES = 128; +const TRUNCATION_SUFFIX = "...[truncated]"; +const USAGE_TOTAL_FIELDS = ["prompt_tokens", "completion_tokens", "total_tokens"] as const; +const USAGE_DETAIL_FIELDS = [ + "audio_tokens", + "cached_tokens", + "reasoning_tokens", + "accepted_prediction_tokens", + "rejected_prediction_tokens", +] as const; +const USAGE_DETAIL_GROUPS = ["prompt_tokens_details", "completion_tokens_details"] as const; + +export interface InferenceCommandResult { + exitCode: number | null; + stdout: string; + stderr: string; +} + +export interface FullE2eInferenceResponseEvidence { + answer: string; + content: string; + finishReason: string | null; + model: string | null; + reasoningContent: string; + usage: Record | null; +} + +export interface FullE2eInferenceAttempt { + answerMatched: boolean; + attempt: number; + maxTokens: number; + parseError?: string; + response?: FullE2eInferenceResponseEvidence; + result: Result; +} + +export type FullE2eInferenceOutcome = + | "passed" + | "command-failure" + | "response-failure" + | "semantic-mismatch"; + +export interface FullE2eInferenceProbeResult { + attempts: FullE2eInferenceAttempt[]; + outcome: FullE2eInferenceOutcome; +} + +export interface FullE2eInferenceAttemptInput { + artifactName: string; + attempt: number; + maxTokens: number; + requestBody: string; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function optionalString(value: unknown): string | null { + return typeof value === "string" ? value : null; +} + +function limitUtf8(value: string, limitBytes: number): string { + if (Buffer.byteLength(value, "utf8") <= limitBytes) return value; + const contentLimit = limitBytes - Buffer.byteLength(TRUNCATION_SUFFIX, "utf8"); + let content = ""; + let contentBytes = 0; + for (const character of value) { + const characterBytes = Buffer.byteLength(character, "utf8"); + if (contentBytes + characterBytes > contentLimit) break; + content += character; + contentBytes += characterBytes; + } + return `${content}${TRUNCATION_SUFFIX}`; +} + +function finiteNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function projectUsageDetails(value: unknown): Record | undefined { + if (!isRecord(value)) return undefined; + const details = Object.fromEntries( + USAGE_DETAIL_FIELDS.flatMap((field) => { + const projected = finiteNumber(value[field]); + return projected === undefined ? [] : [[field, projected]]; + }), + ); + return Object.keys(details).length > 0 ? details : undefined; +} + +function projectUsage(value: Record | null): Record | null { + if (!value) return null; + const projected: Record = {}; + for (const field of USAGE_TOTAL_FIELDS) { + const total = finiteNumber(value[field]); + if (total !== undefined) projected[field] = total; + } + for (const field of USAGE_DETAIL_GROUPS) { + const details = projectUsageDetails(value[field]); + if (details) projected[field] = details; + } + return projected; +} + +export function buildFullE2eInferenceRequest(model: string, maxTokens: number): string { + const maxTokensField = resolveMaxTokensField(model); + return JSON.stringify({ + model, + messages: [{ role: "user", content: ARITHMETIC_PROMPT }], + ...(maxTokensField === "max_tokens" ? { temperature: 0 } : {}), + [maxTokensField]: maxTokens, + stream: false, + }); +} + +export function parseFullE2eInferenceResponse(body: string): FullE2eInferenceResponseEvidence { + let parsed: unknown; + try { + parsed = JSON.parse(body); + } catch (error) { + throw new Error( + `inference.local returned invalid JSON: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + + if (!isRecord(parsed)) throw new Error("inference.local response must be a JSON object"); + const choices = parsed.choices; + if (!Array.isArray(choices) || !isRecord(choices[0])) { + throw new Error("inference.local response must contain choices[0]"); + } + const choice = choices[0]; + if (!isRecord(choice.message)) { + throw new Error("inference.local response must contain choices[0].message"); + } + + const content = optionalString(choice.message.content)?.trim() ?? ""; + const reasoningContent = + ( + optionalString(choice.message.reasoning_content) ?? optionalString(choice.message.reasoning) + )?.trim() ?? ""; + return { + answer: content || reasoningContent, + content, + finishReason: optionalString(choice.finish_reason), + model: optionalString(parsed.model), + reasoningContent, + usage: isRecord(parsed.usage) ? parsed.usage : null, + }; +} + +export async function runFullE2eInferenceProbe( + model: string, + execute: (input: FullE2eInferenceAttemptInput) => Promise, +): Promise> { + const attempts: FullE2eInferenceAttempt[] = []; + + for (const [index, maxTokens] of FULL_E2E_INFERENCE_REPLY_BUDGETS.entries()) { + const attempt = index + 1; + const result = await execute({ + artifactName: `phase-4-sandbox-inference-local-attempt-${String(attempt).padStart(2, "0")}`, + attempt, + maxTokens, + requestBody: buildFullE2eInferenceRequest(model, maxTokens), + }); + if (result.exitCode !== 0) { + attempts.push({ answerMatched: false, attempt, maxTokens, result }); + return { attempts, outcome: "command-failure" }; + } + + let response: FullE2eInferenceResponseEvidence; + try { + response = parseFullE2eInferenceResponse(result.stdout); + } catch (error) { + attempts.push({ + answerMatched: false, + attempt, + maxTokens, + parseError: error instanceof Error ? error.message : String(error), + result, + }); + return { attempts, outcome: "response-failure" }; + } + + if (!response.answer && response.finishReason !== "length") { + attempts.push({ + answerMatched: false, + attempt, + maxTokens, + parseError: + "inference.local response did not contain assistant content or reasoning content", + response, + result, + }); + return { attempts, outcome: "response-failure" }; + } + + const answerMatched = containsInteger42Answer(response.answer); + attempts.push({ answerMatched, attempt, maxTokens, response, result }); + if (answerMatched) return { attempts, outcome: "passed" }; + } + + return { attempts, outcome: "semantic-mismatch" }; +} + +export function fullE2eInferenceProbeEvidence( + probe: FullE2eInferenceProbeResult, +): Record { + const evidence = { + schemaVersion: "nemoclaw.full_e2e_inference.v1", + outcome: probe.outcome, + attempts: probe.attempts.map((attempt) => ({ + answerMatched: attempt.answerMatched, + attempt: attempt.attempt, + exitCode: attempt.result.exitCode, + maxTokens: attempt.maxTokens, + ...(attempt.parseError + ? { parseError: limitUtf8(attempt.parseError, EVIDENCE_PARSE_ERROR_LIMIT_BYTES) } + : {}), + ...(attempt.response + ? { + response: { + content: limitUtf8(attempt.response.content, EVIDENCE_TEXT_LIMIT_BYTES), + finish_reason: + attempt.response.finishReason === null + ? null + : limitUtf8(attempt.response.finishReason, EVIDENCE_FINISH_REASON_LIMIT_BYTES), + model: + attempt.response.model === null + ? null + : limitUtf8(attempt.response.model, EVIDENCE_MODEL_LIMIT_BYTES), + reasoning_content: limitUtf8( + attempt.response.reasoningContent, + EVIDENCE_TEXT_LIMIT_BYTES, + ), + usage: projectUsage(attempt.response.usage), + }, + } + : {}), + })), + }; + const evidenceBytes = Buffer.byteLength(JSON.stringify(evidence), "utf8"); + if (evidenceBytes > FULL_E2E_INFERENCE_EVIDENCE_LIMIT_BYTES) { + throw new Error( + `full E2E inference evidence exceeded ${FULL_E2E_INFERENCE_EVIDENCE_LIMIT_BYTES} bytes after projection`, + ); + } + return evidence; +} diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index bc5a3b16b8..433871a96b 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -4,7 +4,6 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/command.ts"; @@ -36,6 +35,11 @@ import { buildOpenClawFirstTurnLatencyEvidence, extractOpenClawAgentPayloadText, } from "./agent-turn-latency-helpers.ts"; +import { + FULL_E2E_INFERENCE_CAPTURE_LIMIT_BYTES, + fullE2eInferenceProbeEvidence, + runFullE2eInferenceProbe, +} from "./full-e2e-inference-probe.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-full"; const SETUP_MODE = process.env.NEMOCLAW_E2E_SETUP_MODE ?? "source-install"; @@ -124,23 +128,6 @@ async function cleanup(host: HostCliClient, sandbox: SandboxClient): Promise undefined); } -function chatRequest(model: string): string { - return JSON.stringify({ - model, - messages: [ - { - role: "user", - content: "What is 6 multiplied by 7? Reply with only the integer, no extra words.", - }, - ], - max_tokens: 100, - }); -} - -function parseReplyCommand(): string { - return String.raw`python3 -c 'import json,sys; d=json.load(sys.stdin); m=d["choices"][0]["message"]; print((m.get("content") or m.get("reasoning_content") or "").strip())'`; -} - function readAndDeleteTraceWindow(traceFile: string, traceDirectory: string): OnboardTraceWindow { try { return readOnboardTraceWindow(JSON.parse(fs.readFileSync(traceFile, "utf8")) as unknown); @@ -486,22 +473,38 @@ test("full e2e: install, onboard, inference, cli operations, and cleanup", { expect(direct.exitCode, resultText(direct)).toBe(0); expect(resultText(direct)).toContain("data"); - const sandboxInference = await sandbox.exec( - SANDBOX_NAME, - [ - "sh", - "-lc", - `curl -fsS --max-time 90 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' --data '${chatRequest(hosted.model)}' | ${parseReplyCommand()}`, - ], - { - artifactName: "phase-4-sandbox-inference-local", - env: env(), - redactionValues, - timeoutMs: 120_000, - }, + const sandboxInference = await runFullE2eInferenceProbe(hosted.model, async (attempt) => + sandbox.exec( + SANDBOX_NAME, + [ + "curl", + "-fsS", + "--max-time", + "90", + "https://inference.local/v1/chat/completions", + "-H", + "Content-Type: application/json", + "--data-raw", + attempt.requestBody, + ], + { + artifactName: attempt.artifactName, + captureLimitBytes: FULL_E2E_INFERENCE_CAPTURE_LIMIT_BYTES, + env: env(), + redactionValues, + timeoutMs: 120_000, + }, + ), + ); + const sandboxInferenceEvidence = fullE2eInferenceProbeEvidence(sandboxInference); + await artifacts.writeJson( + "phase-4-sandbox-inference-local-attempts.json", + sandboxInferenceEvidence, ); - expect(sandboxInference.exitCode, resultText(sandboxInference)).toBe(0); - expect(containsInteger42Answer(sandboxInference.stdout), resultText(sandboxInference)).toBe(true); + const finalInferenceAttempt = sandboxInference.attempts.at(-1)!; + const sandboxInferenceDiagnostic = `${resultText(finalInferenceAttempt.result)}\n${JSON.stringify(sandboxInferenceEvidence, null, 2)}`; + expect(finalInferenceAttempt.result.exitCode, sandboxInferenceDiagnostic).toBe(0); + expect(sandboxInference.outcome, sandboxInferenceDiagnostic).toBe("passed"); progress.phase("inspect runtime logs and security posture"); const logs = await repoNemoclaw( diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index aab551a2c6..648f1cc97e 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -51,6 +51,7 @@ { "live": "test/e2e/live/full-e2e.test.ts", "fast": [ + "test/e2e/support/full-e2e-inference-probe.test.ts", "test/e2e/support/onboard-performance.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" diff --git a/test/e2e/support/full-e2e-inference-probe.test.ts b/test/e2e/support/full-e2e-inference-probe.test.ts new file mode 100644 index 0000000000..238574806a --- /dev/null +++ b/test/e2e/support/full-e2e-inference-probe.test.ts @@ -0,0 +1,284 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildFullE2eInferenceRequest, + FULL_E2E_INFERENCE_EVIDENCE_LIMIT_BYTES, + type FullE2eInferenceAttemptInput, + fullE2eInferenceProbeEvidence, + type InferenceCommandResult, + parseFullE2eInferenceResponse, + runFullE2eInferenceProbe, +} from "../live/full-e2e-inference-probe.ts"; + +function commandResult(stdout: string, exitCode = 0, stderr = ""): InferenceCommandResult { + return { exitCode, stderr, stdout }; +} + +function completion(content: string, finishReason = "stop"): string { + return JSON.stringify({ + choices: [{ finish_reason: finishReason, message: { content } }], + model: "nvidia/nvidia/nemotron-3-ultra", + usage: { completion_tokens: 1, prompt_tokens: 20, total_tokens: 21 }, + }); +} + +describe("full E2E sandbox inference probe", () => { + it("uses deterministic sampling and the requested reply budget for Nemotron", () => { + expect(JSON.parse(buildFullE2eInferenceRequest("nvidia/nvidia/nemotron-3-ultra", 512))).toEqual( + { + model: "nvidia/nvidia/nemotron-3-ultra", + messages: [ + { + role: "user", + content: "What is 6 multiplied by 7? Reply with only the integer, no extra words.", + }, + ], + temperature: 0, + max_tokens: 512, + stream: false, + }, + ); + }); + + it("uses the compatible reply-budget field for models that reject max_tokens", () => { + const request = JSON.parse(buildFullE2eInferenceRequest("gpt-5.4", 512)); + + expect(request).toMatchObject({ max_completion_tokens: 512, stream: false }); + expect(request).not.toHaveProperty("max_tokens"); + expect(request).not.toHaveProperty("temperature"); + }); + + it("preserves response metadata and prefers final content over reasoning", () => { + const response = parseFullE2eInferenceResponse( + JSON.stringify({ + choices: [ + { + finish_reason: "length", + message: { content: "The", reasoning_content: "6 multiplied by 7 is 42" }, + }, + ], + model: "nvidia/nvidia/nemotron-3-ultra", + usage: { completion_tokens: 512, completion_tokens_details: { reasoning_tokens: 511 } }, + }), + ); + + expect(response).toEqual({ + answer: "The", + content: "The", + finishReason: "length", + model: "nvidia/nvidia/nemotron-3-ultra", + reasoningContent: "6 multiplied by 7 is 42", + usage: { completion_tokens: 512, completion_tokens_details: { reasoning_tokens: 511 } }, + }); + }); + + it("falls back to reasoning content when final content is empty", () => { + const response = parseFullE2eInferenceResponse( + JSON.stringify({ + choices: [{ finish_reason: "stop", message: { reasoning_content: "42" } }], + }), + ); + + expect(response.answer).toBe("42"); + expect(response.content).toBe(""); + expect(response.reasoningContent).toBe("42"); + }); + + it("retries only a successful semantic mismatch with a larger reply budget", async () => { + const requests: FullE2eInferenceAttemptInput[] = []; + const probe = await runFullE2eInferenceProbe( + "nvidia/nvidia/nemotron-3-ultra", + async (input) => { + requests.push(input); + return commandResult(input.attempt === 1 ? completion("The") : completion("42")); + }, + ); + + expect(probe.outcome).toBe("passed"); + expect(requests.map(({ attempt, maxTokens }) => ({ attempt, maxTokens }))).toEqual([ + { attempt: 1, maxTokens: 512 }, + { attempt: 2, maxTokens: 1024 }, + ]); + expect(requests.map(({ artifactName }) => artifactName)).toEqual([ + "phase-4-sandbox-inference-local-attempt-01", + "phase-4-sandbox-inference-local-attempt-02", + ]); + expect(JSON.parse(requests[1]!.requestBody)).toMatchObject({ max_tokens: 1024 }); + expect(fullE2eInferenceProbeEvidence(probe)).toEqual({ + schemaVersion: "nemoclaw.full_e2e_inference.v1", + outcome: "passed", + attempts: [ + { + answerMatched: false, + attempt: 1, + exitCode: 0, + maxTokens: 512, + response: { + content: "The", + finish_reason: "stop", + model: "nvidia/nvidia/nemotron-3-ultra", + reasoning_content: "", + usage: { completion_tokens: 1, prompt_tokens: 20, total_tokens: 21 }, + }, + }, + { + answerMatched: true, + attempt: 2, + exitCode: 0, + maxTokens: 1024, + response: { + content: "42", + finish_reason: "stop", + model: "nvidia/nvidia/nemotron-3-ultra", + reasoning_content: "", + usage: { completion_tokens: 1, prompt_tokens: 20, total_tokens: 21 }, + }, + }, + ], + }); + }); + + it("retains parse failures through the public evidence serializer", async () => { + const probe = await runFullE2eInferenceProbe("nvidia/nvidia/nemotron-3-ultra", async () => + commandResult("not json"), + ); + const parseError = probe.attempts[0]?.parseError; + + expect(parseError).toContain("invalid JSON"); + expect(fullE2eInferenceProbeEvidence(probe)).toEqual({ + schemaVersion: "nemoclaw.full_e2e_inference.v1", + outcome: "response-failure", + attempts: [ + { + answerMatched: false, + attempt: 1, + exitCode: 0, + maxTokens: 512, + parseError, + }, + ], + }); + }); + + it("bounds projected response evidence and drops unreviewed usage fields", async () => { + const oversized = `42${"x".repeat(100_000)}`; + const probe = await runFullE2eInferenceProbe("nvidia/nvidia/nemotron-3-ultra", async () => + commandResult( + JSON.stringify({ + choices: [ + { + finish_reason: "stop", + message: { content: oversized, reasoning_content: oversized }, + }, + ], + model: "nvidia/nvidia/nemotron-3-ultra", + usage: { + completion_tokens: 1, + completion_tokens_details: { reasoning_tokens: 1, unreviewed: oversized }, + unreviewed: oversized, + }, + }), + ), + ); + const evidence = fullE2eInferenceProbeEvidence(probe); + const attempt = (evidence.attempts as Array>)[0]!; + const response = attempt.response as Record; + + expect(Buffer.byteLength(JSON.stringify(evidence), "utf8")).toBeLessThanOrEqual( + FULL_E2E_INFERENCE_EVIDENCE_LIMIT_BYTES, + ); + expect(response.content).toMatch(/\.\.\.\[truncated\]$/); + expect(response.reasoning_content).toMatch(/\.\.\.\[truncated\]$/); + expect(response.usage).toEqual({ + completion_tokens: 1, + completion_tokens_details: { reasoning_tokens: 1 }, + }); + }); + + it("accepts the existing whitespace-tolerant 42 answer on the first attempt", async () => { + let calls = 0; + const probe = await runFullE2eInferenceProbe("nvidia/nvidia/nemotron-3-ultra", async () => { + calls += 1; + return commandResult(completion("4\n2")); + }); + + expect(probe.outcome).toBe("passed"); + expect(calls).toBe(1); + }); + + it("does not retry command, HTTP, or transport failures", async () => { + let calls = 0; + const probe = await runFullE2eInferenceProbe("nvidia/nvidia/nemotron-3-ultra", async () => { + calls += 1; + return commandResult("", 22, "curl: (22) HTTP 403"); + }); + + expect(probe.outcome).toBe("command-failure"); + expect(calls).toBe(1); + }); + + it.each([ + ["invalid JSON", "not json", "invalid JSON"], + ["a missing choice", JSON.stringify({}), "choices[0]"], + [ + "a missing message", + JSON.stringify({ choices: [{ finish_reason: "stop" }] }), + "choices[0].message", + ], + ])("does not retry %s", async (_case, stdout, expectedError) => { + let calls = 0; + const probe = await runFullE2eInferenceProbe("nvidia/nvidia/nemotron-3-ultra", async () => { + calls += 1; + return commandResult(stdout); + }); + + expect(probe.outcome).toBe("response-failure"); + expect(probe.attempts[0]?.parseError).toContain(expectedError); + expect(calls).toBe(1); + }); + + it("retries a structurally valid length-truncated response with no answer", async () => { + let calls = 0; + const probe = await runFullE2eInferenceProbe("nvidia/nvidia/nemotron-3-ultra", async () => { + calls += 1; + return calls === 1 + ? commandResult( + JSON.stringify({ + choices: [{ finish_reason: "length", message: { content: "" } }], + }), + ) + : commandResult(completion("42")); + }); + + expect(probe.outcome).toBe("passed"); + expect(calls).toBe(2); + }); + + it("does not retry a completed response with no answer", async () => { + let calls = 0; + const probe = await runFullE2eInferenceProbe("nvidia/nvidia/nemotron-3-ultra", async () => { + calls += 1; + return commandResult( + JSON.stringify({ choices: [{ finish_reason: "stop", message: { content: "" } }] }), + ); + }); + + expect(probe.outcome).toBe("response-failure"); + expect(probe.attempts[0]?.parseError).toContain("did not contain assistant content"); + expect(calls).toBe(1); + }); + + it("reports a bounded semantic mismatch when neither valid response contains 42", async () => { + let calls = 0; + const probe = await runFullE2eInferenceProbe("nvidia/nvidia/nemotron-3-ultra", async () => { + calls += 1; + return commandResult(completion("The answer is forty-two.")); + }); + + expect(probe.outcome).toBe("semantic-mismatch"); + expect(calls).toBe(2); + }); +});