diff --git a/nemoclaw-blueprint/scripts/nemotron-inference-fix.js b/nemoclaw-blueprint/scripts/nemotron-inference-fix.js index 958e384457c..f3871ce99e0 100644 --- a/nemoclaw-blueprint/scripts/nemotron-inference-fix.js +++ b/nemoclaw-blueprint/scripts/nemotron-inference-fix.js @@ -54,6 +54,38 @@ // provider configuration always sends the required model-specific kwargs (or // NVIDIA Build no longer requires them) and the DeepSeek/Kimi channel latency // validation passes without the monkeypatch. +// +// Source-of-truth / removal contract (NemoClaw#4851): +// Invalid state: nvidia/nemotron-3-ultra-550b plans multi-step tasks +// correctly in `reasoning_content` but silently drops intermediate steps +// from `content` when the request has no execution-capable tools and no +// caller-supplied system message. force_nonempty_content does not address +// this — verified via direct curl with and without that kwarg. +// +// Source boundary: the chat template / vLLM-NIM serving stack for this +// model on NVIDIA Build is the actual origin; the model's emission shape +// is outside this repository. Inside the sandbox we own the preload that +// wraps every chat-completions request on the way out. +// +// Why not fix only at the origin: the NVIDIA Build chat template + Ultra +// weight pairing is owned upstream. NVB#6272828 tracks the upstream fix. +// Until that lands, a sandbox-side preload is the only way to keep the +// Ultra path usable in the niche tool-less configuration that triggers +// the bug. +// +// Regression proof: test/nemotron-inference-fix.test.ts covers the +// inject/skip branches via the http stub AND a real fetch/undici request +// against a local OpenAI-compatible endpoint, asserting both the injected +// system message and the refreshed Content-Length. The runtime model- +// output behavior (acceptance criteria from #4851) is validated against +// integrate.api.nvidia.com via the checked-in runbook at +// test/e2e-runtime/4851-ultra-toolless-validation.md — anyone reviewing +// acceptance can re-run it directly. Re-run when this preload changes +// or when OpenClaw bumps a version that may shift Ultra's chat template. +// +// Removal condition: remove the TOOL_LESS_SYSTEM_PROMPT_RULES entry for +// Ultra 550B once NVB#6272828 ships and a clean Ultra response to the +// #4851 prompt no longer requires the nudge. (function () { 'use strict'; @@ -68,6 +100,94 @@ { pattern: /^moonshotai\/kimi-k2\.6$/i, kwargs: { thinking: false } }, ]; + // #4851: Ultra 550B silently drops intermediate steps from `content` when + // asked to perform multi-step tasks without execution-capable tools — + // reasoning plans all steps but content emits only the final command (or + // empty). chat_template_kwargs.force_nonempty_content doesn't help. When + // the caller hasn't supplied a system message AND has no execution- + // capable tools, inject a one-paragraph nudge so the model emits the full + // code/commands the user would run manually. Skip when a system message + // is already present anywhere in the array (caller's prompt wins) or + // when execution-capable tools are present (the model should use them). + // + // Scope boundary: this preload runs inside NemoClaw-managed sandboxes + // where the chat-completions destination is the OpenShell-gateway + // `inference.local` route bound to NVIDIA Build. The path+model regex + // is the intentional trust boundary; non-sandbox OpenAI-compatible + // callers do not load this preload. + var TOOL_LESS_SYSTEM_PROMPT_RULES = [ + { + pattern: /^nvidia\/nemotron-3-ultra-550b/i, + systemPrompt: + 'You do not have tools to write files or execute commands. When the user asks you to perform such actions, include the complete code or command they would need to run manually. Do not skip steps.', + }, + ]; + + // Tools that signal the agent has execution capability — when any of + // these are in the request, the tool-less nudge above doesn't apply + // because the model should call the tool. Search/web/fetch/describe/read + // tools are intentionally NOT in this set: they don't let the model + // write a file or run a command, so they don't change the "no way to + // perform the asked action" condition that #4851 cares about. + // + // Tight allowlist (not a broad regex) to avoid false positives on + // harmless business tools like `create_ticket`, `run_query`, `save_search`, + // `command_palette` that contain "create"/"run"/"save"/"command" but don't + // give the model the ability to write a file or execute a shell command. + // Match exact known names + the canonical OpenClaw/MCP suffixes. + // + // The bare `write`, `edit`, and `notebook_edit` names mirror + // `nemoclaw/src/index.ts:WRITE_TOOL_NAMES` so this allowlist stays + // aligned with the same write-capable surface OpenClaw scans for + // secrets. `tool_call` is the OpenClaw compact-catalog wrapper from + // `scripts/patch-openclaw-tool-catalog.js` — when it's present we + // can't tell from the request alone which underlying tool will be + // dispatched, so treat it as execution-capable and skip the nudge. + var EXECUTION_TOOL_NAMES = new Set([ + 'bash', + 'bash_execute', + 'exec', + 'execute', + 'execute_command', + 'shell', + 'shell_execute', + 'run_command', + 'run_shell', + 'write', + 'write_file', + 'file_write', + 'edit', + 'edit_file', + 'file_edit', + 'notebook_edit', + 'patch_file', + 'file_patch', + 'create_file', + 'file_create', + 'apply_patch', + 'str_replace_editor', + 'computer', + 'tool_call', + ]); + + function isExecutionCapableTool(tool) { + if (!tool || typeof tool !== 'object') return false; + // OpenAI chat-completions tool shape: { type: 'function', function: { name: ... } } + var name = null; + if (typeof tool.name === 'string') { + name = tool.name; + } else if (tool.function && typeof tool.function.name === 'string') { + name = tool.function.name; + } + if (!name) return false; + return EXECUTION_TOOL_NAMES.has(name.toLowerCase()); + } + + function hasExecutionCapableTool(body) { + if (!Array.isArray(body.tools) || body.tools.length === 0) return false; + return body.tools.some(isExecutionCapableTool); + } + function chatTemplateKwargsForModel(model) { var kwargs = null; CHAT_TEMPLATE_KWARG_RULES.forEach(function (rule) { @@ -101,12 +221,45 @@ return true; } + function toolLessSystemPromptForModel(body) { + if (!body || typeof body.model !== 'string') return null; + var rule = null; + for (var i = 0; i < TOOL_LESS_SYSTEM_PROMPT_RULES.length; i++) { + if (TOOL_LESS_SYSTEM_PROMPT_RULES[i].pattern.test(body.model)) { + rule = TOOL_LESS_SYSTEM_PROMPT_RULES[i]; + break; + } + } + if (!rule) return null; + if (!Array.isArray(body.messages) || body.messages.length === 0) return null; + // Scan ALL messages, not just messages[0]: the OpenAI chat-completions + // contract permits a system message anywhere in the array, and the + // "caller prompt wins" contract should hold for any of those positions. + var hasSystemMessage = body.messages.some(function (msg) { + return msg && typeof msg === 'object' && msg.role === 'system'; + }); + if (hasSystemMessage) return null; + // Skip when execution-capable tools are present; harmless tools like + // tool_search / web fetch / file describe don't change the "no way to + // perform the asked action" condition #4851 cares about. + if (hasExecutionCapableTool(body)) return null; + return rule.systemPrompt; + } + + function applyToolLessSystemPrompt(body) { + var systemPrompt = toolLessSystemPromptForModel(body); + if (!systemPrompt) return false; + body.messages.unshift({ role: 'system', content: systemPrompt }); + return true; + } + function patchJsonBody(raw) { try { var body = JSON.parse(raw.toString('utf-8')); - if (!applyChatTemplateKwargs(body)) { - return null; - } + var changed = false; + if (applyChatTemplateKwargs(body)) changed = true; + if (applyToolLessSystemPrompt(body)) changed = true; + if (!changed) return null; return Buffer.from(JSON.stringify(body), 'utf-8'); } catch (_e) { return null; diff --git a/test/e2e-runtime/4851-ultra-toolless-validation.md b/test/e2e-runtime/4851-ultra-toolless-validation.md new file mode 100644 index 00000000000..c6444f04b6d --- /dev/null +++ b/test/e2e-runtime/4851-ultra-toolless-validation.md @@ -0,0 +1,205 @@ + + +# #4851 runtime validation — Ultra 550B tool-less system-prompt injection + +Repository-verifiable acceptance evidence for [PR #5085](https://github.com/NVIDIA/NemoClaw/pull/5085). + +The unit tests in `test/nemotron-inference-fix.test.ts` prove request mutation, Content-Length refresh, and the 12 inject/skip branches via stubbed http + real fetch/undici. They do not prove the upstream model-output behavior the issue's expected result asks for. That requires a live call to NVIDIA Endpoints, which can't run in unit CI without API-key secret infrastructure. + +This runbook is the maintained runtime-validation path. Anyone reviewing #4851 acceptance can run it directly against `integrate.api.nvidia.com` and confirm the model returns `content` with both file-creation code and the run command after the preload's system message is injected. + +## When to run + +- Before merging any PR that changes `nemoclaw-blueprint/scripts/nemotron-inference-fix.js` or `EXECUTION_TOOL_NAMES`. +- When a NemoClaw release pins a new OpenClaw version that may change the upstream chat template behavior on Ultra 550B. +- If QA reopens #4851. + +## Prerequisites + +- An NVIDIA API key with access to `nvidia/nemotron-3-ultra-550b-a55b` (build.nvidia.com → API Keys). +- `node >= 18`, `curl`, and `jq` (the scenarios below pipe responses through `jq` for readable parsing). Any Linux or macOS host works — this validates upstream model behavior, not local sandbox runtime. + +Export the key once for the session: + +```bash +export NVIDIA_API_KEY="nvapi-..." +``` + +## Scenario A — baseline (no preload, no system message, no tools) + +Demonstrates the bug as filed in the issue body. + +```bash +curl -sS -X POST https://integrate.api.nvidia.com/v1/chat/completions \ + -H "Authorization: Bearer ${NVIDIA_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "nvidia/nemotron-3-ultra-550b-a55b", + "messages": [{"role": "user", "content": "Create a file called hello.py in /tmp with a hello world script, then run it."}], + "max_tokens": 400, + "temperature": 0.0 + }' | jq '{ + finish_reason: .choices[0].finish_reason, + completion_tokens: .usage.completion_tokens, + reasoning_chars: (.choices[0].message.reasoning_content // "" | length), + content_chars: (.choices[0].message.content // "" | length), + content: (.choices[0].message.content // "") + }' +``` + +Expected result with `nemotron-3-ultra-550b-a55b` (matches issue body): + +- `finish_reason: "stop"` +- `reasoning_chars` ≈ 150–300 (model plans 3 steps internally) +- `content_chars` ≈ 0–60 (model drops file-creation step, may emit only the run command or empty) + +## Scenario B — `force_nonempty_content` kwarg only (no preload's system message) + +Demonstrates that the existing Nemotron-family kwarg doesn't fix #4851 by itself. + +```bash +curl -sS -X POST https://integrate.api.nvidia.com/v1/chat/completions \ + -H "Authorization: Bearer ${NVIDIA_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "nvidia/nemotron-3-ultra-550b-a55b", + "messages": [{"role": "user", "content": "Create a file called hello.py in /tmp with a hello world script, then run it."}], + "max_tokens": 400, + "temperature": 0.0, + "chat_template_kwargs": {"force_nonempty_content": true} + }' | jq '.choices[0].message.content | length' +``` + +Expected: still ≈ 0–60 chars. The kwarg doesn't change the failure mode. + +## Scenario C — preload's full mutation (system message + kwarg) + +Demonstrates the fix shipped in this PR. + +```bash +curl -sS -X POST https://integrate.api.nvidia.com/v1/chat/completions \ + -H "Authorization: Bearer ${NVIDIA_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "nvidia/nemotron-3-ultra-550b-a55b", + "messages": [ + {"role": "system", "content": "You do not have tools to write files or execute commands. When the user asks you to perform such actions, include the complete code or command they would need to run manually. Do not skip steps."}, + {"role": "user", "content": "Create a file called hello.py in /tmp with a hello world script, then run it."} + ], + "max_tokens": 400, + "temperature": 0.0, + "chat_template_kwargs": {"force_nonempty_content": true} + }' | jq '{ + finish_reason: .choices[0].finish_reason, + content_chars: (.choices[0].message.content // "" | length), + content: (.choices[0].message.content // "") + }' +``` + +Expected (`#4851` acceptance): + +- `content_chars` ≈ 400–600 +- `content` includes BOTH file creation (heredoc, redirection, or full source of `hello.py`) AND the run command (`python3 /tmp/hello.py` or equivalent) +- `finish_reason: "stop"` + +This satisfies the issue's "Expected Result, Option A" (`Model explains it lacks a file-write tool and shows the full code the user would need to run manually`). + +## Sanitized acceptance transcript + +The transcript below was captured by @cjagwani on 2026-06-09 against `integrate.api.nvidia.com` from a GCP Brev box. Reproduces the bug behavior in Scenarios A/B and the fix behavior in Scenario C. Use this as the durable acceptance baseline; new runs that differ structurally should update this section (and the dated entry below) rather than the unit tests. + +### Scenario A (baseline) — 2026-06-09 + +```text +finish_reason: stop +prompt_tokens: 35 +completion_tokens: 117 +reasoning_chars: 184 +content_chars: 1 +content: " " +``` + +Reasoning content (the model plans 3 steps but emits none of them in `content`): + +```text +The user wants me to: +1. Create a file called hello.py in /tmp +2. Put a hello world script in it +3. Run it + +I'll use the write tool to create the file and then the bash tool to run it. +``` + +### Scenario B (force_nonempty_content only) — 2026-06-09 + +```text +finish_reason: stop +prompt_tokens: 35 +completion_tokens: 131 +reasoning_chars: 184 +content_chars: 1 +content: " " +``` + +Same baseline failure mode — the kwarg alone doesn't address #4851. + +### Scenario C (full preload mutation) — 2026-06-09 + +```text +finish_reason: stop +prompt_tokens: 75 +completion_tokens: 187 +reasoning_chars: 241 +content_chars: 501 +``` + +Content (full text): + +````markdown +I'll provide you with the commands to create and run the hello world script manually. + +## Create the file + +```bash +cat > /tmp/hello.py << 'EOF' +#!/usr/bin/env python3 + +def main(): + print("Hello, World!") + +if __name__ == "__main__": + main() +EOF +``` + +## Make it executable (optional) + +```bash +chmod +x /tmp/hello.py +``` + +## Run it + +```bash +python3 /tmp/hello.py +``` + +**Expected output:** + +```text +Hello, World! +``` + +You can copy and paste these commands into your terminal to create and run the script. +```` + +This satisfies the issue's Option A acceptance condition: model explains it lacks file-write/execute tools and shows the complete code the user would need to run manually, with all 3 planned steps present in `content`. + +## Live verification log + +- 2026-06-09 — verified by @cjagwani on a GCP Brev box against `integrate.api.nvidia.com`. Numbers and content above match this run. + +When you re-run this runbook, add a dated entry here so the next reviewer can see how recently the upstream behavior was last confirmed. If the response shape differs materially from the sanitized transcript above, update both this log and the transcript. diff --git a/test/nemotron-inference-fix.test.ts b/test/nemotron-inference-fix.test.ts index 8e28e3285b4..aa606d8fc95 100644 --- a/test/nemotron-inference-fix.test.ts +++ b/test/nemotron-inference-fix.test.ts @@ -2,11 +2,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { spawnSync } from "node:child_process"; -import { describe, it, expect } from "vitest"; +import { describe, expect, it } from "vitest"; const START_SCRIPT = path.join(import.meta.dirname, "..", "scripts", "nemoclaw-start.sh"); const NEMOTRON_FIX_SOURCE = path.join( @@ -269,6 +269,11 @@ async function main() { model: 'other-model', messages: [{ role: 'user', content: 'ping' }], }, { 'content-type': 'application/json' }); + // #4851: Ultra 550B injection over the real fetch/undici path + await postJson(url, { + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'Create a file and run it.' }], + }, { 'content-type': 'application/json', 'content-length': '999' }); console.log(JSON.stringify(records)); } finally { await close(); @@ -286,7 +291,7 @@ main().catch((err) => { }); expect(result.status, result.stderr).toBe(0); const records = JSON.parse(result.stdout.trim()); - expect(records).toHaveLength(3); + expect(records).toHaveLength(4); const deepSeekBody = JSON.parse(records[0].body); expect(deepSeekBody.chat_template_kwargs).toEqual({ thinking: false }); @@ -300,5 +305,304 @@ main().catch((err) => { const otherBody = JSON.parse(records[2].body); expect(otherBody.chat_template_kwargs).toBeUndefined(); + + // #4851: Ultra 550B injection takes the fetch/undici path too, system + // message is prepended and Content-Length is refreshed for the larger body + const ultraBody = JSON.parse(records[3].body); + expect(ultraBody.messages[0].role).toBe("system"); + expect(ultraBody.messages[0].content).toMatch(/do not have tools/i); + expect(ultraBody.messages[1]).toEqual({ + role: "user", + content: "Create a file and run it.", + }); + expect(records[3].headers["content-length"]).toBe(String(Buffer.byteLength(records[3].body))); + expect(records[3].headers["content-length"]).not.toBe("999"); + }); + + it("preload injects a tool-less system prompt for Ultra 550B without overriding caller intent (#4851)", () => { + const preload = extractStartScriptHeredoc(src, "NEMOTRON_FIX_EOF"); + const harness = ` +const http = require('http'); +const records = []; +http.request = function (options) { + const record = { options, writes: [], headers: {}, removed: [] }; + records.push(record); + return { + write(chunk) { + record.writes.push(Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk)); + return true; + }, + end(cb) { + if (typeof cb === 'function') cb(); + return true; + }, + getHeader(name) { return record.headers[name]; }, + setHeader(name, value) { record.headers[name] = value; }, + removeHeader(name) { record.removed.push(name); delete record.headers[name]; }, + }; +}; +${preload} +function send(body) { + const req = http.request({ method: 'POST', path: '/v1/chat/completions' }); + req.write(body); + req.end(); +} +// case 0: Ultra 550B, no system, no tools — expect injected system message +send(JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'Create a file and run it.' }], +})); +// case 1: Ultra 550B, existing system message — expect NO injection +send(JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [ + { role: 'system', content: 'You are pirate-themed.' }, + { role: 'user', content: 'hi' }, + ], +})); +// case 2: Ultra 550B, with tools — expect NO injection (model should use tools) +send(JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'hi' }], + tools: [{ type: 'function', function: { name: 'exec', parameters: {} } }], +})); +// case 3: non-matching Nemotron, no system, no tools — expect NO injection +send(JSON.stringify({ + model: 'nvidia/nemotron-3-super-120b-a12b', + messages: [{ role: 'user', content: 'hi' }], +})); +// case 4: Ultra 550B with system message at non-zero index — expect NO injection +send(JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [ + { role: 'user', content: 'prior turn' }, + { role: 'assistant', content: 'ok' }, + { role: 'system', content: 'mid-conversation system message' }, + { role: 'user', content: 'hi again' }, + ], +})); +// case 5: Ultra 550B with non-execution tools only (toolSearch, web fetch) — +// expect INJECTION because these tools can't write files or run commands, +// matching the practical end-user config from #4851's repro +send(JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'hi' }], + tools: [ + { type: 'function', function: { name: 'tool_search', parameters: {} } }, + { type: 'function', function: { name: 'web_fetch', parameters: {} } }, + { type: 'function', function: { name: 'tool_describe', parameters: {} } }, + ], +})); +// case 6: Ultra 550B with mixed tools (search + bash_execute) — expect NO +// injection because execution-capable tool is present +send(JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'hi' }], + tools: [ + { type: 'function', function: { name: 'tool_search', parameters: {} } }, + { type: 'function', function: { name: 'bash_execute', parameters: {} } }, + ], +})); +// case 7: Ultra 550B with tools whose names contain broad tokens +// (create/run/save/command) but are NOT actually execution-capable — +// expect INJECTION because the tight allowlist rejects these false +// positives (would have been swallowed by a substring regex match). +send(JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'hi' }], + tools: [ + { type: 'function', function: { name: 'create_ticket', parameters: {} } }, + { type: 'function', function: { name: 'run_query', parameters: {} } }, + { type: 'function', function: { name: 'save_search', parameters: {} } }, + { type: 'function', function: { name: 'command_palette', parameters: {} } }, + ], +})); +// case 8: Ultra 550B with write_file specifically — expect NO injection +send(JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'hi' }], + tools: [{ type: 'function', function: { name: 'write_file', parameters: {} } }], +})); +// case 9: Ultra 550B with bare 'write'/'edit'/'notebook_edit' (mirrors +// nemoclaw/src/index.ts:WRITE_TOOL_NAMES) — expect NO injection +send(JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'hi' }], + tools: [ + { type: 'function', function: { name: 'write', parameters: {} } }, + { type: 'function', function: { name: 'edit', parameters: {} } }, + { type: 'function', function: { name: 'notebook_edit', parameters: {} } }, + ], +})); +// case 10: Ultra 550B with compact-catalog tool_call wrapper — expect NO +// injection because tool_call can dispatch to real exec/write tools. +send(JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'hi' }], + tools: [ + { type: 'function', function: { name: 'tool_search', parameters: {} } }, + { type: 'function', function: { name: 'tool_describe', parameters: {} } }, + { type: 'function', function: { name: 'tool_call', parameters: {} } }, + ], +})); +// case 11: top-level tool.name shape (no nested .function) — CodeRabbit nit. +// Some callers send { name, parameters } at the top level instead of the +// OpenAI nested function shape. +send(JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'hi' }], + tools: [{ name: 'bash_execute', parameters: {} }], +})); +console.log(JSON.stringify(records)); +`; + + const result = spawnSync(process.execPath, ["-e", harness], { + encoding: "utf-8", + timeout: 5000, + }); + expect(result.status, result.stderr).toBe(0); + const records = JSON.parse(result.stdout.trim()); + expect(records).toHaveLength(12); + + // case 0: system message injected at position 0 + const ultraBare = JSON.parse(records[0].writes.join("")); + expect(ultraBare.messages[0].role).toBe("system"); + expect(ultraBare.messages[0].content).toMatch(/do not have tools/i); + expect(ultraBare.messages[1]).toEqual({ + role: "user", + content: "Create a file and run it.", + }); + // kwargs still applied (Nemotron rule) + expect(ultraBare.chat_template_kwargs).toEqual({ force_nonempty_content: true }); + + // case 1: caller's system message preserved, no injection prepended + const ultraWithSystem = JSON.parse(records[1].writes.join("")); + expect(ultraWithSystem.messages).toHaveLength(2); + expect(ultraWithSystem.messages[0].content).toBe("You are pirate-themed."); + + // case 2: execution-capable tool (exec) present, no injection + const ultraWithExecTool = JSON.parse(records[2].writes.join("")); + expect(ultraWithExecTool.messages).toHaveLength(1); + expect(ultraWithExecTool.messages[0].role).toBe("user"); + + // case 3: non-matching Nemotron model, no injection + const superModel = JSON.parse(records[3].writes.join("")); + expect(superModel.messages).toHaveLength(1); + expect(superModel.messages[0].role).toBe("user"); + + // case 4: system message at non-zero index — caller intent preserved + const ultraMidSystem = JSON.parse(records[4].writes.join("")); + expect(ultraMidSystem.messages).toHaveLength(4); + expect(ultraMidSystem.messages[0].role).toBe("user"); + expect(ultraMidSystem.messages[2].role).toBe("system"); + + // case 5: non-execution tools only (toolSearch + web fetch) — injection + // STILL fires because these tools can't satisfy the user's exec request. + // This is the practical end-user config from #4851's repro. + const ultraWithSearchTools = JSON.parse(records[5].writes.join("")); + expect(ultraWithSearchTools.messages[0].role).toBe("system"); + expect(ultraWithSearchTools.messages[0].content).toMatch(/do not have tools/i); + expect(ultraWithSearchTools.tools).toHaveLength(3); + + // case 6: mixed tools (search + bash_execute) — no injection because + // an execution-capable tool is present + const ultraWithMixedTools = JSON.parse(records[6].writes.join("")); + expect(ultraWithMixedTools.messages).toHaveLength(1); + expect(ultraWithMixedTools.messages[0].role).toBe("user"); + + // case 7: harmless business-tool names with broad tokens (create/run/ + // save/command) — injection STILL fires; tight allowlist rejects false + // positives that a substring regex would have swallowed + const ultraWithBusinessTools = JSON.parse(records[7].writes.join("")); + expect(ultraWithBusinessTools.messages[0].role).toBe("system"); + expect(ultraWithBusinessTools.messages[0].content).toMatch(/do not have tools/i); + expect(ultraWithBusinessTools.tools).toHaveLength(4); + + // case 8: write_file — explicit canonical exec tool, no injection + const ultraWithWriteFile = JSON.parse(records[8].writes.join("")); + expect(ultraWithWriteFile.messages).toHaveLength(1); + expect(ultraWithWriteFile.messages[0].role).toBe("user"); + + // case 9: bare write/edit/notebook_edit (mirrors WRITE_TOOL_NAMES) — no injection + const ultraWithBareWriteEdit = JSON.parse(records[9].writes.join("")); + expect(ultraWithBareWriteEdit.messages).toHaveLength(1); + expect(ultraWithBareWriteEdit.messages[0].role).toBe("user"); + + // case 10: tool_call wrapper present — no injection because it can + // dispatch to real exec/write tools + const ultraWithToolCall = JSON.parse(records[10].writes.join("")); + expect(ultraWithToolCall.messages).toHaveLength(1); + expect(ultraWithToolCall.messages[0].role).toBe("user"); + + // case 11: top-level tool.name shape (no nested .function) — predicate + // handles both shapes per OpenAI / non-OpenAI caller variance + const ultraTopLevelName = JSON.parse(records[11].writes.join("")); + expect(ultraTopLevelName.messages).toHaveLength(1); + expect(ultraTopLevelName.messages[0].role).toBe("user"); + }); + + it("preload pins path+model as the intended scope boundary (#4851)", () => { + // Contract test: the Ultra 550B tool-less injection is scoped by HTTP + // path (/v1/chat/completions) + model regex, not by destination host. + // This preload runs inside NemoClaw-managed sandboxes where the only + // chat-completions destination is the inference.local route bound to + // NVIDIA Build. The path+model boundary is the intentional contract. + // This test pins that contract so a future change toward narrower + // (host-aware) gating is a deliberate decision, not silent drift. + const preload = extractStartScriptHeredoc(src, "NEMOTRON_FIX_EOF"); + const harness = ` +const http = require('http'); +const records = []; +http.request = function (options) { + const record = { options, writes: [], headers: {}, removed: [] }; + records.push(record); + return { + write(chunk) { + record.writes.push(Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk)); + return true; + }, + end(cb) { if (typeof cb === 'function') cb(); return true; }, + getHeader(name) { return record.headers[name]; }, + setHeader(name, value) { record.headers[name] = value; }, + removeHeader(name) { record.removed.push(name); delete record.headers[name]; }, + }; +}; +${preload} +function send(host, body) { + const req = http.request({ method: 'POST', host, path: '/v1/chat/completions' }); + req.write(body); + req.end(); +} +// Different upstream hosts — all matching the path+model contract +send('inference.local', JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'hi' }], +})); +send('integrate.api.nvidia.com', JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'hi' }], +})); +send('some-other-openai-compat-host.example.com', JSON.stringify({ + model: 'nvidia/nemotron-3-ultra-550b-a55b', + messages: [{ role: 'user', content: 'hi' }], +})); +console.log(JSON.stringify(records)); +`; + const result = spawnSync(process.execPath, ["-e", harness], { + encoding: "utf-8", + timeout: 5000, + }); + expect(result.status, result.stderr).toBe(0); + const records = JSON.parse(result.stdout.trim()); + expect(records).toHaveLength(3); + + // All three hosts get the injection — host is intentionally NOT part + // of the scope boundary. If this assertion ever changes, the + // documented contract above must change too. + for (const r of records) { + const body = JSON.parse(r.writes.join("")); + expect(body.messages[0].role).toBe("system"); + expect(body.messages[0].content).toMatch(/do not have tools/i); + } }); });