Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 141 additions & 3 deletions nemoclaw-blueprint/scripts/nemotron-inference-fix.js
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,35 @@
// provider configuration always sends the required model-specific kwargs (or
// NVIDIA Build no longer requires them) and the DeepSeek/Kimi channel latency
// validation passes without the monkeypatch.
//
// Source-of-truth / removal contract (NemoClaw#4851):
// Invalid state: nvidia/nemotron-3-ultra-550b plans multi-step tasks
// correctly in `reasoning_content` but silently drops intermediate steps
// from `content` when the request has no execution-capable tools and no
// caller-supplied system message. force_nonempty_content does not address
// this — verified via direct curl with and without that kwarg.
//
// Source boundary: the chat template / vLLM-NIM serving stack for this
// model on NVIDIA Build is the actual origin; the model's emission shape
// is outside this repository. Inside the sandbox we own the preload that
// wraps every chat-completions request on the way out.
//
// Why not fix only at the origin: the NVIDIA Build chat template + Ultra
// weight pairing is owned upstream. NVB#6272828 tracks the upstream fix.
// Until that lands, a sandbox-side preload is the only way to keep the
// Ultra path usable in the niche tool-less configuration that triggers
// the bug.
//
// Regression proof: test/nemotron-inference-fix.test.ts covers the
// inject/skip branches via the http stub AND a real fetch/undici request
// against a local OpenAI-compatible endpoint, asserting both the injected
// system message and the refreshed Content-Length. Live curl verification
// against integrate.api.nvidia.com (in PR #5085 body) demonstrates the
// model-output acceptance criteria from #4851.
//
// Removal condition: remove the TOOL_LESS_SYSTEM_PROMPT_RULES entry for
// Ultra 550B once NVB#6272828 ships and a clean Ultra response to the
// #4851 prompt no longer requires the nudge.

(function () {
'use strict';
Expand All @@ -68,6 +97,82 @@
{ pattern: /^moonshotai\/kimi-k2\.6$/i, kwargs: { thinking: false } },
];

// #4851: Ultra 550B silently drops intermediate steps from `content` when
// asked to perform multi-step tasks without execution-capable tools —
// reasoning plans all steps but content emits only the final command (or
// empty). chat_template_kwargs.force_nonempty_content doesn't help. When
// the caller hasn't supplied a system message AND has no execution-
// capable tools, inject a one-paragraph nudge so the model emits the full
// code/commands the user would run manually. Skip when a system message
// is already present anywhere in the array (caller's prompt wins) or
// when execution-capable tools are present (the model should use them).
//
// Scope boundary: this preload runs inside NemoClaw-managed sandboxes
// where the chat-completions destination is the OpenShell-gateway
// `inference.local` route bound to NVIDIA Build. The path+model regex
// is the intentional trust boundary; non-sandbox OpenAI-compatible
// callers do not load this preload.
var TOOL_LESS_SYSTEM_PROMPT_RULES = [
{
pattern: /^nvidia\/nemotron-3-ultra-550b/i,
systemPrompt:
'You do not have tools to write files or execute commands. When the user asks you to perform such actions, include the complete code or command they would need to run manually. Do not skip steps.',
},
];

// Tools that signal the agent has execution capability — when any of
// these are in the request, the tool-less nudge above doesn't apply
// because the model should call the tool. Search/web/fetch/describe/read
// tools are intentionally NOT in this set: they don't let the model
// write a file or run a command, so they don't change the "no way to
// perform the asked action" condition that #4851 cares about.
//
// Tight allowlist (not a broad regex) to avoid false positives on
// harmless business tools like `create_ticket`, `run_query`, `save_search`,
// `command_palette` that contain "create"/"run"/"save"/"command" but don't
// give the model the ability to write a file or execute a shell command.
// Match exact known names + the canonical OpenClaw/MCP suffixes.
var EXECUTION_TOOL_NAMES = new Set([
'bash',
'bash_execute',
'exec',
'execute',
'execute_command',
'shell',
'shell_execute',
'run_command',
'run_shell',
'write_file',
'file_write',
'edit_file',
'file_edit',
'patch_file',
'file_patch',
'create_file',
'file_create',
'apply_patch',
'str_replace_editor',
'computer',
]);

function isExecutionCapableTool(tool) {
if (!tool || typeof tool !== 'object') return false;
// OpenAI chat-completions tool shape: { type: 'function', function: { name: ... } }
var name = null;
if (typeof tool.name === 'string') {
name = tool.name;
} else if (tool.function && typeof tool.function.name === 'string') {
name = tool.function.name;
}
if (!name) return false;
return EXECUTION_TOOL_NAMES.has(name.toLowerCase());
}

function hasExecutionCapableTool(body) {
if (!Array.isArray(body.tools) || body.tools.length === 0) return false;
return body.tools.some(isExecutionCapableTool);
}

function chatTemplateKwargsForModel(model) {
var kwargs = null;
CHAT_TEMPLATE_KWARG_RULES.forEach(function (rule) {
Expand Down Expand Up @@ -101,12 +206,45 @@
return true;
}

function toolLessSystemPromptForModel(body) {
if (!body || typeof body.model !== 'string') return null;
var rule = null;
for (var i = 0; i < TOOL_LESS_SYSTEM_PROMPT_RULES.length; i++) {
if (TOOL_LESS_SYSTEM_PROMPT_RULES[i].pattern.test(body.model)) {
rule = TOOL_LESS_SYSTEM_PROMPT_RULES[i];
break;
}
}
if (!rule) return null;
if (!Array.isArray(body.messages) || body.messages.length === 0) return null;
// Scan ALL messages, not just messages[0]: the OpenAI chat-completions
// contract permits a system message anywhere in the array, and the
// "caller prompt wins" contract should hold for any of those positions.
var hasSystemMessage = body.messages.some(function (msg) {
return msg && typeof msg === 'object' && msg.role === 'system';
});
if (hasSystemMessage) return null;
// Skip when execution-capable tools are present; harmless tools like
// tool_search / web fetch / file describe don't change the "no way to
// perform the asked action" condition #4851 cares about.
if (hasExecutionCapableTool(body)) return null;
return rule.systemPrompt;
}

function applyToolLessSystemPrompt(body) {
var systemPrompt = toolLessSystemPromptForModel(body);
if (!systemPrompt) return false;
body.messages.unshift({ role: 'system', content: systemPrompt });
return true;
}

function patchJsonBody(raw) {
try {
var body = JSON.parse(raw.toString('utf-8'));
if (!applyChatTemplateKwargs(body)) {
return null;
}
var changed = false;
if (applyChatTemplateKwargs(body)) changed = true;
if (applyToolLessSystemPrompt(body)) changed = true;
if (!changed) return null;
return Buffer.from(JSON.stringify(body), 'utf-8');
} catch (_e) {
return null;
Expand Down
Loading
Loading