fix(inference): inject tool-less system prompt for Ultra 550B (#4851)

nemoclaw-blueprint/scripts/nemotron-inference-fix.js

@@ -54,6 +54,38 @@
 //   provider configuration always sends the required model-specific kwargs (or
 //   NVIDIA Build no longer requires them) and the DeepSeek/Kimi channel latency
 //   validation passes without the monkeypatch.
+//
+// Source-of-truth / removal contract (NemoClaw#4851):
+//   Invalid state: nvidia/nemotron-3-ultra-550b plans multi-step tasks
+//   correctly in `reasoning_content` but silently drops intermediate steps
+//   from `content` when the request has no execution-capable tools and no
+//   caller-supplied system message. force_nonempty_content does not address
+//   this — verified via direct curl with and without that kwarg.
+//
+//   Source boundary: the chat template / vLLM-NIM serving stack for this
+//   model on NVIDIA Build is the actual origin; the model's emission shape
+//   is outside this repository. Inside the sandbox we own the preload that
+//   wraps every chat-completions request on the way out.
+//
+//   Why not fix only at the origin: the NVIDIA Build chat template + Ultra
+//   weight pairing is owned upstream. NVB#6272828 tracks the upstream fix.
+//   Until that lands, a sandbox-side preload is the only way to keep the
+//   Ultra path usable in the niche tool-less configuration that triggers
+//   the bug.
+//
+//   Regression proof: test/nemotron-inference-fix.test.ts covers the
+//   inject/skip branches via the http stub AND a real fetch/undici request
+//   against a local OpenAI-compatible endpoint, asserting both the injected
+//   system message and the refreshed Content-Length. The runtime model-
+//   output behavior (acceptance criteria from #4851) is validated against
+//   integrate.api.nvidia.com via the checked-in runbook at
+//   test/e2e-runtime/4851-ultra-toolless-validation.md — anyone reviewing
+//   acceptance can re-run it directly. Re-run when this preload changes
+//   or when OpenClaw bumps a version that may shift Ultra's chat template.
+//
+//   Removal condition: remove the TOOL_LESS_SYSTEM_PROMPT_RULES entry for
+//   Ultra 550B once NVB#6272828 ships and a clean Ultra response to the
+//   #4851 prompt no longer requires the nudge.
 
 (function () {
   'use strict';
@@ -68,6 +100,94 @@
     { pattern: /^moonshotai\/kimi-k2\.6$/i, kwargs: { thinking: false } },
   ];
 
+  // #4851: Ultra 550B silently drops intermediate steps from `content` when
+  // asked to perform multi-step tasks without execution-capable tools —
+  // reasoning plans all steps but content emits only the final command (or
+  // empty). chat_template_kwargs.force_nonempty_content doesn't help. When
+  // the caller hasn't supplied a system message AND has no execution-
+  // capable tools, inject a one-paragraph nudge so the model emits the full
+  // code/commands the user would run manually. Skip when a system message
+  // is already present anywhere in the array (caller's prompt wins) or
+  // when execution-capable tools are present (the model should use them).
+  //
+  // Scope boundary: this preload runs inside NemoClaw-managed sandboxes
+  // where the chat-completions destination is the OpenShell-gateway
+  // `inference.local` route bound to NVIDIA Build. The path+model regex
+  // is the intentional trust boundary; non-sandbox OpenAI-compatible
+  // callers do not load this preload.
+  var TOOL_LESS_SYSTEM_PROMPT_RULES = [
+    {
+      pattern: /^nvidia\/nemotron-3-ultra-550b/i,
+      systemPrompt:
+        'You do not have tools to write files or execute commands. When the user asks you to perform such actions, include the complete code or command they would need to run manually. Do not skip steps.',
+    },
+  ];
+
+  // Tools that signal the agent has execution capability — when any of
+  // these are in the request, the tool-less nudge above doesn't apply
+  // because the model should call the tool. Search/web/fetch/describe/read
+  // tools are intentionally NOT in this set: they don't let the model
+  // write a file or run a command, so they don't change the "no way to
+  // perform the asked action" condition that #4851 cares about.
+  //
+  // Tight allowlist (not a broad regex) to avoid false positives on
+  // harmless business tools like `create_ticket`, `run_query`, `save_search`,
+  // `command_palette` that contain "create"/"run"/"save"/"command" but don't
+  // give the model the ability to write a file or execute a shell command.
+  // Match exact known names + the canonical OpenClaw/MCP suffixes.
+  //
+  // The bare `write`, `edit`, and `notebook_edit` names mirror
+  // `nemoclaw/src/index.ts:WRITE_TOOL_NAMES` so this allowlist stays
+  // aligned with the same write-capable surface OpenClaw scans for
+  // secrets. `tool_call` is the OpenClaw compact-catalog wrapper from
+  // `scripts/patch-openclaw-tool-catalog.js` — when it's present we
+  // can't tell from the request alone which underlying tool will be
+  // dispatched, so treat it as execution-capable and skip the nudge.
+  var EXECUTION_TOOL_NAMES = new Set([
+    'bash',
+    'bash_execute',
+    'exec',
+    'execute',
+    'execute_command',
+    'shell',
+    'shell_execute',
+    'run_command',
+    'run_shell',
+    'write',
+    'write_file',
+    'file_write',
+    'edit',
+    'edit_file',
+    'file_edit',
+    'notebook_edit',
+    'patch_file',
+    'file_patch',
+    'create_file',
+    'file_create',
+    'apply_patch',
+    'str_replace_editor',
+    'computer',
+    'tool_call',
+  ]);
+
+  function isExecutionCapableTool(tool) {
+    if (!tool || typeof tool !== 'object') return false;
+    // OpenAI chat-completions tool shape: { type: 'function', function: { name: ... } }
+    var name = null;
+    if (typeof tool.name === 'string') {
+      name = tool.name;
+    } else if (tool.function && typeof tool.function.name === 'string') {
+      name = tool.function.name;
+    }
+    if (!name) return false;
+    return EXECUTION_TOOL_NAMES.has(name.toLowerCase());
+  }
+
+  function hasExecutionCapableTool(body) {
+    if (!Array.isArray(body.tools) || body.tools.length === 0) return false;
+    return body.tools.some(isExecutionCapableTool);
+  }
+
   function chatTemplateKwargsForModel(model) {
     var kwargs = null;
     CHAT_TEMPLATE_KWARG_RULES.forEach(function (rule) {
@@ -101,12 +221,45 @@
     return true;
   }
 
+  function toolLessSystemPromptForModel(body) {
+    if (!body || typeof body.model !== 'string') return null;
+    var rule = null;
+    for (var i = 0; i < TOOL_LESS_SYSTEM_PROMPT_RULES.length; i++) {
+      if (TOOL_LESS_SYSTEM_PROMPT_RULES[i].pattern.test(body.model)) {
+        rule = TOOL_LESS_SYSTEM_PROMPT_RULES[i];
+        break;
+      }
+    }
+    if (!rule) return null;
+    if (!Array.isArray(body.messages) || body.messages.length === 0) return null;
+    // Scan ALL messages, not just messages[0]: the OpenAI chat-completions
+    // contract permits a system message anywhere in the array, and the
+    // "caller prompt wins" contract should hold for any of those positions.
+    var hasSystemMessage = body.messages.some(function (msg) {
+      return msg && typeof msg === 'object' && msg.role === 'system';
+    });
+    if (hasSystemMessage) return null;
+    // Skip when execution-capable tools are present; harmless tools like
+    // tool_search / web fetch / file describe don't change the "no way to
+    // perform the asked action" condition #4851 cares about.
+    if (hasExecutionCapableTool(body)) return null;
+    return rule.systemPrompt;
+  }
+
+  function applyToolLessSystemPrompt(body) {
+    var systemPrompt = toolLessSystemPromptForModel(body);
+    if (!systemPrompt) return false;
+    body.messages.unshift({ role: 'system', content: systemPrompt });
+    return true;
+  }
+
   function patchJsonBody(raw) {
     try {
       var body = JSON.parse(raw.toString('utf-8'));
-      if (!applyChatTemplateKwargs(body)) {
-        return null;
-      }
+      var changed = false;
+      if (applyChatTemplateKwargs(body)) changed = true;
+      if (applyToolLessSystemPrompt(body)) changed = true;
+      if (!changed) return null;
       return Buffer.from(JSON.stringify(body), 'utf-8');
     } catch (_e) {
       return null;

test/e2e-runtime/4851-ultra-toolless-validation.md

@@ -0,0 +1,205 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# #4851 runtime validation — Ultra 550B tool-less system-prompt injection
+
+Repository-verifiable acceptance evidence for [PR #5085](https://github.com/NVIDIA/NemoClaw/pull/5085).
+
+The unit tests in `test/nemotron-inference-fix.test.ts` prove request mutation, Content-Length refresh, and the 12 inject/skip branches via stubbed http + real fetch/undici. They do not prove the upstream model-output behavior the issue's expected result asks for. That requires a live call to NVIDIA Endpoints, which can't run in unit CI without API-key secret infrastructure.
+
+This runbook is the maintained runtime-validation path. Anyone reviewing #4851 acceptance can run it directly against `integrate.api.nvidia.com` and confirm the model returns `content` with both file-creation code and the run command after the preload's system message is injected.
+
+## When to run
+
+- Before merging any PR that changes `nemoclaw-blueprint/scripts/nemotron-inference-fix.js` or `EXECUTION_TOOL_NAMES`.
+- When a NemoClaw release pins a new OpenClaw version that may change the upstream chat template behavior on Ultra 550B.
+- If QA reopens #4851.
+
+## Prerequisites
+
+- An NVIDIA API key with access to `nvidia/nemotron-3-ultra-550b-a55b` (build.nvidia.com → API Keys).
+- `node >= 18`, `curl`, and `jq` (the scenarios below pipe responses through `jq` for readable parsing). Any Linux or macOS host works — this validates upstream model behavior, not local sandbox runtime.
+
+Export the key once for the session:
+
+```bash
+export NVIDIA_API_KEY="nvapi-..."
+```
+
+## Scenario A — baseline (no preload, no system message, no tools)
+
+Demonstrates the bug as filed in the issue body.
+
+```bash
+curl -sS -X POST https://integrate.api.nvidia.com/v1/chat/completions \
+  -H "Authorization: Bearer ${NVIDIA_API_KEY}" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "nvidia/nemotron-3-ultra-550b-a55b",
+    "messages": [{"role": "user", "content": "Create a file called hello.py in /tmp with a hello world script, then run it."}],
+    "max_tokens": 400,
+    "temperature": 0.0
+  }' | jq '{
+    finish_reason: .choices[0].finish_reason,
+    completion_tokens: .usage.completion_tokens,
+    reasoning_chars: (.choices[0].message.reasoning_content // "" | length),
+    content_chars: (.choices[0].message.content // "" | length),
+    content: (.choices[0].message.content // "")
+  }'
+```
+
+Expected result with `nemotron-3-ultra-550b-a55b` (matches issue body):
+
+- `finish_reason: "stop"`
+- `reasoning_chars` ≈ 150–300 (model plans 3 steps internally)
+- `content_chars` ≈ 0–60 (model drops file-creation step, may emit only the run command or empty)
+
+## Scenario B — `force_nonempty_content` kwarg only (no preload's system message)
+
+Demonstrates that the existing Nemotron-family kwarg doesn't fix #4851 by itself.
+
+```bash
+curl -sS -X POST https://integrate.api.nvidia.com/v1/chat/completions \
+  -H "Authorization: Bearer ${NVIDIA_API_KEY}" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "nvidia/nemotron-3-ultra-550b-a55b",
+    "messages": [{"role": "user", "content": "Create a file called hello.py in /tmp with a hello world script, then run it."}],
+    "max_tokens": 400,
+    "temperature": 0.0,
+    "chat_template_kwargs": {"force_nonempty_content": true}
+  }' | jq '.choices[0].message.content | length'
+```
+
+Expected: still ≈ 0–60 chars. The kwarg doesn't change the failure mode.
+
+## Scenario C — preload's full mutation (system message + kwarg)
+
+Demonstrates the fix shipped in this PR.
+
+```bash
+curl -sS -X POST https://integrate.api.nvidia.com/v1/chat/completions \
+  -H "Authorization: Bearer ${NVIDIA_API_KEY}" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "nvidia/nemotron-3-ultra-550b-a55b",
+    "messages": [
+      {"role": "system", "content": "You do not have tools to write files or execute commands. When the user asks you to perform such actions, include the complete code or command they would need to run manually. Do not skip steps."},
+      {"role": "user", "content": "Create a file called hello.py in /tmp with a hello world script, then run it."}
+    ],
+    "max_tokens": 400,
+    "temperature": 0.0,
+    "chat_template_kwargs": {"force_nonempty_content": true}
+  }' | jq '{
+    finish_reason: .choices[0].finish_reason,
+    content_chars: (.choices[0].message.content // "" | length),
+    content: (.choices[0].message.content // "")
+  }'
+```
+
+Expected (`#4851` acceptance):
+
+- `content_chars` ≈ 400–600
+- `content` includes BOTH file creation (heredoc, redirection, or full source of `hello.py`) AND the run command (`python3 /tmp/hello.py` or equivalent)
+- `finish_reason: "stop"`
+
+This satisfies the issue's "Expected Result, Option A" (`Model explains it lacks a file-write tool and shows the full code the user would need to run manually`).
+
+## Sanitized acceptance transcript
+
+The transcript below was captured by @cjagwani on 2026-06-09 against `integrate.api.nvidia.com` from a GCP Brev box. Reproduces the bug behavior in Scenarios A/B and the fix behavior in Scenario C. Use this as the durable acceptance baseline; new runs that differ structurally should update this section (and the dated entry below) rather than the unit tests.
+
+### Scenario A (baseline) — 2026-06-09
+
+```text
+finish_reason:     stop
+prompt_tokens:     35
+completion_tokens: 117
+reasoning_chars:   184
+content_chars:     1
+content:           " "
+```
+
+Reasoning content (the model plans 3 steps but emits none of them in `content`):
+
+```text
+The user wants me to:
+1. Create a file called hello.py in /tmp
+2. Put a hello world script in it
+3. Run it
+
+I'll use the write tool to create the file and then the bash tool to run it.
+```
+
+### Scenario B (force_nonempty_content only) — 2026-06-09
+
+```text
+finish_reason:     stop
+prompt_tokens:     35
+completion_tokens: 131
+reasoning_chars:   184
+content_chars:     1
+content:           " "
+```
+
+Same baseline failure mode — the kwarg alone doesn't address #4851.
+
+### Scenario C (full preload mutation) — 2026-06-09
+
+```text
+finish_reason:     stop
+prompt_tokens:     75
+completion_tokens: 187
+reasoning_chars:   241
+content_chars:     501
+```
+
+Content (full text):
+
+````markdown
+I'll provide you with the commands to create and run the hello world script manually.
+
+## Create the file
+
+```bash
+cat > /tmp/hello.py << 'EOF'
+#!/usr/bin/env python3
+
+def main():
+    print("Hello, World!")
+
+if __name__ == "__main__":
+    main()
+EOF
+```
+
+## Make it executable (optional)
+
+```bash
+chmod +x /tmp/hello.py
+```
+
+## Run it
+
+```bash
+python3 /tmp/hello.py
+```
+
+**Expected output:**
+
+```text
+Hello, World!
+```
+
+You can copy and paste these commands into your terminal to create and run the script.
+````
+
+This satisfies the issue's Option A acceptance condition: model explains it lacks file-write/execute tools and shows the complete code the user would need to run manually, with all 3 planned steps present in `content`.
+
+## Live verification log
+
+- 2026-06-09 — verified by @cjagwani on a GCP Brev box against `integrate.api.nvidia.com`. Numbers and content above match this run.
+
+When you re-run this runbook, add a dated entry here so the next reviewer can see how recently the upstream behavior was last confirmed. If the response shape differs materially from the sanitized transcript above, update both this log and the transcript.