fix(onboard): prompt for multimodal model inputs (Fixes #3850)

src/lib/onboard.ts

@@ -16,9 +16,8 @@ const {
   setOnboardBrandingAgent,
 }: typeof import("./onboard/branding") = require("./onboard/branding");
 const { createSelectOnboardAgent }: typeof import("./onboard/agent-selection") = require("./onboard/agent-selection");
-const {
-  createInferenceSelectionValidationHelpers,
-}: typeof import("./onboard/inference-selection-validation") = require("./onboard/inference-selection-validation");
+const { createInferenceSelectionValidationHelpers }: typeof import("./onboard/inference-selection-validation") = require("./onboard/inference-selection-validation");
+const inferenceInputCapability = require("./onboard/inference-input-capability");
 const { cleanupTempDir }: typeof import("./onboard/temp-files") = require("./onboard/temp-files");
 const { abortNonInteractive }: typeof import("./onboard/non-interactive-abort") = require("./onboard/non-interactive-abort");
 const { stopStaleDashboardListenersForSandbox } = require("./onboard/stale-gateway-cleanup");
@@ -5160,6 +5159,7 @@ async function setupNim(
   }
 
   const selectedModel = isBackToSelection(model) ? null : model;
+  await inferenceInputCapability.maybePromptForInferenceInputCapability(selectedModel, { isNonInteractive, prompt });
   return {
     model: selectedModel,
     provider,

src/lib/onboard/inference-input-capability.ts

@@ -0,0 +1,52 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+type InferenceInputCapabilityDeps = {
+  env?: NodeJS.ProcessEnv;
+  isNonInteractive: () => boolean;
+  prompt: (message: string) => Promise<string>;
+};
+
+const VALID_INFERENCE_INPUTS = new Set(["text", "image"]);
+const MULTIMODAL_MODEL_HINT_PATTERN = /(^|[\/:_\-.])(omni|vision|vl|image|multimodal)([\/:_\-.]|$)/i;
+
+export function isValidInferenceInputsOverride(value: string | undefined): boolean {
+  if (!value) return false;
+  const tokens = value.split(",");
+  return tokens.every((token) => VALID_INFERENCE_INPUTS.has(token)) && new Set(tokens).size === tokens.length;
+}
+
+export function shouldPromptForInferenceInputCapability(
+  model: string | null | undefined,
+): boolean {
+  return !!model && MULTIMODAL_MODEL_HINT_PATTERN.test(model);
+}
+
+export async function maybePromptForInferenceInputCapability(
+  model: string | null,
+  deps: InferenceInputCapabilityDeps,
+): Promise<void> {
+  const env = deps.env || process.env;
+  if (
+    deps.isNonInteractive() ||
+    !shouldPromptForInferenceInputCapability(model) ||
+    isValidInferenceInputsOverride(env.NEMOCLAW_INFERENCE_INPUTS)
+  ) {
+    return;
+  }
+
+  console.log("");
+  console.log(`  Selected model: ${model}`);
+  console.log("");
+  console.log("  Input capability:");
+  console.log("    1) Text only");
+  console.log("    2) Text + Image");
+  console.log("");
+  const choice = await deps.prompt("  Choose input capability [1]: ");
+  if ((choice || "1").trim() === "2") {
+    env.NEMOCLAW_INFERENCE_INPUTS = "text,image";
+    console.log("  ✓ Model input capability set to text + image.");
+    return;
+  }
+  env.NEMOCLAW_INFERENCE_INPUTS = "text";
+}

test/onboard.test.ts

@@ -8,6 +8,11 @@ import os from "node:os";
 import path from "node:path";
 import { describe, expect, it } from "vitest";
 import { appendHostProxyEnvArgs } from "../dist/lib/onboard/host-proxy-env.js";
+import {
+  isValidInferenceInputsOverride,
+  maybePromptForInferenceInputCapability,
+  shouldPromptForInferenceInputCapability,
+} from "../dist/lib/onboard/inference-input-capability.js";
 import { stageOptimizedSandboxBuildContext } from "../dist/lib/sandbox/build-context.js";
 import { testTimeoutOptions } from "./helpers/timeouts";
 
@@ -505,6 +510,40 @@ startGateway(null).catch(() => {});
     }
   });
 
+  it("prompts for input capability only on likely multimodal model names", () => {
+    expect(shouldPromptForInferenceInputCapability("nvidia/nemotron-3-nano-omni-30b-a3b")).toBe(
+      true,
+    );
+    expect(shouldPromptForInferenceInputCapability("qwen2.5-vl-72b")).toBe(true);
+    expect(shouldPromptForInferenceInputCapability("moonshotai/kimi-k2.6")).toBe(false);
+    expect(shouldPromptForInferenceInputCapability(null)).toBe(false);
+  });
+
+  it("accepts only supported inference input capability overrides", () => {
+    expect(isValidInferenceInputsOverride("text")).toBe(true);
+    expect(isValidInferenceInputsOverride("image")).toBe(true);
+    expect(isValidInferenceInputsOverride("text,image")).toBe(true);
+    expect(isValidInferenceInputsOverride("image,text")).toBe(true);
+    expect(isValidInferenceInputsOverride("text,text")).toBe(false);
+    expect(isValidInferenceInputsOverride("image,image")).toBe(false);
+    expect(isValidInferenceInputsOverride("text, image")).toBe(false);
+    expect(isValidInferenceInputsOverride("audio")).toBe(false);
+  });
+
+  it("normalizes invalid inference input capability overrides when choosing text only", async () => {
+    const env = {
+      NEMOCLAW_INFERENCE_INPUTS: "audio",
+    } as NodeJS.ProcessEnv;
+
+    await maybePromptForInferenceInputCapability("nvidia/nemotron-3-nano-omni-30b-a3b", {
+      env,
+      isNonInteractive: () => false,
+      prompt: async () => "",
+    });
+
+    expect(env.NEMOCLAW_INFERENCE_INPUTS).toBe("text");
+  });
+
   it("detects resume conflicts for explicit provider and model changes", () => {
     const previousProvider = process.env.NEMOCLAW_PROVIDER;
     const previousModel = process.env.NEMOCLAW_MODEL;