diff --git a/src/lib/inference/serving/requested-profile-model.test.ts b/src/lib/inference/serving/requested-profile-model.test.ts new file mode 100644 index 00000000000..de5a1f820d9 --- /dev/null +++ b/src/lib/inference/serving/requested-profile-model.test.ts @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { loadServingCatalog } from "./catalog-loader"; +import { + resolveRequestedServingProfileModel, + servingProfileModel, +} from "./requested-profile-model"; + +describe("requested serving profile model", () => { + it("reports both names the shipped catalog gives a profile", () => { + const catalog = loadServingCatalog(); + const preset = catalog.presets[0]!; + const recipe = catalog.recipes.find( + ({ metadata }) => metadata.id === preset.spec.plan.recipeRef, + )!; + + expect(servingProfileModel(catalog, preset.metadata.id)).toEqual({ + presetId: preset.metadata.id, + backend: recipe.spec.backend, + servedName: recipe.spec.model.servedName, + modelId: recipe.spec.model.id, + }); + }); + + it("returns null for a profile the catalog does not carry", () => { + expect(servingProfileModel(loadServingCatalog(), "vllm.absent.profile")).toBeNull(); + }); + + it("reads the preset the current run requested", () => { + const catalog = loadServingCatalog(); + const preset = catalog.presets[0]!; + + expect( + resolveRequestedServingProfileModel({ NEMOCLAW_SERVING_PRESET: preset.metadata.id }, catalog), + ).toEqual(servingProfileModel(catalog, preset.metadata.id)); + }); + + it("returns null when no profile was requested", () => { + expect(resolveRequestedServingProfileModel({}, loadServingCatalog())).toBeNull(); + expect(resolveRequestedServingProfileModel({ NEMOCLAW_SERVING_PRESET: " " })).toBeNull(); + }); + + it("does not fail a selection when the catalog cannot be read", () => { + expect( + resolveRequestedServingProfileModel({ NEMOCLAW_SERVING_PRESET: "any" }, { + get presets(): never { + throw new Error("catalog unreadable"); + }, + } as never), + ).toBeNull(); + }); +}); diff --git a/src/lib/inference/serving/requested-profile-model.ts b/src/lib/inference/serving/requested-profile-model.ts new file mode 100644 index 00000000000..79e77c20991 --- /dev/null +++ b/src/lib/inference/serving/requested-profile-model.ts @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { loadServingCatalog } from "./catalog-loader.js"; +import { NEMOCLAW_SERVING_PRESET_ENV } from "./managed-cluster-discovery.js"; +import type { CompiledServingCatalog } from "./types.js"; + +/** The identifiers under which a running endpoint can report a requested profile's model. */ +export interface RequestedServingProfileModel { + readonly presetId: string; + readonly backend: string; + /** Alias the recipe pins with --served-model-name, so what /v1/models reports. */ + readonly servedName: string; + /** Weights the recipe downloads, so what a reporting endpoint gives as the root. */ + readonly modelId: string; +} + +export function servingProfileModel( + catalog: CompiledServingCatalog, + presetId: string, +): RequestedServingProfileModel | null { + const presets = catalog.presets.filter(({ metadata }) => metadata.id === presetId); + if (presets.length !== 1) return null; + const recipes = catalog.recipes.filter( + ({ metadata }) => metadata.id === presets[0]!.spec.plan.recipeRef, + ); + if (recipes.length !== 1) return null; + const spec = recipes[0]!.spec; + const servedName = typeof spec.model.servedName === "string" ? spec.model.servedName.trim() : ""; + return servedName + ? { presetId, backend: spec.backend, servedName, modelId: spec.model.id } + : null; +} + +/** + * Model the serving profile requested for this run declares. + * + * Provider selection reads the preset from the environment rather than from the + * resolved provenance, so this reads the same place. `--profile` sets that + * variable for the run, but an operator can also export it directly, in which + * case no flag validation has run against it — hence the backend on the result, + * which callers use to reject a preset their own selection cannot serve. + * + * Returns null rather than throwing. An unreadable catalog then leaves the + * caller's model check unarmed instead of stopping onboarding. + */ +export function resolveRequestedServingProfileModel( + env: NodeJS.ProcessEnv = process.env, + catalog?: CompiledServingCatalog, +): RequestedServingProfileModel | null { + const presetId = String(env[NEMOCLAW_SERVING_PRESET_ENV] ?? "").trim(); + if (!presetId) return null; + try { + return servingProfileModel(catalog ?? loadServingCatalog(), presetId); + } catch { + return null; + } +} diff --git a/src/lib/onboard/setup-nim-flow-serving-profile.test.ts b/src/lib/onboard/setup-nim-flow-serving-profile.test.ts new file mode 100644 index 00000000000..ef842d72e80 --- /dev/null +++ b/src/lib/onboard/setup-nim-flow-serving-profile.test.ts @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { VllmProfile } from "../inference/vllm"; +import { makeDeps, makeHostState } from "./__test-helpers__/setup-nim-flow"; +import { createSetupNim, type SetupNimFlowDeps } from "./setup-nim-flow"; + +const servingProfileModel = { + presetId: "vllm.dgx-spark-gb10.single.muse-glimmer-30b-nvfp4-w4a4", + backend: "vllm", + servedName: "muse-glimmer", + modelId: "Inferact/Muse-Glimmer-30B-NVFP4-W4A4", +}; + +const routeGuard = () => ({ + requiredModel: null, + requiredEndpointUrl: null, + requiredInferenceApi: null, +}); + +function runningVllmHostState() { + return makeHostState({ + vllmRunning: true, + vllmProfile: { name: "DGX Spark" } as VllmProfile, + hasVllmImage: true, + vllmEntries: [{ key: "vllm", label: "Local vLLM (localhost:8000) — running (suggested)" }], + }); +} + +function acceptVllmSelection() { + return vi.fn(async (state) => { + state.provider = "vllm"; + state.model = "muse-glimmer"; + state.endpointUrl = "http://127.0.0.1:8000/v1"; + state.credentialEnv = null; + state.preferredInferenceApi = "openai-completions"; + return "selected"; + }); +} + +async function selectAgainstRunningVllm( + handleVllmSelection: ReturnType, + resolveRequestedServingProfileModel: SetupNimFlowDeps["resolveRequestedServingProfileModel"], +) { + const setupNim = createSetupNim( + makeDeps({ + isNonInteractive: () => true, + getNonInteractiveProvider: () => "install-vllm", + detectInferenceProviderHostState: () => runningVllmHostState(), + handleVllmSelection, + resolveRequestedServingProfileModel, + }), + ); + const sparkGpu = { platform: "spark" } as unknown as Parameters[0]; + return await setupNim(sparkGpu, null, null, true, null, "nemoclaw", routeGuard); +} + +describe("serving profile onboarding against a running vLLM", () => { + it("passes the requested profile's model to the running-server selection (#9563)", async () => { + // `--profile` exports NEMOCLAW_PROVIDER=install-vllm, but a running server + // leaves only the `vllm` entry, so the request collapses onto a deployment + // the profile never selected. The test leaves `installVllm` at the shared + // `unexpected` guard, so a call would fail the test: nothing installs on + // this path, which is why the flow must pass the profile's model on. + const handleVllmSelection = acceptVllmSelection(); + + const result = await selectAgainstRunningVllm(handleVllmSelection, () => servingProfileModel); + + expect(handleVllmSelection).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ managedInstall: false, servingProfileModel }), + ); + expect(result).toMatchObject({ provider: "vllm", model: "muse-glimmer" }); + }); + + it("passes no profile model when the run requested no profile", async () => { + const handleVllmSelection = acceptVllmSelection(); + + await selectAgainstRunningVllm(handleVllmSelection, () => null); + + expect(handleVllmSelection).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ servingProfileModel: null }), + ); + }); + + it("does not compare a preset that another backend serves (#9563)", async () => { + const handleVllmSelection = acceptVllmSelection(); + + await selectAgainstRunningVllm(handleVllmSelection, () => ({ + ...servingProfileModel, + presetId: "llama-cpp.dgx-spark-gb10.single.muse-glimmer-30b", + backend: "install-llama-cpp", + })); + + expect(handleVllmSelection).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ servingProfileModel: null }), + ); + }); +}); diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index a68471fe4b9..ca9ac9f7a8e 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -843,7 +843,7 @@ describe("createSetupNim", () => { expect(handleVllmSelection).toHaveBeenCalledOnce(); expect(handleVllmSelection).toHaveBeenCalledWith( expect.objectContaining({ model: "vllm-model" }), - { managedInstall: true, sparkHost: false }, + { managedInstall: true, sparkHost: false, servingProfileModel: null }, ); expect(result).toMatchObject({ model: "vllm-model", @@ -1046,7 +1046,7 @@ describe("createSetupNim", () => { }); expect(handleVllmSelection).toHaveBeenCalledWith( expect.objectContaining({ model: servedModel }), - { managedInstall: true, sparkHost: false }, + { managedInstall: true, sparkHost: false, servingProfileModel: null }, ); expect(result).toMatchObject({ model: servedModel, diff --git a/src/lib/onboard/setup-nim-flow.ts b/src/lib/onboard/setup-nim-flow.ts index 44b6b17212c..ebd34498ccb 100644 --- a/src/lib/onboard/setup-nim-flow.ts +++ b/src/lib/onboard/setup-nim-flow.ts @@ -25,6 +25,10 @@ import { resolveManagedLlamaCppSelection, } from "../inference/llama-cpp/managed-selection"; import { getOllamaContextWindowFloorForAgent } from "../inference/ollama-runtime-context"; +import { + type RequestedServingProfileModel, + resolveRequestedServingProfileModel, +} from "../inference/serving/requested-profile-model"; import type { VllmProfile } from "../inference/vllm"; import { promptManualModelId } from "../inference/model-prompts"; import { isBackToSelection } from "../navigation"; @@ -204,8 +208,15 @@ export interface SetupNimFlowDeps { ): Promise<{ ok: boolean }>; handleVllmSelection( state: SetupNimSelectionState, - options?: { managedInstall?: boolean; sparkHost?: boolean }, + options?: { + managedInstall?: boolean; + sparkHost?: boolean; + servingProfileModel?: RequestedServingProfileModel | null; + }, ): Promise; + resolveRequestedServingProfileModel?( + env?: NodeJS.ProcessEnv, + ): RequestedServingProfileModel | null; handleRoutedSelection(state: SetupNimSelectionState): Promise; coerceAgentInferenceApi( agent: AgentDefinition | null, @@ -521,6 +532,19 @@ function vllmPortConflictMessage( return "vLLM is already running on this host. Select Local vLLM, or stop the existing server before selecting the managed install path."; } +/** + * Model a requested serving profile declares, when a vLLM selection can serve it. + * + * A preset for another backend reaches this selection through the environment, + * and no vLLM server can answer it, so it is left out rather than compared. + */ +function requestedVllmServingProfileModel( + resolve: SetupNimFlowDeps["resolveRequestedServingProfileModel"], +): RequestedServingProfileModel | null { + const requested = (resolve ?? resolveRequestedServingProfileModel)(); + return requested?.backend === "vllm" ? requested : null; +} + async function resolveFreshHermesPortableOllamaSelection(input: { deps: SetupNimFlowDeps; agent: AgentDefinition | null; @@ -1082,9 +1106,18 @@ export function createSetupNim( if (selected.key === "vllm") { const state = preparedVllmState ?? createSelectionState(); state.model = preparedVllmState?.model ?? requestedModel ?? recoveredModel; + // A requested profile reaches this branch two ways: its own install + // finished, or `install-vllm` collapsed onto a server that was already + // listening. The second path runs no install, so nothing seeds a + // required model and the endpoint's own report becomes the route. + // Comparing the profile's model is the only check that the server + // serves what the profile declares. const result = await deps.handleVllmSelection(state, { managedInstall: preparedVllmState !== null, sparkHost: gpu?.spark === true, + servingProfileModel: requestedVllmServingProfileModel( + deps.resolveRequestedServingProfileModel, + ), }); ({ model, diff --git a/src/lib/onboard/setup-nim-vllm.test.ts b/src/lib/onboard/setup-nim-vllm.test.ts index e99d9d68216..fd826c1f944 100644 --- a/src/lib/onboard/setup-nim-vllm.test.ts +++ b/src/lib/onboard/setup-nim-vllm.test.ts @@ -834,3 +834,106 @@ describe("DGX Spark existing vLLM headroom warning", () => { ).toBeNull(); }); }); + + +describe("setupNim vLLM requested serving profile", () => { + const profile = { + presetId: "vllm.dgx-spark-gb10.single.muse-glimmer-30b-nvfp4-w4a4", + backend: "vllm", + servedName: "muse-glimmer", + modelId: "Inferact/Muse-Glimmer-30B-NVFP4-W4A4", + }; + + it("refuses a running server that reports a different model (#9563)", async () => { + const validateOpenAiLikeSelection = vi.fn(async () => ({ + ok: true as const, + api: "openai-completions", + })); + const handler = createSetupNimVllmHandler( + deps({ + runCapture: () => JSON.stringify({ data: [{ id: "nvidia/Qwen3.6-35B-A3B-NVFP4" }] }), + validateOpenAiLikeSelection, + }), + ); + + await expect(handler(state(null), { servingProfileModel: profile })).rejects.toThrow("exit 1"); + expect(console.error).toHaveBeenCalledWith( + " Serving profile 'vllm.dgx-spark-gb10.single.muse-glimmer-30b-nvfp4-w4a4' serves " + + "'Inferact/Muse-Glimmer-30B-NVFP4-W4A4' as 'muse-glimmer', but vLLM on localhost:8000 " + + "reports 'nvidia/Qwen3.6-35B-A3B-NVFP4'.", + ); + expect(console.error).toHaveBeenCalledWith( + " Stop the existing vLLM server on localhost:8000, then rerun the original " + + "install/onboard command.", + ); + expect(validateOpenAiLikeSelection).not.toHaveBeenCalled(); + }); + + it("clears the exported preset in the keep-the-detected-model path (#9563)", async () => { + const handler = createSetupNimVllmHandler( + deps({ + runCapture: () => JSON.stringify({ data: [{ id: "nvidia/Qwen3.6-35B-A3B-NVFP4" }] }), + }), + ); + + await expect(handler(state(null), { servingProfileModel: profile })).rejects.toThrow("exit 1"); + expect(console.error).toHaveBeenCalledWith( + " unset NEMOCLAW_SERVING_PRESET NEMOCLAW_PROVIDER", + ); + }); + + it("names the managed endpoint rather than loopback for a managed binding (#9563)", async () => { + const handler = createSetupNimVllmHandler( + deps({ + getManagedVllmProviderBinding: () => ({ + baseUrl: "http://10.40.0.1:8000/v1", + apiKey: "a".repeat(64), + }), + queryVllmModels: () => JSON.stringify({ data: [{ id: "nvidia/Qwen3.6-35B-A3B-NVFP4" }] }), + }), + ); + + await expect(handler(state(null), { servingProfileModel: profile })).rejects.toThrow("exit 1"); + expect(console.error).toHaveBeenCalledWith( + " Serving profile 'vllm.dgx-spark-gb10.single.muse-glimmer-30b-nvfp4-w4a4' serves " + + "'Inferact/Muse-Glimmer-30B-NVFP4-W4A4' as 'muse-glimmer', but the managed vLLM endpoint " + + "reports 'nvidia/Qwen3.6-35B-A3B-NVFP4'.", + ); + expect(console.error).toHaveBeenCalledWith( + " Stop the managed vLLM deployment, then rerun the original install/onboard command.", + ); + }); + + it("accepts the served alias the profile pins", async () => { + const selection = state(null); + const handler = createSetupNimVllmHandler( + deps({ runCapture: () => JSON.stringify({ data: [{ id: "muse-glimmer" }] }) }), + ); + + await expect(handler(selection, { servingProfileModel: profile })).resolves.toBe("selected"); + expect(selection.model).toBe("muse-glimmer"); + }); + + it("accepts an alias whose reported root is the model the profile declares", async () => { + const handler = createSetupNimVllmHandler( + deps({ + runCapture: () => + JSON.stringify({ + data: [{ id: "local-alias", root: "Inferact/Muse-Glimmer-30B-NVFP4-W4A4" }], + }), + }), + ); + + await expect(handler(state(null), { servingProfileModel: profile })).resolves.toBe("selected"); + }); + + it("accepts a different served model when the run requested no profile", async () => { + const handler = createSetupNimVllmHandler( + deps({ + runCapture: () => JSON.stringify({ data: [{ id: "nvidia/Qwen3.6-35B-A3B-NVFP4" }] }), + }), + ); + + await expect(handler(state(null), { servingProfileModel: null })).resolves.toBe("selected"); + }); +}); diff --git a/src/lib/onboard/setup-nim-vllm.ts b/src/lib/onboard/setup-nim-vllm.ts index 2129be5c504..61178ecf1c6 100644 --- a/src/lib/onboard/setup-nim-vllm.ts +++ b/src/lib/onboard/setup-nim-vllm.ts @@ -6,6 +6,7 @@ import { isTrustedPrivateEndpointCapability, type TrustedPrivateEndpointCapability, } from "../inference/endpoint-ssrf-preflight"; +import type { RequestedServingProfileModel } from "../inference/serving/requested-profile-model"; import { VLLM_MODELS } from "../inference/vllm-models"; import { isLoopbackHostname } from "../private-networks"; import { cliName } from "./branding"; @@ -25,6 +26,13 @@ export interface SetupNimVllmSelectionOptions { managedInstall?: boolean; /** True when the already-detected GPU confirms DGX Spark (covers firmware-unknown GB10 hosts). */ sparkHost?: boolean; + /** + * Model the serving profile requested for this run declares, when one was and + * its backend is vLLM. A running server answers that request only if it serves + * that model; otherwise onboarding stores a recorded route the profile does not + * describe while the review screen still shows the profile (#9563). + */ + servingProfileModel?: RequestedServingProfileModel | null; } export interface SetupNimVllmDeps { @@ -152,6 +160,21 @@ function reportedModelMatchesRequest( return root.toLowerCase() === (registeredModel?.id ?? requestedModel).toLowerCase(); } +/** + * A running endpoint answers a requested profile under the alias the recipe pins, + * or under any alias whose reported root is the declared model. An endpoint that + * reports no safe root answers only under the pinned alias. + */ +function reportedModelMatchesServingProfile( + models: VllmModels, + detectedModel: string, + profile: RequestedServingProfileModel, +): boolean { + if (detectedModel === profile.servedName) return true; + const root = reportedModelRoot(findVllmModelEntry(models, detectedModel)); + return root !== null && root.toLowerCase() === profile.modelId.toLowerCase(); +} + /** Preserve the checkpoint identity proven by the vLLM model response. */ function validatedVllmModelIdentity( models: VllmModels, @@ -315,6 +338,37 @@ export function createSetupNimVllmHandler( console.error(" Detected vLLM model ID contains invalid characters."); deps.exitProcess(1); } + const servingProfile = options.servingProfileModel ?? null; + if ( + servingProfile && + !reportedModelMatchesServingProfile(models, detectedModel, servingProfile) + ) { + const declared = `serves '${servingProfile.modelId}' as '${servingProfile.servedName}'`; + console.error( + managedEndpoint + ? ` Serving profile '${servingProfile.presetId}' ${declared}, but the managed vLLM ` + + `endpoint reports '${detectedModel}'.` + : ` Serving profile '${servingProfile.presetId}' ${declared}, but vLLM on ` + + `localhost:${deps.VLLM_PORT} reports '${detectedModel}'.`, + ); + console.error( + " Onboarding would store that model as the sandbox's recorded route, so the agent " + + "would use a model the profile does not declare.", + ); + console.error( + managedEndpoint + ? " Stop the managed vLLM deployment, then rerun the original install/onboard command." + : ` Stop the existing vLLM server on localhost:${deps.VLLM_PORT}, then rerun the ` + + "original install/onboard command.", + ); + console.error( + ` To keep '${detectedModel}' instead, start detailed setup without a profile:`, + ); + console.error(" unset NEMOCLAW_SERVING_PRESET NEMOCLAW_PROVIDER"); + console.error(` ${cliName()} onboard --fresh`); + console.error(" Then select Local vLLM when prompted."); + deps.exitProcess(1); + } if ( requiredModel && detectedModel !== requiredModel && diff --git a/src/lib/onboard/summary.test.ts b/src/lib/onboard/summary.test.ts index 22346b01ba1..0d07c7dd20f 100644 --- a/src/lib/onboard/summary.test.ts +++ b/src/lib/onboard/summary.test.ts @@ -115,6 +115,37 @@ describe("onboard summary helpers", () => { assert.match(summary, /Downloads:.*image 1\.0 GiB, model 2\.0 GiB/u); }); + it("names the declared model beside the served alias the route carries (#9563)", () => { + const summary = formatOnboardConfigSummary({ + provider: "vllm-local", + model: "muse-glimmer", + webSearchConfig: null, + sandboxName: "profile-test", + servingProfileProvenance: { + schemaVersion: 1, + catalogDigest: `sha256:${"1".repeat(64)}`, + preset: { + id: "vllm.dgx-spark-gb10.single.muse-glimmer-30b-nvfp4-w4a4", + digest: `sha256:${"2".repeat(64)}`, + displayName: "Muse Glimmer 30B NVFP4 W4A4 on one DGX Spark", + supportState: "experimental", + }, + recipe: { + id: "vllm.muse-glimmer-30b-nvfp4-w4a4.spark-single.v1", + digest: `sha256:${"3".repeat(64)}`, + backend: "vllm", + }, + model: { id: "Inferact/Muse-Glimmer-30B-NVFP4-W4A4", revision: "revision-1" }, + runtimeImage: `example.invalid/vllm@sha256:${"4".repeat(64)}`, + estimatedImageDownloadBytes: 1024 ** 3, + estimatedModelDownloadBytes: 2 * 1024 ** 3, + }, + }); + + assert.match(summary, /Model: {9}Inferact\/Muse-Glimmer-30B-NVFP4-W4A4/u); + assert.match(summary, /Served model: {2}muse-glimmer/u); + }); + it("formatSandboxBuildEstimateNote warns when runtime is under-provisioned (#2514)", () => { const note = formatSandboxBuildEstimateNote({ isContainerRuntimeUnderProvisioned: true, diff --git a/src/lib/onboard/summary.ts b/src/lib/onboard/summary.ts index 5a692355dd5..d4d8ff2a4e0 100644 --- a/src/lib/onboard/summary.ts +++ b/src/lib/onboard/summary.ts @@ -113,9 +113,11 @@ export function formatOnboardConfigSummary({ const noteLines = (Array.isArray(notes) ? notes : []) .filter((note) => typeof note === "string" && note.length > 0) .map((note) => ` Note: ${note}`); + const reviewModel = servingProfileProvenance?.model.id ?? model; const profileLines = servingProfileProvenance ? [ ` Profile: ${servingProfileProvenance.preset.displayName} (${servingProfileProvenance.preset.id})`, + ` Served model: ${model ?? "(unset)"}`, ` Recipe: ${servingProfileProvenance.recipe.id}`, ` Support: ${servingProfileProvenance.preset.supportState}`, ` Runtime image: ${servingProfileProvenance.runtimeImage ?? "(not declared)"}`, @@ -128,7 +130,7 @@ export function formatOnboardConfigSummary({ " Review configuration", bar, ` Provider: ${provider ?? "(unset)"}`, - ` Model: ${model ?? "(unset)"}`, + ` Model: ${reviewModel ?? "(unset)"}`, ...profileLines, apiKeyLine, ` Web search: ${webSearch}`,