diff --git a/src/lib/inference/serving/managed-cluster-installer.test.ts b/src/lib/inference/serving/managed-cluster-installer.test.ts index a7e98c81120..4d904f1ea0e 100644 --- a/src/lib/inference/serving/managed-cluster-installer.test.ts +++ b/src/lib/inference/serving/managed-cluster-installer.test.ts @@ -564,6 +564,89 @@ describe("managed-cluster vLLM installer selection", () => { expect(installEffects.downloadModel).not.toHaveBeenCalled(); }); + it("refuses a resumed model the cluster preset does not select, before any effect", async () => { + const capability = readyCapability(); + const installEffects = effects(); + const promptFn = vi.fn(async () => "yes"); + const assertGatedModelAccess = vi.fn(); + const revalidateCapability = vi.fn(); + const claimCapability = vi.fn(); + const checkpointInstallIntent = vi.fn(); + const beforeInstall = vi.fn(); + const error = vi.fn(); + + const result = await tryInstallManagedClusterManagedVllm( + { + platform: "spark", + env: {}, + nonInteractive: true, + promptFn, + checkpointInstallIntent, + beforeInstall, + resumedPresetModel: "nvidia/Qwen3.6-35B-A3B-NVFP4", + }, + installEffects, + { + probeCapability: () => capability, + resolveSelection: () => fixtureManagedClusterSelection(), + assertGatedModelAccess, + revalidateCapability, + claimCapability, + log: vi.fn(), + error, + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(error).toHaveBeenCalledWith( + expect.stringContaining("the resumed model 'nvidia/Qwen3.6-35B-A3B-NVFP4' does not match"), + ); + // Nothing may be claimed, recorded, pulled, staged, or created first. + expect(assertGatedModelAccess).not.toHaveBeenCalled(); + expect(promptFn).not.toHaveBeenCalled(); + expect(revalidateCapability).not.toHaveBeenCalled(); + expect(claimCapability).not.toHaveBeenCalled(); + expect(checkpointInstallIntent).not.toHaveBeenCalled(); + expect(beforeInstall).not.toHaveBeenCalled(); + expect(installEffects.prerequisites).not.toHaveBeenCalled(); + expect(installEffects.pullImage).not.toHaveBeenCalled(); + expect(installEffects.downloadModel).not.toHaveBeenCalled(); + }); + + it("accepts a resumed model the cluster preset selects", async () => { + const capability = readyCapability(); + const assertGatedModelAccess = vi.fn(); + const error = vi.fn(); + + const result = await tryInstallManagedClusterManagedVllm( + { + platform: "spark", + env: {}, + nonInteractive: true, + promptFn: vi.fn(async () => "yes"), + // The served name is one of the aliases the preset's model answers to. + resumedPresetModel: "deepseek-v4-flash-0731", + }, + effects(), + { + probeCapability: () => capability, + resolveSelection: () => fixtureManagedClusterSelection(), + assertGatedModelAccess, + revalidateCapability: vi.fn(() => { + throw new Error("stop after the resumed-model check"); + }), + log: vi.fn(), + error, + }, + ); + + // The resumed model passed, so the run reached the gated-access preflight + // and the later stages instead of being refused up front. + expect(assertGatedModelAccess).toHaveBeenCalledOnce(); + expect(error).not.toHaveBeenCalledWith(expect.stringContaining("does not match")); + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + }); + it("stages both exact nodes, launches, persists ownership, and retires temporary binding state", async () => { const capability = readyCapability(); const confirmed = confirmedCapability(capability); diff --git a/src/lib/inference/serving/managed-cluster-installer.ts b/src/lib/inference/serving/managed-cluster-installer.ts index 74275fa4a14..9c8d898d6f8 100644 --- a/src/lib/inference/serving/managed-cluster-installer.ts +++ b/src/lib/inference/serving/managed-cluster-installer.ts @@ -5,7 +5,12 @@ import { resolveVllmPort } from "../../core/vllm-port.js"; import { isAffirmativeAnswer } from "../../onboard/prompt-helpers.js"; import type { VllmProfile } from "../vllm.js"; import { ensureManagedVllmApiKey } from "../vllm-api-key.js"; -import { assertGatedModelAccess, VLLM_EXTRA_ARGS_ENV, type VllmModelDef } from "../vllm-models.js"; +import { + assertGatedModelAccess, + VLLM_EXTRA_ARGS_ENV, + type VllmModelDef, + vllmModelMatchesAlias, +} from "../vllm-models.js"; import { imageStorageRequirementBytes, modelStorageRequirementBytes } from "../vllm-storage.js"; import { claimManagedClusterManagedServingCapability, @@ -54,6 +59,13 @@ export interface ManagedClusterInstallerOptions { readonly promptFn: (question: string) => Promise; readonly beforeInstall?: (modelId: string) => void; readonly checkpointInstallIntent?: (modelId: string) => void; + /** + * Model recorded by an interrupted managed install, carried separately from + * the environment because a serving preset owns model selection here. The + * caller keeps it out of `NEMOCLAW_VLLM_MODEL` so NemoClaw's own checkpoint + * is not mistaken for an operator override (#11148). + */ + readonly resumedPresetModel?: string; } export interface ManagedClusterInstallerEffects { @@ -453,8 +465,24 @@ export async function tryInstallManagedClusterManagedVllm( deps.error(` Managed-cluster vLLM setup stopped: ${(error as Error).message}`); return { kind: "handled", result: { ok: false } }; } + // Revalidate the interrupted run's checkpoint against the model this + // preset resolves to, before the capability claim, the checkpoint write, + // the image pull, model staging, or any container creation. The resumed + // model is no longer part of the selection intent, so without this the + // cluster path would install the preset's model over a mismatched + // checkpoint (#11148). + const resumedPresetModel = String(options.resumedPresetModel ?? "").trim(); + const previewModel = managedModel(previewPlan, previewResolution.recipe); + if (resumedPresetModel && !vllmModelMatchesAlias(previewModel, resumedPresetModel)) { + deps.error( + ` Managed-cluster vLLM setup stopped: the resumed model '${resumedPresetModel}' does not match ` + + `'${previewModel.envValue}', which ${NEMOCLAW_SERVING_PRESET_ENV} selects. ` + + "Re-run onboarding with --fresh to discard the interrupted session.", + ); + return { kind: "handled", result: { ok: false } }; + } try { - deps.assertGatedModelAccess(managedModel(previewPlan, previewResolution.recipe), env); + deps.assertGatedModelAccess(previewModel, env); } catch (error) { deps.error(` Managed-cluster vLLM setup stopped: ${(error as Error).message}`); return { kind: "handled", result: { ok: false } }; diff --git a/src/lib/inference/vllm-fixed-catalog-install.test.ts b/src/lib/inference/vllm-fixed-catalog-install.test.ts index dd34ee78a5b..7bc46535df2 100644 --- a/src/lib/inference/vllm-fixed-catalog-install.test.ts +++ b/src/lib/inference/vllm-fixed-catalog-install.test.ts @@ -307,6 +307,113 @@ describe("fixed catalog vLLM installs", () => { expect(mocks.dockerRunDetached).toHaveBeenCalledOnce(); }); + /** + * Resume replays NemoClaw's own checkpoint, so the preset-driven install + * paths below run the real selection guard rather than a canned result. + */ + async function withActualSelectionGuard( + readinessReports: ReturnType, + ): Promise { + const actualSelection = await vi.importActual< + typeof import("./serving/host-local-vllm-selection") + >("./serving/host-local-vllm-selection"); + mocks.resolveHostLocalVllmSelection.mockImplementation((base, env, options) => + actualSelection.resolveHostLocalVllmSelection(base, env, { + ...options, + readinessReports, + }), + ); + } + + it("resumes a checkpointed model under an explicitly selected serving preset", async () => { + const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; + const modelIntent = "muse-glimmer-30b"; + const selected = await resolveActualHostLocalSelection( + profile, + { NEMOCLAW_VLLM_MODEL: modelIntent }, + modelIntent, + ); + process.env.NEMOCLAW_SERVING_PRESET = selected.presetId; + const readinessReports = vllmInstallTestReadiness(profile, modelIntent); + await withActualSelectionGuard(readinessReports); + const servedModelId = selected.model.servedModelId ?? selected.model.id; + mockSuccessfulVllmInstall(mocks, selected.profile.containerName); + mockSuccessfulAuthenticatedReadiness(servedModelId); + const beforeInstall = vi.fn(); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + modelIntent, + readinessReports, + beforeInstall, + resolveManagedBridgeHost: () => "172.18.0.1", + }); + + expect(spies.errSpy).not.toHaveBeenCalledWith( + expect.stringContaining("NEMOCLAW_SERVING_PRESET conflicts with NEMOCLAW_VLLM_MODEL"), + ); + expect(result).toEqual({ ok: true }); + // The preset stays the model authority, so the install that onboarding + // records is the one the preset selects. + expect(beforeInstall).toHaveBeenCalledWith(servedModelId); + }); + + it("rejects a resumed model the serving preset does not select", async () => { + const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; + const presetModel = "muse-glimmer-30b"; + const selected = await resolveActualHostLocalSelection( + profile, + { NEMOCLAW_VLLM_MODEL: presetModel }, + presetModel, + ); + process.env.NEMOCLAW_SERVING_PRESET = selected.presetId; + const readinessReports = vllmInstallTestReadiness(profile, presetModel); + await withActualSelectionGuard(readinessReports); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + modelIntent: "qwen3.6-35b-a3b-nvfp4", + readinessReports, + }); + + expect(result).toEqual({ ok: false }); + expect(spies.errSpy).toHaveBeenCalledWith( + expect.stringContaining("the resumed model 'qwen3.6-35b-a3b-nvfp4' does not match"), + ); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + }); + + it("still rejects an operator model override against a serving preset", async () => { + const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; + const presetModel = "muse-glimmer-30b"; + const selected = await resolveActualHostLocalSelection( + profile, + { NEMOCLAW_VLLM_MODEL: presetModel }, + presetModel, + ); + process.env.NEMOCLAW_SERVING_PRESET = selected.presetId; + process.env.NEMOCLAW_VLLM_MODEL = "qwen3.6-35b-a3b-nvfp4"; + const readinessReports = vllmInstallTestReadiness(profile, presetModel); + await withActualSelectionGuard(readinessReports); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + readinessReports, + }); + + expect(result).toEqual({ ok: false }); + expect(spies.errSpy).toHaveBeenCalledWith( + expect.stringContaining("NEMOCLAW_SERVING_PRESET conflicts with NEMOCLAW_VLLM_MODEL"), + ); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + }); + it("defers non-interactive custom arguments to the established installer", async () => { process.env.NEMOCLAW_VLLM_MODEL = "qwen3.6-35b-a3b-nvfp4"; process.env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON = '["--max-model-len","32768"]'; diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index ba88d52bbc2..02366d005d0 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -577,20 +577,24 @@ export function modelsForPlatform(platform: VllmPlatform): readonly VllmModelDef const HF_TOKEN_ENV_KEYS = ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"] as const; export const VLLM_EXTRA_ARGS_ENV = "NEMOCLAW_VLLM_EXTRA_ARGS_JSON"; -/** Resolve any unique model name owned by the managed inference catalog. */ -export function resolveVllmModelAlias(value: string): VllmModelDef | null { +/** True when `value` names this model by slug, Hugging Face ID, or served name. */ +export function vllmModelMatchesAlias(model: VllmModelDef, value: string): boolean { const requested = value.trim().toLowerCase(); - if (!requested) return null; + if (!requested) return false; return ( - VLLM_MODELS.find( - (model) => - model.envValue.toLowerCase() === requested || - model.id.toLowerCase() === requested || - model.servedModelId?.toLowerCase() === requested, - ) ?? null + model.envValue.toLowerCase() === requested || + model.id.toLowerCase() === requested || + model.servedModelId?.toLowerCase() === requested ); } +/** Resolve any unique model name owned by the managed inference catalog. */ +export function resolveVllmModelAlias(value: string): VllmModelDef | null { + const requested = value.trim(); + if (!requested) return null; + return VLLM_MODELS.find((model) => vllmModelMatchesAlias(model, requested)) ?? null; +} + /** * Look up the requested express-vLLM model from `NEMOCLAW_VLLM_MODEL`. * Returns `null` when the env var is empty so the caller can fall back to diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 8fd63e1ccfc..a455c88fc93 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -68,6 +68,7 @@ import { VLLM_EXTRA_ARGS_ENV, VLLM_MODELS, vllmModelForOrchestration, + vllmModelMatchesAlias, vllmModelUsesOrchestration, vllmPlatformSpecificity, type VllmModelDef, @@ -1846,7 +1847,12 @@ interface ServingPortProbe { } type VllmInstallSelectionEnv = - | { readonly ok: true; readonly env: NodeJS.ProcessEnv; readonly explicitModel: string } + | { + readonly ok: true; + readonly env: NodeJS.ProcessEnv; + readonly explicitModel: string; + readonly resumedPresetModel: string; + } | { readonly ok: false }; function resolveVllmInstallSelectionEnv( @@ -1862,11 +1868,20 @@ function resolveVllmInstallSelectionEnv( ) { return { ok: false }; } - const selectionEnv = resumedModel ? { ...env, NEMOCLAW_VLLM_MODEL: resumedModel } : env; + // A serving preset already names the model to install, so the preset stays + // authoritative and the resumed checkpoint is only verified against it. + // Feeding that checkpoint back through NEMOCLAW_VLLM_MODEL made NemoClaw's + // own record look like a competing operator override, and resuming a + // preset-driven install was refused as a preset/model conflict even though + // the operator had set neither variable (#11148). + const presetSelected = String(env[NEMOCLAW_SERVING_PRESET_ENV] ?? "").trim().length > 0; + const selectionEnv = + resumedModel && !presetSelected ? { ...env, NEMOCLAW_VLLM_MODEL: resumedModel } : env; return { ok: true, env: selectionEnv, explicitModel: String(selectionEnv.NEMOCLAW_VLLM_MODEL ?? "").trim(), + resumedPresetModel: presetSelected ? resumedModel : "", }; } @@ -1875,6 +1890,7 @@ type VllmInstallRequestEnv = readonly ok: true; readonly env: NodeJS.ProcessEnv; readonly explicitModel: string; + readonly resumedPresetModel: string; readonly requestedGpuDevice: string | null; readonly configuredPeer: string; readonly configuredManagedClusterPeers: string; @@ -2009,6 +2025,7 @@ async function runVllmInstall( const { env: selectionEnv, explicitModel, + resumedPresetModel, requestedGpuDevice, configuredPeer, configuredManagedClusterPeers, @@ -2047,6 +2064,9 @@ async function runVllmInstall( promptFn: opts.promptFn, beforeInstall: opts.beforeInstall, checkpointInstallIntent: opts.checkpointInstallIntent, + // This branch returns before the host-local revalidation below, so the + // resumed checkpoint has to travel with it (#11148). + resumedPresetModel, }, { prerequisites: dockerPrereqsOk, @@ -2150,6 +2170,18 @@ async function runVllmInstall( }); } if (!resolved) return { ok: false }; + // The preset chose the model above; this is the receipt check that the + // interrupted run had committed to the same one. It covers every branch that + // produced `resolved` — preset selection, a fixed catalog profile, and the + // Station pair — because each of them is reachable on resume (#11148). + if (resumedPresetModel && !vllmModelMatchesAlias(resolved.model, resumedPresetModel)) { + console.error( + ` vLLM install failed: the resumed model '${resumedPresetModel}' does not match ` + + `'${resolved.model.envValue}', which ${NEMOCLAW_SERVING_PRESET_ENV} selects. ` + + `Re-run onboarding with --fresh to discard the interrupted session.`, + ); + return { ok: false }; + } if ( !hostLocalSelection && resolved.source === "picker" && diff --git a/src/lib/onboard/local-model-profile/onboarder.ts b/src/lib/onboard/local-model-profile/onboarder.ts index 105d27913c3..fcfba160b21 100644 --- a/src/lib/onboard/local-model-profile/onboarder.ts +++ b/src/lib/onboard/local-model-profile/onboarder.ts @@ -4,7 +4,7 @@ import { materializeHostLocalVllmSelection } from "../../inference/serving/host-local-vllm-selection"; import type { ResolvedHostLocalInferenceSelection } from "../../inference/serving/types"; import type { VllmProfile } from "../../inference/vllm"; -import { VLLM_EXTRA_ARGS_ENV } from "../../inference/vllm-models"; +import { VLLM_EXTRA_ARGS_ENV, vllmModelMatchesAlias } from "../../inference/vllm-models"; import type { SetupNimSelectionResult, SetupNimSelectionState } from "../setup-nim-flow"; import { vllmInstallRecoveryOptions } from "../provider-recovery"; import type { LocalModelProfilePlan } from "./plan"; @@ -87,13 +87,8 @@ export function createLocalModelProfileOnboarder(deps: LocalModelProfileOnboarde return "retry-selection"; } const recovery = vllmInstallRecoveryOptions(deps); - const resumedModel = recovery.modelIntent?.trim().toLowerCase(); - if ( - resumedModel && - ![materialized.model.id, materialized.model.envValue, materialized.model.servedModelId].some( - (candidate) => candidate?.toLowerCase() === resumedModel, - ) - ) { + const resumedModel = recovery.modelIntent?.trim(); + if (resumedModel && !vllmModelMatchesAlias(materialized.model, resumedModel)) { deps.error( ` The resumed vLLM model conflicts with the ${materialized.model.envValue} local model profile.`, );