Skip to content
Merged
10 changes: 9 additions & 1 deletion src/lib/inference/serving/adapter-registry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import {
getManagedInferencePreparationDescriptor,
getManagedInferenceRecipeRegistrationError,
getManagedInferenceTopologyQualificationDescriptor,
HOST_LOCAL_VLLM_LIFECYCLE_REF,
HOST_LOCAL_VLLM_MATERIALIZER_REF,
listManagedInferenceLifecycleDescriptors,
listManagedInferenceMaterializerDescriptors,
listManagedInferencePreparationDescriptors,
Expand All @@ -19,12 +21,12 @@ import {
SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF,
} from "./adapter-registry.js";
import { loadManagedInferenceCatalog } from "./catalog-loader.js";
import type { ManagedInferenceServingRecipe } from "./types.js";
import { fixtureManagedClusterSelection } from "./managed-cluster-fixture.test-support.js";
import {
MANAGED_CLUSTER_TOPOLOGY_ID,
MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION,
} from "./managed-cluster-topology.js";
import type { ManagedInferenceServingRecipe } from "./types.js";

function shippedRecipe(): ManagedInferenceServingRecipe {
const recipe = loadManagedInferenceCatalog().recipes.find(
Expand All @@ -45,9 +47,11 @@ describe("managed inference adapter registries", () => {
]);
expect(listManagedInferenceMaterializerDescriptors()).toMatchObject([
{ ref: MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, backend: "vllm" },
{ ref: HOST_LOCAL_VLLM_MATERIALIZER_REF, backend: "vllm" },
]);
expect(listManagedInferenceLifecycleDescriptors()).toMatchObject([
{ ref: MANAGED_CLUSTER_VLLM_LIFECYCLE_REF, backend: "vllm" },
{ ref: HOST_LOCAL_VLLM_LIFECYCLE_REF, backend: "vllm" },
]);
expect(listManagedInferencePreparationDescriptors()).toEqual(
expect.arrayContaining([
Expand All @@ -73,10 +77,14 @@ describe("managed inference adapter registries", () => {
expect(
getManagedInferenceMaterializerDescriptor(MANAGED_CLUSTER_VLLM_MATERIALIZER_REF),
).toBeDefined();
expect(
getManagedInferenceMaterializerDescriptor(HOST_LOCAL_VLLM_MATERIALIZER_REF),
).toBeDefined();
expect(getManagedInferenceMaterializerDescriptor("unknown.materializer/v1")).toBeUndefined();
expect(
getManagedInferenceLifecycleDescriptor(MANAGED_CLUSTER_VLLM_LIFECYCLE_REF),
).toBeDefined();
expect(getManagedInferenceLifecycleDescriptor(HOST_LOCAL_VLLM_LIFECYCLE_REF)).toBeDefined();
expect(getManagedInferenceLifecycleDescriptor("unknown.lifecycle/v1")).toBeUndefined();
expect(
getManagedInferencePreparationDescriptor(
Expand Down
80 changes: 78 additions & 2 deletions src/lib/inference/serving/adapter-registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION,
} from "./managed-cluster-topology.js";
import type {
HostLocalInferenceServingRecipe,
ManagedInferenceServingRecipe,
ManagedInferenceTopologyQualification,
ServingCatalogRegistries,
Expand All @@ -16,6 +17,8 @@ import type {

export const MANAGED_CLUSTER_VLLM_MATERIALIZER_REF = "vllm.managed-cluster/v1" as const;
export const MANAGED_CLUSTER_VLLM_LIFECYCLE_REF = "vllm.managed-cluster.lifecycle/v1" as const;
export const HOST_LOCAL_VLLM_MATERIALIZER_REF = "vllm.host-local/v1" as const;
export const HOST_LOCAL_VLLM_LIFECYCLE_REF = "vllm.host-local.lifecycle/v1" as const;
export const LLAMA_CPP_HOST_LOCAL_RECEIPT_REF = "llama-cpp.host-local.receipt/v1" as const;
export const LLAMA_CPP_HOST_LOCAL_MATERIALIZER_REF = "llama-cpp.host-local/v1" as const;
export const LLAMA_CPP_HOST_LOCAL_LIFECYCLE_REF = "llama-cpp.host-local.lifecycle/v1" as const;
Expand Down Expand Up @@ -44,7 +47,7 @@ export interface ManagedInferenceMaterializerDescriptor {
readonly ref: string;
readonly backend: string;
readonly outputPlanSchema: string;
readonly topology: {
readonly topology?: {
readonly qualificationId: string;
readonly schemaVersion: number;
readonly outputSchema: string;
Expand Down Expand Up @@ -250,6 +253,57 @@ function validateManagedClusterLifecycleRecipe(
: "recipe does not select the managed cluster lifecycle";
}

function validateHostLocalVllmMaterializerRecipe(
recipe: HostLocalInferenceServingRecipe,
): string | undefined {
if (recipe.spec.backend !== "vllm") return "host-local vLLM materializer requires backend vllm";
if (recipe.spec.execution.materializerRef !== HOST_LOCAL_VLLM_MATERIALIZER_REF) {
return "recipe does not select the host-local vLLM materializer";
}
const execution = recipe.spec.execution;
if (
execution.topologyBinding !== undefined ||
execution.nodeCount !== undefined ||
execution.tensorParallelSize !== undefined ||
execution.pipelineParallelSize !== undefined ||
execution.distributedExecutorBackend !== undefined ||
execution.rendezvousPort !== undefined
) {
return "host-local vLLM materializer does not accept distributed execution settings";
}
const bindings = recipe.spec.bindings;
if (bindings !== undefined) {
return "host-local vLLM materializer does not accept topology bindings";
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
const runtime = recipe.spec.runtime;
if (runtime.architecture !== "arm64") {
return "host-local vLLM materializer requires an arm64 runtime";
}
if (recipe.spec.model.preparation?.ref !== NO_PREPARATION_REF) {
return "host-local vLLM materializer currently requires an empty preparation operation";
}
if (recipe.spec.readiness.expectedModel !== recipe.spec.model.servedName) {
return "host-local vLLM readiness must expect the recipe served model";
}
return undefined;
}

function validateHostLocalVllmLifecycleRecipe(
recipe: HostLocalInferenceServingRecipe,
): string | undefined {
const materializerError = validateHostLocalVllmMaterializerRecipe(recipe);
if (materializerError) return materializerError;
return recipe.spec.execution.lifecycleRef === HOST_LOCAL_VLLM_LIFECYCLE_REF
? undefined
: "recipe does not select the host-local vLLM lifecycle";
}

export function isHostLocalInferenceServingRecipe(
recipe: ManagedInferenceServingRecipe,
): recipe is HostLocalInferenceServingRecipe {
return recipe.spec.execution.materializerRef === HOST_LOCAL_VLLM_MATERIALIZER_REF;
}

interface SnapshotPreparationInput {
readonly ref: typeof SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF;
readonly snapshotCopy: {
Expand Down Expand Up @@ -383,6 +437,15 @@ const MATERIALIZER_DESCRIPTORS = [
},
validateRecipe: validateManagedClusterMaterializerRecipe,
},
{
ref: HOST_LOCAL_VLLM_MATERIALIZER_REF,
backend: "vllm",
outputPlanSchema: "nemoclaw.nvidia.com/host-local-vllm-plan/v1",
validateRecipe: (recipe) =>
isHostLocalInferenceServingRecipe(recipe)
? validateHostLocalVllmMaterializerRecipe(recipe)
: "recipe does not select the host-local vLLM materializer",
},
] as const satisfies readonly ManagedInferenceMaterializerDescriptor[];

const LIFECYCLE_DESCRIPTORS = [
Expand All @@ -394,6 +457,17 @@ const LIFECYCLE_DESCRIPTORS = [
secretHandlePermissions: ["sshBinding"],
validateRecipe: validateManagedClusterLifecycleRecipe,
},
{
ref: HOST_LOCAL_VLLM_LIFECYCLE_REF,
backend: "vllm",
acceptedMaterializerRefs: [HOST_LOCAL_VLLM_MATERIALIZER_REF],
acceptedPlanSchemas: ["nemoclaw.nvidia.com/host-local-vllm-plan/v1"],
secretHandlePermissions: [],
validateRecipe: (recipe) =>
isHostLocalInferenceServingRecipe(recipe)
? validateHostLocalVllmLifecycleRecipe(recipe)
: "recipe does not select the host-local vLLM lifecycle",
},
] as const satisfies readonly ManagedInferenceLifecycleDescriptor[];

const PREPARATION_DESCRIPTORS = [
Expand Down Expand Up @@ -546,7 +620,9 @@ export function getManagedInferenceServingCatalogRegistries(): ServingCatalogReg
validateRecipe: (recipe: ServingRecipe) => {
if (
recipe.spec.execution.materializerRef !== MANAGED_CLUSTER_VLLM_MATERIALIZER_REF &&
recipe.spec.execution.lifecycleRef !== MANAGED_CLUSTER_VLLM_LIFECYCLE_REF
recipe.spec.execution.lifecycleRef !== MANAGED_CLUSTER_VLLM_LIFECYCLE_REF &&
recipe.spec.execution.materializerRef !== HOST_LOCAL_VLLM_MATERIALIZER_REF &&
recipe.spec.execution.lifecycleRef !== HOST_LOCAL_VLLM_LIFECYCLE_REF
) {
return undefined;
}
Expand Down
46 changes: 46 additions & 0 deletions src/lib/inference/serving/catalog-loader.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

import { describe, expect, it } from "vitest";
import {
HOST_LOCAL_VLLM_LIFECYCLE_REF,
HOST_LOCAL_VLLM_MATERIALIZER_REF,
MANAGED_CLUSTER_VLLM_LIFECYCLE_REF,
MANAGED_CLUSTER_VLLM_MATERIALIZER_REF,
} from "./adapter-registry";
Expand Down Expand Up @@ -130,6 +132,50 @@ describe("managed inference catalog loader", () => {
expect(catalogDigest).toBe(servingCatalogDigest(catalogContents));
});

it("retains registered host-local vLLM definitions (#8246)", () => {
const servingCatalog = loadServingCatalog();
const sourceRecipe = servingCatalog.recipes.find(({ spec }) => spec.backend === "vllm")!;
const sourcePreset = servingCatalog.presets.find(
({ spec }) => spec.plan.recipeRef === sourceRecipe.metadata.id,
)!;
const { bindings: _bindings, ...hostLocalSpec } = sourceRecipe.spec;
const execution = {
materializerRef: HOST_LOCAL_VLLM_MATERIALIZER_REF,
lifecycleRef: HOST_LOCAL_VLLM_LIFECYCLE_REF,
};
const recipe = {
...sourceRecipe,
metadata: { id: "test.vllm-host-local-recipe" },
spec: {
...hostLocalSpec,
model: { ...sourceRecipe.spec.model, preparation: { ref: "none/v1" } },
execution,
},
} as unknown as ServingRecipe;
const preset = {
...sourcePreset,
metadata: { id: "test.vllm-host-local-preset" },
spec: {
...sourcePreset.spec,
selection: "explicit-only",
requirements: {
all: sourcePreset.spec.requirements!.all.filter(
(requirement) => "readiness" in requirement,
),
},
plan: { backend: "vllm", recipeRef: recipe.metadata.id },
},
} as ServingPreset;
const projected = managedInferenceCatalogFromServingCatalog({
...EMPTY_CATALOG,
recipes: [recipe],
presets: [preset],
});

expect(projected.recipes.map(({ metadata }) => metadata.id)).toEqual([recipe.metadata.id]);
expect(projected.presets.map(({ metadata }) => metadata.id)).toEqual([preset.metadata.id]);
});

it("retains managed vLLM definitions while projecting the mixed serving catalog (#8173)", () => {
const servingCatalog = loadServingCatalog();

Expand Down
14 changes: 8 additions & 6 deletions src/lib/inference/serving/catalog-loader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import { join } from "node:path";

import {
getManagedInferenceRecipeRegistrationError,
HOST_LOCAL_VLLM_LIFECYCLE_REF,
HOST_LOCAL_VLLM_MATERIALIZER_REF,
MANAGED_CLUSTER_VLLM_LIFECYCLE_REF,
MANAGED_CLUSTER_VLLM_MATERIALIZER_REF,
} from "./adapter-registry.js";
Expand Down Expand Up @@ -60,10 +62,8 @@ function assertManagedRecipe(
function assertManagedPreset(
preset: ServingPreset,
): asserts preset is ManagedInferenceServingPreset {
if (!preset.spec.requirements || !preset.spec.plan.bindings) {
throw new Error(
`Managed inference preset ${preset.metadata.id} must declare requirements and topology bindings`,
);
if (!preset.spec.requirements) {
throw new Error(`Managed inference preset ${preset.metadata.id} must declare requirements`);
}
}

Expand Down Expand Up @@ -97,11 +97,13 @@ export function parseCompiledManagedInferenceCatalogJson(
function isManagedClusterRecipeCandidate(recipe: ServingRecipe): boolean {
return (
recipe.spec.execution.materializerRef === MANAGED_CLUSTER_VLLM_MATERIALIZER_REF ||
recipe.spec.execution.lifecycleRef === MANAGED_CLUSTER_VLLM_LIFECYCLE_REF
recipe.spec.execution.lifecycleRef === MANAGED_CLUSTER_VLLM_LIFECYCLE_REF ||
recipe.spec.execution.materializerRef === HOST_LOCAL_VLLM_MATERIALIZER_REF ||
recipe.spec.execution.lifecycleRef === HOST_LOCAL_VLLM_LIFECYCLE_REF
);
}

/** The managed-cluster runtime accepts only recipes backed by its vLLM materializer and lifecycle. */
/** Project recipes backed by a registered NemoClaw-managed vLLM runtime. */
export function managedInferenceCatalogFromServingCatalog(
catalog: CompiledServingCatalog,
): CompiledManagedInferenceCatalog {
Expand Down
Loading
Loading