From 89ca24ed79e20683e566b8cb488d98ec6fe6e9c2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 05:55:44 -0700 Subject: [PATCH 1/4] feat(inference): add Spark llama.cpp catalog profile Signed-off-by: Aaron Erickson --- ...k-gb10.single.nemotron-3-nano-30b-a3b.yaml | 105 ++++++++++++++++ ...motron-3-nano-30b-a3b.spark-single.v1.yaml | 116 ++++++++++++++++++ managed-inference/schemas/recipe.schema.json | 59 +++++++++ src/lib/inference/serving/adapter-registry.ts | 30 ++++- .../inference/serving/catalog-loader.test.ts | 44 ++++++- src/lib/inference/serving/catalog-loader.ts | 56 +++++++-- src/lib/inference/serving/catalog.test.ts | 56 ++++++++- src/lib/inference/serving/catalog.ts | 27 +++- src/lib/inference/serving/types.ts | 13 +- ...managed-inference-catalog-compiler.test.ts | 64 +++++++++- 10 files changed, 549 insertions(+), 21 deletions(-) create mode 100644 managed-inference/presets/llama-cpp.dgx-spark-gb10.single.nemotron-3-nano-30b-a3b.yaml create mode 100644 managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml diff --git a/managed-inference/presets/llama-cpp.dgx-spark-gb10.single.nemotron-3-nano-30b-a3b.yaml b/managed-inference/presets/llama-cpp.dgx-spark-gb10.single.nemotron-3-nano-30b-a3b.yaml new file mode 100644 index 00000000000..73b988f11c4 --- /dev/null +++ b/managed-inference/presets/llama-cpp.dgx-spark-gb10.single.nemotron-3-nano-30b-a3b.yaml @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.nvidia.com/managed-inference/v1 +kind: ServingPreset + +metadata: + id: llama-cpp.dgx-spark-gb10.single.nemotron-3-nano-30b-a3b + displayName: NVIDIA Nemotron 3 Nano 30B-A3B on one DGX Spark + +spec: + selection: explicit-only + priority: 450 + + requirements: + all: + - readiness: + scope: everyNode + kind: qualification + id: host.platform.dgx_spark + status: qualified + - readiness: + scope: everyNode + kind: capability + id: host.platform.supported + state: present + - readiness: + scope: everyNode + kind: capability + id: host.platform.dgx_spark + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.daemon_reachable + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.runtime_supported + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.storage_compatible + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.nvidia_available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.container_toolkit_available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.cdi_healthy + state: present + - readiness: + scope: everyNode + kind: observation + id: host.os.platform + comparison: + operator: equals + value: linux + - readiness: + scope: everyNode + kind: observation + id: host.os.architecture + comparison: + operator: equals + value: arm64 + - readiness: + scope: everyNode + kind: observation + id: host.docker.runtime + comparison: + operator: equals + value: docker + - readiness: + scope: everyNode + kind: observation + id: host.gpu.count + comparison: + operator: at-least + value: 1 + - readiness: + scope: everyNode + kind: observation + id: host.gpu.driver_version + comparison: + operator: version-at-least + value: 580.65.06 + + plan: + backend: install-llama-cpp + recipeRef: llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1 diff --git a/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml b/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml new file mode 100644 index 00000000000..758bd6b4855 --- /dev/null +++ b/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.nvidia.com/managed-inference/v1 +kind: ServingRecipe + +metadata: + id: llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1 + displayName: NVIDIA Nemotron 3 Nano 30B-A3B with llama.cpp + +spec: + backend: install-llama-cpp + providerId: llama-cpp-local + + server: + technology: llama.cpp + source: + repository: ggml-org/llama.cpp + revision: 22dc605c4ead20e36f447cc67b55ef87e523bd55 + + model: + id: unsloth/Nemotron-3-Nano-30B-A3B-GGUF + revision: 9ad8b366c308f931b2a96b9306f0b41aef9cd405 + servedName: nvidia-nemotron-3-nano-30b-a3b + files: + - path: Nemotron-3-Nano-30B-A3B-UD-Q4_K_XL.gguf + digest: sha256:627f5b04aedc97f967332f331bd75b7a4ed2f33ca83e6ee74b44235cc1887890 + sizeBytes: 22833947424 + format: gguf + quantization: UD-Q4_K_XL + license: NVIDIA-Open-Model-License + + runtime: + image: ghcr.io/ggml-org/llama.cpp@sha256:866ad568474de9e835e487ae841ad6ace1a494b5eab4f292cbd45adb6180f711 + imageDownloadSizeBytes: 2181958990 + platforms: + - linux/amd64 + - linux/arm64 + containerRuntime: docker + networkExposure: loopback + hosts: 1 + cuda: + baseImage: docker.io/nvidia/cuda@sha256:789e629e49401647e22b7054ae9c6c4f6427dba68010ba428deb4cc6b063676e + minimumDriverVersion: 580.65.06 + gpu: + vendor: nvidia + count: 1 + offload: full + cpuFallback: reject + resources: + memoryBytes: 51539607552 + writableStorageBytes: 42949672960 + pidsLimit: 256 + + execution: + receiptRef: llama-cpp.host-local.receipt/v1 + materializerRef: llama-cpp.host-local/v1 + lifecycleRef: llama-cpp.host-local.lifecycle/v1 + + serve: + protocol: openai-completions + authentication: bearer + port: 8081 + chatTemplate: nemotron-v3-embedded + contextSize: 262144 + slots: 1 + idleSleepSeconds: -1 + batchSize: 2048 + microBatchSize: 512 + flashAttention: enabled + kvCache: + key: f16 + value: f16 + speculativeDecoding: disabled + limits: + maxRequestBodyBytes: 16777216 + maxPromptTokens: 253952 + maxCompletionTokens: 8192 + requestTimeoutSeconds: 900 + + readiness: + contractRef: llama-cpp.server-readiness/v1 + timeoutSeconds: 1800 + expectedModel: nvidia-nemotron-3-nano-30b-a3b + probes: + models: true + health: true + properties: true + metrics: true + + policy: + egress: disabled + modelSource: verified-local + modelDownloads: disabled + + surfaces: + ui: disabled + slotInspection: disabled + router: disabled + mcpProxy: disabled + serverTools: disabled + agentMode: disabled + multimodalProjection: disabled + + capabilities: + agents: [] + protocols: + - openai-completions + streaming: true + toolCalls: true + structuredOutputs: true + parallelToolCalls: false + responsesApi: false + embeddings: false + reranking: false + multimodal: false diff --git a/managed-inference/schemas/recipe.schema.json b/managed-inference/schemas/recipe.schema.json index 1cf5f6dd5b2..d43836a2015 100644 --- a/managed-inference/schemas/recipe.schema.json +++ b/managed-inference/schemas/recipe.schema.json @@ -256,6 +256,9 @@ "containerRuntime": { "const": "docker" }, + "networkExposure": { + "const": "loopback" + }, "hosts": { "const": 1 }, @@ -429,6 +432,44 @@ "minimum": -1, "maximum": 86400 }, + "batchSize": { + "type": "integer", + "minimum": 1, + "maximum": 65536 + }, + "microBatchSize": { + "type": "integer", + "minimum": 1, + "maximum": 65536 + }, + "flashAttention": { + "const": "enabled" + }, + "kvCache": { + "type": "object", + "required": [ + "key", + "value" + ], + "properties": { + "key": { + "enum": [ + "f16", + "q8_0" + ] + }, + "value": { + "enum": [ + "f16", + "q8_0" + ] + } + }, + "additionalProperties": false + }, + "speculativeDecoding": { + "const": "disabled" + }, "limits": { "type": "object", "required": [ @@ -1044,8 +1085,10 @@ "type": "object", "required": [ "image", + "imageDownloadSizeBytes", "platforms", "containerRuntime", + "networkExposure", "hosts", "cuda", "gpu", @@ -1053,8 +1096,10 @@ ], "properties": { "image": {}, + "imageDownloadSizeBytes": {}, "platforms": {}, "containerRuntime": {}, + "networkExposure": {}, "hosts": {}, "cuda": {}, "gpu": {}, @@ -1100,15 +1145,24 @@ "type": "object", "required": [ "protocol", + "authentication", "port", "chatTemplate", "contextSize", "slots", "idleSleepSeconds", + "batchSize", + "microBatchSize", + "flashAttention", + "kvCache", + "speculativeDecoding", "limits" ], "properties": { "protocol": {}, + "authentication": { + "const": "bearer" + }, "port": { "const": 8081 }, @@ -1120,6 +1174,11 @@ "idleSleepSeconds": { "const": -1 }, + "batchSize": {}, + "microBatchSize": {}, + "flashAttention": {}, + "kvCache": {}, + "speculativeDecoding": {}, "limits": {} }, "not": { diff --git a/src/lib/inference/serving/adapter-registry.ts b/src/lib/inference/serving/adapter-registry.ts index 910011bf69f..401e125751e 100644 --- a/src/lib/inference/serving/adapter-registry.ts +++ b/src/lib/inference/serving/adapter-registry.ts @@ -9,13 +9,17 @@ import { import type { ManagedInferenceServingRecipe, ManagedInferenceTopologyQualification, - ReadinessEntityKind, ServingCatalogRegistries, + ServingReadinessRegistryValue, ServingRecipe, } from "./types.js"; export const MANAGED_CLUSTER_VLLM_MATERIALIZER_REF = "vllm.managed-cluster/v1" as const; export const MANAGED_CLUSTER_VLLM_LIFECYCLE_REF = "vllm.managed-cluster.lifecycle/v1" as const; +export const LLAMA_CPP_HOST_LOCAL_RECEIPT_REF = "llama-cpp.host-local.receipt/v1" as const; +export const LLAMA_CPP_HOST_LOCAL_MATERIALIZER_REF = "llama-cpp.host-local/v1" as const; +export const LLAMA_CPP_HOST_LOCAL_LIFECYCLE_REF = "llama-cpp.host-local.lifecycle/v1" as const; +export const LLAMA_CPP_SERVER_READINESS_REF = "llama-cpp.server-readiness/v1" as const; export const SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF = "snapshot-copy-and-exact-text-replacement/v1" as const; export const NO_PREPARATION_REF = "none/v1" as const; @@ -495,8 +499,16 @@ export function getManagedInferenceRecipeRegistrationError( const SERVING_READINESS_REGISTRY: ServingCatalogRegistries["readiness"] = new Map< string, - ReadinessEntityKind | ReadonlySet + ServingReadinessRegistryValue >([ + ["host.os.platform", { kind: "observation", valueType: "string", role: "operating-system" }], + ["host.os.architecture", { kind: "observation", valueType: "string", role: "architecture" }], + ["host.docker.runtime", { kind: "observation", valueType: "string", role: "container-runtime" }], + ["host.gpu.count", { kind: "observation", valueType: "number", role: "gpu-count" }], + [ + "host.gpu.driver_version", + { kind: "observation", valueType: "version", role: "driver-version" }, + ], ["host.platform.dgx_spark", new Set(["qualification", "capability"] as const)], ["host.platform.supported", "capability"], ["host.docker.available", "capability"], @@ -510,10 +522,16 @@ const SERVING_READINESS_REGISTRY: ServingCatalogRegistries["readiness"] = new Ma export function getManagedInferenceServingCatalogRegistries(): ServingCatalogRegistries { return { - receipts: new Set(), - materializers: new Set(MATERIALIZER_DESCRIPTORS.map(({ ref }) => ref)), - lifecycles: new Set(LIFECYCLE_DESCRIPTORS.map(({ ref }) => ref)), - readinessContracts: new Set(), + receipts: new Set([LLAMA_CPP_HOST_LOCAL_RECEIPT_REF]), + materializers: new Set([ + ...MATERIALIZER_DESCRIPTORS.map(({ ref }) => ref), + LLAMA_CPP_HOST_LOCAL_MATERIALIZER_REF, + ]), + lifecycles: new Set([ + ...LIFECYCLE_DESCRIPTORS.map(({ ref }) => ref), + LLAMA_CPP_HOST_LOCAL_LIFECYCLE_REF, + ]), + readinessContracts: new Set([LLAMA_CPP_SERVER_READINESS_REF]), readiness: SERVING_READINESS_REGISTRY, facts: new Set(["cluster.nodeCount"]), topologyQualifications: new Map( diff --git a/src/lib/inference/serving/catalog-loader.test.ts b/src/lib/inference/serving/catalog-loader.test.ts index 5255c4197af..d9688bef6fa 100644 --- a/src/lib/inference/serving/catalog-loader.test.ts +++ b/src/lib/inference/serving/catalog-loader.test.ts @@ -6,7 +6,10 @@ import { MANAGED_CLUSTER_VLLM_LIFECYCLE_REF, MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, } from "./adapter-registry"; -import { assertManagedInferenceCatalog } from "./catalog-loader"; +import { + assertManagedInferenceCatalog, + managedInferenceCatalogFromServingCatalog, +} from "./catalog-loader"; import type { CompiledServingCatalog, ServingPreset, ServingRecipe } from "./types"; const EMPTY_CATALOG: CompiledServingCatalog = { @@ -45,6 +48,31 @@ const INCOMPLETE_MANAGED_PRESET: ServingPreset = { }, }; +const HOST_LOCAL_RECIPE: ServingRecipe = { + apiVersion: "nemoclaw.nvidia.com/managed-inference/v1", + kind: "ServingRecipe", + metadata: { id: "test.host-local-recipe" }, + spec: { + backend: "install-llama-cpp", + model: { id: "test/model", revision: "d".repeat(40) }, + execution: { + materializerRef: "llama-cpp.host-local/v1", + lifecycleRef: "llama-cpp.host-local.lifecycle/v1", + }, + }, +}; + +const HOST_LOCAL_PRESET: ServingPreset = { + apiVersion: "nemoclaw.nvidia.com/managed-inference/v1", + kind: "ServingPreset", + metadata: { id: "test.host-local-preset" }, + spec: { + selection: "explicit-only", + priority: 1, + plan: { backend: "install-llama-cpp", recipeRef: HOST_LOCAL_RECIPE.metadata.id }, + }, +}; + function managedCatalogValidationError(catalog: CompiledServingCatalog): string | undefined { try { assertManagedInferenceCatalog(catalog); @@ -67,4 +95,18 @@ describe("managed inference catalog loader", () => { ] as const)("rejects an incomplete managed %s", (_label, catalog) => { expect(managedCatalogValidationError(catalog)).toMatch(/Managed inference (preset|recipe)/u); }); + + it("projects host-local definitions out of the managed-cluster runtime catalog (#8173)", () => { + const servingCatalog: CompiledServingCatalog = { + ...EMPTY_CATALOG, + recipes: [HOST_LOCAL_RECIPE], + presets: [HOST_LOCAL_PRESET], + }; + + const managedCatalog = managedInferenceCatalogFromServingCatalog(servingCatalog); + + expect(managedCatalog.recipes).toEqual([]); + expect(managedCatalog.presets).toEqual([]); + expect(managedCatalog.catalogDigest).toBe(EMPTY_CATALOG.catalogDigest); + }); }); diff --git a/src/lib/inference/serving/catalog-loader.ts b/src/lib/inference/serving/catalog-loader.ts index c8e7ac78ab6..c1dabbb88b4 100644 --- a/src/lib/inference/serving/catalog-loader.ts +++ b/src/lib/inference/serving/catalog-loader.ts @@ -4,7 +4,11 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { getManagedInferenceRecipeRegistrationError } from "./adapter-registry.js"; +import { + getManagedInferenceRecipeRegistrationError, + MANAGED_CLUSTER_VLLM_LIFECYCLE_REF, + MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, +} from "./adapter-registry.js"; import { parseCompiledServingCatalogJson } from "./catalog.js"; import { immutableManagedInferenceCopy } from "./catalog-integrity.js"; import type { @@ -34,7 +38,8 @@ function loadSchemas(rootDir: string): ServingCatalogSchemas { }; } -let loadedCatalog: CompiledManagedInferenceCatalog | undefined; +let loadedServingCatalog: CompiledServingCatalog | undefined; +let loadedManagedCatalog: CompiledManagedInferenceCatalog | undefined; function assertManagedRecipe( recipe: ServingRecipe, @@ -89,14 +94,51 @@ export function parseCompiledManagedInferenceCatalogJson( return catalog; } -export function loadManagedInferenceCatalog(): CompiledManagedInferenceCatalog { - if (loadedCatalog) return loadedCatalog; +function isManagedClusterRecipeCandidate(recipe: ServingRecipe): boolean { + return ( + recipe.spec.execution.materializerRef === MANAGED_CLUSTER_VLLM_MATERIALIZER_REF || + recipe.spec.execution.lifecycleRef === MANAGED_CLUSTER_VLLM_LIFECYCLE_REF + ); +} + +/** Project the backend-polymorphic serving catalog onto the managed-cluster runtime. */ +export function managedInferenceCatalogFromServingCatalog( + catalog: CompiledServingCatalog, +): CompiledManagedInferenceCatalog { + const recipes = catalog.recipes.filter(isManagedClusterRecipeCandidate); + const recipeIds = new Set(recipes.map(({ metadata }) => metadata.id)); + const presets = catalog.presets.filter( + (preset) => recipeIds.has(preset.spec.plan.recipeRef) || preset.spec.plan.backend === "vllm", + ); + const managedCatalog = { ...catalog, recipes, presets }; + assertManagedInferenceCatalog(managedCatalog); + return managedCatalog; +} + +export function loadServingCatalog(): CompiledServingCatalog { + if (loadedServingCatalog) return loadedServingCatalog; const rootDir = repositoryRoot(); const source = readFileSync(join(rootDir, "dist", "managed-inference", "catalog.json"), "utf8"); - loadedCatalog = immutableManagedInferenceCopy( - parseCompiledManagedInferenceCatalogJson(source, loadSchemas(rootDir)), + loadedServingCatalog = immutableManagedInferenceCopy( + parseCompiledServingCatalogJson(source, loadSchemas(rootDir)), ); - return loadedCatalog; + return loadedServingCatalog; +} + +export function loadManagedInferenceCatalog(): CompiledManagedInferenceCatalog { + if (loadedManagedCatalog) return loadedManagedCatalog; + loadedManagedCatalog = immutableManagedInferenceCopy( + managedInferenceCatalogFromServingCatalog(loadServingCatalog()), + ); + return loadedManagedCatalog; +} + +export function getCompiledServingPreset(id: string): ServingPreset | undefined { + return loadServingCatalog().presets.find(({ metadata }) => metadata.id === id); +} + +export function getCompiledServingRecipe(id: string): ServingRecipe | undefined { + return loadServingCatalog().recipes.find(({ metadata }) => metadata.id === id); } export function getManagedInferenceCompiledPreset( diff --git a/src/lib/inference/serving/catalog.test.ts b/src/lib/inference/serving/catalog.test.ts index 1d3453894aa..0d6ccbf6007 100644 --- a/src/lib/inference/serving/catalog.test.ts +++ b/src/lib/inference/serving/catalog.test.ts @@ -143,10 +143,12 @@ spec: license: Apache-2.0 runtime: image: registry.example/test/llama-server@sha256:${"2".repeat(64)} + imageDownloadSizeBytes: 1073741824 platforms: - linux/amd64 - linux/arm64 containerRuntime: docker + networkExposure: loopback hosts: 1 cuda: baseImage: registry.example/nvidia/cuda@sha256:${"3".repeat(64)} @@ -166,11 +168,19 @@ spec: lifecycleRef: test.lifecycle/v1 serve: protocol: openai-completions + authentication: bearer port: 8081 chatTemplate: test-chat contextSize: 32768 slots: 1 idleSleepSeconds: -1 + batchSize: 2048 + microBatchSize: 512 + flashAttention: enabled + kvCache: + key: f16 + value: f16 + speculativeDecoding: disabled limits: maxRequestBodyBytes: 1048576 maxPromptTokens: 24576 @@ -326,10 +336,22 @@ describe("managed inference serving catalog compiler", () => { providerId: "llama-cpp-local", server: { technology: "llama.cpp" }, runtime: { + imageDownloadSizeBytes: 1073741824, platforms: ["linux/amd64", "linux/arm64"], + networkExposure: "loopback", gpu: { count: 1, cpuFallback: "reject" }, }, - serve: { protocol: "openai-completions", port: 8081, slots: 1 }, + serve: { + protocol: "openai-completions", + authentication: "bearer", + port: 8081, + slots: 1, + batchSize: 2048, + microBatchSize: 512, + flashAttention: "enabled", + kvCache: { key: "f16", value: "f16" }, + speculativeDecoding: "disabled", + }, capabilities: { agents: [], protocols: ["openai-completions"] }, }); expect(first.presets[0]?.spec.selection).toBe("explicit-only"); @@ -357,6 +379,7 @@ describe("managed inference serving catalog compiler", () => { " baseImage: registry.example/nvidia/cuda:12.8", ], ["incomplete platform coverage", " - linux/arm64\n", ""], + ["a missing image byte size", " imageDownloadSizeBytes: 1073741824\n", ""], ["missing process limits", " pidsLimit: 256\n", ""], [ "an embedded credential", @@ -372,6 +395,15 @@ describe("managed inference serving catalog compiler", () => { " protocol: openai-responses", ], ["CPU fallback", " cpuFallback: reject", " cpuFallback: allow"], + ["external network exposure", " networkExposure: loopback", " networkExposure: lan"], + ["missing authentication", " authentication: bearer\n", ""], + ["an unsafe KV cache type", " key: f16", " key: q4_0"], + ["disabled flash attention", " flashAttention: enabled", " flashAttention: disabled"], + [ + "enabled speculative decoding", + " speculativeDecoding: disabled", + " speculativeDecoding: enabled", + ], ["server egress", " egress: disabled", " egress: enabled"], ["a remote model source", " modelSource: verified-local", " modelSource: remote"], [ @@ -412,6 +444,18 @@ describe("managed inference serving catalog compiler", () => { ); }); + it("rejects a llama.cpp micro-batch larger than its batch (#8173)", () => { + const invalidBatching = replaceSource( + llamaCppRecipeSource(), + " microBatchSize: 512", + " microBatchSize: 4096", + ); + + expect(() => compile([invalidBatching])).toThrow( + "serve.microBatchSize cannot exceed serve.batchSize", + ); + }); + it.each([ ["a backend that does not match llama.cpp", " backend: install-llama-cpp", " backend: vllm"], [ @@ -483,6 +527,16 @@ describe("managed inference serving catalog compiler", () => { ); }); + it("allows an explicit llama.cpp preset to narrow a multiarch image to arm64 (#8173)", () => { + const arm64Preset = replaceSource( + llamaCppPresetSource(), + " operator: one-of\n values:\n - amd64\n - arm64", + " operator: equals\n value: arm64", + ); + + expect(() => compile([llamaCppRecipeSource(), arm64Preset])).not.toThrow(); + }); + it.each([ ["operating-system", " value: linux", " value: windows"], ["architecture", " - arm64", " - riscv64"], diff --git a/src/lib/inference/serving/catalog.ts b/src/lib/inference/serving/catalog.ts index 1ebcac0f811..a849dccddbd 100644 --- a/src/lib/inference/serving/catalog.ts +++ b/src/lib/inference/serving/catalog.ts @@ -10,6 +10,7 @@ import { parseDocument } from "yaml"; import type { CompiledServingCatalog, CompiledServingCatalogPayload, + LlamaCppServingRecipe, ServingCatalogRegistries, ServingCatalogSchemas, ServingCatalogSource, @@ -144,8 +145,6 @@ function definitionIdentity( return { kind, id }; } -type LlamaCppServingRecipe = Extract; - function isLlamaCppServingRecipe(recipe: ServingRecipe): recipe is LlamaCppServingRecipe { return recipe.spec.providerId === "llama-cpp-local"; } @@ -248,6 +247,11 @@ function validateRecipeSemantics( `Recipe ${recipe.metadata.id} request token limits exceed serve.contextSize.`, ); } + if (serve.microBatchSize > serve.batchSize) { + throw new ServingCatalogValidationError( + `Recipe ${recipe.metadata.id} serve.microBatchSize cannot exceed serve.batchSize.`, + ); + } const agents = new Set(); for (const agent of recipe.spec.capabilities.agents) { if (agents.has(agent.id)) { @@ -369,6 +373,23 @@ function canonicalReadinessComparison(comparison: ServingReadinessComparison): s }); } +function llamaCppReadinessComparisonMatches( + recipe: LlamaCppServingRecipe, + role: ServingReadinessObservationRole, + actual: ServingReadinessComparison, + expected: ServingReadinessComparison, +): boolean { + if (role !== "architecture" || actual.operator !== "equals") { + return canonicalReadinessComparison(actual) === canonicalReadinessComparison(expected); + } + return ( + typeof actual.value === "string" && + recipe.spec.runtime.platforms.includes( + `linux/${actual.value}` as LlamaCppServingRecipe["spec"]["runtime"]["platforms"][number], + ) + ); +} + function validateLlamaCppPreset( recipe: LlamaCppServingRecipe, preset: ServingPreset, @@ -412,7 +433,7 @@ function validateLlamaCppPreset( seenRoles.add(registered.role); if ( !("comparison" in readiness) || - canonicalReadinessComparison(readiness.comparison) !== canonicalReadinessComparison(expected) + !llamaCppReadinessComparisonMatches(recipe, registered.role, readiness.comparison, expected) ) { throw new ServingCatalogValidationError( `Preset ${preset.metadata.id} must require ${registered.role} matching llama.cpp recipe ${recipe.metadata.id}.`, diff --git a/src/lib/inference/serving/types.ts b/src/lib/inference/serving/types.ts index d666260fb3b..34b255df244 100644 --- a/src/lib/inference/serving/types.ts +++ b/src/lib/inference/serving/types.ts @@ -164,7 +164,7 @@ interface GenericServingRecipe extends ServingRecipeEnvelope { }; } -interface LlamaCppServingRecipe extends ServingRecipeEnvelope { +export interface LlamaCppServingRecipe extends ServingRecipeEnvelope { readonly spec: { readonly backend: "install-llama-cpp"; readonly providerId: "llama-cpp-local"; @@ -188,8 +188,10 @@ interface LlamaCppServingRecipe extends ServingRecipeEnvelope { }; readonly runtime: { readonly image: string; + readonly imageDownloadSizeBytes: number; readonly platforms: readonly ("linux/amd64" | "linux/arm64")[]; readonly containerRuntime: "docker"; + readonly networkExposure: "loopback"; readonly hosts: 1; readonly cuda: { readonly baseImage: string; readonly minimumDriverVersion: string }; readonly gpu: { @@ -212,11 +214,20 @@ interface LlamaCppServingRecipe extends ServingRecipeEnvelope { }; readonly serve: { readonly protocol: "openai-completions"; + readonly authentication: "bearer"; readonly port: 8081; readonly chatTemplate: string; readonly contextSize: number; readonly slots: 1; readonly idleSleepSeconds: -1; + readonly batchSize: number; + readonly microBatchSize: number; + readonly flashAttention: "enabled"; + readonly kvCache: { + readonly key: "f16" | "q8_0"; + readonly value: "f16" | "q8_0"; + }; + readonly speculativeDecoding: "disabled"; readonly limits: { readonly maxRequestBodyBytes: number; readonly maxPromptTokens: number; diff --git a/test/managed-inference-catalog-compiler.test.ts b/test/managed-inference-catalog-compiler.test.ts index 907c002041a..5028b9fb097 100644 --- a/test/managed-inference-catalog-compiler.test.ts +++ b/test/managed-inference-catalog-compiler.test.ts @@ -15,6 +15,12 @@ import type { ServingCatalogSource } from "../src/lib/inference/serving/types.js const REPOSITORY_ROOT = path.join(import.meta.dirname, ".."); const PROFILE_ID = "vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731"; const RECIPE_ID = "vllm.deepseek-v4-flash-0731.spark-dual.v1"; +const LLAMA_CPP_PROFILE_ID = "llama-cpp.dgx-spark-gb10.single.nemotron-3-nano-30b-a3b"; +const LLAMA_CPP_RECIPE_ID = "llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1"; +const LLAMA_CPP_IMAGE = + "ghcr.io/ggml-org/llama.cpp@sha256:866ad568474de9e835e487ae841ad6ace1a494b5eab4f292cbd45adb6180f711"; +const LLAMA_CPP_MODEL_DIGEST = + "sha256:627f5b04aedc97f967332f331bd75b7a4ed2f33ca83e6ee74b44235cc1887890"; function catalogSources(): ServingCatalogSource[] { return (["presets", "recipes"] as const).flatMap((kind) => { @@ -59,8 +65,8 @@ describe("managed inference YAML profile contract", () => { it("accepts another compatible profile as YAML-only catalog additions (#8129)", () => { const sources = catalogSources(); - const recipeSource = sources.find(({ path: sourcePath }) => sourcePath.includes("/recipes/"))!; - const presetSource = sources.find(({ path: sourcePath }) => sourcePath.includes("/presets/"))!; + const recipeSource = sources.find(({ contents }) => contents.includes(`id: ${RECIPE_ID}`))!; + const presetSource = sources.find(({ contents }) => contents.includes(`id: ${PROFILE_ID}`))!; const syntheticRecipeId = "vllm.synthetic.managed-cluster.v1"; const syntheticPresetId = "vllm.synthetic.managed-cluster"; const catalog = compile([ @@ -82,6 +88,58 @@ describe("managed inference YAML profile contract", () => { expect(catalog.presets.some(({ metadata }) => metadata.id === syntheticPresetId)).toBe(true); }); + it("compiles the explicit single-Spark llama.cpp proof from YAML (#8173)", () => { + const catalog = compile(catalogSources()); + const preset = catalog.presets.find(({ metadata }) => metadata.id === LLAMA_CPP_PROFILE_ID); + const recipe = catalog.recipes.find(({ metadata }) => metadata.id === LLAMA_CPP_RECIPE_ID); + + expect(preset?.spec).toMatchObject({ + selection: "explicit-only", + plan: { backend: "install-llama-cpp", recipeRef: LLAMA_CPP_RECIPE_ID }, + }); + expect(preset?.spec.requirements?.all).toContainEqual({ + readiness: { + scope: "everyNode", + kind: "observation", + id: "host.os.architecture", + comparison: { operator: "equals", value: "arm64" }, + }, + }); + expect(recipe?.spec).toMatchObject({ + backend: "install-llama-cpp", + providerId: "llama-cpp-local", + model: { + servedName: "nvidia-nemotron-3-nano-30b-a3b", + files: [{ digest: LLAMA_CPP_MODEL_DIGEST, sizeBytes: 22833947424 }], + }, + runtime: { + image: LLAMA_CPP_IMAGE, + imageDownloadSizeBytes: 2181958990, + networkExposure: "loopback", + }, + serve: { + authentication: "bearer", + contextSize: 262144, + batchSize: 2048, + microBatchSize: 512, + flashAttention: "enabled", + kvCache: { key: "f16", value: "f16" }, + speculativeDecoding: "disabled", + }, + }); + }); + + it("keeps vLLM automatic while llama.cpp remains explicit-only (#8173)", () => { + const catalog = compile(catalogSources()); + const vllmPreset = catalog.presets.find(({ metadata }) => metadata.id === PROFILE_ID); + const llamaCppPreset = catalog.presets.find( + ({ metadata }) => metadata.id === LLAMA_CPP_PROFILE_ID, + ); + + expect(vllmPreset?.spec.selection).toBe("automatic"); + expect(llamaCppPreset?.spec.selection).toBe("explicit-only"); + }); + it("does not enable arbitrary remote model code in shipped managed recipes (#8129)", () => { const unsafeRecipes = compile(catalogSources()) .recipes.filter((recipe) => { @@ -106,5 +164,7 @@ describe("managed inference YAML profile contract", () => { expect(productionSources).not.toContain(PROFILE_ID); expect(productionSources).not.toContain(RECIPE_ID); + expect(productionSources).not.toContain(LLAMA_CPP_PROFILE_ID); + expect(productionSources).not.toContain(LLAMA_CPP_RECIPE_ID); }); }); From 3f113d9ad05520a3394f8f7fad4e8cf234fe436a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 06:00:47 -0700 Subject: [PATCH 2/4] docs(inference): clarify llama.cpp catalog scope Signed-off-by: Aaron Erickson --- src/lib/inference/serving/catalog-loader.test.ts | 2 +- src/lib/inference/serving/catalog-loader.ts | 2 +- test/managed-inference-catalog-compiler.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/inference/serving/catalog-loader.test.ts b/src/lib/inference/serving/catalog-loader.test.ts index d9688bef6fa..6575754bbce 100644 --- a/src/lib/inference/serving/catalog-loader.test.ts +++ b/src/lib/inference/serving/catalog-loader.test.ts @@ -96,7 +96,7 @@ describe("managed inference catalog loader", () => { expect(managedCatalogValidationError(catalog)).toMatch(/Managed inference (preset|recipe)/u); }); - it("projects host-local definitions out of the managed-cluster runtime catalog (#8173)", () => { + it("excludes host-local definitions from the managed-cluster runtime catalog (#8173)", () => { const servingCatalog: CompiledServingCatalog = { ...EMPTY_CATALOG, recipes: [HOST_LOCAL_RECIPE], diff --git a/src/lib/inference/serving/catalog-loader.ts b/src/lib/inference/serving/catalog-loader.ts index c1dabbb88b4..ae062b03ec6 100644 --- a/src/lib/inference/serving/catalog-loader.ts +++ b/src/lib/inference/serving/catalog-loader.ts @@ -101,7 +101,7 @@ function isManagedClusterRecipeCandidate(recipe: ServingRecipe): boolean { ); } -/** Project the backend-polymorphic serving catalog onto the managed-cluster runtime. */ +/** The managed-cluster runtime accepts only recipes backed by its vLLM materializer and lifecycle. */ export function managedInferenceCatalogFromServingCatalog( catalog: CompiledServingCatalog, ): CompiledManagedInferenceCatalog { diff --git a/test/managed-inference-catalog-compiler.test.ts b/test/managed-inference-catalog-compiler.test.ts index 5028b9fb097..f004cbed6f7 100644 --- a/test/managed-inference-catalog-compiler.test.ts +++ b/test/managed-inference-catalog-compiler.test.ts @@ -88,7 +88,7 @@ describe("managed inference YAML profile contract", () => { expect(catalog.presets.some(({ metadata }) => metadata.id === syntheticPresetId)).toBe(true); }); - it("compiles the explicit single-Spark llama.cpp proof from YAML (#8173)", () => { + it("compiles the explicit DGX Spark llama.cpp profile from YAML (#8173)", () => { const catalog = compile(catalogSources()); const preset = catalog.presets.find(({ metadata }) => metadata.id === LLAMA_CPP_PROFILE_ID); const recipe = catalog.recipes.find(({ metadata }) => metadata.id === LLAMA_CPP_RECIPE_ID); From aedbeb286320c053eed4481cb9abe8de470b8e02 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 06:58:19 -0700 Subject: [PATCH 3/4] fix(inference): preserve managed catalog integrity Signed-off-by: Aaron Erickson --- managed-inference/schemas/recipe.schema.json | 80 ++++++++++++------- .../inference/serving/catalog-loader.test.ts | 32 +++++++- src/lib/inference/serving/catalog-loader.ts | 12 +-- src/lib/inference/serving/catalog.test.ts | 43 ++++++++++ 4 files changed, 132 insertions(+), 35 deletions(-) diff --git a/managed-inference/schemas/recipe.schema.json b/managed-inference/schemas/recipe.schema.json index d43836a2015..263833537c7 100644 --- a/managed-inference/schemas/recipe.schema.json +++ b/managed-inference/schemas/recipe.schema.json @@ -1079,6 +1079,16 @@ } } } + }, + "propertyNames": { + "not": { + "enum": [ + "downloadSizeBytes", + "gated", + "installFastSafetensors", + "preparation" + ] + } } }, "runtime": { @@ -1105,27 +1115,23 @@ "gpu": {}, "resources": {} }, - "not": { - "anyOf": [ - { - "type": "object", - "properties": { - "architecture": {} - }, - "required": [ - "architecture" - ] - }, - { - "type": "object", - "properties": { - "components": {} - }, - "required": [ - "components" - ] - } - ] + "propertyNames": { + "not": { + "enum": [ + "pullTimeoutSeconds", + "architecture", + "networkMode", + "ipcMode", + "sharedMemoryBytes", + "gpuRequest", + "devices", + "ulimits", + "modelCache", + "temporaryFilesystems", + "environment", + "components" + ] + } } }, "execution": { @@ -1139,6 +1145,18 @@ "receiptRef": {}, "materializerRef": {}, "lifecycleRef": {} + }, + "propertyNames": { + "not": { + "enum": [ + "topologyBinding", + "nodeCount", + "tensorParallelSize", + "pipelineParallelSize", + "distributedExecutorBackend", + "rendezvousPort" + ] + } } }, "serve": { @@ -1181,14 +1199,13 @@ "speculativeDecoding": {}, "limits": {} }, - "not": { - "type": "object", - "properties": { - "arguments": {} - }, - "required": [ - "arguments" - ] + "propertyNames": { + "not": { + "enum": [ + "executable", + "arguments" + ] + } } }, "readiness": { @@ -1215,6 +1232,11 @@ "capabilities": { "type": "object" } + }, + "propertyNames": { + "not": { + "const": "bindings" + } } }, "else": { diff --git a/src/lib/inference/serving/catalog-loader.test.ts b/src/lib/inference/serving/catalog-loader.test.ts index 6575754bbce..0811aa13e67 100644 --- a/src/lib/inference/serving/catalog-loader.test.ts +++ b/src/lib/inference/serving/catalog-loader.test.ts @@ -6,8 +6,11 @@ import { MANAGED_CLUSTER_VLLM_LIFECYCLE_REF, MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, } from "./adapter-registry"; +import { servingCatalogDigest } from "./catalog"; import { assertManagedInferenceCatalog, + loadManagedInferenceCatalog, + loadServingCatalog, managedInferenceCatalogFromServingCatalog, } from "./catalog-loader"; import type { CompiledServingCatalog, ServingPreset, ServingRecipe } from "./types"; @@ -82,6 +85,16 @@ function managedCatalogValidationError(catalog: CompiledServingCatalog): string } } +function expectOnlyManagedVllmDefinitions(catalog: CompiledServingCatalog): void { + const { catalogDigest, ...catalogContents } = catalog; + + expect(catalogDigest).toBe(servingCatalogDigest(catalogContents)); + expect(catalog.recipes.length).toBeGreaterThan(0); + expect(catalog.recipes.every(({ spec }) => spec.backend === "vllm")).toBe(true); + expect(catalog.presets.length).toBeGreaterThan(0); + expect(catalog.presets.every(({ spec }) => spec.plan.backend === "vllm")).toBe(true); +} + describe("managed inference catalog loader", () => { it("accepts an empty managed catalog", () => { assertManagedInferenceCatalog(EMPTY_CATALOG); @@ -107,6 +120,23 @@ describe("managed inference catalog loader", () => { expect(managedCatalog.recipes).toEqual([]); expect(managedCatalog.presets).toEqual([]); - expect(managedCatalog.catalogDigest).toBe(EMPTY_CATALOG.catalogDigest); + expect(managedCatalog.catalogDigest).not.toBe(EMPTY_CATALOG.catalogDigest); + const { catalogDigest, ...catalogContents } = managedCatalog; + expect(catalogDigest).toBe(servingCatalogDigest(catalogContents)); + }); + + it("retains managed vLLM definitions while projecting the mixed serving catalog (#8173)", () => { + const servingCatalog = loadServingCatalog(); + + expect(servingCatalog.recipes.some(({ spec }) => spec.backend === "vllm")).toBe(true); + expect(servingCatalog.recipes.some(({ spec }) => spec.backend === "install-llama-cpp")).toBe( + true, + ); + + expectOnlyManagedVllmDefinitions(managedInferenceCatalogFromServingCatalog(servingCatalog)); + }); + + it("excludes host-local definitions through the public managed loader (#8173)", () => { + expectOnlyManagedVllmDefinitions(loadManagedInferenceCatalog()); }); }); diff --git a/src/lib/inference/serving/catalog-loader.ts b/src/lib/inference/serving/catalog-loader.ts index ae062b03ec6..c370407ae50 100644 --- a/src/lib/inference/serving/catalog-loader.ts +++ b/src/lib/inference/serving/catalog-loader.ts @@ -9,7 +9,7 @@ import { MANAGED_CLUSTER_VLLM_LIFECYCLE_REF, MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, } from "./adapter-registry.js"; -import { parseCompiledServingCatalogJson } from "./catalog.js"; +import { parseCompiledServingCatalogJson, servingCatalogDigest } from "./catalog.js"; import { immutableManagedInferenceCopy } from "./catalog-integrity.js"; import type { CompiledManagedInferenceCatalog, @@ -107,10 +107,12 @@ export function managedInferenceCatalogFromServingCatalog( ): CompiledManagedInferenceCatalog { const recipes = catalog.recipes.filter(isManagedClusterRecipeCandidate); const recipeIds = new Set(recipes.map(({ metadata }) => metadata.id)); - const presets = catalog.presets.filter( - (preset) => recipeIds.has(preset.spec.plan.recipeRef) || preset.spec.plan.backend === "vllm", - ); - const managedCatalog = { ...catalog, recipes, presets }; + const presets = catalog.presets.filter((preset) => recipeIds.has(preset.spec.plan.recipeRef)); + const definitionIds = new Set([...recipeIds, ...presets.map(({ metadata }) => metadata.id)]); + const sources = catalog.sources.filter(({ id }) => definitionIds.has(id)); + const { catalogDigest: _catalogDigest, ...catalogContents } = catalog; + const payload = { ...catalogContents, recipes, presets, sources }; + const managedCatalog = { ...payload, catalogDigest: servingCatalogDigest(payload) }; assertManagedInferenceCatalog(managedCatalog); return managedCatalog; } diff --git a/src/lib/inference/serving/catalog.test.ts b/src/lib/inference/serving/catalog.test.ts index 0d6ccbf6007..1ba3549361a 100644 --- a/src/lib/inference/serving/catalog.test.ts +++ b/src/lib/inference/serving/catalog.test.ts @@ -420,6 +420,49 @@ describe("managed inference serving catalog compiler", () => { expect(() => compile([recipe])).toThrow("does not satisfy the ServingRecipe schema"); }); + it.each([ + [ + "top-level topology bindings", + " backend: install-llama-cpp", + ` backend: install-llama-cpp + bindings: + cluster: + type: topologyQualificationOutput + qualificationId: test.qualification + schemaVersion: 1 + outputSchema: test.output`, + ], + [ + "managed model preparation", + " servedName: test-model", + ` servedName: test-model + preparation: + ref: none/v1`, + ], + [ + "managed runtime networking", + " networkExposure: loopback", + ` networkExposure: loopback + networkMode: host`, + ], + [ + "managed cluster execution sizing", + " receiptRef: test.receipt/v1", + ` receiptRef: test.receipt/v1 + nodeCount: 1`, + ], + [ + "an executable override", + " authentication: bearer", + ` authentication: bearer + executable: /usr/local/bin/llama-server`, + ], + ])("rejects generic-only %s in a llama.cpp recipe (#8173)", (_case, expected, replacement) => { + const recipe = replaceSource(llamaCppRecipeSource(), expected, replacement); + + expect(() => compile([recipe])).toThrow("does not satisfy the ServingRecipe schema"); + }); + it("rejects a llama.cpp readiness model that differs from the served name (#8181)", () => { const wrongExpectedModel = replaceSource( llamaCppRecipeSource(), From df13c021ef3db76434920b247994ac7d51f2b75e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 07:17:06 -0700 Subject: [PATCH 4/4] test(inference): assert exact catalog projection Signed-off-by: Aaron Erickson --- .../inference/serving/catalog-loader.test.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/lib/inference/serving/catalog-loader.test.ts b/src/lib/inference/serving/catalog-loader.test.ts index 0811aa13e67..7040b94c3d7 100644 --- a/src/lib/inference/serving/catalog-loader.test.ts +++ b/src/lib/inference/serving/catalog-loader.test.ts @@ -25,6 +25,12 @@ const EMPTY_CATALOG: CompiledServingCatalog = { sources: [], catalogDigest: `sha256:${"b".repeat(64)}`, }; +const EXPECTED_MANAGED_RECIPE_IDS = ["vllm.deepseek-v4-flash-0731.spark-dual.v1"]; +const EXPECTED_MANAGED_PRESET_IDS = ["vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731"]; +const EXPECTED_MANAGED_SOURCE_IDS = [ + "vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731", + "vllm.deepseek-v4-flash-0731.spark-dual.v1", +]; const INCOMPLETE_MANAGED_RECIPE: ServingRecipe = { apiVersion: "nemoclaw.nvidia.com/managed-inference/v1", @@ -89,10 +95,9 @@ function expectOnlyManagedVllmDefinitions(catalog: CompiledServingCatalog): void const { catalogDigest, ...catalogContents } = catalog; expect(catalogDigest).toBe(servingCatalogDigest(catalogContents)); - expect(catalog.recipes.length).toBeGreaterThan(0); - expect(catalog.recipes.every(({ spec }) => spec.backend === "vllm")).toBe(true); - expect(catalog.presets.length).toBeGreaterThan(0); - expect(catalog.presets.every(({ spec }) => spec.plan.backend === "vllm")).toBe(true); + expect(catalog.recipes.map(({ metadata }) => metadata.id)).toEqual(EXPECTED_MANAGED_RECIPE_IDS); + expect(catalog.presets.map(({ metadata }) => metadata.id)).toEqual(EXPECTED_MANAGED_PRESET_IDS); + expect(catalog.sources.map(({ id }) => id)).toEqual(EXPECTED_MANAGED_SOURCE_IDS); } describe("managed inference catalog loader", () => { @@ -137,6 +142,10 @@ describe("managed inference catalog loader", () => { }); it("excludes host-local definitions through the public managed loader (#8173)", () => { - expectOnlyManagedVllmDefinitions(loadManagedInferenceCatalog()); + const projectedCatalog = managedInferenceCatalogFromServingCatalog(loadServingCatalog()); + const loadedCatalog = loadManagedInferenceCatalog(); + + expect(loadedCatalog).toEqual(projectedCatalog); + expectOnlyManagedVllmDefinitions(loadedCatalog); }); });