diff --git a/managed-inference/schemas/catalog.schema.json b/managed-inference/schemas/catalog.schema.json index ba141726798..ee2eeeae281 100644 --- a/managed-inference/schemas/catalog.schema.json +++ b/managed-inference/schemas/catalog.schema.json @@ -16,7 +16,7 @@ ], "properties": { "schemaVersion": { "const": "1.0.0" }, - "compilerVersion": { "const": "1.0.0" }, + "compilerVersion": { "const": "1.1.0" }, "sourceRevision": { "type": "string", "pattern": "^[0-9a-f]{40,64}$" }, "readinessSchemaRef": { "const": "https://github.com/NVIDIA/NemoClaw/schemas/system-readiness.schema.json" diff --git a/managed-inference/schemas/preset.schema.json b/managed-inference/schemas/preset.schema.json index 6ce98de367d..7bfc98587a2 100644 --- a/managed-inference/schemas/preset.schema.json +++ b/managed-inference/schemas/preset.schema.json @@ -76,6 +76,66 @@ }, "additionalProperties": false }, + { + "type": "object", + "required": ["scope", "kind", "id", "comparison"], + "properties": { + "scope": { "enum": ["controller", "everyNode"] }, + "kind": { "const": "observation" }, + "id": { "$ref": "#/$defs/stableId" }, + "comparison": { + "oneOf": [ + { + "type": "object", + "required": ["operator", "value"], + "properties": { + "operator": { "const": "equals" }, + "value": { "$ref": "#/$defs/comparisonScalar" } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["operator", "values"], + "properties": { + "operator": { "const": "one-of" }, + "values": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "items": { "$ref": "#/$defs/comparisonScalar" } + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["operator", "value"], + "properties": { + "operator": { "const": "at-least" }, + "value": { + "type": "number", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["operator", "value"], + "properties": { + "operator": { "const": "version-at-least" }, + "value": { "$ref": "#/$defs/version" } + }, + "additionalProperties": false + } + ] + } + }, + "additionalProperties": false + }, { "type": "object", "required": ["scope", "kind", "id", "status"], @@ -91,6 +151,23 @@ } }, "additionalProperties": false + }, + "comparisonScalar": { + "oneOf": [ + { "$ref": "#/$defs/comparisonString" }, + { "type": "number", "minimum": 0, "maximum": 9007199254740991 }, + { "type": "boolean" } + ] + }, + "comparisonString": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._+:-]{0,127}$", + "maxLength": 128 + }, + "version": { + "type": "string", + "pattern": "^[0-9]+(?:\\.[0-9]+){1,3}(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$", + "maxLength": 64 } } } diff --git a/managed-inference/schemas/recipe.schema.json b/managed-inference/schemas/recipe.schema.json index 3b38c19a9f6..77866674d83 100644 --- a/managed-inference/schemas/recipe.schema.json +++ b/managed-inference/schemas/recipe.schema.json @@ -14,6 +14,24 @@ "required": ["backend", "model", "execution"], "properties": { "backend": { "$ref": "#/$defs/backendId" }, + "providerId": { "$ref": "#/$defs/backendId" }, + "server": { + "type": "object", + "required": ["technology", "source"], + "properties": { + "technology": { "const": "llama.cpp" }, + "source": { + "type": "object", + "required": ["repository", "revision"], + "properties": { + "repository": { "$ref": "#/$defs/repositoryId" }, + "revision": { "$ref": "#/$defs/sourceRevision" } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, "model": { "type": "object", "required": ["id", "revision"], @@ -38,7 +56,15 @@ "maxLength": 512, "pattern": "^(?!\\.{1,2}(?:/|$))(?!.*\\/\\.{1,2}(?:/|$))[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*$" }, - "digest": { "$ref": "#/$defs/sha256" } + "digest": { "$ref": "#/$defs/sha256" }, + "sizeBytes": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "format": { "const": "gguf" }, + "quantization": { "$ref": "#/$defs/artifactLabel" }, + "license": { "$ref": "#/$defs/artifactLabel" } }, "additionalProperties": false } @@ -54,6 +80,57 @@ "pattern": "^(?:[a-z0-9]+(?:[._-][a-z0-9]+)*(?::[0-9]+)?/)?(?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)*[a-z0-9]+(?:[._-][a-z0-9]+)*@sha256:[0-9a-f]{64}$" }, "architecture": { "enum": ["amd64", "arm64"] }, + "platforms": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "uniqueItems": true, + "items": { "enum": ["linux/amd64", "linux/arm64"] }, + "allOf": [ + { "contains": { "const": "linux/amd64" } }, + { "contains": { "const": "linux/arm64" } } + ] + }, + "containerRuntime": { "const": "docker" }, + "hosts": { "const": 1 }, + "cuda": { + "type": "object", + "required": ["baseImage", "minimumDriverVersion"], + "properties": { + "baseImage": { "$ref": "#/$defs/imageDigest" }, + "minimumDriverVersion": { "$ref": "#/$defs/version" } + }, + "additionalProperties": false + }, + "gpu": { + "type": "object", + "required": ["vendor", "count", "offload", "cpuFallback"], + "properties": { + "vendor": { "const": "nvidia" }, + "count": { "const": 1 }, + "offload": { "const": "full" }, + "cpuFallback": { "const": "reject" } + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "required": ["memoryBytes", "writableStorageBytes", "pidsLimit"], + "properties": { + "memoryBytes": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "writableStorageBytes": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "pidsLimit": { "type": "integer", "minimum": 1, "maximum": 1048576 } + }, + "additionalProperties": false + }, "components": { "type": "object", "maxProperties": 32, @@ -71,6 +148,7 @@ "type": "object", "required": ["materializerRef", "lifecycleRef"], "properties": { + "receiptRef": { "$ref": "#/$defs/adapterId" }, "materializerRef": { "$ref": "#/$defs/adapterId" }, "lifecycleRef": { "$ref": "#/$defs/adapterId" } }, @@ -79,6 +157,28 @@ "serve": { "type": "object", "properties": { + "protocol": { "const": "openai-completions" }, + "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "chatTemplate": { "$ref": "#/$defs/artifactLabel" }, + "contextSize": { "type": "integer", "minimum": 1, "maximum": 1048576 }, + "slots": { "type": "integer", "minimum": 1, "maximum": 1024 }, + "idleSleepSeconds": { "type": "integer", "minimum": -1, "maximum": 86400 }, + "limits": { + "type": "object", + "required": [ + "maxRequestBodyBytes", + "maxPromptTokens", + "maxCompletionTokens", + "requestTimeoutSeconds" + ], + "properties": { + "maxRequestBodyBytes": { "type": "integer", "minimum": 1, "maximum": 1073741824 }, + "maxPromptTokens": { "type": "integer", "minimum": 1, "maximum": 1048576 }, + "maxCompletionTokens": { "type": "integer", "minimum": 1, "maximum": 1048576 }, + "requestTimeoutSeconds": { "type": "integer", "minimum": 1, "maximum": 86400 } + }, + "additionalProperties": false + }, "arguments": { "type": "array", "maxItems": 128, @@ -102,13 +202,104 @@ "readiness": { "type": "object", "properties": { + "contractRef": { "$ref": "#/$defs/adapterId" }, "timeoutSeconds": { "type": "integer", "minimum": 1, "maximum": 86400 }, - "expectedModel": { "$ref": "#/$defs/stableId" } + "expectedModel": { "$ref": "#/$defs/stableId" }, + "probes": { + "type": "object", + "required": ["models", "health", "properties", "metrics"], + "properties": { + "models": { "const": true }, + "health": { "const": true }, + "properties": { "const": true }, + "metrics": { "const": true } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "policy": { + "type": "object", + "required": ["egress", "modelSource", "modelDownloads"], + "properties": { + "egress": { "const": "disabled" }, + "modelSource": { "const": "verified-local" }, + "modelDownloads": { "const": "disabled" } + }, + "additionalProperties": false + }, + "surfaces": { + "type": "object", + "required": [ + "ui", + "slotInspection", + "router", + "mcpProxy", + "serverTools", + "agentMode", + "multimodalProjection" + ], + "properties": { + "ui": { "const": "disabled" }, + "slotInspection": { "const": "disabled" }, + "router": { "const": "disabled" }, + "mcpProxy": { "const": "disabled" }, + "serverTools": { "const": "disabled" }, + "agentMode": { "const": "disabled" }, + "multimodalProjection": { "const": "disabled" } + }, + "additionalProperties": false + }, + "capabilities": { + "type": "object", + "required": [ + "agents", + "protocols", + "streaming", + "toolCalls", + "structuredOutputs", + "parallelToolCalls", + "responsesApi", + "embeddings", + "reranking", + "multimodal" + ], + "properties": { + "agents": { + "type": "array", + "maxItems": 16, + "uniqueItems": true, + "items": { + "type": "object", + "required": ["id", "qualificationRef"], + "properties": { + "id": { "$ref": "#/$defs/nonPathStableId" }, + "qualificationRef": { "$ref": "#/$defs/stableId" } + }, + "additionalProperties": false + } + }, + "protocols": { + "type": "array", + "minItems": 1, + "maxItems": 1, + "items": { "const": "openai-completions" } + }, + "streaming": { "type": "boolean" }, + "toolCalls": { "type": "boolean" }, + "structuredOutputs": { "type": "boolean" }, + "parallelToolCalls": { "const": false }, + "responsesApi": { "const": false }, + "embeddings": { "const": false }, + "reranking": { "const": false }, + "multimodal": { "const": false } }, "additionalProperties": false } }, - "additionalProperties": false + "additionalProperties": false, + "allOf": [{ "$ref": "#/$defs/llamaCppRecipeSpec" }] } }, "additionalProperties": false, @@ -127,10 +318,38 @@ "pattern": "^[a-z][a-z0-9]*(?:[._/-][a-z0-9]+)+$", "maxLength": 160 }, + "nonPathStableId": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+$", + "maxLength": 160 + }, "backendId": { "type": "string", "pattern": "^[a-z][a-z0-9-]{0,63}$" }, + "repositoryId": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._-]{0,127}$", + "maxLength": 192 + }, + "sourceRevision": { + "type": "string", + "pattern": "^[0-9a-f]{40,64}$" + }, + "artifactLabel": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._+-]{0,127}$", + "maxLength": 128 + }, + "version": { + "type": "string", + "pattern": "^[0-9]+(?:\\.[0-9]+){1,3}(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$", + "maxLength": 64 + }, + "imageDigest": { + "type": "string", + "pattern": "^(?:[a-z0-9]+(?:[._-][a-z0-9]+)*(?::[0-9]+)?/)?(?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)*[a-z0-9]+(?:[._-][a-z0-9]+)*@sha256:[0-9a-f]{64}$" + }, "adapterId": { "type": "string", "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+/v[1-9][0-9]*$", @@ -146,6 +365,183 @@ { "type": "number" }, { "type": "boolean" } ] + }, + "llamaCppRecipeSpec": { + "if": { + "type": "object", + "anyOf": [ + { + "properties": { "backend": { "const": "install-llama-cpp" } }, + "required": ["backend"] + }, + { + "properties": { "providerId": { "const": "llama-cpp-local" } }, + "required": ["providerId"] + }, + { + "properties": { + "server": { + "type": "object", + "properties": { "technology": { "const": "llama.cpp" } }, + "required": ["technology"] + } + }, + "required": ["server"] + } + ] + }, + "then": { + "type": "object", + "required": [ + "providerId", + "server", + "model", + "runtime", + "execution", + "serve", + "readiness", + "policy", + "surfaces", + "capabilities" + ], + "properties": { + "backend": { "const": "install-llama-cpp" }, + "providerId": { "const": "llama-cpp-local" }, + "server": { + "type": "object", + "properties": { "technology": { "const": "llama.cpp" } }, + "required": ["technology"] + }, + "model": { + "type": "object", + "required": ["id", "revision", "servedName", "files"], + "properties": { + "id": { "$ref": "#/$defs/repositoryId" }, + "revision": { "$ref": "#/$defs/sourceRevision" }, + "servedName": { "$ref": "#/$defs/nonPathStableId" }, + "files": { + "type": "array", + "minItems": 1, + "maxItems": 1, + "items": { + "type": "object", + "required": ["path", "digest", "sizeBytes", "format", "quantization", "license"], + "properties": { + "path": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*\\.gguf$", + "maxLength": 255 + }, + "digest": {}, + "sizeBytes": {}, + "format": {}, + "quantization": {}, + "license": {} + } + } + } + } + }, + "runtime": { + "type": "object", + "required": [ + "image", + "platforms", + "containerRuntime", + "hosts", + "cuda", + "gpu", + "resources" + ], + "properties": { + "image": {}, + "platforms": {}, + "containerRuntime": {}, + "hosts": {}, + "cuda": {}, + "gpu": {}, + "resources": {} + }, + "not": { + "anyOf": [ + { + "type": "object", + "properties": { "architecture": {} }, + "required": ["architecture"] + }, + { + "type": "object", + "properties": { "components": {} }, + "required": ["components"] + } + ] + } + }, + "execution": { + "type": "object", + "required": ["receiptRef", "materializerRef", "lifecycleRef"], + "properties": { "receiptRef": {}, "materializerRef": {}, "lifecycleRef": {} } + }, + "serve": { + "type": "object", + "required": [ + "protocol", + "port", + "chatTemplate", + "contextSize", + "slots", + "idleSleepSeconds", + "limits" + ], + "properties": { + "protocol": {}, + "port": { "const": 8081 }, + "chatTemplate": {}, + "contextSize": {}, + "slots": { "const": 1 }, + "idleSleepSeconds": { "const": -1 }, + "limits": {} + }, + "not": { + "type": "object", + "properties": { "arguments": {} }, + "required": ["arguments"] + } + }, + "readiness": { + "type": "object", + "required": ["contractRef", "timeoutSeconds", "expectedModel", "probes"], + "properties": { + "contractRef": {}, + "timeoutSeconds": {}, + "expectedModel": {}, + "probes": {} + } + }, + "policy": { "type": "object" }, + "surfaces": { "type": "object" }, + "capabilities": { "type": "object" } + } + }, + "else": { + "type": "object", + "not": { + "anyOf": [ + { + "type": "object", + "properties": { "providerId": {} }, + "required": ["providerId"] + }, + { "type": "object", "properties": { "policy": {} }, "required": ["policy"] }, + { "type": "object", "properties": { "surfaces": {} }, "required": ["surfaces"] }, + { + "type": "object", + "properties": { "capabilities": {} }, + "required": ["capabilities"] + } + ] + } + } } } } diff --git a/src/lib/inference/serving/catalog.test.ts b/src/lib/inference/serving/catalog.test.ts index 9f90b7c6181..1d3453894aa 100644 --- a/src/lib/inference/serving/catalog.test.ts +++ b/src/lib/inference/serving/catalog.test.ts @@ -31,11 +31,22 @@ const SCHEMAS: ServingCatalogSchemas = { recipe: recipeSchema, }; const REGISTRIES: ServingCatalogRegistries = { + receipts: new Set(["test.receipt/v1"]), materializers: new Set(["test.materializer/v1"]), lifecycles: new Set(["test.lifecycle/v1"]), + readinessContracts: new Set(["test.readiness/v1"]), readiness: new Map([ - ["test.runtime.present", "capability"], - ["test.runtime.other", "capability"], + ["test.runtime.present", { kind: "capability" }], + ["test.runtime.other", { kind: "capability" }], + ["test.os", { kind: "observation", valueType: "string", role: "operating-system" }], + ["test.architecture", { kind: "observation", valueType: "string", role: "architecture" }], + [ + "test.container-runtime", + { kind: "observation", valueType: "string", role: "container-runtime" }, + ], + ["test.gpu-count", { kind: "observation", valueType: "number", role: "gpu-count" }], + ["test.driver-version", { kind: "observation", valueType: "version", role: "driver-version" }], + ["test.agent.qualified", { kind: "qualification" }], ]), }; @@ -103,6 +114,172 @@ spec: }; } +function llamaCppRecipeSource(): ServingCatalogSource { + return { + path: "managed-inference/recipes/test/test.llama.recipe.yaml", + contents: ` +apiVersion: nemoclaw.nvidia.com/managed-inference/v1 +kind: ServingRecipe +metadata: + id: test.llama.recipe +spec: + backend: install-llama-cpp + providerId: llama-cpp-local + server: + technology: llama.cpp + source: + repository: test/llama.cpp + revision: ${"e".repeat(40)} + model: + id: test/model + revision: ${"f".repeat(40)} + servedName: test-model + files: + - path: test-model.Q4_K_M.gguf + digest: sha256:${"1".repeat(64)} + sizeBytes: 4294967296 + format: gguf + quantization: Q4_K_M + license: Apache-2.0 + runtime: + image: registry.example/test/llama-server@sha256:${"2".repeat(64)} + platforms: + - linux/amd64 + - linux/arm64 + containerRuntime: docker + hosts: 1 + cuda: + baseImage: registry.example/nvidia/cuda@sha256:${"3".repeat(64)} + minimumDriverVersion: 570.0.0 + gpu: + vendor: nvidia + count: 1 + offload: full + cpuFallback: reject + resources: + memoryBytes: 34359738368 + writableStorageBytes: 1073741824 + pidsLimit: 256 + execution: + receiptRef: test.receipt/v1 + materializerRef: test.materializer/v1 + lifecycleRef: test.lifecycle/v1 + serve: + protocol: openai-completions + port: 8081 + chatTemplate: test-chat + contextSize: 32768 + slots: 1 + idleSleepSeconds: -1 + limits: + maxRequestBodyBytes: 1048576 + maxPromptTokens: 24576 + maxCompletionTokens: 8192 + requestTimeoutSeconds: 120 + readiness: + contractRef: test.readiness/v1 + timeoutSeconds: 120 + expectedModel: test-model + probes: + models: true + health: true + properties: true + metrics: true + policy: + egress: disabled + modelSource: verified-local + modelDownloads: disabled + surfaces: + ui: disabled + slotInspection: disabled + router: disabled + mcpProxy: disabled + serverTools: disabled + agentMode: disabled + multimodalProjection: disabled + capabilities: + agents: [] + protocols: + - openai-completions + streaming: true + toolCalls: true + structuredOutputs: true + parallelToolCalls: false + responsesApi: false + embeddings: false + reranking: false + multimodal: false +`, + }; +} + +function llamaCppPresetSource(selection = "explicit-only"): ServingCatalogSource { + return { + path: "managed-inference/presets/test/test.llama.preset.yaml", + contents: ` +apiVersion: nemoclaw.nvidia.com/managed-inference/v1 +kind: ServingPreset +metadata: + id: test.llama.preset +spec: + selection: ${selection} + priority: 100 + requirements: + all: + - readiness: + scope: everyNode + kind: observation + id: test.os + comparison: + operator: equals + value: linux + - readiness: + scope: everyNode + kind: observation + id: test.architecture + comparison: + operator: one-of + values: + - amd64 + - arm64 + - readiness: + scope: everyNode + kind: observation + id: test.container-runtime + comparison: + operator: equals + value: docker + - readiness: + scope: everyNode + kind: observation + id: test.gpu-count + comparison: + operator: at-least + value: 1 + - readiness: + scope: everyNode + kind: observation + id: test.driver-version + comparison: + operator: version-at-least + value: 570.0.0 + plan: + backend: install-llama-cpp + recipeRef: test.llama.recipe +`, + }; +} + +function replaceSource( + source: ServingCatalogSource, + expected: string | RegExp, + replacement: string, +): ServingCatalogSource { + const contents = source.contents.replace(expected, replacement); + expect(contents).not.toBe(source.contents); + return { ...source, contents }; +} + function compile(sources: readonly ServingCatalogSource[], registries = REGISTRIES) { return compileTrustedServingCatalog({ sources, @@ -116,14 +293,6 @@ function requireValidationFailure(run: () => void, expectedMessage: string): voi expect(run).toThrow(expectedMessage); } -function recipeWithModelRevision(revision: string): ServingCatalogSource { - const source = recipeSource(); - return { - ...source, - contents: source.contents.replace(`revision: ${MODEL_REVISION}`, `revision: ${revision}`), - }; -} - describe("managed inference serving catalog compiler", () => { it("compiles managed-inference YAML to deterministic canonical JSON (#8144)", () => { const recipe = recipeSource(); @@ -136,6 +305,7 @@ describe("managed inference serving catalog compiler", () => { expect(first.readinessSchemaRef).toBe( "https://github.com/NVIDIA/NemoClaw/schemas/system-readiness.schema.json", ); + expect(first.compilerVersion).toBe("1.1.0"); expect(first.recipes.map((definition) => definition.metadata.id)).toEqual(["test.recipe.v1"]); expect(first.presets.map((definition) => definition.metadata.id)).toEqual(["test.preset.auto"]); expect(first.sources.map((source) => source.path)).toEqual([ @@ -144,6 +314,311 @@ describe("managed inference serving catalog compiler", () => { ]); }); + it("compiles a complete synthetic llama.cpp recipe and explicit-only preset (#8181)", () => { + const recipe = llamaCppRecipeSource(); + const preset = llamaCppPresetSource(); + + const first = compile([recipe, preset]); + const second = compile([preset, recipe]); + + expect(serializeCompiledServingCatalog(first)).toBe(serializeCompiledServingCatalog(second)); + expect(first.recipes[0]?.spec).toMatchObject({ + providerId: "llama-cpp-local", + server: { technology: "llama.cpp" }, + runtime: { + platforms: ["linux/amd64", "linux/arm64"], + gpu: { count: 1, cpuFallback: "reject" }, + }, + serve: { protocol: "openai-completions", port: 8081, slots: 1 }, + capabilities: { agents: [], protocols: ["openai-completions"] }, + }); + expect(first.presets[0]?.spec.selection).toBe("explicit-only"); + expect(first.sources.every((source) => /^sha256:[0-9a-f]{64}$/.test(source.digest))).toBe(true); + expect(first.catalogDigest).toMatch(/^sha256:[0-9a-f]{64}$/); + }); + + it.each([ + ["a missing server source revision", ` revision: ${"e".repeat(40)}\n`, ""], + [ + "a mutable server source revision", + ` revision: ${"e".repeat(40)}`, + " revision: main", + ], + ["a mutable model revision", ` revision: ${"f".repeat(40)}`, " revision: latest"], + ["a missing GGUF byte size", " sizeBytes: 4294967296\n", ""], + [ + "a mutable server image", + ` image: registry.example/test/llama-server@sha256:${"2".repeat(64)}`, + " image: registry.example/test/llama-server:latest", + ], + [ + "a mutable CUDA base image", + ` baseImage: registry.example/nvidia/cuda@sha256:${"3".repeat(64)}`, + " baseImage: registry.example/nvidia/cuda:12.8", + ], + ["incomplete platform coverage", " - linux/arm64\n", ""], + ["missing process limits", " pidsLimit: 256\n", ""], + [ + "an embedded credential", + ` revision: ${"e".repeat(40)}`, + ` revision: ${"e".repeat(40)}\n credential: test-secret`, + ], + ["a host model path", " - path: test-model.Q4_K_M.gguf", " - path: /tmp/model.gguf"], + ["shell syntax", " chatTemplate: test-chat", " chatTemplate: $(id)"], + ["environment expansion", " chatTemplate: test-chat", " chatTemplate: \${HOME}"], + [ + "an unsupported protocol", + " protocol: openai-completions", + " protocol: openai-responses", + ], + ["CPU fallback", " cpuFallback: reject", " cpuFallback: allow"], + ["server egress", " egress: disabled", " egress: enabled"], + ["a remote model source", " modelSource: verified-local", " modelSource: remote"], + [ + "an arbitrary launch argument", + " protocol: openai-completions", + " protocol: openai-completions\n arguments:\n - name: --unsafe", + ], + ["an unsupported capability", " responsesApi: false", " responsesApi: true"], + ["a malformed agent qualification", " agents: []", " agents: [openclaw]"], + ["a path-based served model name", " servedName: test-model", " servedName: test/model"], + ])("rejects %s in a llama.cpp recipe (#8181)", (_case, expected, replacement) => { + const recipe = replaceSource(llamaCppRecipeSource(), expected, replacement); + + expect(() => compile([recipe])).toThrow("does not satisfy the ServingRecipe schema"); + }); + + it("rejects a llama.cpp readiness model that differs from the served name (#8181)", () => { + const wrongExpectedModel = replaceSource( + llamaCppRecipeSource(), + " expectedModel: test-model", + " expectedModel: other-model", + ); + + expect(() => compile([wrongExpectedModel])).toThrow( + "readiness expectedModel must match model servedName", + ); + }); + + it("rejects llama.cpp request token limits that exceed serve.contextSize (#8181)", () => { + const excessiveTokenLimits = replaceSource( + llamaCppRecipeSource(), + " maxCompletionTokens: 8192", + " maxCompletionTokens: 8193", + ); + + expect(() => compile([excessiveTokenLimits])).toThrow( + "request token limits exceed serve.contextSize", + ); + }); + + it.each([ + ["a backend that does not match llama.cpp", " backend: install-llama-cpp", " backend: vllm"], + [ + "a GGUF byte size above Number.MAX_SAFE_INTEGER", + " sizeBytes: 4294967296", + " sizeBytes: 9007199254740992", + ], + ])("rejects %s in the typed llama.cpp contract (#8181)", (_case, expected, replacement) => { + const recipe = replaceSource(llamaCppRecipeSource(), expected, replacement); + + expect(() => compile([recipe])).toThrow("does not satisfy the ServingRecipe schema"); + }); + + it("reserves the install-llama-cpp backend for typed llama.cpp recipes (#8181)", () => { + const escapedRecipe = replaceSource( + recipeSource(), + " backend: test", + " backend: install-llama-cpp", + ); + + expect(() => compile([escapedRecipe])).toThrow("does not satisfy the ServingRecipe schema"); + }); + + it.each([ + ["receipt contract", { receipts: new Set() }, "unknown receipt contract"], + ["materializer", { materializers: new Set() }, "unknown materializer"], + ["lifecycle adapter", { lifecycles: new Set() }, "unknown lifecycle adapter"], + ["readiness contract", { readinessContracts: new Set() }, "unknown readiness contract"], + ])("rejects a llama.cpp recipe with an unknown %s reference (#8181)", (_name, registry, message) => { + expect(() => compile([llamaCppRecipeSource()], { ...REGISTRIES, ...registry })).toThrow( + message, + ); + }); + + it("validates optional receipt and readiness contracts on generic recipes (#8181)", () => { + const receiptRecipe = recipeSource("test.recipe.receipt", { + execution: + " receiptRef: test.receipt/v1\n materializerRef: test.materializer/v1\n lifecycleRef: test.lifecycle/v1", + }); + const readinessRecipe = replaceSource( + recipeSource("test.recipe.readiness"), + " readiness:\n", + " readiness:\n contractRef: test.readiness/v1\n", + ); + + expect(() => compile([receiptRecipe], { ...REGISTRIES, receipts: new Set() })).toThrow( + "unknown receipt contract test.receipt/v1", + ); + expect(() => + compile([readinessRecipe], { ...REGISTRIES, readinessContracts: new Set() }), + ).toThrow("unknown readiness contract test.readiness/v1"); + }); + + it("rejects automatic selection for a llama.cpp recipe (#8181)", () => { + expect(() => compile([llamaCppRecipeSource(), llamaCppPresetSource("automatic")])).toThrow( + "must use explicit-only selection for llama.cpp recipe", + ); + }); + + it("requires readiness requirements for an explicit llama.cpp preset (#8181)", () => { + const presetWithoutReadiness = replaceSource( + llamaCppPresetSource(), + / requirements:[\s\S]*? plan:/u, + " plan:", + ); + + expect(() => compile([llamaCppRecipeSource(), presetWithoutReadiness])).toThrow( + "must declare readiness requirements for llama.cpp recipe", + ); + }); + + it.each([ + ["operating-system", " value: linux", " value: windows"], + ["architecture", " - arm64", " - riscv64"], + ["container-runtime", " value: docker", " value: podman"], + ["gpu-count", " value: 1", " value: 0"], + ["driver-version", " value: 570.0.0", " value: 1.0.0"], + ])("rejects a llama.cpp preset whose %s contradicts its recipe (#8181)", (role, expected, replacement) => { + const preset = replaceSource(llamaCppPresetSource(), expected, replacement); + + expect(() => compile([llamaCppRecipeSource(), preset])).toThrow( + `must require ${role} matching llama.cpp recipe`, + ); + }); + + it.each([ + ["shell syntax", "$(id)"], + ["environment expansion", "${HOME}"], + ["an absolute path", "/tmp/model.gguf"], + ])("rejects %s in a readiness comparison string (#8181)", (_case, value) => { + const preset = replaceSource( + llamaCppPresetSource(), + " value: linux", + ` value: ${value}`, + ); + + expect(() => compile([llamaCppRecipeSource(), preset])).toThrow( + "does not satisfy the ServingPreset schema", + ); + }); + + it("rejects a negative readiness threshold (#8181)", () => { + const preset = replaceSource( + llamaCppPresetSource(), + " value: 1", + " value: -1", + ); + + expect(() => compile([llamaCppRecipeSource(), preset])).toThrow( + "does not satisfy the ServingPreset schema", + ); + }); + + it("requires declared llama.cpp agents to reference a qualification entity (#8181)", () => { + const declaredAgent = replaceSource( + llamaCppRecipeSource(), + " agents: []", + " agents:\n - id: test.agent\n qualificationRef: test.agent.qualified", + ); + const unknownQualification = replaceSource( + declaredAgent, + " qualificationRef: test.agent.qualified", + " qualificationRef: test.agent.unknown", + ); + + expect(() => compile([declaredAgent])).not.toThrow(); + expect(() => compile([unknownQualification])).toThrow( + "references unknown agent qualification test.agent.unknown for test.agent", + ); + }); + + it("gates declared llama.cpp agents on preset qualification readiness (#8181)", () => { + const declaredAgent = replaceSource( + llamaCppRecipeSource(), + " agents: []", + " agents:\n - id: test.agent\n qualificationRef: test.agent.qualified", + ); + const preset = llamaCppPresetSource(); + const qualifiedPreset = replaceSource( + preset, + " plan:\n", + ` - readiness: + scope: everyNode + kind: qualification + id: test.agent.qualified + status: qualified + plan: +`, + ); + + expect(() => compile([declaredAgent, preset])).toThrow( + "must require qualified status for test.agent.qualified", + ); + expect(() => compile([declaredAgent, qualifiedPreset])).not.toThrow(); + }); + + it("rejects duplicate llama.cpp agent capability declarations (#8181)", () => { + const duplicateAgent = replaceSource( + llamaCppRecipeSource(), + " agents: []", + ` agents: + - id: test.agent + qualificationRef: test.agent.qualified + - id: test.agent + qualificationRef: test.other.qualified`, + ); + + expect(() => compile([duplicateAgent])).toThrow("repeats agent capability test.agent"); + }); + + it("rejects duplicate and contradictory readiness requirements (#8181)", () => { + const repeatedRequirement = ` - readiness: + scope: everyNode + kind: observation + id: test.os + comparison: + operator: equals + value: linux +`; + const duplicate = replaceSource( + llamaCppPresetSource(), + " plan:\n", + `${repeatedRequirement} plan:\n`, + ); + const contradiction = replaceSource( + duplicate, + " value: linux\n plan:", + " value: windows\n plan:", + ); + + expect(() => compile([llamaCppRecipeSource(), duplicate])).toThrow( + "repeats readiness requirement", + ); + expect(() => compile([llamaCppRecipeSource(), contradiction])).toThrow( + "has contradictory readiness requirements", + ); + }); + + it("rejects a readiness comparison whose value type conflicts with its registry entry (#8181)", () => { + const readiness = new Map(REGISTRIES.readiness); + readiness.set("test.gpu-count", { kind: "observation", valueType: "string" }); + + expect(() => + compile([llamaCppRecipeSource(), llamaCppPresetSource()], { ...REGISTRIES, readiness }), + ).toThrow("compares test.gpu-count as number, but the readiness registry declares string"); + }); + it("rejects duplicate definition IDs and dangling recipe references (#8144)", () => { const duplicate = { ...recipeSource(), @@ -180,12 +655,18 @@ describe("managed inference serving catalog compiler", () => { it("accepts only exact immutable model revision forms (#8144)", () => { for (const revision of ["e".repeat(40), "e".repeat(64), `sha256:${"e".repeat(64)}`]) { - expect(() => compile([recipeWithModelRevision(revision)])).not.toThrow(); + expect(() => + compile([ + replaceSource(recipeSource(), `revision: ${MODEL_REVISION}`, `revision: ${revision}`), + ]), + ).not.toThrow(); } for (const revision of ["e".repeat(41), "e".repeat(63)]) { - expect(() => compile([recipeWithModelRevision(revision)])).toThrow( - "does not satisfy the ServingRecipe schema", - ); + expect(() => + compile([ + replaceSource(recipeSource(), `revision: ${MODEL_REVISION}`, `revision: ${revision}`), + ]), + ).toThrow("does not satisfy the ServingRecipe schema"); } }); @@ -206,24 +687,18 @@ describe("managed inference serving catalog compiler", () => { }); it("rejects duplicate structured arguments and model files (#8144)", () => { - const argumentRecipe = recipeSource("test.recipe.duplicate-argument"); - const duplicateArgument = { - ...argumentRecipe, - contents: argumentRecipe.contents.replace( - " - name: --port\n value: 8081", - " - name: --port\n value: 8081\n - name: --port\n value: 8082", - ), - }; + const duplicateArgument = replaceSource( + recipeSource("test.recipe.duplicate-argument"), + " - name: --port\n value: 8081", + " - name: --port\n value: 8081\n - name: --port\n value: 8082", + ); expect(() => compile([duplicateArgument])).toThrow("repeats structured argument --port"); - const modelRecipe = recipeSource("test.recipe.duplicate-model-file"); - const duplicateModelFile = { - ...modelRecipe, - contents: modelRecipe.contents.replace( - ` - path: model.gguf\n digest: sha256:${"d".repeat(64)}`, - ` - path: model.gguf\n digest: sha256:${"d".repeat(64)}\n - path: model.gguf\n digest: sha256:${"e".repeat(64)}`, - ), - }; + const duplicateModelFile = replaceSource( + recipeSource("test.recipe.duplicate-model-file"), + ` - path: model.gguf\n digest: sha256:${"d".repeat(64)}`, + ` - path: model.gguf\n digest: sha256:${"d".repeat(64)}\n - path: model.gguf\n digest: sha256:${"e".repeat(64)}`, + ); expect(() => compile([duplicateModelFile])).toThrow("repeats model file model.gguf"); for (const path of [ @@ -234,14 +709,9 @@ describe("managed inference serving catalog compiler", () => { "models\\model.gguf", "C:\\model.gguf", ]) { - const pathRecipe = recipeSource("test.recipe.noncanonical-model-file"); - const noncanonicalModelFile = { - ...pathRecipe, - contents: pathRecipe.contents.replace("path: model.gguf", `path: ${path}`), - }; - expect(() => compile([noncanonicalModelFile])).toThrow( - "does not satisfy the ServingRecipe schema", - ); + expect(() => + compile([replaceSource(recipeSource(), "path: model.gguf", `path: ${path}`)]), + ).toThrow("does not satisfy the ServingRecipe schema"); } }); @@ -273,25 +743,6 @@ describe("managed inference serving catalog compiler", () => { ).toThrow( "uses test.runtime.present as observation, but the readiness registry declares capability", ); - - const duplicateRequirement = presetSource("test.preset.duplicate-requirement"); - const readinessRequirement = ` - readiness: - scope: everyNode - kind: capability - id: test.runtime.present - state: present`; - expect(() => - compile([ - recipeSource(), - { - ...duplicateRequirement, - contents: duplicateRequirement.contents.replace( - readinessRequirement, - `${readinessRequirement}\n${readinessRequirement}`, - ), - }, - ]), - ).toThrow("repeats readiness requirement"); }); it("rejects duplicate automatic selectors at one priority (#8144)", () => { diff --git a/src/lib/inference/serving/catalog.ts b/src/lib/inference/serving/catalog.ts index aaffa5655da..81279302760 100644 --- a/src/lib/inference/serving/catalog.ts +++ b/src/lib/inference/serving/catalog.ts @@ -15,12 +15,15 @@ import type { ServingCatalogSource, ServingDefinitionKind, ServingPreset, + ServingReadinessComparison, + ServingReadinessObservationRole, + ServingReadinessRequirement, ServingRecipe, } from "./types"; const API_VERSION = "nemoclaw.nvidia.com/managed-inference/v1" as const; const CATALOG_SCHEMA_VERSION = "1.0.0" as const; -const COMPILER_VERSION = "1.0.0" as const; +const COMPILER_VERSION = "1.1.0" as const; const READINESS_SCHEMA_REF = "https://github.com/NVIDIA/NemoClaw/schemas/system-readiness.schema.json" as const; const RECIPE_SCHEMA_ID = @@ -144,11 +147,22 @@ function definitionIdentity( return { kind, id }; } +type LlamaCppServingRecipe = Extract; + +function isLlamaCppServingRecipe(recipe: ServingRecipe): recipe is LlamaCppServingRecipe { + return recipe.spec.providerId === "llama-cpp-local"; +} + function validateRecipeSemantics( recipe: ServingRecipe, registries: ServingCatalogRegistries, ): void { - const { materializerRef, lifecycleRef } = recipe.spec.execution; + const { receiptRef, materializerRef, lifecycleRef } = recipe.spec.execution; + if (receiptRef && !registries.receipts.has(receiptRef)) { + throw new ServingCatalogValidationError( + `Recipe ${recipe.metadata.id} references unknown receipt contract ${receiptRef}.`, + ); + } if (!registries.materializers.has(materializerRef)) { throw new ServingCatalogValidationError( `Recipe ${recipe.metadata.id} references unknown materializer ${materializerRef}.`, @@ -159,15 +173,22 @@ function validateRecipeSemantics( `Recipe ${recipe.metadata.id} references unknown lifecycle adapter ${lifecycleRef}.`, ); } - - const argumentNames = new Set(); - for (const argument of recipe.spec.serve?.arguments ?? []) { - if (argumentNames.has(argument.name)) { - throw new ServingCatalogValidationError( - `Recipe ${recipe.metadata.id} repeats structured argument ${argument.name}.`, - ); + const readinessContractRef = recipe.spec.readiness?.contractRef; + if (readinessContractRef && !registries.readinessContracts.has(readinessContractRef)) { + throw new ServingCatalogValidationError( + `Recipe ${recipe.metadata.id} references unknown readiness contract ${readinessContractRef}.`, + ); + } + if (!isLlamaCppServingRecipe(recipe)) { + const argumentNames = new Set(); + for (const argument of recipe.spec.serve?.arguments ?? []) { + if (argumentNames.has(argument.name)) { + throw new ServingCatalogValidationError( + `Recipe ${recipe.metadata.id} repeats structured argument ${argument.name}.`, + ); + } + argumentNames.add(argument.name); } - argumentNames.add(argument.name); } const modelFiles = new Set(); @@ -185,31 +206,177 @@ function validateRecipeSemantics( } modelFiles.add(normalizedPath); } + + if (isLlamaCppServingRecipe(recipe)) { + const servedName = recipe.spec.model.servedName; + if (recipe.spec.readiness.expectedModel !== servedName) { + throw new ServingCatalogValidationError( + `Recipe ${recipe.metadata.id} readiness expectedModel must match model servedName.`, + ); + } + const { serve } = recipe.spec; + if (serve.limits.maxPromptTokens + serve.limits.maxCompletionTokens > serve.contextSize) { + throw new ServingCatalogValidationError( + `Recipe ${recipe.metadata.id} request token limits exceed serve.contextSize.`, + ); + } + const agents = new Set(); + for (const agent of recipe.spec.capabilities.agents) { + if (agents.has(agent.id)) { + throw new ServingCatalogValidationError( + `Recipe ${recipe.metadata.id} repeats agent capability ${agent.id}.`, + ); + } + agents.add(agent.id); + const qualification = registries.readiness.get(agent.qualificationRef); + if (qualification?.kind !== "qualification") { + throw new ServingCatalogValidationError( + `Recipe ${recipe.metadata.id} references unknown agent qualification ${agent.qualificationRef} for ${agent.id}.`, + ); + } + } + } +} + +function comparisonValueType( + comparison: ServingReadinessComparison, +): "boolean" | "number" | "string" | "version" | "mixed" { + if (comparison.operator === "at-least") return "number"; + if (comparison.operator === "version-at-least") return "version"; + const values = comparison.operator === "one-of" ? comparison.values : [comparison.value]; + const types = new Set(values.map((value) => typeof value)); + if (types.size !== 1) return "mixed"; + const [type] = types; + return type === "boolean" || type === "number" || type === "string" ? type : "mixed"; +} + +function readinessRequirementKey(requirement: ServingReadinessRequirement): string { + const readiness = requirement.readiness; + return `${readiness.scope}:${readiness.kind}:${readiness.id}`; } function validatePresetReadiness( preset: ServingPreset, registries: ServingCatalogRegistries, ): void { - const requirements = new Set(); + const requirementsByEntity = new Map(); for (const requirement of preset.spec.requirements?.all ?? []) { - const requirementKey = canonicalServingCatalogJson(requirement); - if (requirements.has(requirementKey)) { + const readiness = requirement.readiness; + const registered = registries.readiness.get(readiness.id); + if (registered === undefined) { throw new ServingCatalogValidationError( - `Preset ${preset.metadata.id} repeats readiness requirement ${requirementKey}.`, + `Preset ${preset.metadata.id} references unknown readiness entity ${readiness.id}.`, ); } - requirements.add(requirementKey); + if (registered.kind !== readiness.kind) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} uses ${readiness.id} as ${readiness.kind}, but the readiness registry declares ${registered.kind}.`, + ); + } + + if ("comparison" in readiness) { + if (registered.kind !== "observation") { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} cannot compare readiness entity ${readiness.id}.`, + ); + } + const comparisonType = comparisonValueType(readiness.comparison); + if (!registered.valueType || comparisonType !== registered.valueType) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} compares ${readiness.id} as ${comparisonType}, but the readiness registry declares ${registered.valueType ?? "no value type"}.`, + ); + } + } - const registeredKind = registries.readiness.get(requirement.readiness.id); - if (registeredKind === undefined) { + const key = readinessRequirementKey(requirement); + const canonicalRequirement = canonicalServingCatalogJson(requirement); + const previous = requirementsByEntity.get(key); + if (previous === canonicalRequirement) { throw new ServingCatalogValidationError( - `Preset ${preset.metadata.id} references unknown readiness entity ${requirement.readiness.id}.`, + `Preset ${preset.metadata.id} repeats readiness requirement ${key}.`, ); } - if (registeredKind !== requirement.readiness.kind) { + if (previous !== undefined) { throw new ServingCatalogValidationError( - `Preset ${preset.metadata.id} uses ${requirement.readiness.id} as ${requirement.readiness.kind}, but the readiness registry declares ${registeredKind}.`, + `Preset ${preset.metadata.id} has contradictory readiness requirements for ${key}.`, + ); + } + requirementsByEntity.set(key, canonicalRequirement); + } +} + +function canonicalReadinessComparison(comparison: ServingReadinessComparison): string { + if (comparison.operator !== "one-of") { + return canonicalServingCatalogJson(comparison); + } + return canonicalServingCatalogJson({ + ...comparison, + values: [...comparison.values].sort((left, right) => + compareCanonicalText(canonicalServingCatalogJson(left), canonicalServingCatalogJson(right)), + ), + }); +} + +function validateLlamaCppPreset( + recipe: LlamaCppServingRecipe, + preset: ServingPreset, + registries: ServingCatalogRegistries, +): void { + const expectedComparisons = new Map([ + ["operating-system", { operator: "equals", value: "linux" }], + [ + "architecture", + { + operator: "one-of", + values: recipe.spec.runtime.platforms.map((platform) => platform.slice("linux/".length)), + }, + ], + ["container-runtime", { operator: "equals", value: recipe.spec.runtime.containerRuntime }], + ["gpu-count", { operator: "at-least", value: recipe.spec.runtime.gpu.count }], + [ + "driver-version", + { operator: "version-at-least", value: recipe.spec.runtime.cuda.minimumDriverVersion }, + ], + ]); + const seenRoles = new Set(); + const qualified = new Set(); + + for (const requirement of preset.spec.requirements?.all ?? []) { + const readiness = requirement.readiness; + if (readiness.kind === "qualification" && readiness.status === "qualified") { + qualified.add(readiness.id); + } + const registered = registries.readiness.get(readiness.id); + if (registered?.kind !== "observation" || registered.role === undefined) continue; + const expected = expectedComparisons.get(registered.role); + if (!expected) continue; + if (seenRoles.has(registered.role)) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} repeats llama.cpp readiness role ${registered.role}.`, + ); + } + seenRoles.add(registered.role); + if ( + !("comparison" in readiness) || + canonicalReadinessComparison(readiness.comparison) !== canonicalReadinessComparison(expected) + ) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} must require ${registered.role} matching llama.cpp recipe ${recipe.metadata.id}.`, + ); + } + } + + for (const role of expectedComparisons.keys()) { + if (!seenRoles.has(role)) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} must require ${role} matching llama.cpp recipe ${recipe.metadata.id}.`, + ); + } + } + for (const agent of recipe.spec.capabilities.agents) { + if (!qualified.has(agent.qualificationRef)) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} must require qualified status for ${agent.qualificationRef} before selecting llama.cpp recipe ${recipe.metadata.id}.`, ); } } @@ -218,6 +385,7 @@ function validatePresetReadiness( function validateCatalogSemantics( recipes: readonly ServingRecipe[], presets: readonly ServingPreset[], + registries: ServingCatalogRegistries, ): void { const recipesById = new Map(recipes.map((recipe) => [recipe.metadata.id, recipe])); for (const preset of presets) { @@ -232,6 +400,19 @@ function validateCatalogSemantics( `Preset ${preset.metadata.id} selects backend ${preset.spec.plan.backend}, but recipe ${recipe.metadata.id} uses ${recipe.spec.backend}.`, ); } + if (isLlamaCppServingRecipe(recipe)) { + if (preset.spec.selection !== "explicit-only") { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} must use explicit-only selection for llama.cpp recipe ${recipe.metadata.id}.`, + ); + } + if ((preset.spec.requirements?.all.length ?? 0) === 0) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} must declare readiness requirements for llama.cpp recipe ${recipe.metadata.id}.`, + ); + } + validateLlamaCppPreset(recipe, preset, registries); + } } const automaticSelectors = new Map(); @@ -320,7 +501,7 @@ export function compileTrustedServingCatalog( recipes.sort((left, right) => compareCanonicalText(left.metadata.id, right.metadata.id)); presets.sort((left, right) => compareCanonicalText(left.metadata.id, right.metadata.id)); - validateCatalogSemantics(recipes, presets); + validateCatalogSemantics(recipes, presets, options.registries); const payload: CompiledServingCatalogPayload = { schemaVersion: CATALOG_SCHEMA_VERSION, diff --git a/src/lib/inference/serving/generate-catalog.ts b/src/lib/inference/serving/generate-catalog.ts index b780ea59afd..fe3a56b900d 100644 --- a/src/lib/inference/serving/generate-catalog.ts +++ b/src/lib/inference/serving/generate-catalog.ts @@ -15,8 +15,10 @@ import type { // Bootstrap only: valid while the catalog has no definitions. Production definitions must // inject populated registries; tracked by https://github.com/NVIDIA/NemoClaw/issues/8144. const EMPTY_REGISTRIES: ServingCatalogRegistries = { + receipts: new Set(), materializers: new Set(), lifecycles: new Set(), + readinessContracts: new Set(), readiness: new Map(), }; diff --git a/src/lib/inference/serving/types.ts b/src/lib/inference/serving/types.ts index 00f82e774d8..c2d5af58e58 100644 --- a/src/lib/inference/serving/types.ts +++ b/src/lib/inference/serving/types.ts @@ -4,18 +4,30 @@ export type ServingDefinitionKind = "ServingRecipe" | "ServingPreset"; export type ServingSelectionPolicy = "automatic" | "explicit-only" | "disabled"; export type ReadinessEntityKind = "observation" | "capability" | "qualification"; +export type ReadinessValueType = "boolean" | "number" | "string" | "version"; +export type ServingReadinessObservationRole = + | "operating-system" + | "architecture" + | "container-runtime" + | "gpu-count" + | "driver-version"; export interface ServingMetadata { id: string; displayName?: string; } -export interface ServingRecipe { +interface ServingRecipeEnvelope { apiVersion: "nemoclaw.nvidia.com/managed-inference/v1"; kind: "ServingRecipe"; metadata: ServingMetadata; +} + +interface GenericServingRecipe extends ServingRecipeEnvelope { spec: { backend: string; + providerId?: never; + server?: never; model: { id: string; revision: string; @@ -28,6 +40,7 @@ export interface ServingRecipe { components?: Record; }; execution: { + receiptRef?: string; materializerRef: string; lifecycleRef: string; }; @@ -35,12 +48,105 @@ export interface ServingRecipe { arguments?: Array<{ name: string; value?: string | number | boolean }>; }; readiness?: { + contractRef?: string; timeoutSeconds?: number; expectedModel?: string; }; }; } +interface LlamaCppServingRecipe extends ServingRecipeEnvelope { + spec: { + backend: "install-llama-cpp"; + providerId: "llama-cpp-local"; + server: { + technology: "llama.cpp"; + source: { repository: string; revision: string }; + }; + model: { + id: string; + revision: string; + servedName: string; + files: Array<{ + path: string; + digest: string; + sizeBytes: number; + format: "gguf"; + quantization: string; + license: string; + }>; + }; + runtime: { + image: string; + platforms: Array<"linux/amd64" | "linux/arm64">; + containerRuntime: "docker"; + hosts: 1; + cuda: { baseImage: string; minimumDriverVersion: string }; + gpu: { vendor: "nvidia"; count: 1; offload: "full"; cpuFallback: "reject" }; + resources: { memoryBytes: number; writableStorageBytes: number; pidsLimit: number }; + }; + execution: { + receiptRef: string; + materializerRef: string; + lifecycleRef: string; + }; + serve: { + protocol: "openai-completions"; + port: 8081; + chatTemplate: string; + contextSize: number; + slots: 1; + idleSleepSeconds: -1; + limits: { + maxRequestBodyBytes: number; + maxPromptTokens: number; + maxCompletionTokens: number; + requestTimeoutSeconds: number; + }; + }; + readiness: { + contractRef: string; + timeoutSeconds: number; + expectedModel: string; + probes: { models: true; health: true; properties: true; metrics: true }; + }; + policy: { + egress: "disabled"; + modelSource: "verified-local"; + modelDownloads: "disabled"; + }; + surfaces: { + ui: "disabled"; + slotInspection: "disabled"; + router: "disabled"; + mcpProxy: "disabled"; + serverTools: "disabled"; + agentMode: "disabled"; + multimodalProjection: "disabled"; + }; + capabilities: { + agents: Array<{ id: string; qualificationRef: string }>; + protocols: ["openai-completions"]; + streaming: boolean; + toolCalls: boolean; + structuredOutputs: boolean; + parallelToolCalls: false; + responsesApi: false; + embeddings: false; + reranking: false; + multimodal: false; + }; + }; +} + +export type ServingRecipe = GenericServingRecipe | LlamaCppServingRecipe; + +export type ServingReadinessComparison = + | { operator: "equals"; value: string | number | boolean } + | { operator: "one-of"; values: Array } + | { operator: "at-least"; value: number } + | { operator: "version-at-least"; value: string }; + export interface ServingReadinessRequirement { readiness: | { @@ -49,6 +155,12 @@ export interface ServingReadinessRequirement { id: string; state: "present" | "absent"; } + | { + scope: "controller" | "everyNode"; + kind: "observation"; + id: string; + comparison: ServingReadinessComparison; + } | { scope: "controller" | "everyNode"; kind: "qualification"; @@ -81,7 +193,7 @@ export interface ServingCatalogSourceProvenance { export interface CompiledServingCatalogPayload { schemaVersion: "1.0.0"; - compilerVersion: "1.0.0"; + compilerVersion: "1.1.0"; sourceRevision: string; readinessSchemaRef: "https://github.com/NVIDIA/NemoClaw/schemas/system-readiness.schema.json"; recipes: ServingRecipe[]; @@ -105,7 +217,17 @@ export interface ServingCatalogSchemas { } export interface ServingCatalogRegistries { + receipts: ReadonlySet; materializers: ReadonlySet; lifecycles: ReadonlySet; - readiness: ReadonlyMap; + readinessContracts: ReadonlySet; + readiness: ReadonlyMap< + string, + | { + kind: "observation"; + valueType?: ReadinessValueType; + role?: ServingReadinessObservationRole; + } + | { kind: "capability" | "qualification" } + >; }