diff --git a/managed-inference/images/llama-cpp/image.yaml b/managed-inference/images/llama-cpp/image.yaml index fd6ce5d3cd0..447855e1c5d 100644 --- a/managed-inference/images/llama-cpp/image.yaml +++ b/managed-inference/images/llama-cpp/image.yaml @@ -56,7 +56,29 @@ spec: cpuFallback: reject probes: - health - - completion + - models + - synchronous-chat + - streaming-chat + - usage + - structured-output + - tool-call + - tool-result-continuation + - context-window + - authentication + - malformed-request + - cancellation + - client-timeout + probeBounds: + cancellationMaxTokens: 4096 + clientTimeoutMilliseconds: 250 + maxResponseBytes: 16777216 + maxStreamEvents: 512 + maxTokens: + synchronousChat: 16 + streamingChat: 32 + structuredOutput: 64 + toolCall: 256 + toolResultContinuation: 64 source: repository: https://github.com/ggml-org/llama.cpp diff --git a/scripts/checks/export-llama-cpp-image-config.mts b/scripts/checks/export-llama-cpp-image-config.mts index 957b4c5215c..7766e615b31 100644 --- a/scripts/checks/export-llama-cpp-image-config.mts +++ b/scripts/checks/export-llama-cpp-image-config.mts @@ -9,6 +9,7 @@ import Ajv2020 from "ajv/dist/2020.js"; import YAML from "yaml"; import { + LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES, llamaCppDgxSparkExecutionPlanSha256, parseLlamaCppDgxSparkExecutionPlan, } from "./llama-cpp-dgx-spark-qualification-contract.mts"; @@ -50,6 +51,7 @@ type ServerImageManifest = { gpu?: unknown; model?: unknown; platform?: unknown; + probeBounds?: unknown; probes?: unknown; profile?: unknown; recipeRef?: unknown; @@ -332,6 +334,7 @@ export function loadLlamaCppImageConfig( "gpu", "model", "platform", + "probeBounds", "probes", "profile", "recipeRef", @@ -439,7 +442,7 @@ export function loadLlamaCppImageConfig( fullOffload: true, vendor: "nvidia", }) || - JSON.stringify(qualification?.probes) !== JSON.stringify(["health", "completion"]) + JSON.stringify(qualification?.probes) !== JSON.stringify(LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES) ) { throw new Error("invalid llama.cpp image publication contract"); } @@ -536,6 +539,7 @@ export function loadLlamaCppImageConfig( id: qualificationModel?.id, }, platform: qualification?.platform, + probeBounds: qualification?.probeBounds, probes: qualification?.probes, profile: qualification?.profile, recipeRef: qualificationRecipeRef, @@ -669,7 +673,12 @@ export function loadLlamaCppImageConfig( revision: spec?.source?.revision, }, }, + qualification: { + probeBounds: qualification?.probeBounds, + probes: qualification?.probes, + }, recipe: { + capabilities: recipe.spec.capabilities, id: recipe.metadata.id, model: { acquisition: { diff --git a/scripts/checks/llama-cpp-dgx-spark-protocol-qualification.mts b/scripts/checks/llama-cpp-dgx-spark-protocol-qualification.mts new file mode 100644 index 00000000000..0ae39f4749f --- /dev/null +++ b/scripts/checks/llama-cpp-dgx-spark-protocol-qualification.mts @@ -0,0 +1,815 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES, + type LlamaCppDgxSparkExecutionPlan, + type LlamaCppDgxSparkQualificationReceipt, +} from "./llama-cpp-dgx-spark-qualification-contract.mts"; + +type ProtocolProbe = (typeof LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES)[number]; + +type JsonRecord = Record; +type ProtocolEvidence = LlamaCppDgxSparkQualificationReceipt["probes"]; +type FetchImplementation = typeof fetch; + +type ToolCall = { + readonly arguments: JsonRecord; + readonly id: string; + readonly name: "get_current_weather"; + readonly raw: JsonRecord; +}; + +const weatherTool = { + type: "function", + function: { + name: "get_current_weather", + description: "Return the current weather for one location.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + location: { type: "string" }, + }, + required: ["location"], + }, + }, +} as const; + +function isRecord(value: unknown): value is JsonRecord { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function assertLoopbackBaseUrl(value: string): string { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new Error("protocol qualification base URL is invalid"); + } + if ( + parsed.protocol !== "http:" || + parsed.hostname !== "127.0.0.1" || + parsed.username !== "" || + parsed.password !== "" || + parsed.pathname !== "/" || + parsed.search !== "" || + parsed.hash !== "" || + !/^[1-9][0-9]{0,4}$/u.test(parsed.port) + ) { + throw new Error("protocol qualification must target an explicit loopback port"); + } + return parsed.toString().replace(/\/$/u, ""); +} + +function assertAuthorization(value: string): string { + if (!/^Bearer [0-9a-f]{64}$/u.test(value)) { + throw new Error("protocol qualification authorization is invalid"); + } + return value; +} + +function contentLength(response: Response): number | undefined { + const value = response.headers.get("content-length"); + if (value === null) return undefined; + if (!/^(?:0|[1-9][0-9]*)$/u.test(value)) { + throw new Error("protocol probe returned an invalid content length"); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new Error("protocol probe returned an invalid content length"); + } + return parsed; +} + +async function readBoundedBytes(response: Response, maximumBytes: number): Promise { + const declaredLength = contentLength(response); + if (declaredLength !== undefined && declaredLength > maximumBytes) { + throw new Error("protocol probe response exceeded its declarative byte bound"); + } + if (!response.body) return new Uint8Array(); + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + if (totalBytes > maximumBytes) { + await reader.cancel(); + throw new Error("protocol probe response exceeded its declarative byte bound"); + } + chunks.push(value); + } + const bytes = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return bytes; +} + +async function readJson( + response: Response, + expectedStatus: number, + maximumBytes: number, + label: string, +): Promise { + if (response.status !== expectedStatus) { + throw new Error(`${label} returned an unexpected HTTP status`); + } + if (!response.headers.get("content-type")?.toLowerCase().includes("application/json")) { + throw new Error(`${label} did not return JSON`); + } + const bytes = await readBoundedBytes(response, maximumBytes); + try { + return JSON.parse(new TextDecoder("utf8", { fatal: true }).decode(bytes)) as unknown; + } catch { + throw new Error(`${label} returned invalid JSON`); + } +} + +async function expectStatus( + response: Response, + expectedStatus: number, + maximumBytes: number, + label: string, +): Promise { + if (response.status !== expectedStatus) { + throw new Error(`${label} returned an unexpected HTTP status`); + } + await readBoundedBytes(response, maximumBytes); +} + +function requestSignal(timeoutMilliseconds: number): AbortSignal { + return AbortSignal.timeout(timeoutMilliseconds); +} + +function jsonRequest( + authorization: string, + body: unknown, + timeoutMilliseconds: number, + signal = requestSignal(timeoutMilliseconds), +): RequestInit { + return { + body: JSON.stringify(body), + headers: { + Authorization: authorization, + "Content-Type": "application/json", + }, + method: "POST", + signal, + }; +} + +function usageFrom(value: unknown): ProtocolEvidence["usage"] { + if (!isRecord(value)) throw new Error("chat usage was not returned"); + const promptTokens = value.prompt_tokens; + const completionTokens = value.completion_tokens; + const totalTokens = value.total_tokens; + if ( + !Number.isSafeInteger(promptTokens) || + Number(promptTokens) < 1 || + !Number.isSafeInteger(completionTokens) || + Number(completionTokens) < 1 || + !Number.isSafeInteger(totalTokens) || + Number(totalTokens) !== Number(promptTokens) + Number(completionTokens) + ) { + throw new Error("chat usage did not satisfy the token accounting contract"); + } + return { + completionTokens: Number(completionTokens), + ok: true, + promptTokens: Number(promptTokens), + totalTokens: Number(totalTokens), + }; +} + +export function validateModelsResponse(value: unknown, expectedModel: string): void { + if ( + !isRecord(value) || + value.object !== "list" || + !Array.isArray(value.data) || + value.data.length !== 1 || + !isRecord(value.data[0]) || + value.data[0].id !== expectedModel + ) { + throw new Error("models probe did not return the exact served model identity"); + } +} + +export function validateChatCompletionResponse(value: unknown, expectedModel: string): void { + if ( + !isRecord(value) || + value.object !== "chat.completion" || + value.model !== expectedModel || + !Array.isArray(value.choices) || + value.choices.length !== 1 || + !isRecord(value.choices[0]) || + !isRecord(value.choices[0].message) || + value.choices[0].message.role !== "assistant" || + typeof value.choices[0].message.content !== "string" || + value.choices[0].message.content.length < 1 || + value.choices[0].message.content.length > 4096 + ) { + throw new Error("authenticated chat completion probe failed its response contract"); + } +} + +export function validatePropertiesResponse( + value: unknown, + expectedContextSize: number, +): ProtocolEvidence["contextWindow"] { + if ( + !isRecord(value) || + value.total_slots !== 1 || + !isRecord(value.default_generation_settings) || + value.default_generation_settings.n_ctx !== expectedContextSize + ) { + throw new Error("properties probe did not prove the declarative context window"); + } + return { contextSize: expectedContextSize, ok: true, slots: 1 }; +} + +export function validateStructuredOutputResponse(value: unknown, expectedModel: string): void { + validateChatCompletionResponse(value, expectedModel); + const choice = (value as JsonRecord).choices as JsonRecord[]; + const message = choice[0]?.message as JsonRecord; + let content: unknown; + try { + content = JSON.parse(String(message.content)) as unknown; + } catch { + throw new Error("structured-output probe did not return valid JSON"); + } + if (!isRecord(content) || content.status !== "ready" || Object.keys(content).length !== 1) { + throw new Error("structured-output probe did not satisfy its JSON schema"); + } +} + +export function validateToolResultContinuationResponse( + value: unknown, + expectedModel: string, +): void { + validateChatCompletionResponse(value, expectedModel); + const choice = (value as JsonRecord).choices as JsonRecord[]; + const message = choice[0]?.message as JsonRecord; + let content: unknown; + try { + content = JSON.parse(String(message.content)) as unknown; + } catch { + throw new Error("tool-result continuation did not return valid JSON"); + } + if ( + !isRecord(content) || + content.conditions !== "clear" || + content.temperature_c !== 21 || + Object.keys(content).length !== 2 + ) { + throw new Error("tool-result continuation did not consume the supplied result"); + } +} + +export function validateToolCallResponse(value: unknown, expectedModel: string): ToolCall { + if ( + !isRecord(value) || + value.object !== "chat.completion" || + value.model !== expectedModel || + !Array.isArray(value.choices) || + value.choices.length !== 1 || + !isRecord(value.choices[0]) || + value.choices[0].finish_reason !== "tool_calls" || + !isRecord(value.choices[0].message) || + value.choices[0].message.role !== "assistant" || + !Array.isArray(value.choices[0].message.tool_calls) || + value.choices[0].message.tool_calls.length !== 1 || + !isRecord(value.choices[0].message.tool_calls[0]) + ) { + throw new Error("tool-call probe did not return one structured tool call"); + } + const raw = value.choices[0].message.tool_calls[0]; + if ( + raw.type !== "function" || + typeof raw.id !== "string" || + !/^[A-Za-z0-9_.:-]{1,256}$/u.test(raw.id) || + !isRecord(raw.function) || + raw.function.name !== "get_current_weather" + ) { + throw new Error("tool-call probe returned an invalid function identity"); + } + let args: unknown = raw.function.arguments; + if (typeof args === "string") { + if (args.length < 2 || args.length > 4096) { + throw new Error("tool-call probe returned invalid arguments"); + } + try { + args = JSON.parse(args) as unknown; + } catch { + throw new Error("tool-call probe returned invalid arguments"); + } + } + if ( + !isRecord(args) || + Object.keys(args).length !== 1 || + typeof args.location !== "string" || + args.location.length < 1 || + args.location.length > 256 + ) { + throw new Error("tool-call probe returned arguments outside the declared schema"); + } + return { arguments: args, id: raw.id, name: "get_current_weather", raw }; +} + +export function validateStreamingChatResponse( + source: string, + expectedModel: ProtocolEvidence["streamingChat"]["model"], + maximumEvents: number, +): Pick & { + usage: ProtocolEvidence["usage"]; +} { + let content = ""; + let done = false; + let events = 0; + let sawFinish = false; + let usage: ProtocolEvidence["usage"] | undefined; + for (const line of source.split(/\r?\n/u)) { + if (line === "") continue; + if (line.startsWith(":")) continue; + if (!line.startsWith("data: ")) { + throw new Error("streaming chat returned a non-SSE event"); + } + const data = line.slice("data: ".length); + if (data === "[DONE]") { + done = true; + continue; + } + events += 1; + if (events > maximumEvents) { + throw new Error("streaming chat exceeded its declarative event bound"); + } + let value: unknown; + try { + value = JSON.parse(data) as unknown; + } catch { + throw new Error("streaming chat returned invalid event JSON"); + } + if ( + !isRecord(value) || + value.object !== "chat.completion.chunk" || + value.model !== expectedModel || + !Array.isArray(value.choices) + ) { + throw new Error("streaming chat event failed its response contract"); + } + if (value.usage !== undefined && value.usage !== null) usage = usageFrom(value.usage); + for (const choice of value.choices) { + if (!isRecord(choice) || !isRecord(choice.delta)) { + throw new Error("streaming chat choice failed its response contract"); + } + if (choice.finish_reason === "stop" || choice.finish_reason === "length") sawFinish = true; + if (choice.delta.content !== undefined && choice.delta.content !== null) { + if (typeof choice.delta.content !== "string") { + throw new Error("streaming chat content was invalid"); + } + content += choice.delta.content; + } + } + } + if (!done || !sawFinish || content.length < 1 || content.length > 4096 || !usage) { + throw new Error("streaming chat did not complete with content, usage, and a terminal event"); + } + return { done: true, events, model: expectedModel, ok: true, usage }; +} + +function isAbortError(value: unknown): boolean { + return value instanceof Error && (value.name === "AbortError" || value.name === "TimeoutError"); +} + +async function recoveryCompletion( + fetchImpl: FetchImplementation, + url: string, + authorization: string, + model: string, + maxTokens: number, + timeoutMilliseconds: number, + maximumBytes: number, +): Promise { + const deadline = Date.now() + timeoutMilliseconds; + while (Date.now() < deadline) { + let response: Response; + try { + response = await fetchImpl( + url, + jsonRequest( + authorization, + { + max_tokens: maxTokens, + messages: [{ content: "Reply with one token.", role: "user" }], + model, + temperature: 0, + }, + timeoutMilliseconds, + ), + ); + } catch (error) { + if (!isAbortError(error) && !(error instanceof TypeError)) throw error; + await new Promise((resolve) => setTimeout(resolve, 250)); + continue; + } + if (response.status === 200) { + validateChatCompletionResponse( + await readJson(response, 200, maximumBytes, "recovery completion"), + model, + ); + return; + } + if (response.status !== 429 && response.status !== 503) { + throw new Error("serving slot recovery returned an unexpected HTTP status"); + } + await readBoundedBytes(response, maximumBytes); + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error("cancelled protocol request did not release the serving slot"); +} + +function assertExecutedProbeInventory( + plannedProbes: readonly ProtocolProbe[], + executedProbes: ReadonlySet, +): void { + const plannedProbeSet = new Set(plannedProbes); + if ( + plannedProbeSet.size !== plannedProbes.length || + executedProbes.size !== plannedProbeSet.size || + plannedProbes.some((probe) => !executedProbes.has(probe)) + ) { + throw new Error("protocol qualification did not execute every declarative probe"); + } +} + +async function cancellationProbe( + fetchImpl: FetchImplementation, + url: string, + authorization: string, + plan: LlamaCppDgxSparkExecutionPlan, + timeoutMilliseconds: number, +): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMilliseconds); + try { + const response = await fetchImpl( + url, + jsonRequest( + authorization, + { + max_tokens: plan.qualification.probeBounds.cancellationMaxTokens, + messages: [{ content: "Count upward without stopping.", role: "user" }], + model: plan.recipe.model.servedName, + stream: true, + temperature: 0, + }, + timeoutMilliseconds, + controller.signal, + ), + ); + if (response.status !== 200 || !response.body) { + throw new Error("cancellation probe did not start a streaming response"); + } + const reader = response.body.getReader(); + const first = await reader.read(); + if (first.done || !first.value || first.value.byteLength < 1) { + throw new Error("cancellation probe ended before cancellation"); + } + controller.abort(); + try { + await reader.read(); + } catch (error) { + if (!isAbortError(error)) throw error; + } + if (!controller.signal.aborted) throw new Error("cancellation probe was not aborted"); + } finally { + clearTimeout(timer); + } + await recoveryCompletion( + fetchImpl, + url, + authorization, + plan.recipe.model.servedName, + plan.qualification.probeBounds.maxTokens.synchronousChat, + timeoutMilliseconds, + plan.qualification.probeBounds.maxResponseBytes, + ); +} + +async function clientTimeoutProbe( + fetchImpl: FetchImplementation, + url: string, + authorization: string, + plan: LlamaCppDgxSparkExecutionPlan, + timeoutMilliseconds: number, +): Promise { + let aborted = false; + try { + const response = await fetchImpl( + url, + jsonRequest( + authorization, + { + max_tokens: plan.qualification.probeBounds.cancellationMaxTokens, + messages: [{ content: "Count upward without stopping.", role: "user" }], + model: plan.recipe.model.servedName, + stream: true, + temperature: 0, + }, + plan.qualification.probeBounds.clientTimeoutMilliseconds, + ), + ); + await readBoundedBytes(response, plan.qualification.probeBounds.maxResponseBytes); + } catch (error) { + if (!isAbortError(error)) throw error; + aborted = true; + } + if (!aborted) throw new Error("client-timeout probe completed outside its deadline contract"); + await recoveryCompletion( + fetchImpl, + url, + authorization, + plan.recipe.model.servedName, + plan.qualification.probeBounds.maxTokens.synchronousChat, + timeoutMilliseconds, + plan.qualification.probeBounds.maxResponseBytes, + ); +} + +export async function runLlamaCppDgxSparkProtocolQualification(options: { + readonly authorization: string; + readonly baseUrl: string; + readonly fetchImpl?: FetchImplementation; + readonly plan: LlamaCppDgxSparkExecutionPlan; +}): Promise { + const baseUrl = assertLoopbackBaseUrl(options.baseUrl); + const authorization = assertAuthorization(options.authorization); + const fetchImpl = options.fetchImpl ?? fetch; + const { plan } = options; + const model = plan.recipe.model.servedName; + const bounds = plan.qualification.probeBounds; + const timeoutMilliseconds = plan.recipe.serve.limits.requestTimeoutSeconds * 1000; + const chatUrl = `${baseUrl}/v1/chat/completions`; + const executedProbes = new Set(); + + const healthResponse = await fetchImpl(`${baseUrl}/health`, { + headers: { Authorization: authorization }, + signal: requestSignal(timeoutMilliseconds), + }); + await expectStatus(healthResponse, 200, bounds.maxResponseBytes, "health probe"); + executedProbes.add("health"); + + const modelsResponse = await fetchImpl(`${baseUrl}/v1/models`, { + headers: { Authorization: authorization }, + signal: requestSignal(timeoutMilliseconds), + }); + validateModelsResponse( + await readJson(modelsResponse, 200, bounds.maxResponseBytes, "models probe"), + model, + ); + executedProbes.add("models"); + + const propertiesResponse = await fetchImpl(`${baseUrl}/props`, { + headers: { Authorization: authorization }, + signal: requestSignal(timeoutMilliseconds), + }); + const contextWindow = validatePropertiesResponse( + await readJson(propertiesResponse, 200, bounds.maxResponseBytes, "properties probe"), + plan.recipe.serve.contextSize, + ); + executedProbes.add("context-window"); + + const authenticationResponse = await fetchImpl( + chatUrl, + jsonRequest( + `${authorization.slice(0, -1)}${authorization.endsWith("0") ? "1" : "0"}`, + { + max_tokens: 1, + messages: [{ content: "This request must be rejected.", role: "user" }], + model, + }, + timeoutMilliseconds, + ), + ); + await expectStatus(authenticationResponse, 401, bounds.maxResponseBytes, "authentication probe"); + executedProbes.add("authentication"); + + const malformedResponse = await fetchImpl(chatUrl, { + body: "{", + headers: { + Authorization: authorization, + "Content-Type": "application/json", + }, + method: "POST", + signal: requestSignal(timeoutMilliseconds), + }); + await expectStatus(malformedResponse, 400, bounds.maxResponseBytes, "malformed-request probe"); + executedProbes.add("malformed-request"); + + const synchronousResponse = await fetchImpl( + chatUrl, + jsonRequest( + authorization, + { + max_tokens: bounds.maxTokens.synchronousChat, + messages: [{ content: "Return one short readiness token.", role: "user" }], + model, + temperature: 0, + }, + timeoutMilliseconds, + ), + ); + const synchronousValue = await readJson( + synchronousResponse, + 200, + bounds.maxResponseBytes, + "synchronous chat probe", + ); + validateChatCompletionResponse(synchronousValue, model); + executedProbes.add("synchronous-chat"); + const usage = usageFrom(isRecord(synchronousValue) ? synchronousValue.usage : undefined); + executedProbes.add("usage"); + + const streamingResponse = await fetchImpl( + chatUrl, + jsonRequest( + authorization, + { + max_tokens: bounds.maxTokens.streamingChat, + messages: [{ content: "Reply with exactly: ready", role: "user" }], + model, + stream: true, + stream_options: { include_usage: true }, + temperature: 0, + }, + timeoutMilliseconds, + ), + ); + if ( + streamingResponse.status !== 200 || + !streamingResponse.headers.get("content-type")?.toLowerCase().includes("text/event-stream") + ) { + throw new Error("streaming chat did not return an SSE response"); + } + const streamingSource = new TextDecoder("utf8", { fatal: true }).decode( + await readBoundedBytes(streamingResponse, bounds.maxResponseBytes), + ); + const streaming = validateStreamingChatResponse(streamingSource, model, bounds.maxStreamEvents); + executedProbes.add("streaming-chat"); + + const structuredResponse = await fetchImpl( + chatUrl, + jsonRequest( + authorization, + { + max_tokens: bounds.maxTokens.structuredOutput, + messages: [ + { + content: "Report the requested qualification status.", + role: "user", + }, + ], + model, + response_format: { + type: "json_schema", + json_schema: { + name: "qualification_status", + strict: true, + schema: { + type: "object", + additionalProperties: false, + properties: { status: { const: "ready", type: "string" } }, + required: ["status"], + }, + }, + }, + temperature: 0, + }, + timeoutMilliseconds, + ), + ); + const structuredValue = await readJson( + structuredResponse, + 200, + bounds.maxResponseBytes, + "structured-output probe", + ); + validateStructuredOutputResponse(structuredValue, model); + executedProbes.add("structured-output"); + + const toolMessages = [ + { + content: "Use the available tool to get the weather in Seattle.", + role: "user", + }, + ]; + const toolResponse = await fetchImpl( + chatUrl, + jsonRequest( + authorization, + { + max_tokens: bounds.maxTokens.toolCall, + messages: toolMessages, + model, + parallel_tool_calls: false, + temperature: 0, + tool_choice: "required", + tools: [weatherTool], + }, + timeoutMilliseconds, + ), + ); + const toolValue = await readJson(toolResponse, 200, bounds.maxResponseBytes, "tool-call probe"); + const toolCall = validateToolCallResponse(toolValue, model); + executedProbes.add("tool-call"); + + const continuationResponse = await fetchImpl( + chatUrl, + jsonRequest( + authorization, + { + max_tokens: bounds.maxTokens.toolResultContinuation, + messages: [ + ...toolMessages, + { content: null, role: "assistant", tool_calls: [toolCall.raw] }, + { + content: JSON.stringify({ conditions: "clear", temperature_c: 21 }), + role: "tool", + tool_call_id: toolCall.id, + }, + ], + model, + response_format: { + type: "json_schema", + json_schema: { + name: "weather_result", + strict: true, + schema: { + type: "object", + additionalProperties: false, + properties: { + conditions: { const: "clear", type: "string" }, + temperature_c: { const: 21, type: "number" }, + }, + required: ["conditions", "temperature_c"], + }, + }, + }, + temperature: 0, + tool_choice: "none", + tools: [weatherTool], + }, + timeoutMilliseconds, + ), + ); + const continuationValue = await readJson( + continuationResponse, + 200, + bounds.maxResponseBytes, + "tool-result continuation probe", + ); + validateToolResultContinuationResponse(continuationValue, model); + executedProbes.add("tool-result-continuation"); + + await cancellationProbe(fetchImpl, chatUrl, authorization, plan, timeoutMilliseconds); + executedProbes.add("cancellation"); + await clientTimeoutProbe(fetchImpl, chatUrl, authorization, plan, timeoutMilliseconds); + executedProbes.add("client-timeout"); + assertExecutedProbeInventory(plan.qualification.probes, executedProbes); + + return { + authentication: { httpStatus: 401, ok: true }, + cancellation: { aborted: true, ok: true, recovered: true }, + contextWindow, + health: { httpStatus: 200, ok: true }, + malformedRequest: { httpStatus: 400, ok: true }, + models: { httpStatus: 200, model, ok: true }, + clientTimeout: { + aborted: true, + limitMilliseconds: bounds.clientTimeoutMilliseconds, + ok: true, + recovered: true, + }, + streamingChat: { + done: streaming.done, + events: streaming.events, + httpStatus: 200, + model, + ok: true, + }, + structuredOutput: { httpStatus: 200, model, ok: true, schemaMatched: true }, + synchronousChat: { httpStatus: 200, model, ok: true }, + toolCall: { + argumentsValid: true, + httpStatus: 200, + name: "get_current_weather", + ok: true, + }, + toolResultContinuation: { httpStatus: 200, model, ok: true }, + usage, + }; +} diff --git a/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts b/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts index abc825d6119..ecce5141a41 100644 --- a/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts +++ b/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts @@ -36,6 +36,27 @@ export const LLAMA_CPP_DGX_SPARK_CUDA_RUNTIME_BASE = export const LLAMA_CPP_DGX_SPARK_TOOL_IMAGE = "nvcr.io/nvidia/vllm@sha256:9204569b17ee4c0eff75194b8e6e458479c8aee18953b5ab9cf359fcdac659e2" as const; export const LLAMA_CPP_DGX_SPARK_MINIMUM_DRIVER_VERSION = "580.65.06" as const; +export const LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES = [ + "health", + "models", + "synchronous-chat", + "streaming-chat", + "usage", + "structured-output", + "tool-call", + "tool-result-continuation", + "context-window", + "authentication", + "malformed-request", + "cancellation", + "client-timeout", +] as const; + +const LLAMA_CPP_DGX_SPARK_CLIENT_TIMEOUT_RANGE = { maximum: 10_000, minimum: 10 } as const; +const LLAMA_CPP_DGX_SPARK_CONTEXT_SIZE_RANGE = { + maximum: 1024 * 1024, + minimum: 1024, +} as const; export const LLAMA_CPP_DGX_SPARK_SHA_PATTERN = /^[a-f0-9]{40}$/u; export const LLAMA_CPP_DGX_SPARK_DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u; @@ -69,7 +90,20 @@ export type LlamaCppDgxSparkQualificationPlan = { readonly id: typeof LLAMA_CPP_DGX_SPARK_MODEL_ID; }; readonly platform: typeof LLAMA_CPP_DGX_SPARK_QUALIFICATION_PLATFORM; - readonly probes: readonly ["health", "completion"]; + readonly probeBounds: { + readonly cancellationMaxTokens: number; + readonly clientTimeoutMilliseconds: number; + readonly maxResponseBytes: number; + readonly maxStreamEvents: number; + readonly maxTokens: { + readonly streamingChat: number; + readonly structuredOutput: number; + readonly synchronousChat: number; + readonly toolCall: number; + readonly toolResultContinuation: number; + }; + }; + readonly probes: typeof LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES; readonly profile: typeof LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE; readonly recipeRef: typeof LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE; readonly required: true; @@ -113,7 +147,23 @@ export type LlamaCppDgxSparkExecutionPlan = { readonly revision: typeof LLAMA_CPP_DGX_SPARK_SOURCE_REVISION; }; }; + readonly qualification: { + readonly probeBounds: LlamaCppDgxSparkQualificationPlan["probeBounds"]; + readonly probes: typeof LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES; + }; readonly recipe: { + readonly capabilities: { + readonly agents: readonly []; + readonly embeddings: false; + readonly multimodal: false; + readonly parallelToolCalls: false; + readonly protocols: readonly ["openai-completions"]; + readonly reranking: false; + readonly responsesApi: false; + readonly streaming: true; + readonly structuredOutputs: true; + readonly toolCalls: true; + }; readonly id: typeof LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE; readonly model: { readonly acquisition: { @@ -239,15 +289,74 @@ export type LlamaCppDgxSparkQualificationReceipt = { readonly id: typeof LLAMA_CPP_DGX_SPARK_MODEL_ID; }; readonly probes: { - readonly completion: { + readonly authentication: { + readonly httpStatus: 401; + readonly ok: true; + }; + readonly cancellation: { + readonly aborted: true; + readonly ok: true; + readonly recovered: true; + }; + readonly contextWindow: { + readonly contextSize: number; + readonly ok: true; + readonly slots: 1; + }; + readonly health: { + readonly httpStatus: 200; + readonly ok: true; + }; + readonly malformedRequest: { + readonly httpStatus: 400; + readonly ok: true; + }; + readonly models: { readonly httpStatus: 200; readonly model: typeof LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID; readonly ok: true; }; - readonly health: { + readonly clientTimeout: { + readonly aborted: true; + readonly limitMilliseconds: number; + readonly ok: true; + readonly recovered: true; + }; + readonly streamingChat: { + readonly done: true; + readonly events: number; + readonly httpStatus: 200; + readonly model: typeof LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID; + readonly ok: true; + }; + readonly structuredOutput: { + readonly httpStatus: 200; + readonly model: typeof LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID; + readonly ok: true; + readonly schemaMatched: true; + }; + readonly synchronousChat: { + readonly httpStatus: 200; + readonly model: typeof LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID; + readonly ok: true; + }; + readonly toolCall: { + readonly argumentsValid: true; readonly httpStatus: 200; + readonly name: "get_current_weather"; readonly ok: true; }; + readonly toolResultContinuation: { + readonly httpStatus: 200; + readonly model: typeof LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID; + readonly ok: true; + }; + readonly usage: { + readonly completionTokens: number; + readonly ok: true; + readonly promptTokens: number; + readonly totalTokens: number; + }; }; readonly repository: "NVIDIA/NemoClaw"; readonly run: { @@ -318,6 +427,82 @@ function boundedInteger(value: unknown, label: string, minimum: number, maximum: return Number(value); } +function parseProtocolProbeBounds( + value: unknown, +): LlamaCppDgxSparkQualificationPlan["probeBounds"] { + const probeBounds = record(value, "llama.cpp DGX Spark qualification probe bounds"); + requireExactKeys( + probeBounds, + [ + "cancellationMaxTokens", + "clientTimeoutMilliseconds", + "maxResponseBytes", + "maxStreamEvents", + "maxTokens", + ], + "qualification probe bounds", + ); + const maxTokens = record(probeBounds.maxTokens, "qualification probe token bounds"); + requireExactKeys( + maxTokens, + ["streamingChat", "structuredOutput", "synchronousChat", "toolCall", "toolResultContinuation"], + "qualification probe token bounds", + ); + return { + cancellationMaxTokens: boundedInteger( + probeBounds.cancellationMaxTokens, + "qualification cancellation token bound", + 128, + 32_768, + ), + clientTimeoutMilliseconds: boundedInteger( + probeBounds.clientTimeoutMilliseconds, + "qualification client timeout", + LLAMA_CPP_DGX_SPARK_CLIENT_TIMEOUT_RANGE.minimum, + LLAMA_CPP_DGX_SPARK_CLIENT_TIMEOUT_RANGE.maximum, + ), + maxResponseBytes: boundedInteger( + probeBounds.maxResponseBytes, + "qualification response byte bound", + 64 * 1024, + 64 * 1024 * 1024, + ), + maxStreamEvents: boundedInteger( + probeBounds.maxStreamEvents, + "qualification stream event bound", + 8, + 4096, + ), + maxTokens: { + streamingChat: boundedInteger( + maxTokens.streamingChat, + "qualification streaming token bound", + 1, + 512, + ), + structuredOutput: boundedInteger( + maxTokens.structuredOutput, + "qualification structured-output token bound", + 1, + 512, + ), + synchronousChat: boundedInteger( + maxTokens.synchronousChat, + "qualification synchronous token bound", + 1, + 256, + ), + toolCall: boundedInteger(maxTokens.toolCall, "qualification tool-call token bound", 1, 1024), + toolResultContinuation: boundedInteger( + maxTokens.toolResultContinuation, + "qualification tool-result token bound", + 1, + 512, + ), + }, + }; +} + function requiredSha(value: unknown, label: string): string { if (typeof value !== "string" || !LLAMA_CPP_DGX_SPARK_SHA_PATTERN.test(value)) { throw new Error(`${label} is invalid`); @@ -409,6 +594,7 @@ export function parseLlamaCppDgxSparkQualificationPlan( "gpu", "model", "platform", + "probeBounds", "probes", "profile", "recipeRef", @@ -425,6 +611,7 @@ export function parseLlamaCppDgxSparkQualificationPlan( requireExactKeys(gpu, ["cpuFallback", "fullOffload", "vendor"], "qualification GPU"); const model = record(plan.model, "llama.cpp DGX Spark qualification model"); requireExactKeys(model, ["digest", "hostPath", "id"], "qualification model"); + const parsedProbeBounds = parseProtocolProbeBounds(plan.probeBounds); if ( plan.required !== true || plan.profile !== LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE || @@ -435,7 +622,7 @@ export function parseLlamaCppDgxSparkQualificationPlan( gpu.cpuFallback !== "reject" || model.id !== LLAMA_CPP_DGX_SPARK_MODEL_ID || model.digest !== LLAMA_CPP_DGX_SPARK_MODEL_DIGEST || - JSON.stringify(plan.probes) !== JSON.stringify(["health", "completion"]) + JSON.stringify(plan.probes) !== JSON.stringify(LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES) ) { throw new Error("llama.cpp DGX Spark qualification plan is invalid"); } @@ -449,7 +636,8 @@ export function parseLlamaCppDgxSparkQualificationPlan( id: LLAMA_CPP_DGX_SPARK_MODEL_ID, }, platform: LLAMA_CPP_DGX_SPARK_QUALIFICATION_PLATFORM, - probes: ["health", "completion"], + probeBounds: parsedProbeBounds, + probes: LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES, profile: LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE, recipeRef: LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE, required: true, @@ -464,13 +652,22 @@ export function parseLlamaCppDgxSparkExecutionPlan( const plan = record(value, "compiled llama.cpp DGX Spark qualification plan"); requireExactKeys( plan, - ["contractVersion", "imageBuild", "recipe"], + ["contractVersion", "imageBuild", "qualification", "recipe"], "compiled llama.cpp DGX Spark qualification plan", ); if (plan.contractVersion !== 1) { throw new Error("compiled llama.cpp DGX Spark qualification plan version is invalid"); } + const qualification = record(plan.qualification, "compiled protocol qualification"); + requireExactKeys(qualification, ["probeBounds", "probes"], "compiled protocol qualification"); + if ( + JSON.stringify(qualification.probes) !== JSON.stringify(LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES) + ) { + throw new Error("compiled llama.cpp DGX Spark protocol probes are invalid"); + } + const protocolProbeBounds = parseProtocolProbeBounds(qualification.probeBounds); + const imageBuild = record(plan.imageBuild, "compiled qualification image build"); requireExactKeys( imageBuild, @@ -522,9 +719,50 @@ export function parseLlamaCppDgxSparkExecutionPlan( const recipe = record(plan.recipe, "compiled llama.cpp DGX Spark qualification recipe"); requireExactKeys( recipe, - ["id", "model", "policy", "readiness", "runtime", "serve", "server", "surfaces"], + [ + "capabilities", + "id", + "model", + "policy", + "readiness", + "runtime", + "serve", + "server", + "surfaces", + ], "compiled llama.cpp DGX Spark qualification recipe", ); + const capabilities = record(recipe.capabilities, "compiled qualification capabilities"); + requireExactKeys( + capabilities, + [ + "agents", + "embeddings", + "multimodal", + "parallelToolCalls", + "protocols", + "reranking", + "responsesApi", + "streaming", + "structuredOutputs", + "toolCalls", + ], + "compiled qualification capabilities", + ); + if ( + JSON.stringify(capabilities.agents) !== "[]" || + JSON.stringify(capabilities.protocols) !== JSON.stringify(["openai-completions"]) || + capabilities.streaming !== true || + capabilities.toolCalls !== true || + capabilities.structuredOutputs !== true || + capabilities.parallelToolCalls !== false || + capabilities.responsesApi !== false || + capabilities.embeddings !== false || + capabilities.reranking !== false || + capabilities.multimodal !== false + ) { + throw new Error("compiled llama.cpp DGX Spark capability claims are invalid"); + } const model = record(recipe.model, "compiled qualification recipe model"); requireExactKeys( model, @@ -692,8 +930,8 @@ export function parseLlamaCppDgxSparkExecutionPlan( const contextSize = boundedInteger( serve.contextSize, "compiled qualification context size", - 1024, - 1024 * 1024, + LLAMA_CPP_DGX_SPARK_CONTEXT_SIZE_RANGE.minimum, + LLAMA_CPP_DGX_SPARK_CONTEXT_SIZE_RANGE.maximum, ); const batchSize = boundedInteger(serve.batchSize, "compiled qualification batch size", 1, 8192); const microBatchSize = boundedInteger( @@ -782,7 +1020,23 @@ export function parseLlamaCppDgxSparkExecutionPlan( revision: LLAMA_CPP_DGX_SPARK_SOURCE_REVISION, }, }, + qualification: { + probeBounds: protocolProbeBounds, + probes: LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES, + }, recipe: { + capabilities: { + agents: [], + protocols: ["openai-completions"], + streaming: true, + toolCalls: true, + structuredOutputs: true, + parallelToolCalls: false, + responsesApi: false, + embeddings: false, + reranking: false, + multimodal: false, + }, id: LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE, model: { acquisition: { downloaderImage: LLAMA_CPP_DGX_SPARK_TOOL_IMAGE }, @@ -1003,17 +1257,137 @@ export function parseLlamaCppDgxSparkQualificationReceipt( } const probes = record(receipt.probes, "llama.cpp DGX Spark qualification receipt probes"); - requireExactKeys(probes, ["completion", "health"], "receipt probes"); + requireExactKeys( + probes, + [ + "authentication", + "cancellation", + "contextWindow", + "health", + "malformedRequest", + "models", + "clientTimeout", + "streamingChat", + "structuredOutput", + "synchronousChat", + "toolCall", + "toolResultContinuation", + "usage", + ], + "receipt probes", + ); const health = record(probes.health, "llama.cpp DGX Spark health probe"); requireExactKeys(health, ["httpStatus", "ok"], "health probe"); - const completion = record(probes.completion, "llama.cpp DGX Spark completion probe"); - requireExactKeys(completion, ["httpStatus", "model", "ok"], "completion probe"); + const models = record(probes.models, "llama.cpp DGX Spark models probe"); + requireExactKeys(models, ["httpStatus", "model", "ok"], "models probe"); + const synchronousChat = record(probes.synchronousChat, "synchronous chat probe"); + requireExactKeys(synchronousChat, ["httpStatus", "model", "ok"], "synchronous chat probe"); + const streamingChat = record(probes.streamingChat, "streaming chat probe"); + requireExactKeys( + streamingChat, + ["done", "events", "httpStatus", "model", "ok"], + "streaming chat probe", + ); + const streamingEvents = boundedInteger( + streamingChat.events, + "streaming chat event count", + 1, + 4096, + ); + const usage = record(probes.usage, "chat usage probe"); + requireExactKeys( + usage, + ["completionTokens", "ok", "promptTokens", "totalTokens"], + "chat usage probe", + ); + const promptTokens = safeInteger(usage.promptTokens, "prompt token count", 1024 * 1024); + const completionTokens = safeInteger( + usage.completionTokens, + "completion token count", + 1024 * 1024, + ); + const totalTokens = safeInteger(usage.totalTokens, "total token count", 2 * 1024 * 1024); + const structuredOutput = record(probes.structuredOutput, "structured-output probe"); + requireExactKeys( + structuredOutput, + ["httpStatus", "model", "ok", "schemaMatched"], + "structured-output probe", + ); + const toolCall = record(probes.toolCall, "tool-call probe"); + requireExactKeys(toolCall, ["argumentsValid", "httpStatus", "name", "ok"], "tool-call probe"); + const toolResultContinuation = record( + probes.toolResultContinuation, + "tool-result continuation probe", + ); + requireExactKeys( + toolResultContinuation, + ["httpStatus", "model", "ok"], + "tool-result continuation probe", + ); + const contextWindow = record(probes.contextWindow, "context-window probe"); + requireExactKeys(contextWindow, ["contextSize", "ok", "slots"], "context-window probe"); + const contextSize = boundedInteger( + contextWindow.contextSize, + "qualified context size", + LLAMA_CPP_DGX_SPARK_CONTEXT_SIZE_RANGE.minimum, + LLAMA_CPP_DGX_SPARK_CONTEXT_SIZE_RANGE.maximum, + ); + const authentication = record(probes.authentication, "authentication probe"); + requireExactKeys(authentication, ["httpStatus", "ok"], "authentication probe"); + const malformedRequest = record(probes.malformedRequest, "malformed-request probe"); + requireExactKeys(malformedRequest, ["httpStatus", "ok"], "malformed-request probe"); + const cancellation = record(probes.cancellation, "cancellation probe"); + requireExactKeys(cancellation, ["aborted", "ok", "recovered"], "cancellation probe"); + const clientTimeout = record(probes.clientTimeout, "client-timeout probe"); + requireExactKeys( + clientTimeout, + ["aborted", "limitMilliseconds", "ok", "recovered"], + "client-timeout probe", + ); + const clientTimeoutMilliseconds = boundedInteger( + clientTimeout.limitMilliseconds, + "qualification client timeout", + LLAMA_CPP_DGX_SPARK_CLIENT_TIMEOUT_RANGE.minimum, + LLAMA_CPP_DGX_SPARK_CLIENT_TIMEOUT_RANGE.maximum, + ); if ( health.ok !== true || health.httpStatus !== 200 || - completion.ok !== true || - completion.httpStatus !== 200 || - completion.model !== LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID + models.ok !== true || + models.httpStatus !== 200 || + models.model !== LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID || + synchronousChat.ok !== true || + synchronousChat.httpStatus !== 200 || + synchronousChat.model !== LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID || + streamingChat.ok !== true || + streamingChat.httpStatus !== 200 || + streamingChat.model !== LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID || + streamingChat.done !== true || + usage.ok !== true || + totalTokens !== promptTokens + completionTokens || + structuredOutput.ok !== true || + structuredOutput.httpStatus !== 200 || + structuredOutput.model !== LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID || + structuredOutput.schemaMatched !== true || + toolCall.ok !== true || + toolCall.httpStatus !== 200 || + toolCall.name !== "get_current_weather" || + toolCall.argumentsValid !== true || + toolResultContinuation.ok !== true || + toolResultContinuation.httpStatus !== 200 || + toolResultContinuation.model !== LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID || + contextWindow.ok !== true || + contextWindow.slots !== 1 || + authentication.ok !== true || + authentication.httpStatus !== 401 || + malformedRequest.ok !== true || + malformedRequest.httpStatus !== 400 || + cancellation.ok !== true || + cancellation.aborted !== true || + cancellation.recovered !== true || + clientTimeout.ok !== true || + clientTimeout.aborted !== true || + clientTimeout.recovered !== true ) { throw new Error("llama.cpp DGX Spark qualification probes did not pass"); } @@ -1067,12 +1441,52 @@ export function parseLlamaCppDgxSparkQualificationReceipt( id: LLAMA_CPP_DGX_SPARK_MODEL_ID, }, probes: { - completion: { + authentication: { httpStatus: 401, ok: true }, + cancellation: { aborted: true, ok: true, recovered: true }, + contextWindow: { contextSize, ok: true, slots: 1 }, + health: { httpStatus: 200, ok: true }, + malformedRequest: { httpStatus: 400, ok: true }, + models: { httpStatus: 200, model: LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, ok: true, }, - health: { httpStatus: 200, ok: true }, + clientTimeout: { + aborted: true, + limitMilliseconds: clientTimeoutMilliseconds, + ok: true, + recovered: true, + }, + streamingChat: { + done: true, + events: streamingEvents, + httpStatus: 200, + model: LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, + ok: true, + }, + structuredOutput: { + httpStatus: 200, + model: LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, + ok: true, + schemaMatched: true, + }, + synchronousChat: { + httpStatus: 200, + model: LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, + ok: true, + }, + toolCall: { + argumentsValid: true, + httpStatus: 200, + name: "get_current_weather", + ok: true, + }, + toolResultContinuation: { + httpStatus: 200, + model: LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, + ok: true, + }, + usage: { completionTokens, ok: true, promptTokens, totalTokens }, }, repository: "NVIDIA/NemoClaw", run: { attempt: expected.runAttempt, id: expected.runId }, diff --git a/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts b/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts index 74d0c9f280d..bceb1779e8f 100644 --- a/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts +++ b/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts @@ -12,6 +12,11 @@ import { buildLlamaCppHostLocalDockerArgv, type VerifiedLocalModelArtifact, } from "../../src/lib/inference/llama-cpp/host-local-runtime.ts"; +import { + runLlamaCppDgxSparkProtocolQualification, + validateChatCompletionResponse, + validateModelsResponse, +} from "./llama-cpp-dgx-spark-protocol-qualification.mts"; import { LLAMA_CPP_DGX_SPARK_MODEL_PATH_PATTERN, LLAMA_CPP_DGX_SPARK_QUALIFICATION_IMAGE_REPOSITORY, @@ -21,6 +26,8 @@ import { parseLlamaCppDgxSparkExecutionPlan, } from "./llama-cpp-dgx-spark-qualification-contract.mts"; +export { validateChatCompletionResponse, validateModelsResponse }; + const sha256Pattern = /^sha256:[0-9a-f]{64}$/u; const gitShaPattern = /^[0-9a-f]{40}$/u; const runIdPattern = /^[1-9][0-9]{0,19}$/u; @@ -384,36 +391,6 @@ export function parseNvidiaSmi( return { count: 1, driverVersion, name: "NVIDIA GB10" }; } -export function validateModelsResponse(value: unknown, expectedModel: string): void { - if ( - !isRecord(value) || - value.object !== "list" || - !Array.isArray(value.data) || - value.data.length !== 1 || - !isRecord(value.data[0]) || - value.data[0].id !== expectedModel - ) { - throw new Error("models probe did not return the exact served model identity"); - } -} - -export function validateChatCompletionResponse(value: unknown, expectedModel: string): void { - if ( - !isRecord(value) || - value.object !== "chat.completion" || - value.model !== expectedModel || - !Array.isArray(value.choices) || - value.choices.length < 1 || - !isRecord(value.choices[0]) || - !isRecord(value.choices[0].message) || - typeof value.choices[0].message.content !== "string" || - value.choices[0].message.content.length < 1 || - value.choices[0].message.content.length > 4096 - ) { - throw new Error("authenticated chat completion probe failed its response contract"); - } -} - function runCommand( command: string, args: string[], @@ -771,25 +748,6 @@ function resolveLoopbackPort(containerName: string, containerPort: number): numb return requiredInteger(Number.parseInt(match[1] ?? "0", 10), "loopback port", 1024, 65_535); } -async function requestJson( - url: string, - init: RequestInit, - timeoutMilliseconds: number, - maximumBytes: number, -): Promise { - const response = await fetch(url, { ...init, signal: AbortSignal.timeout(timeoutMilliseconds) }); - if (!response.ok) throw new Error("qualification HTTP probe returned a non-success status"); - const body = await response.text(); - if (Buffer.byteLength(body) < 1 || Buffer.byteLength(body) > maximumBytes) { - throw new Error("qualification HTTP probe response exceeded its bound"); - } - try { - return JSON.parse(body) as unknown; - } catch { - throw new Error("qualification HTTP probe did not return JSON"); - } -} - async function waitForHealth(port: number, timeoutSeconds: number): Promise { const deadline = Date.now() + timeoutSeconds * 1000; while (Date.now() < deadline) { @@ -920,29 +878,11 @@ async function runQualification( ); await waitForHealth(loopbackPort, plan.recipe.readiness.timeoutSeconds); const authorization = `Bearer ${apiKey}`; - const models = await requestJson( - `http://127.0.0.1:${loopbackPort}/v1/models`, - { headers: { Authorization: authorization } }, - 10_000, - 1024 * 1024, - ); - validateModelsResponse(models, plan.recipe.readiness.expectedModel); - const completion = await requestJson( - `http://127.0.0.1:${loopbackPort}/v1/chat/completions`, - { - body: JSON.stringify({ - max_tokens: 16, - messages: [{ content: "Return one short readiness token.", role: "user" }], - model: plan.recipe.model.servedName, - temperature: 0, - }), - headers: { Authorization: authorization, "Content-Type": "application/json" }, - method: "POST", - }, - plan.recipe.serve.limits.requestTimeoutSeconds * 1000, - 16 * 1024 * 1024, - ); - validateChatCompletionResponse(completion, plan.recipe.model.servedName); + const probes = await runLlamaCppDgxSparkProtocolQualification({ + authorization, + baseUrl: `http://127.0.0.1:${loopbackPort}`, + plan, + }); const offload = validateStartupLog( runCommand("docker", ["logs", "--tail", "20000", names.containerName], { maximumBytes: 16 * 1024 * 1024, @@ -980,14 +920,7 @@ async function runQualification( fullOffload: true, ...offload, }, - probes: { - completion: { - httpStatus: 200, - model: plan.recipe.model.servedName, - ok: true, - }, - health: { httpStatus: 200, ok: true }, - }, + probes, }; } catch (error) { failure = error; diff --git a/test/llama-cpp-dgx-spark-protocol-qualification.test.ts b/test/llama-cpp-dgx-spark-protocol-qualification.test.ts new file mode 100644 index 00000000000..0fce416b23d --- /dev/null +++ b/test/llama-cpp-dgx-spark-protocol-qualification.test.ts @@ -0,0 +1,374 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { loadLlamaCppImageConfig } from "../scripts/checks/export-llama-cpp-image-config.mts"; +import { + runLlamaCppDgxSparkProtocolQualification, + validatePropertiesResponse, + validateStreamingChatResponse, + validateStructuredOutputResponse, + validateToolCallResponse, + validateToolResultContinuationResponse, +} from "../scripts/checks/llama-cpp-dgx-spark-protocol-qualification.mts"; +import { parseLlamaCppDgxSparkExecutionPlan } from "../scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts"; + +const AUTHORIZATION = `Bearer ${"a".repeat(64)}`; +const MODEL = "nvidia-nemotron-3-nano-30b-a3b"; + +const config = loadLlamaCppImageConfig(); +const compiledPlan = parseLlamaCppDgxSparkExecutionPlan( + JSON.parse(config.publication_qualification_plan) as unknown, + config.publication_qualification_plan_sha256, +); +const plan = { + ...compiledPlan, + qualification: { + ...compiledPlan.qualification, + probeBounds: { + ...compiledPlan.qualification.probeBounds, + clientTimeoutMilliseconds: 10, + }, + }, +}; + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + headers: { "Content-Type": "application/json" }, + status, + }); +} + +function completion(content = "ready", usage = true): JsonObject { + return { + choices: [{ message: { content, role: "assistant" } }], + model: MODEL, + object: "chat.completion", + ...(usage ? { usage: { completion_tokens: 2, prompt_tokens: 5, total_tokens: 7 } } : {}), + }; +} + +type JsonObject = Record; + +function hangingStream(signal: AbortSignal | null | undefined): Response { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('data: {"started":true}\n\n')); + signal?.addEventListener( + "abort", + () => controller.error(new DOMException("aborted", "AbortError")), + { once: true }, + ); + }, + }); + return new Response(stream, { + headers: { "Content-Type": "text/event-stream" }, + status: 200, + }); +} + +describe("llama.cpp DGX Spark protocol qualification", () => { + it("accepts SSE keepalives and exact streaming, tool, and context evidence (#8144)", () => { + const stream = [ + ": keepalive", + `data: ${JSON.stringify({ + choices: [ + { + delta: { content: "ready", role: "assistant" }, + finish_reason: null, + }, + ], + model: MODEL, + object: "chat.completion.chunk", + })}`, + `data: ${JSON.stringify({ + choices: [{ delta: {}, finish_reason: "stop" }], + model: MODEL, + object: "chat.completion.chunk", + })}`, + `data: ${JSON.stringify({ + choices: [], + model: MODEL, + object: "chat.completion.chunk", + usage: { completion_tokens: 1, prompt_tokens: 2, total_tokens: 3 }, + })}`, + "data: [DONE]", + "", + ].join("\n"); + expect(validateStreamingChatResponse(stream, MODEL, 8)).toMatchObject({ + done: true, + events: 3, + usage: { completionTokens: 1, promptTokens: 2, totalTokens: 3 }, + }); + expect(() => + validateStructuredOutputResponse(completion('{"status":"ready"}', false), MODEL), + ).not.toThrow(); + expect( + validateToolCallResponse( + { + choices: [ + { + finish_reason: "tool_calls", + message: { + role: "assistant", + tool_calls: [ + { + function: { + arguments: '{"location":"Seattle"}', + name: "get_current_weather", + }, + id: "call_1", + type: "function", + }, + ], + }, + }, + ], + model: MODEL, + object: "chat.completion", + }, + MODEL, + ), + ).toMatchObject({ arguments: { location: "Seattle" }, id: "call_1" }); + expect(() => + validateToolResultContinuationResponse( + completion('{"conditions":"clear","temperature_c":21}', false), + MODEL, + ), + ).not.toThrow(); + expect( + validatePropertiesResponse( + { default_generation_settings: { n_ctx: 262144 }, total_slots: 1 }, + 262144, + ), + ).toEqual({ + contextSize: 262144, + ok: true, + slots: 1, + }); + }); + + it("rejects malformed or unbounded protocol evidence without exposing response content (#8144)", () => { + expect(() => + validateStreamingChatResponse( + [ + `data: ${JSON.stringify({ choices: [], model: MODEL, object: "chat.completion.chunk" })}`, + `data: ${JSON.stringify({ choices: [], model: MODEL, object: "chat.completion.chunk" })}`, + "data: [DONE]", + ].join("\n"), + MODEL, + 1, + ), + ).toThrow("event bound"); + expect(() => + validateStructuredOutputResponse(completion('{"status":"wrong"}', false), MODEL), + ).toThrow("JSON schema"); + expect(() => + validatePropertiesResponse( + { default_generation_settings: { n_ctx: 131072 }, total_slots: 1 }, + 262144, + ), + ).toThrow("context window"); + expect(() => + validateToolCallResponse( + { + choices: [ + { + finish_reason: "tool_calls", + message: { + role: "assistant", + tool_calls: [ + { + function: { + arguments: '{"location":"Seattle","shell":"id"}', + name: "get_current_weather", + }, + id: "call_1", + type: "function", + }, + ], + }, + }, + ], + model: MODEL, + object: "chat.completion", + }, + MODEL, + ), + ).toThrow("declared schema"); + expect(() => + validateToolResultContinuationResponse( + completion('{"conditions":"rain","temperature_c":21}', false), + MODEL, + ), + ).toThrow("supplied result"); + }); + + it("stops reading an oversized probe response at the declarative byte bound (#8144)", async () => { + const boundedPlan = { + ...plan, + qualification: { + ...plan.qualification, + probeBounds: { ...plan.qualification.probeBounds, maxResponseBytes: 64 * 1024 }, + }, + }; + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(64 * 1024)); + controller.enqueue(new TextEncoder().encode("sensitive-response-body")); + controller.close(); + }, + }); + const fetchImpl = vi.fn( + async () => new Response(stream, { status: 200 }), + ) as unknown as typeof fetch; + + await expect( + runLlamaCppDgxSparkProtocolQualification({ + authorization: AUTHORIZATION, + baseUrl: "http://127.0.0.1:18081", + fetchImpl, + plan: boundedPlan, + }), + ).rejects.toThrow("declarative byte bound"); + }); + + it("drives every YAML-selected probe with declarative bounds and returns sanitized evidence (#8144)", async () => { + const requestedMaxTokens: number[] = []; + let longRequest = 0; + const fetchImpl = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + switch (url) { + case "http://127.0.0.1:18081/health": + return new Response("{}", { status: 200 }); + case "http://127.0.0.1:18081/v1/models": + return jsonResponse({ data: [{ id: MODEL }], object: "list" }); + case "http://127.0.0.1:18081/props": + return jsonResponse({ + default_generation_settings: { + n_ctx: plan.recipe.serve.contextSize, + }, + total_slots: 1, + }); + default: + expect(url).toBe("http://127.0.0.1:18081/v1/chat/completions"); + } + const body = String(init?.body ?? ""); + const authorization = new Headers(init?.headers).get("authorization"); + switch (true) { + case authorization !== AUTHORIZATION: + return jsonResponse({ error: {} }, 401); + case body === "{": + return jsonResponse({ error: {} }, 400); + } + + const request = JSON.parse(body) as JsonObject; + const maxTokens = Number(request.max_tokens); + requestedMaxTokens.push(maxTokens); + switch (true) { + case maxTokens === plan.qualification.probeBounds.cancellationMaxTokens: + longRequest += 1; + return hangingStream(init?.signal); + case request.stream_options !== undefined: + return new Response( + [ + `data: ${JSON.stringify({ + choices: [ + { + delta: { content: "ready", role: "assistant" }, + finish_reason: null, + }, + ], + model: MODEL, + object: "chat.completion.chunk", + })}`, + `data: ${JSON.stringify({ + choices: [{ delta: {}, finish_reason: "stop" }], + model: MODEL, + object: "chat.completion.chunk", + })}`, + `data: ${JSON.stringify({ + choices: [], + model: MODEL, + object: "chat.completion.chunk", + usage: { + completion_tokens: 1, + prompt_tokens: 2, + total_tokens: 3, + }, + })}`, + "data: [DONE]", + "", + ].join("\n"), + { headers: { "Content-Type": "text/event-stream" }, status: 200 }, + ); + case request.tool_choice === "none": + return jsonResponse(completion('{"conditions":"clear","temperature_c":21}', false)); + case request.response_format !== undefined: + return jsonResponse(completion('{"status":"ready"}', false)); + case request.tool_choice === "required": + return jsonResponse({ + choices: [ + { + finish_reason: "tool_calls", + message: { + role: "assistant", + tool_calls: [ + { + function: { + arguments: '{"location":"Seattle"}', + name: "get_current_weather", + }, + id: "call_1", + type: "function", + }, + ], + }, + }, + ], + model: MODEL, + object: "chat.completion", + }); + default: + return jsonResponse(completion()); + } + }) as unknown as typeof fetch; + + const evidence = await runLlamaCppDgxSparkProtocolQualification({ + authorization: AUTHORIZATION, + baseUrl: "http://127.0.0.1:18081", + fetchImpl, + plan, + }); + + expect(evidence).toMatchObject({ + authentication: { httpStatus: 401, ok: true }, + cancellation: { aborted: true, ok: true, recovered: true }, + contextWindow: { contextSize: plan.recipe.serve.contextSize, slots: 1 }, + malformedRequest: { httpStatus: 400, ok: true }, + clientTimeout: { limitMilliseconds: 10, recovered: true }, + streamingChat: { done: true, events: 3, model: MODEL }, + structuredOutput: { schemaMatched: true }, + toolCall: { argumentsValid: true, name: "get_current_weather" }, + toolResultContinuation: { model: MODEL }, + usage: { completionTokens: 2, promptTokens: 5, totalTokens: 7 }, + }); + expect(longRequest).toBe(2); + expect(requestedMaxTokens).toEqual( + expect.arrayContaining([ + plan.qualification.probeBounds.maxTokens.synchronousChat, + plan.qualification.probeBounds.maxTokens.streamingChat, + plan.qualification.probeBounds.maxTokens.structuredOutput, + plan.qualification.probeBounds.maxTokens.toolCall, + plan.qualification.probeBounds.maxTokens.toolResultContinuation, + plan.qualification.probeBounds.cancellationMaxTokens, + ]), + ); + const serializedEvidence = JSON.stringify(evidence); + expect(serializedEvidence).not.toContain(AUTHORIZATION.slice("Bearer ".length)); + expect(serializedEvidence).not.toContain("Seattle"); + expect(serializedEvidence).not.toContain("conditions"); + expect(serializedEvidence).not.toContain("temperature_c"); + }); +}); diff --git a/test/llama-cpp-dgx-spark-qualification-contract.test.ts b/test/llama-cpp-dgx-spark-qualification-contract.test.ts index 84d1a2ae769..13d96845de8 100644 --- a/test/llama-cpp-dgx-spark-qualification-contract.test.ts +++ b/test/llama-cpp-dgx-spark-qualification-contract.test.ts @@ -15,6 +15,7 @@ import { LLAMA_CPP_DGX_SPARK_MODEL_ID, LLAMA_CPP_DGX_SPARK_MODEL_PATH_PATTERN, LLAMA_CPP_DGX_SPARK_OWNED_IMAGE_REPOSITORY, + LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES, LLAMA_CPP_DGX_SPARK_QUALIFICATION_ACTIVATION_PATH, LLAMA_CPP_DGX_SPARK_QUALIFICATION_IMAGE_REPOSITORY, LLAMA_CPP_DGX_SPARK_QUALIFICATION_JOB_ID, @@ -43,6 +44,22 @@ const WORKFLOW_SHA = "c".repeat(40); const IMAGE_DIGEST = `sha256:${"d".repeat(64)}`; const MODEL_HOST_PATH = "/var/lib/nemoclaw/models/Nemotron-3-Nano-30B-A3B-UD-Q4_K_XL.gguf"; +function probeBounds() { + return { + cancellationMaxTokens: 4096, + clientTimeoutMilliseconds: 250, + maxResponseBytes: 16777216, + maxStreamEvents: 512, + maxTokens: { + streamingChat: 32, + structuredOutput: 64, + synchronousChat: 16, + toolCall: 256, + toolResultContinuation: 64, + }, + }; +} + function activation() { return { contractVersion: 1, @@ -63,7 +80,8 @@ function disabledPlan() { id: LLAMA_CPP_DGX_SPARK_MODEL_ID, }, platform: LLAMA_CPP_DGX_SPARK_QUALIFICATION_PLATFORM, - probes: ["health", "completion"], + probeBounds: probeBounds(), + probes: LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES, profile: LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE, recipeRef: LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE, required: true, @@ -126,8 +144,52 @@ function receipt() { id: LLAMA_CPP_DGX_SPARK_MODEL_ID, }, probes: { - completion: { httpStatus: 200, model: LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, ok: true }, + authentication: { httpStatus: 401, ok: true }, + cancellation: { aborted: true, ok: true, recovered: true }, + contextWindow: { contextSize: 262144, ok: true, slots: 1 }, health: { httpStatus: 200, ok: true }, + malformedRequest: { httpStatus: 400, ok: true }, + models: { + httpStatus: 200, + model: LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, + ok: true, + }, + clientTimeout: { + aborted: true, + limitMilliseconds: 250, + ok: true, + recovered: true, + }, + streamingChat: { + done: true, + events: 4, + httpStatus: 200, + model: LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, + ok: true, + }, + structuredOutput: { + httpStatus: 200, + model: LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, + ok: true, + schemaMatched: true, + }, + synchronousChat: { + httpStatus: 200, + model: LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, + ok: true, + }, + toolCall: { + argumentsValid: true, + httpStatus: 200, + name: "get_current_weather", + ok: true, + }, + toolResultContinuation: { + httpStatus: 200, + model: LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, + ok: true, + }, + usage: { completionTokens: 2, ok: true, promptTokens: 5, totalTokens: 7 }, }, repository: "NVIDIA/NemoClaw", run: { attempt: 2, id: 42 }, @@ -154,7 +216,23 @@ function executionPlan() { revision: LLAMA_CPP_DGX_SPARK_SOURCE_REVISION, }, }, + qualification: { + probeBounds: probeBounds(), + probes: LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES, + }, recipe: { + capabilities: { + agents: [], + protocols: ["openai-completions"], + streaming: true, + toolCalls: true, + structuredOutputs: true, + parallelToolCalls: false, + responsesApi: false, + embeddings: false, + reranking: false, + multimodal: false, + }, id: LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE, model: { acquisition: { downloaderImage: LLAMA_CPP_DGX_SPARK_TOOL_IMAGE }, @@ -330,6 +408,24 @@ describe("llama.cpp DGX Spark qualification contract", () => { expect(() => parseLlamaCppDgxSparkQualificationPlan({ ...enabledPlan(), arguments: ["--shell"] }), ).toThrow("unexpected fields"); + expect(() => + parseLlamaCppDgxSparkQualificationPlan({ + ...enabledPlan(), + probes: LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES.filter((probe) => probe !== "tool-call"), + }), + ).toThrow("qualification plan is invalid"); + expect(() => + parseLlamaCppDgxSparkQualificationPlan({ + ...enabledPlan(), + probes: ["health", "completion"], + }), + ).toThrow("qualification plan is invalid"); + expect(() => + parseLlamaCppDgxSparkQualificationPlan({ + ...enabledPlan(), + probeBounds: { ...probeBounds(), clientTimeoutMilliseconds: 0 }, + }), + ).toThrow("client timeout is invalid"); }); it("validates the exact workflow evidence identity before receipt parsing (#8260)", () => { @@ -375,6 +471,7 @@ describe("llama.cpp DGX Spark qualification contract", () => { const value = executionPlan(); const reordered = { recipe: value.recipe, + qualification: value.qualification, imageBuild: value.imageBuild, contractVersion: value.contractVersion, }; @@ -469,6 +566,24 @@ describe("llama.cpp DGX Spark qualification contract", () => { }, }), ).toThrow("surfaces are not disabled"); + expect(() => + parseLlamaCppDgxSparkExecutionPlan({ + ...value, + recipe: { + ...value.recipe, + capabilities: { ...value.recipe.capabilities, toolCalls: false }, + }, + }), + ).toThrow("capability claims are invalid"); + expect(() => + parseLlamaCppDgxSparkExecutionPlan({ + ...value, + qualification: { + ...value.qualification, + probes: LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES.slice(0, -1), + }, + }), + ).toThrow("protocol probes are invalid"); }); it("accepts one bounded receipt with only allowlisted workflow, image, model, and Spark evidence (#8260)", () => { @@ -496,6 +611,18 @@ describe("llama.cpp DGX Spark qualification contract", () => { evidenceIdentity(), ), ).toThrow("unexpected fields"); + expect(() => + parseLlamaCppDgxSparkQualificationReceipt( + { + ...receipt(), + probes: { + ...receipt().probes, + completion: { httpStatus: 200, ok: true }, + }, + }, + evidenceIdentity(), + ), + ).toThrow("unexpected fields"); }); it("rejects mutable, mismatched, or wrong-platform image identity (#8260)", () => { @@ -592,7 +719,34 @@ describe("llama.cpp DGX Spark qualification contract", () => { ...receipt(), probes: { ...receipt().probes, - completion: { ...receipt().probes.completion, model: "/models/private.gguf" }, + usage: { ...receipt().probes.usage, totalTokens: 8 }, + }, + }, + evidenceIdentity(), + ), + ).toThrow("probes did not pass"); + expect(() => + parseLlamaCppDgxSparkQualificationReceipt( + { + ...receipt(), + probes: { + ...receipt().probes, + cancellation: { + ...receipt().probes.cancellation, + recovered: false, + }, + }, + }, + evidenceIdentity(), + ), + ).toThrow("probes did not pass"); + expect(() => + parseLlamaCppDgxSparkQualificationReceipt( + { + ...receipt(), + probes: { + ...receipt().probes, + models: { ...receipt().probes.models, model: "/models/private.gguf" }, }, }, evidenceIdentity(), diff --git a/test/llama-cpp-dgx-spark-qualification-plan.test.ts b/test/llama-cpp-dgx-spark-qualification-plan.test.ts index 94e8411d507..f9af76e3c2d 100644 --- a/test/llama-cpp-dgx-spark-qualification-plan.test.ts +++ b/test/llama-cpp-dgx-spark-qualification-plan.test.ts @@ -10,6 +10,7 @@ import YAML from "yaml"; import { exportLlamaCppDgxSparkQualificationPlan } from "../scripts/checks/export-llama-cpp-dgx-spark-qualification-plan.mts"; import { + LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES, parseLlamaCppDgxSparkExecutionPlan, parseLlamaCppDgxSparkQualificationPlan, } from "../scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts"; @@ -94,7 +95,34 @@ describe("llama.cpp DGX Spark qualification plan export (#8260)", () => { }); expect( parseLlamaCppDgxSparkExecutionPlan(JSON.parse(output.plan), output.plan_sha256), - ).toMatchObject({ contractVersion: 1 }); + ).toMatchObject({ + contractVersion: 1, + qualification: { + probeBounds: { + cancellationMaxTokens: 4096, + clientTimeoutMilliseconds: 250, + maxResponseBytes: 16777216, + maxStreamEvents: 512, + maxTokens: { + streamingChat: 32, + structuredOutput: 64, + synchronousChat: 16, + toolCall: 256, + toolResultContinuation: 64, + }, + }, + probes: LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES, + }, + recipe: { + capabilities: { + agents: [], + protocols: ["openai-completions"], + streaming: true, + structuredOutputs: true, + toolCalls: true, + }, + }, + }); }); it("exports the checked-in main qualification as enabled", () => { diff --git a/test/llama-cpp-image.test.ts b/test/llama-cpp-image.test.ts index 31c0fb64a35..caa9a54ac11 100644 --- a/test/llama-cpp-image.test.ts +++ b/test/llama-cpp-image.test.ts @@ -371,6 +371,12 @@ describe("declarative llama.cpp server image", () => { recipeSource.replace("offload: full", "offload: partial"), ), ).toThrow(); + expect(() => + loadLlamaCppImageConfig( + manifestSource, + recipeSource.replace("toolCalls: true", "toolCalls: false"), + ), + ).toThrow("capability claims are invalid"); expect(() => loadLlamaCppImageConfig( manifestSource,