From ac05d39f546ceb95526a3b44e4a685b2d0a8ad73 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas <34834085+prekshivyas@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:11:46 -0700 Subject: [PATCH] fix(inference): bound Spark llama.cpp request bodies --- managed-inference/images/llama-cpp/Dockerfile | 6 +- managed-inference/images/llama-cpp/image.yaml | 11 +- .../images/llama-cpp/request-guard/main.go | 44 ++++--- .../llama-cpp/request-guard/main_test.go | 2 + ...-cpp.muse-glimmer-30b.spark-single.v1.yaml | 2 +- ...motron-3-nano-30b-a3b.spark-single.v1.yaml | 4 +- .../checks/export-llama-cpp-image-config.mts | 6 +- ...a-cpp-dgx-spark-protocol-qualification.mts | 124 ++++++++++++++++++ ...a-cpp-dgx-spark-qualification-contract.mts | 55 +++++++- ...p-dgx-spark-protocol-qualification.test.ts | 32 +++++ ...p-dgx-spark-qualification-contract.test.ts | 29 +++- ...managed-inference-catalog-compiler.test.ts | 2 + 12 files changed, 279 insertions(+), 38 deletions(-) diff --git a/managed-inference/images/llama-cpp/Dockerfile b/managed-inference/images/llama-cpp/Dockerfile index 9084fe7ceb0..e1a16186ff1 100644 --- a/managed-inference/images/llama-cpp/Dockerfile +++ b/managed-inference/images/llama-cpp/Dockerfile @@ -29,11 +29,11 @@ RUN apt-get update \ build-essential=12.10ubuntu1 \ ca-certificates=20260601~24.04.1 \ cmake=3.28.3-1build7 \ - curl=8.5.0-2ubuntu10.11 \ + curl=8.5.0-2ubuntu10.12 \ g++-14=14.2.0-4ubuntu2~24.04.1 \ gcc-14=14.2.0-4ubuntu2~24.04.1 \ golang-go=2:1.22~2build1 \ - libcurl4-openssl-dev=8.5.0-2ubuntu10.11 \ + libcurl4-openssl-dev=8.5.0-2ubuntu10.12 \ libssl-dev=3.0.13-0ubuntu3.12 \ && rm -rf /var/lib/apt/lists/* @@ -123,7 +123,7 @@ RUN test -n "${CUDA_DEV_IMAGE}" \ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates=20260601~24.04.1 \ - libcurl4t64=8.5.0-2ubuntu10.11 \ + libcurl4t64=8.5.0-2ubuntu10.12 \ libgomp1=14.2.0-4ubuntu2~24.04.1 \ && groupadd --gid "${RUNTIME_GID}" nemoclaw-llama \ && useradd --uid "${RUNTIME_UID}" --gid "${RUNTIME_GID}" \ diff --git a/managed-inference/images/llama-cpp/image.yaml b/managed-inference/images/llama-cpp/image.yaml index 25ca1a91fa6..f54f5255f98 100644 --- a/managed-inference/images/llama-cpp/image.yaml +++ b/managed-inference/images/llama-cpp/image.yaml @@ -72,6 +72,7 @@ spec: - context-window - authentication - malformed-request + - request-body-limit - cancellation - client-timeout - log-redaction @@ -89,8 +90,8 @@ spec: source: repository: https://github.com/ggml-org/llama.cpp - revision: 22dc605c4ead20e36f447cc67b55ef87e523bd55 - archiveSha256: sha256:975f70723e053785e894f4e1d9cf770f2f1a7bc762fd3af174ff5635014108b6 + revision: 8e7f22b67ef4667b4ddd50230771287f328cfb3f + archiveSha256: sha256:45a24299e7a24410624489d19924d492bc71a120fa17d9b7cb32f6d5c4f1aed0 cuda: developmentBase: docker.io/nvidia/cuda@sha256:ef2203909e80b8b976cfc672f7e2ae2b00bc0e25c404ee86d89e10a3802f1c52 @@ -115,11 +116,11 @@ spec: build-essential: 12.10ubuntu1 ca-certificates: 20260601~24.04.1 cmake: 3.28.3-1build7 - curl: 8.5.0-2ubuntu10.11 + curl: 8.5.0-2ubuntu10.12 g++-14: 14.2.0-4ubuntu2~24.04.1 gcc-14: 14.2.0-4ubuntu2~24.04.1 golang-go: 2:1.22~2build1 - libcurl4-openssl-dev: 8.5.0-2ubuntu10.11 + libcurl4-openssl-dev: 8.5.0-2ubuntu10.12 libssl-dev: 3.0.13-0ubuntu3.12 cmake: ggmlBackendDl: true @@ -162,7 +163,7 @@ spec: - /usr/bin/sh packages: ca-certificates: 20260601~24.04.1 - libcurl4t64: 8.5.0-2ubuntu10.11 + libcurl4t64: 8.5.0-2ubuntu10.12 libgomp1: 14.2.0-4ubuntu2~24.04.1 writablePaths: - /tmp diff --git a/managed-inference/images/llama-cpp/request-guard/main.go b/managed-inference/images/llama-cpp/request-guard/main.go index 8c61ed81303..e154804ae69 100644 --- a/managed-inference/images/llama-cpp/request-guard/main.go +++ b/managed-inference/images/llama-cpp/request-guard/main.go @@ -97,27 +97,29 @@ func requireExactCommandMarker(command []string, option string) error { func validateSupportedCommandOptions(command []string) error { allowed := map[string]bool{ - "--alias": true, - "--api-key-file": true, - "--batch-size": true, - "--cache-type-k": true, - "--cache-type-v": true, - "--ctx-size": true, - "--flash-attn": true, - "--gpu-layers": true, - "--host": true, - "--metrics": false, - "--model": true, - "--no-agent": false, - "--no-mmproj": false, - "--no-slots": false, - "--no-ui": false, - "--n-predict": true, - "--parallel": true, - "--port": true, - "--sleep-idle-seconds": true, - "--timeout": true, - "--ubatch-size": true, + "--alias": true, + "--api-key-file": true, + "--batch-size": true, + "--cache-type-k": true, + "--cache-type-v": true, + "--chat-template-kwargs": true, + "--ctx-size": true, + "--flash-attn": true, + "--gpu-layers": true, + "--host": true, + "--jinja": false, + "--metrics": false, + "--model": true, + "--no-agent": false, + "--no-mmproj": false, + "--no-slots": false, + "--no-ui": false, + "--n-predict": true, + "--parallel": true, + "--port": true, + "--sleep-idle-seconds": true, + "--timeout": true, + "--ubatch-size": true, } seen := make(map[string]bool, len(allowed)) for index := 0; index < len(command); index++ { diff --git a/managed-inference/images/llama-cpp/request-guard/main_test.go b/managed-inference/images/llama-cpp/request-guard/main_test.go index 89b09c1cfe9..f74018299aa 100644 --- a/managed-inference/images/llama-cpp/request-guard/main_test.go +++ b/managed-inference/images/llama-cpp/request-guard/main_test.go @@ -119,6 +119,8 @@ func TestParseConfigRequiresEveryDeclaredValue(t *testing.T) { "--port", "8082", "--api-key-file", llamaServerAPIKeyPath, "--n-predict", "4096", + "--jinja", + "--chat-template-kwargs", `{"reasoning_strength":"low"}`, "--no-ui", "--no-slots", "--no-mmproj", diff --git a/managed-inference/recipes/llama-cpp.muse-glimmer-30b.spark-single.v1.yaml b/managed-inference/recipes/llama-cpp.muse-glimmer-30b.spark-single.v1.yaml index 0d25d5546ca..65248161cb1 100644 --- a/managed-inference/recipes/llama-cpp.muse-glimmer-30b.spark-single.v1.yaml +++ b/managed-inference/recipes/llama-cpp.muse-glimmer-30b.spark-single.v1.yaml @@ -88,7 +88,7 @@ spec: value: f16 speculativeDecoding: disabled limits: - maxRequestBodyBytes: 1048576 + maxRequestBodyBytes: 16384 maxRequestHeaderBytes: 32768 maxOutputTokens: 4096 requestTimeoutSeconds: 900 diff --git a/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml b/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml index 437ff556676..6cbb67f1f9a 100644 --- a/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml +++ b/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml @@ -16,7 +16,7 @@ spec: technology: llama.cpp source: repository: ggml-org/llama.cpp - revision: 22dc605c4ead20e36f447cc67b55ef87e523bd55 + revision: 8e7f22b67ef4667b4ddd50230771287f328cfb3f model: id: unsloth/Nemotron-3-Nano-30B-A3B-GGUF @@ -86,7 +86,7 @@ spec: value: f16 speculativeDecoding: disabled limits: - maxRequestBodyBytes: 1048576 + maxRequestBodyBytes: 32768 maxRequestHeaderBytes: 32768 maxOutputTokens: 4096 requestTimeoutSeconds: 900 diff --git a/scripts/checks/export-llama-cpp-image-config.mts b/scripts/checks/export-llama-cpp-image-config.mts index 63bea90d52c..951622cce8b 100644 --- a/scripts/checks/export-llama-cpp-image-config.mts +++ b/scripts/checks/export-llama-cpp-image-config.mts @@ -625,11 +625,11 @@ export function loadLlamaCppImageConfig( "build-essential": "12.10ubuntu1", "ca-certificates": "20260601~24.04.1", cmake: "3.28.3-1build7", - curl: "8.5.0-2ubuntu10.11", + curl: "8.5.0-2ubuntu10.12", "g++-14": "14.2.0-4ubuntu2~24.04.1", "gcc-14": "14.2.0-4ubuntu2~24.04.1", "golang-go": "2:1.22~2build1", - "libcurl4-openssl-dev": "8.5.0-2ubuntu10.11", + "libcurl4-openssl-dev": "8.5.0-2ubuntu10.12", "libssl-dev": "3.0.13-0ubuntu3.12", }; const expectedCompiler = { @@ -639,7 +639,7 @@ export function loadLlamaCppImageConfig( }; const expectedRuntimePackages = { "ca-certificates": "20260601~24.04.1", - libcurl4t64: "8.5.0-2ubuntu10.11", + libcurl4t64: "8.5.0-2ubuntu10.12", libgomp1: "14.2.0-4ubuntu2~24.04.1", }; const expectedRequiredPaths = [ diff --git a/scripts/checks/llama-cpp-dgx-spark-protocol-qualification.mts b/scripts/checks/llama-cpp-dgx-spark-protocol-qualification.mts index c25d6d70a8b..32b1eba7704 100644 --- a/scripts/checks/llama-cpp-dgx-spark-protocol-qualification.mts +++ b/scripts/checks/llama-cpp-dgx-spark-protocol-qualification.mts @@ -3,6 +3,7 @@ import { LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES, + LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES, LLAMA_CPP_DGX_SPARK_REQUIRED_METRIC_SERIES, type LlamaCppDgxSparkExecutionPlan, type LlamaCppDgxSparkQualificationReceipt, @@ -164,6 +165,51 @@ function jsonRequest( }; } +function exactSizeChatRequest(model: string, targetBytes: number): string { + const request = { + max_tokens: 1, + messages: [{ content: "", role: "user" }], + model, + temperature: 0, + }; + const emptyBody = JSON.stringify(request); + const contentBytes = targetBytes - new TextEncoder().encode(emptyBody).byteLength; + if (contentBytes < 0) throw new Error("request-body probe target is too small"); + request.messages[0].content = "x".repeat(contentBytes); + const body = JSON.stringify(request); + if (new TextEncoder().encode(body).byteLength !== targetBytes) { + throw new Error("request-body probe did not construct the exact declared size"); + } + return body; +} + +function exactSizeJsonRequest( + authorization: string, + body: string, + timeoutMilliseconds: number, +): RequestInit { + return { + body, + headers: { + Authorization: authorization, + "Content-Type": "application/json", + }, + method: "POST", + signal: requestSignal(timeoutMilliseconds), + }; +} + +function validateRequestBodyLimitError(value: unknown): void { + if ( + !isRecord(value) || + !isRecord(value.error) || + value.error.code !== "request_body_too_large" || + value.error.type !== "invalid_request_error" + ) { + throw new Error("request-body limit probe did not return the declared error contract"); + } +} + function usageFrom(value: unknown): ProtocolEvidence["usage"] { if (!isRecord(value)) throw new Error("chat usage was not returned"); const promptTokens = value.prompt_tokens; @@ -727,6 +773,73 @@ export async function runLlamaCppDgxSparkProtocolQualification(options: { await expectStatus(malformedResponse, 400, bounds.maxResponseBytes, "malformed-request probe"); executedProbes.add("malformed-request"); + const acceptedRequestBytes = plan.recipe.serve.limits.maxRequestBodyBytes; + const acceptedResponse = await fetchImpl( + chatUrl, + exactSizeJsonRequest( + authorization, + exactSizeChatRequest(model, acceptedRequestBytes), + timeoutMilliseconds, + ), + ); + await expectStatus( + acceptedResponse, + 200, + bounds.maxResponseBytes, + "request-body boundary probe", + ); + + const rejectedResponse = await fetchImpl( + chatUrl, + exactSizeJsonRequest( + authorization, + exactSizeChatRequest(model, LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES), + timeoutMilliseconds, + ), + ); + validateRequestBodyLimitError( + await readJson( + rejectedResponse, + 413, + bounds.maxResponseBytes, + "oversized request-body probe", + ), + ); + + const continuationHealthResponse = await fetchImpl(`${baseUrl}/health`, { + headers: { Authorization: authorization }, + signal: requestSignal(bounds.clientTimeoutMilliseconds), + }); + await expectStatus( + continuationHealthResponse, + 200, + bounds.maxResponseBytes, + "request-body continuation health probe", + ); + const bodyLimitContinuationResponse = await fetchImpl( + chatUrl, + jsonRequest( + authorization, + { + max_tokens: bounds.maxTokens.synchronousChat, + messages: [{ content: "Return one short continuation token.", role: "user" }], + model, + temperature: 0, + }, + timeoutMilliseconds, + ), + ); + validateChatCompletionResponse( + await readJson( + bodyLimitContinuationResponse, + 200, + bounds.maxResponseBytes, + "request-body continuation completion probe", + ), + model, + ); + executedProbes.add("request-body-limit"); + const synchronousResponse = await fetchImpl( chatUrl, jsonRequest( @@ -914,6 +1027,17 @@ export async function runLlamaCppDgxSparkProtocolQualification(options: { }, health: { httpStatus: 200, ok: true }, malformedRequest: { httpStatus: 400, ok: true }, + requestBodyLimit: { + acceptedBytes: acceptedRequestBytes, + acceptedHttpStatus: 200, + continuationHealthHttpStatus: 200, + continuationHttpStatus: 200, + errorCode: "request_body_too_large", + errorType: "invalid_request_error", + ok: true, + rejectedBytes: LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES, + rejectedHttpStatus: 413, + }, models: { httpStatus: 200, model, ok: true }, metrics, properties: propertiesEvidence.properties, diff --git a/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts b/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts index c425aaffa2b..5022d5e4758 100644 --- a/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts +++ b/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts @@ -23,7 +23,7 @@ export const LLAMA_CPP_DGX_SPARK_MODEL_DIGEST = "sha256:627f5b04aedc97f967332f331bd75b7a4ed2f33ca83e6ee74b44235cc1887890" as const; export const LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID = "nvidia-nemotron-3-nano-30b-a3b" as const; export const LLAMA_CPP_DGX_SPARK_SOURCE_REVISION = - "22dc605c4ead20e36f447cc67b55ef87e523bd55" as const; + "8e7f22b67ef4667b4ddd50230771287f328cfb3f" as const; export const LLAMA_CPP_DGX_SPARK_QUALIFICATION_IMAGE_REPOSITORY = "localhost:5000/nemoclaw-llama-cpp-dgx-spark/llama-cpp-server" as const; export const LLAMA_CPP_DGX_SPARK_OWNED_IMAGE_REPOSITORY = @@ -31,7 +31,7 @@ export const LLAMA_CPP_DGX_SPARK_OWNED_IMAGE_REPOSITORY = export const LLAMA_CPP_DGX_SPARK_SOURCE_REPOSITORY = "https://github.com/ggml-org/llama.cpp" as const; export const LLAMA_CPP_DGX_SPARK_SOURCE_ARCHIVE_SHA256 = - "sha256:975f70723e053785e894f4e1d9cf770f2f1a7bc762fd3af174ff5635014108b6" as const; + "sha256:45a24299e7a24410624489d19924d492bc71a120fa17d9b7cb32f6d5c4f1aed0" as const; export const LLAMA_CPP_DGX_SPARK_CUDA_DEVELOPMENT_BASE = "docker.io/nvidia/cuda@sha256:ef2203909e80b8b976cfc672f7e2ae2b00bc0e25c404ee86d89e10a3802f1c52" as const; export const LLAMA_CPP_DGX_SPARK_CUDA_RUNTIME_BASE = @@ -39,6 +39,7 @@ export const LLAMA_CPP_DGX_SPARK_CUDA_RUNTIME_BASE = export const LLAMA_CPP_DGX_SPARK_TOOL_IMAGE = "nvcr.io/nvidia/vllm@sha256:94e21552f644e0c1627464ba89d2f7a4ce7442e196f72afa0bb5d7fba23cbb03" as const; export const LLAMA_CPP_DGX_SPARK_MINIMUM_DRIVER_VERSION = "580.65.06" as const; +export const LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES = 50_000 as const; export const LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES = [ "health", "models", @@ -54,6 +55,7 @@ export const LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES = [ "context-window", "authentication", "malformed-request", + "request-body-limit", "cancellation", "client-timeout", ] as const; @@ -469,6 +471,17 @@ export type LlamaCppDgxSparkQualificationReceipt = { readonly httpStatus: 400; readonly ok: true; }; + readonly requestBodyLimit: { + readonly acceptedBytes: number; + readonly acceptedHttpStatus: 200; + readonly continuationHealthHttpStatus: 200; + readonly continuationHttpStatus: 200; + readonly errorCode: "request_body_too_large"; + readonly errorType: "invalid_request_error"; + readonly ok: true; + readonly rejectedBytes: typeof LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES; + readonly rejectedHttpStatus: 413; + }; readonly models: { readonly httpStatus: 200; readonly model: typeof LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID; @@ -1341,6 +1354,7 @@ export function parseLlamaCppDgxSparkExecutionPlan( serve.idleSleepSeconds !== -1 || serve.flashAttention !== "enabled" || serve.speculativeDecoding !== "disabled" || + maxRequestBodyBytes !== 32_768 || upstreamPort === serve.port || typeof kvCache.key !== "string" || !allowedKvTypes.has(kvCache.key) || @@ -1799,6 +1813,7 @@ export function parseLlamaCppDgxSparkQualificationReceipt( "health", "logRedaction", "malformedRequest", + "requestBodyLimit", "metrics", "models", "properties", @@ -1909,6 +1924,22 @@ export function parseLlamaCppDgxSparkQualificationReceipt( requireExactKeys(authentication, ["httpStatus", "ok"], "authentication probe"); const malformedRequest = record(probes.malformedRequest, "malformed-request probe"); requireExactKeys(malformedRequest, ["httpStatus", "ok"], "malformed-request probe"); + const requestBodyLimit = record(probes.requestBodyLimit, "request-body limit probe"); + requireExactKeys( + requestBodyLimit, + [ + "acceptedBytes", + "acceptedHttpStatus", + "continuationHealthHttpStatus", + "continuationHttpStatus", + "errorCode", + "errorType", + "ok", + "rejectedBytes", + "rejectedHttpStatus", + ], + "request-body limit probe", + ); const cancellation = record(probes.cancellation, "cancellation probe"); requireExactKeys(cancellation, ["aborted", "ok", "recovered"], "cancellation probe"); const clientTimeout = record(probes.clientTimeout, "client-timeout probe"); @@ -1973,6 +2004,15 @@ export function parseLlamaCppDgxSparkQualificationReceipt( authentication.httpStatus !== 401 || malformedRequest.ok !== true || malformedRequest.httpStatus !== 400 || + requestBodyLimit.ok !== true || + requestBodyLimit.acceptedBytes !== expectedPlan.recipe.serve.limits.maxRequestBodyBytes || + requestBodyLimit.acceptedHttpStatus !== 200 || + requestBodyLimit.rejectedBytes !== LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES || + requestBodyLimit.rejectedHttpStatus !== 413 || + requestBodyLimit.errorCode !== "request_body_too_large" || + requestBodyLimit.errorType !== "invalid_request_error" || + requestBodyLimit.continuationHealthHttpStatus !== 200 || + requestBodyLimit.continuationHttpStatus !== 200 || cancellation.ok !== true || cancellation.aborted !== true || cancellation.recovered !== true || @@ -2049,6 +2089,17 @@ export function parseLlamaCppDgxSparkQualificationReceipt( health: { httpStatus: 200, ok: true }, logRedaction: { ok: true }, malformedRequest: { httpStatus: 400, ok: true }, + requestBodyLimit: { + acceptedBytes: expectedPlan.recipe.serve.limits.maxRequestBodyBytes, + acceptedHttpStatus: 200, + continuationHealthHttpStatus: 200, + continuationHttpStatus: 200, + errorCode: "request_body_too_large", + errorType: "invalid_request_error", + ok: true, + rejectedBytes: LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES, + rejectedHttpStatus: 413, + }, metrics: { httpStatus: 200, ok: true, diff --git a/test/llama-cpp-dgx-spark-protocol-qualification.test.ts b/test/llama-cpp-dgx-spark-protocol-qualification.test.ts index 68dbc1301f3..80d2afefa45 100644 --- a/test/llama-cpp-dgx-spark-protocol-qualification.test.ts +++ b/test/llama-cpp-dgx-spark-protocol-qualification.test.ts @@ -267,11 +267,14 @@ describe("llama.cpp DGX Spark protocol qualification", () => { it("drives every YAML-selected probe with declarative bounds and returns sanitized evidence (#8144)", async () => { const requestedMaxTokens: number[] = []; + const requestBodySizes: number[] = []; + let healthProbes = 0; let longRequest = 0; const fetchImpl = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { const url = String(input); switch (url) { case "http://127.0.0.1:18081/health": + healthProbes += 1; return new Response("{}", { status: 200 }); case "http://127.0.0.1:18081/v1/models": return jsonResponse({ data: [{ id: MODEL }], object: "list" }); @@ -295,12 +298,25 @@ describe("llama.cpp DGX Spark protocol qualification", () => { expect(url).toBe("http://127.0.0.1:18081/v1/chat/completions"); } const body = String(init?.body ?? ""); + const requestBodyBytes = new TextEncoder().encode(body).byteLength; + requestBodySizes.push(requestBodyBytes); const authorization = new Headers(init?.headers).get("authorization"); switch (true) { case authorization !== AUTHORIZATION: return jsonResponse({ error: {} }, 401); case body === "{": return jsonResponse({ error: {} }, 400); + case requestBodyBytes === 50_000: + return jsonResponse( + { + error: { + code: "request_body_too_large", + message: "Request body exceeds the declared limit.", + type: "invalid_request_error", + }, + }, + 413, + ); } const request = JSON.parse(body) as JsonObject; @@ -395,6 +411,16 @@ describe("llama.cpp DGX Spark protocol qualification", () => { uiHttpStatus: 404, }, malformedRequest: { httpStatus: 400, ok: true }, + requestBodyLimit: { + acceptedBytes: 32768, + acceptedHttpStatus: 200, + continuationHealthHttpStatus: 200, + continuationHttpStatus: 200, + errorCode: "request_body_too_large", + errorType: "invalid_request_error", + rejectedBytes: 50000, + rejectedHttpStatus: 413, + }, metrics: { requiredSeries: 11, unauthenticatedHttpStatus: 401 }, properties: { metrics: true, model: MODEL, modelPath: MODEL_FILE }, clientTimeout: { limitMilliseconds: 10, recovered: true }, @@ -405,6 +431,12 @@ describe("llama.cpp DGX Spark protocol qualification", () => { usage: { completionTokens: 2, promptTokens: 5, totalTokens: 7 }, }); expect(longRequest).toBe(2); + expect(healthProbes).toBe(2); + expect( + requestBodySizes.filter( + (size) => size >= plan.recipe.serve.limits.maxRequestBodyBytes, + ), + ).toEqual([32768, 50000]); expect(requestedMaxTokens).toEqual( expect.arrayContaining([ plan.qualification.probeBounds.maxTokens.synchronousChat, diff --git a/test/llama-cpp-dgx-spark-qualification-contract.test.ts b/test/llama-cpp-dgx-spark-qualification-contract.test.ts index 85bb7fb2a36..ffaa9722008 100644 --- a/test/llama-cpp-dgx-spark-qualification-contract.test.ts +++ b/test/llama-cpp-dgx-spark-qualification-contract.test.ts @@ -27,6 +27,7 @@ import { LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROBES, LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE, LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE, + LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES, LLAMA_CPP_DGX_SPARK_RUNNER_PATTERN, LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID, LLAMA_CPP_DGX_SPARK_SOURCE_ARCHIVE_SHA256, @@ -208,6 +209,17 @@ function receipt() { health: { httpStatus: 200, ok: true }, logRedaction: { ok: true }, malformedRequest: { httpStatus: 400, ok: true }, + requestBodyLimit: { + acceptedBytes: 32768, + acceptedHttpStatus: 200, + continuationHealthHttpStatus: 200, + continuationHttpStatus: 200, + errorCode: "request_body_too_large", + errorType: "invalid_request_error", + ok: true, + rejectedBytes: LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES, + rejectedHttpStatus: 413, + }, metrics: { httpStatus: 200, ok: true, @@ -361,7 +373,7 @@ function executionPlan() { kvCache: { key: "f16", value: "f16" }, speculativeDecoding: "disabled", limits: { - maxRequestBodyBytes: 1048576, + maxRequestBodyBytes: 32768, maxRequestHeaderBytes: 32768, maxOutputTokens: 4096, requestTimeoutSeconds: 900, @@ -896,6 +908,21 @@ describe("llama.cpp DGX Spark qualification contract", () => { evidenceIdentity(), ), ).toThrow("probes did not pass"); + expect(() => + parseLlamaCppDgxSparkQualificationReceipt( + { + ...receipt(), + probes: { + ...receipt().probes, + requestBodyLimit: { + ...receipt().probes.requestBodyLimit, + continuationHealthHttpStatus: 503, + }, + }, + }, + evidenceIdentity(), + ), + ).toThrow("probes did not pass"); expect(() => parseLlamaCppDgxSparkQualificationReceipt( { diff --git a/test/managed-inference-catalog-compiler.test.ts b/test/managed-inference-catalog-compiler.test.ts index 9b385006fb4..a67f762c88f 100644 --- a/test/managed-inference-catalog-compiler.test.ts +++ b/test/managed-inference-catalog-compiler.test.ts @@ -141,6 +141,7 @@ describe("managed inference YAML profile contract", () => { serve: { authentication: "bearer", contextSize: 262144, + limits: { maxRequestBodyBytes: 32768 }, batchSize: 2048, microBatchSize: 512, flashAttention: "enabled", @@ -197,6 +198,7 @@ describe("managed inference YAML profile contract", () => { chatTemplate: "model-embedded-jinja", chatTemplateArguments: { reasoningStrength: "low" }, contextSize: 131072, + limits: { maxRequestBodyBytes: 16384 }, slots: 1, speculativeDecoding: "disabled", },