fix(inference): bound Spark llama.cpp request bodies

managed-inference/images/llama-cpp/image.yaml

@@ -72,6 +72,7 @@ spec:
         - context-window
         - authentication
         - malformed-request
+        - request-body-limit
         - cancellation
         - client-timeout
         - log-redaction
@@ -89,8 +90,8 @@ spec:
 
   source:
     repository: https://github.com/ggml-org/llama.cpp
-    revision: 22dc605c4ead20e36f447cc67b55ef87e523bd55
-    archiveSha256: sha256:975f70723e053785e894f4e1d9cf770f2f1a7bc762fd3af174ff5635014108b6
+    revision: 8e7f22b67ef4667b4ddd50230771287f328cfb3f
+    archiveSha256: sha256:45a24299e7a24410624489d19924d492bc71a120fa17d9b7cb32f6d5c4f1aed0
 
   cuda:
     developmentBase: docker.io/nvidia/cuda@sha256:ef2203909e80b8b976cfc672f7e2ae2b00bc0e25c404ee86d89e10a3802f1c52

managed-inference/images/llama-cpp/request-guard/main.go

@@ -97,27 +97,29 @@ func requireExactCommandMarker(command []string, option string) error {
 
 func validateSupportedCommandOptions(command []string) error {
 	allowed := map[string]bool{
-		"--alias":              true,
-		"--api-key-file":       true,
-		"--batch-size":         true,
-		"--cache-type-k":       true,
-		"--cache-type-v":       true,
-		"--ctx-size":           true,
-		"--flash-attn":         true,
-		"--gpu-layers":         true,
-		"--host":               true,
-		"--metrics":            false,
-		"--model":              true,
-		"--no-agent":           false,
-		"--no-mmproj":          false,
-		"--no-slots":           false,
-		"--no-ui":              false,
-		"--n-predict":          true,
-		"--parallel":           true,
-		"--port":               true,
-		"--sleep-idle-seconds": true,
-		"--timeout":            true,
-		"--ubatch-size":        true,
+		"--alias":                true,
+		"--api-key-file":         true,
+		"--batch-size":           true,
+		"--cache-type-k":         true,
+		"--cache-type-v":         true,
+		"--chat-template-kwargs": true,
+		"--ctx-size":             true,
+		"--flash-attn":           true,
+		"--gpu-layers":           true,
+		"--host":                 true,
+		"--jinja":                false,
+		"--metrics":              false,
+		"--model":                true,
+		"--no-agent":             false,
+		"--no-mmproj":            false,
+		"--no-slots":             false,
+		"--no-ui":                false,
+		"--n-predict":            true,
+		"--parallel":             true,
+		"--port":                 true,
+		"--sleep-idle-seconds":   true,
+		"--timeout":              true,
+		"--ubatch-size":          true,
 	}
 	seen := make(map[string]bool, len(allowed))
 	for index := 0; index < len(command); index++ {

managed-inference/images/llama-cpp/request-guard/main_test.go

@@ -119,6 +119,8 @@ func TestParseConfigRequiresEveryDeclaredValue(t *testing.T) {
 		"--port", "8082",
 		"--api-key-file", llamaServerAPIKeyPath,
 		"--n-predict", "4096",
+		"--jinja",
+		"--chat-template-kwargs", `{"reasoning_strength":"low"}`,
 		"--no-ui",
 		"--no-slots",
 		"--no-mmproj",

managed-inference/recipes/llama-cpp.muse-glimmer-30b.spark-single.v1.yaml

@@ -88,7 +88,7 @@ spec:
       value: f16
     speculativeDecoding: disabled
     limits:
-      maxRequestBodyBytes: 1048576
+      maxRequestBodyBytes: 16384
       maxRequestHeaderBytes: 32768
       maxOutputTokens: 4096
       requestTimeoutSeconds: 900

managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml

@@ -16,7 +16,7 @@ spec:
     technology: llama.cpp
     source:
       repository: ggml-org/llama.cpp
-      revision: 22dc605c4ead20e36f447cc67b55ef87e523bd55
+      revision: 8e7f22b67ef4667b4ddd50230771287f328cfb3f
 
   model:
     id: unsloth/Nemotron-3-Nano-30B-A3B-GGUF
@@ -86,7 +86,7 @@ spec:
       value: f16
     speculativeDecoding: disabled
     limits:
-      maxRequestBodyBytes: 1048576
+      maxRequestBodyBytes: 32768
       maxRequestHeaderBytes: 32768
       maxOutputTokens: 4096
       requestTimeoutSeconds: 900

scripts/checks/llama-cpp-dgx-spark-protocol-qualification.mts

@@ -3,6 +3,7 @@
 
 import {
   LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES,
+  LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES,
   LLAMA_CPP_DGX_SPARK_REQUIRED_METRIC_SERIES,
   type LlamaCppDgxSparkExecutionPlan,
   type LlamaCppDgxSparkQualificationReceipt,
@@ -164,6 +165,51 @@ function jsonRequest(
   };
 }
 
+function exactSizeChatRequest(model: string, targetBytes: number): string {
+  const request = {
+    max_tokens: 1,
+    messages: [{ content: "", role: "user" }],
+    model,
+    temperature: 0,
+  };
+  const emptyBody = JSON.stringify(request);
+  const contentBytes = targetBytes - new TextEncoder().encode(emptyBody).byteLength;
+  if (contentBytes < 0) throw new Error("request-body probe target is too small");
+  request.messages[0].content = "x".repeat(contentBytes);
+  const body = JSON.stringify(request);
+  if (new TextEncoder().encode(body).byteLength !== targetBytes) {
+    throw new Error("request-body probe did not construct the exact declared size");
+  }
+  return body;
+}
+
+function exactSizeJsonRequest(
+  authorization: string,
+  body: string,
+  timeoutMilliseconds: number,
+): RequestInit {
+  return {
+    body,
+    headers: {
+      Authorization: authorization,
+      "Content-Type": "application/json",
+    },
+    method: "POST",
+    signal: requestSignal(timeoutMilliseconds),
+  };
+}
+
+function validateRequestBodyLimitError(value: unknown): void {
+  if (
+    !isRecord(value) ||
+    !isRecord(value.error) ||
+    value.error.code !== "request_body_too_large" ||
+    value.error.type !== "invalid_request_error"
+  ) {
+    throw new Error("request-body limit probe did not return the declared error contract");
+  }
+}
+
 function usageFrom(value: unknown): ProtocolEvidence["usage"] {
   if (!isRecord(value)) throw new Error("chat usage was not returned");
   const promptTokens = value.prompt_tokens;
@@ -727,6 +773,73 @@ export async function runLlamaCppDgxSparkProtocolQualification(options: {
   await expectStatus(malformedResponse, 400, bounds.maxResponseBytes, "malformed-request probe");
   executedProbes.add("malformed-request");
 
+  const acceptedRequestBytes = plan.recipe.serve.limits.maxRequestBodyBytes;
+  const acceptedResponse = await fetchImpl(
+    chatUrl,
+    exactSizeJsonRequest(
+      authorization,
+      exactSizeChatRequest(model, acceptedRequestBytes),
+      timeoutMilliseconds,
+    ),
+  );
+  await expectStatus(
+    acceptedResponse,
+    200,
+    bounds.maxResponseBytes,
+    "request-body boundary probe",
+  );
+
+  const rejectedResponse = await fetchImpl(
+    chatUrl,
+    exactSizeJsonRequest(
+      authorization,
+      exactSizeChatRequest(model, LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES),
+      timeoutMilliseconds,
+    ),
+  );
+  validateRequestBodyLimitError(
+    await readJson(
+      rejectedResponse,
+      413,
+      bounds.maxResponseBytes,
+      "oversized request-body probe",
+    ),
+  );
+
+  const continuationHealthResponse = await fetchImpl(`${baseUrl}/health`, {
+    headers: { Authorization: authorization },
+    signal: requestSignal(bounds.clientTimeoutMilliseconds),
+  });
+  await expectStatus(
+    continuationHealthResponse,
+    200,
+    bounds.maxResponseBytes,
+    "request-body continuation health probe",
+  );
+  const bodyLimitContinuationResponse = await fetchImpl(
+    chatUrl,
+    jsonRequest(
+      authorization,
+      {
+        max_tokens: bounds.maxTokens.synchronousChat,
+        messages: [{ content: "Return one short continuation token.", role: "user" }],
+        model,
+        temperature: 0,
+      },
+      timeoutMilliseconds,
+    ),
+  );
+  validateChatCompletionResponse(
+    await readJson(
+      bodyLimitContinuationResponse,
+      200,
+      bounds.maxResponseBytes,
+      "request-body continuation completion probe",
+    ),
+    model,
+  );
+  executedProbes.add("request-body-limit");
+
   const synchronousResponse = await fetchImpl(
     chatUrl,
     jsonRequest(
@@ -914,6 +1027,17 @@ export async function runLlamaCppDgxSparkProtocolQualification(options: {
     },
     health: { httpStatus: 200, ok: true },
     malformedRequest: { httpStatus: 400, ok: true },
+    requestBodyLimit: {
+      acceptedBytes: acceptedRequestBytes,
+      acceptedHttpStatus: 200,
+      continuationHealthHttpStatus: 200,
+      continuationHttpStatus: 200,
+      errorCode: "request_body_too_large",
+      errorType: "invalid_request_error",
+      ok: true,
+      rejectedBytes: LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES,
+      rejectedHttpStatus: 413,
+    },
     models: { httpStatus: 200, model, ok: true },
     metrics,
     properties: propertiesEvidence.properties,

scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts

@@ -23,22 +23,23 @@ export const LLAMA_CPP_DGX_SPARK_MODEL_DIGEST =
   "sha256:627f5b04aedc97f967332f331bd75b7a4ed2f33ca83e6ee74b44235cc1887890" as const;
 export const LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID = "nvidia-nemotron-3-nano-30b-a3b" as const;
 export const LLAMA_CPP_DGX_SPARK_SOURCE_REVISION =
-  "22dc605c4ead20e36f447cc67b55ef87e523bd55" as const;
+  "8e7f22b67ef4667b4ddd50230771287f328cfb3f" as const;
 export const LLAMA_CPP_DGX_SPARK_QUALIFICATION_IMAGE_REPOSITORY =
   "localhost:5000/nemoclaw-llama-cpp-dgx-spark/llama-cpp-server" as const;
 export const LLAMA_CPP_DGX_SPARK_OWNED_IMAGE_REPOSITORY =
   "ghcr.io/nvidia/nemoclaw/llama-cpp-server" as const;
 export const LLAMA_CPP_DGX_SPARK_SOURCE_REPOSITORY =
   "https://github.com/ggml-org/llama.cpp" as const;
 export const LLAMA_CPP_DGX_SPARK_SOURCE_ARCHIVE_SHA256 =
-  "sha256:975f70723e053785e894f4e1d9cf770f2f1a7bc762fd3af174ff5635014108b6" as const;
+  "sha256:45a24299e7a24410624489d19924d492bc71a120fa17d9b7cb32f6d5c4f1aed0" as const;
 export const LLAMA_CPP_DGX_SPARK_CUDA_DEVELOPMENT_BASE =
   "docker.io/nvidia/cuda@sha256:ef2203909e80b8b976cfc672f7e2ae2b00bc0e25c404ee86d89e10a3802f1c52" as const;
 export const LLAMA_CPP_DGX_SPARK_CUDA_RUNTIME_BASE =
   "docker.io/nvidia/cuda@sha256:789e629e49401647e22b7054ae9c6c4f6427dba68010ba428deb4cc6b063676e" as const;
 export const LLAMA_CPP_DGX_SPARK_TOOL_IMAGE =
   "nvcr.io/nvidia/vllm@sha256:94e21552f644e0c1627464ba89d2f7a4ce7442e196f72afa0bb5d7fba23cbb03" as const;
 export const LLAMA_CPP_DGX_SPARK_MINIMUM_DRIVER_VERSION = "580.65.06" as const;
+export const LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES = 50_000 as const;
 export const LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES = [
   "health",
   "models",
@@ -54,6 +55,7 @@ export const LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES = [
   "context-window",
   "authentication",
   "malformed-request",
+  "request-body-limit",
   "cancellation",
   "client-timeout",
 ] as const;
@@ -469,6 +471,17 @@ export type LlamaCppDgxSparkQualificationReceipt = {
       readonly httpStatus: 400;
       readonly ok: true;
     };
+    readonly requestBodyLimit: {
+      readonly acceptedBytes: number;
+      readonly acceptedHttpStatus: 200;
+      readonly continuationHealthHttpStatus: 200;
+      readonly continuationHttpStatus: 200;
+      readonly errorCode: "request_body_too_large";
+      readonly errorType: "invalid_request_error";
+      readonly ok: true;
+      readonly rejectedBytes: typeof LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES;
+      readonly rejectedHttpStatus: 413;
+    };
     readonly models: {
       readonly httpStatus: 200;
       readonly model: typeof LLAMA_CPP_DGX_SPARK_SERVED_MODEL_ID;
@@ -1341,6 +1354,7 @@ export function parseLlamaCppDgxSparkExecutionPlan(
     serve.idleSleepSeconds !== -1 ||
     serve.flashAttention !== "enabled" ||
     serve.speculativeDecoding !== "disabled" ||
+    maxRequestBodyBytes !== 32_768 ||
     upstreamPort === serve.port ||
     typeof kvCache.key !== "string" ||
     !allowedKvTypes.has(kvCache.key) ||
@@ -1799,6 +1813,7 @@ export function parseLlamaCppDgxSparkQualificationReceipt(
       "health",
       "logRedaction",
       "malformedRequest",
+      "requestBodyLimit",
       "metrics",
       "models",
       "properties",
@@ -1909,6 +1924,22 @@ export function parseLlamaCppDgxSparkQualificationReceipt(
   requireExactKeys(authentication, ["httpStatus", "ok"], "authentication probe");
   const malformedRequest = record(probes.malformedRequest, "malformed-request probe");
   requireExactKeys(malformedRequest, ["httpStatus", "ok"], "malformed-request probe");
+  const requestBodyLimit = record(probes.requestBodyLimit, "request-body limit probe");
+  requireExactKeys(
+    requestBodyLimit,
+    [
+      "acceptedBytes",
+      "acceptedHttpStatus",
+      "continuationHealthHttpStatus",
+      "continuationHttpStatus",
+      "errorCode",
+      "errorType",
+      "ok",
+      "rejectedBytes",
+      "rejectedHttpStatus",
+    ],
+    "request-body limit probe",
+  );
   const cancellation = record(probes.cancellation, "cancellation probe");
   requireExactKeys(cancellation, ["aborted", "ok", "recovered"], "cancellation probe");
   const clientTimeout = record(probes.clientTimeout, "client-timeout probe");
@@ -1973,6 +2004,15 @@ export function parseLlamaCppDgxSparkQualificationReceipt(
     authentication.httpStatus !== 401 ||
     malformedRequest.ok !== true ||
     malformedRequest.httpStatus !== 400 ||
+    requestBodyLimit.ok !== true ||
+    requestBodyLimit.acceptedBytes !== expectedPlan.recipe.serve.limits.maxRequestBodyBytes ||
+    requestBodyLimit.acceptedHttpStatus !== 200 ||
+    requestBodyLimit.rejectedBytes !== LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES ||
+    requestBodyLimit.rejectedHttpStatus !== 413 ||
+    requestBodyLimit.errorCode !== "request_body_too_large" ||
+    requestBodyLimit.errorType !== "invalid_request_error" ||
+    requestBodyLimit.continuationHealthHttpStatus !== 200 ||
+    requestBodyLimit.continuationHttpStatus !== 200 ||
     cancellation.ok !== true ||
     cancellation.aborted !== true ||
     cancellation.recovered !== true ||
@@ -2049,6 +2089,17 @@ export function parseLlamaCppDgxSparkQualificationReceipt(
       health: { httpStatus: 200, ok: true },
       logRedaction: { ok: true },
       malformedRequest: { httpStatus: 400, ok: true },
+      requestBodyLimit: {
+        acceptedBytes: expectedPlan.recipe.serve.limits.maxRequestBodyBytes,
+        acceptedHttpStatus: 200,
+        continuationHealthHttpStatus: 200,
+        continuationHttpStatus: 200,
+        errorCode: "request_body_too_large",
+        errorType: "invalid_request_error",
+        ok: true,
+        rejectedBytes: LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES,
+        rejectedHttpStatus: 413,
+      },
       metrics: {
         httpStatus: 200,
         ok: true,