feat(inference): bind llama.cpp request guard qualification

managed-inference/images/llama-cpp/image.yaml

@@ -43,6 +43,7 @@ spec:
     qualification:
       required: true
       execution: enabled
+      requestGuard: required
       profile: dgx-spark-gb10-single
       recipeRef: llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1
       platform: linux/arm64

managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml

@@ -86,7 +86,13 @@ spec:
       value: f16
     speculativeDecoding: disabled
     limits:
+      maxRequestBodyBytes: 1048576
+      maxRequestHeaderBytes: 32768
+      maxOutputTokens: 4096
       requestTimeoutSeconds: 900
+      shutdownTimeoutSeconds: 25
+    requestGuard:
+      upstreamPort: 8082
 
   readiness:
     contractRef: llama-cpp.server-readiness/v1

managed-inference/schemas/recipe.schema.json

@@ -482,13 +482,54 @@
             "limits": {
               "type": "object",
               "required": [
-                "requestTimeoutSeconds"
+                "maxRequestBodyBytes",
+                "maxRequestHeaderBytes",
+                "maxOutputTokens",
+                "requestTimeoutSeconds",
+                "shutdownTimeoutSeconds"
               ],
               "properties": {
+                "maxRequestBodyBytes": {
+                  "type": "integer",
+                  "minimum": 1,
+                  "maximum": 67108864
+                },
+                "maxRequestHeaderBytes": {
+                  "type": "integer",
+                  "minimum": 1,
+                  "maximum": 1048576
+                },
+                "maxOutputTokens": {
+                  "type": "integer",
+                  "minimum": 1,
+                  "maximum": 1048576
+                },
                 "requestTimeoutSeconds": {
                   "type": "integer",
                   "minimum": 1,
                   "maximum": 86400
+                },
+                "shutdownTimeoutSeconds": {
+                  "type": "integer",
+                  "minimum": 1,
+                  "maximum": 86400
+                }
+              },
+              "additionalProperties": false
+            },
+            "requestGuard": {
+              "type": "object",
+              "required": [
+                "upstreamPort"
+              ],
+              "properties": {
+                "upstreamPort": {
+                  "type": "integer",
+                  "minimum": 1,
+                  "maximum": 65535,
+                  "not": {
+                    "const": 8081
+                  }
                 }
               },
               "additionalProperties": false
@@ -1239,7 +1280,8 @@
               "flashAttention",
               "kvCache",
               "speculativeDecoding",
-              "limits"
+              "limits",
+              "requestGuard"
             ],
             "properties": {
               "protocol": {},
@@ -1264,7 +1306,8 @@
               "flashAttention": {},
               "kvCache": {},
               "speculativeDecoding": {},
-              "limits": {}
+              "limits": {},
+              "requestGuard": {}
             },
             "propertyNames": {
               "not": {

scripts/checks/export-llama-cpp-image-config.mts

@@ -59,6 +59,7 @@ type ServerImageManifest = {
         probes?: unknown;
         profile?: unknown;
         recipeRef?: unknown;
+        requestGuard?: unknown;
         required?: unknown;
         runner?: unknown;
       };
@@ -136,8 +137,13 @@ type LlamaCppQualificationRecipe = {
       kvCache: { key: string; value: string };
       speculativeDecoding: string;
       limits: {
+        maxRequestBodyBytes: number;
+        maxRequestHeaderBytes: number;
+        maxOutputTokens: number;
         requestTimeoutSeconds: number;
+        shutdownTimeoutSeconds: number;
       };
+      requestGuard: { upstreamPort: number };
     };
     readiness: {
       contractRef: string;
@@ -381,6 +387,7 @@ export function loadLlamaCppImageConfig(
     "probes",
     "profile",
     "recipeRef",
+    "requestGuard",
     "required",
     "runner",
   ]);
@@ -426,6 +433,11 @@ export function loadLlamaCppImageConfig(
     "qualification recipe reference",
     /^llama-cpp\.nemotron-3-nano-30b-a3b\.spark-single\.v1$/u,
   );
+  const qualificationRequestGuard = requiredString(
+    qualification?.requestGuard,
+    "qualification request guard",
+    /^required$/u,
+  );
   const qualificationModel = qualification?.model as
     | { digest?: unknown; hostPath?: unknown; id?: unknown }
     | undefined;
@@ -587,6 +599,7 @@ export function loadLlamaCppImageConfig(
     probes: qualification?.probes,
     profile: qualification?.profile,
     recipeRef: qualificationRecipeRef,
+    requestGuard: qualificationRequestGuard,
     required: qualification?.required,
     runner: qualificationRunner,
   };
@@ -724,6 +737,7 @@ export function loadLlamaCppImageConfig(
       agentQualification,
       probeBounds: qualification?.probeBounds,
       probes: qualification?.probes,
+      requestGuard: qualificationRequestGuard,
     },
     recipe: {
       capabilities: recipe.spec.capabilities,

scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts

@@ -131,6 +131,7 @@ export type LlamaCppDgxSparkQualificationPlan = {
   readonly probes: typeof LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES;
   readonly profile: typeof LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE;
   readonly recipeRef: typeof LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE;
+  readonly requestGuard: "required";
   readonly required: true;
   readonly runner: string | null;
 };
@@ -176,6 +177,7 @@ export type LlamaCppDgxSparkExecutionPlan = {
     readonly agentQualification: LlamaCppDgxSparkAgentQualificationPlan;
     readonly probeBounds: LlamaCppDgxSparkQualificationPlan["probeBounds"];
     readonly probes: typeof LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES;
+    readonly requestGuard: "required";
   };
   readonly recipe: {
     readonly capabilities: {
@@ -254,7 +256,14 @@ export type LlamaCppDgxSparkExecutionPlan = {
         readonly value: "f16" | "q8_0" | "q4_0";
       };
       readonly limits: {
+        readonly maxRequestBodyBytes: number;
+        readonly maxRequestHeaderBytes: number;
+        readonly maxOutputTokens: number;
         readonly requestTimeoutSeconds: number;
+        readonly shutdownTimeoutSeconds: number;
+      };
+      readonly requestGuard: {
+        readonly upstreamPort: number;
       };
       readonly microBatchSize: number;
       readonly port: 8081;
@@ -869,6 +878,7 @@ export function parseLlamaCppDgxSparkQualificationPlan(
       "probes",
       "profile",
       "recipeRef",
+      "requestGuard",
       "required",
       "runner",
     ],
@@ -887,6 +897,7 @@ export function parseLlamaCppDgxSparkQualificationPlan(
     plan.required !== true ||
     plan.profile !== LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE ||
     plan.recipeRef !== LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE ||
+    plan.requestGuard !== "required" ||
     plan.platform !== LLAMA_CPP_DGX_SPARK_QUALIFICATION_PLATFORM ||
     gpu.vendor !== "nvidia" ||
     gpu.fullOffload !== true ||
@@ -911,6 +922,7 @@ export function parseLlamaCppDgxSparkQualificationPlan(
     probes: LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES,
     profile: LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE,
     recipeRef: LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE,
+    requestGuard: "required",
     required: true,
     runner: infrastructure.runner,
   };
@@ -933,14 +945,17 @@ export function parseLlamaCppDgxSparkExecutionPlan(
   const qualification = record(plan.qualification, "compiled protocol qualification");
   requireExactKeys(
     qualification,
-    ["agentQualification", "probeBounds", "probes"],
+    ["agentQualification", "probeBounds", "probes", "requestGuard"],
     "compiled protocol qualification",
   );
   if (
     JSON.stringify(qualification.probes) !== JSON.stringify(LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES)
   ) {
     throw new Error("compiled llama.cpp DGX Spark protocol probes are invalid");
   }
+  if (qualification.requestGuard !== "required") {
+    throw new Error("compiled llama.cpp DGX Spark request-guard activation is invalid");
+  }
   const protocolProbeBounds = parseProtocolProbeBounds(qualification.probeBounds);
   const agentQualification = parseAgentQualification(qualification.agentQualification);
 
@@ -1198,6 +1213,7 @@ export function parseLlamaCppDgxSparkExecutionPlan(
       "microBatchSize",
       "port",
       "protocol",
+      "requestGuard",
       "slots",
       "speculativeDecoding",
     ],
@@ -1220,13 +1236,55 @@ export function parseLlamaCppDgxSparkExecutionPlan(
   requireExactKeys(kvCache, ["key", "value"], "compiled qualification KV cache");
   const allowedKvTypes = new Set(["f16", "q8_0", "q4_0"]);
   const limits = record(serve.limits, "compiled qualification request limits");
-  requireExactKeys(limits, ["requestTimeoutSeconds"], "compiled qualification request limits");
+  requireExactKeys(
+    limits,
+    [
+      "maxOutputTokens",
+      "maxRequestBodyBytes",
+      "maxRequestHeaderBytes",
+      "requestTimeoutSeconds",
+      "shutdownTimeoutSeconds",
+    ],
+    "compiled qualification request limits",
+  );
+  const maxRequestBodyBytes = boundedInteger(
+    limits.maxRequestBodyBytes,
+    "compiled qualification maximum request body bytes",
+    1,
+    64 * 1024 * 1024,
+  );
+  const maxRequestHeaderBytes = boundedInteger(
+    limits.maxRequestHeaderBytes,
+    "compiled qualification maximum request header bytes",
+    1,
+    1024 * 1024,
+  );
+  const maxOutputTokens = boundedInteger(
+    limits.maxOutputTokens,
+    "compiled qualification maximum output tokens",
+    1,
+    contextSize,
+  );
   const requestTimeoutSeconds = boundedInteger(
     limits.requestTimeoutSeconds,
     "compiled qualification request timeout",
     1,
     3600,
   );
+  const shutdownTimeoutSeconds = boundedInteger(
+    limits.shutdownTimeoutSeconds,
+    "compiled qualification shutdown timeout",
+    1,
+    3600,
+  );
+  const requestGuard = record(serve.requestGuard, "compiled qualification request guard");
+  requireExactKeys(requestGuard, ["upstreamPort"], "compiled qualification request guard");
+  const upstreamPort = boundedInteger(
+    requestGuard.upstreamPort,
+    "compiled qualification request-guard upstream port",
+    1,
+    65535,
+  );
   if (
     serve.protocol !== "openai-completions" ||
     serve.authentication !== "bearer" ||
@@ -1236,6 +1294,7 @@ export function parseLlamaCppDgxSparkExecutionPlan(
     serve.idleSleepSeconds !== -1 ||
     serve.flashAttention !== "enabled" ||
     serve.speculativeDecoding !== "disabled" ||
+    upstreamPort === serve.port ||
     typeof kvCache.key !== "string" ||
     !allowedKvTypes.has(kvCache.key) ||
     typeof kvCache.value !== "string" ||
@@ -1300,6 +1359,7 @@ export function parseLlamaCppDgxSparkExecutionPlan(
       agentQualification,
       probeBounds: protocolProbeBounds,
       probes: LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES,
+      requestGuard: "required",
     },
     recipe: {
       capabilities: {
@@ -1372,8 +1432,13 @@ export function parseLlamaCppDgxSparkExecutionPlan(
         },
         speculativeDecoding: "disabled",
         limits: {
+          maxRequestBodyBytes,
+          maxRequestHeaderBytes,
+          maxOutputTokens,
           requestTimeoutSeconds,
+          shutdownTimeoutSeconds,
         },
+        requestGuard: { upstreamPort },
       },
       server: {
         technology: "llama.cpp",

scripts/checks/run-llama-cpp-dgx-spark-qualification.mts

@@ -9,7 +9,7 @@ import path from "node:path";
 import { fileURLToPath } from "node:url";
 
 import {
-  buildLlamaCppHostLocalDockerArgv,
+  buildLlamaCppRequestGuardDockerArgv,
   type VerifiedLocalModelArtifact,
 } from "../../src/lib/inference/llama-cpp/host-local-runtime.ts";
 import {
@@ -332,7 +332,10 @@ export function buildServerContainerArgv(
     hostPort?: number;
   },
 ): string[] {
-  return buildLlamaCppHostLocalDockerArgv(plan.recipe, {
+  if (plan.qualification.requestGuard !== "required") {
+    throw new Error("llama.cpp qualification requires the declarative request guard");
+  }
+  return buildLlamaCppRequestGuardDockerArgv(plan.recipe, {
     apiKeyHostPath: options.apiKeyHostPath,
     containerName: options.containerName,
     imageReference: options.imageReference,