diff --git a/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts b/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts index d83c7fcd053..606913e3576 100644 --- a/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts +++ b/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts @@ -8,6 +8,10 @@ import net from "node:net"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { + buildLlamaCppHostLocalDockerArgv, + type VerifiedLocalModelArtifact, +} from "../../src/lib/inference/llama-cpp/host-local-runtime.ts"; import { LLAMA_CPP_DGX_SPARK_MODEL_PATH_PATTERN, LLAMA_CPP_DGX_SPARK_QUALIFICATION_IMAGE_REPOSITORY, @@ -27,8 +31,6 @@ const registryImage = const registryOwnerLabel = "io.nvidia.nemoclaw.llama-cpp-qualification-owner"; const localImageRepository = LLAMA_CPP_DGX_SPARK_QUALIFICATION_IMAGE_REPOSITORY; const expectedWorkflowRef = "NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main"; -const containerModelPath = "/models/Nemotron-3-Nano-30B-A3B-UD-Q4_K_XL.gguf"; -const containerApiKeyPath = "/run/secrets/llama-cpp-api-key"; const maximumPlanBytes = 1024 * 1024; const maximumDockerfileBytes = 1024 * 1024; const maximumModelBytes = 128 * 1024 * 1024 * 1024; @@ -303,86 +305,23 @@ export function buildServerContainerArgv( apiKeyHostPath: string; containerName: string; imageReference: string; - modelHostPath: string; + model: VerifiedLocalModelArtifact; networkName: string; registryOwner: string; runtimeGid: number; runtimeUid: number; }, ): string[] { - const { resources } = plan.recipe.runtime; - const { serve } = plan.recipe; - const runtimeUid = requiredInteger(options.runtimeUid, "runtime uid", 1, 2_147_483_647); - const runtimeGid = requiredInteger(options.runtimeGid, "runtime gid", 1, 2_147_483_647); - return [ - "run", - "--detach", - "--name", - options.containerName, - "--label", - `${registryOwnerLabel}=${options.registryOwner}`, - "--network", - options.networkName, - "--user", - `${runtimeUid}:${runtimeGid}`, - "--publish", - `127.0.0.1::${serve.port}`, - "--read-only", - "--cap-drop", - "ALL", - "--security-opt", - "no-new-privileges=true", - "--memory", - `${resources.memoryBytes}b`, - "--memory-swap", - `${resources.memoryBytes}b`, - "--pids-limit", - String(resources.pidsLimit), - "--gpus", - "1", - "--tmpfs", - `/tmp:rw,noexec,nosuid,nodev,size=${resources.writableStorageBytes},uid=${runtimeUid},gid=${runtimeGid},mode=1777`, - "--mount", - `type=bind,source=${options.modelHostPath},target=${containerModelPath},readonly`, - "--mount", - `type=bind,source=${options.apiKeyHostPath},target=${containerApiKeyPath},readonly`, - options.imageReference, - "--model", - containerModelPath, - "--alias", - plan.recipe.model.servedName, - "--host", - "0.0.0.0", - "--port", - String(serve.port), - "--gpu-layers", - "all", - "--ctx-size", - String(serve.contextSize), - "--parallel", - String(serve.slots), - "--sleep-idle-seconds", - String(serve.idleSleepSeconds), - "--batch-size", - String(serve.batchSize), - "--ubatch-size", - String(serve.microBatchSize), - "--cache-type-k", - serve.kvCache.key, - "--cache-type-v", - serve.kvCache.value, - "--flash-attn", - "on", - "--timeout", - String(serve.limits.requestTimeoutSeconds), - "--api-key-file", - containerApiKeyPath, - "--metrics", - "--no-ui", - "--no-slots", - "--no-mmproj", - "--no-agent", - ]; + return buildLlamaCppHostLocalDockerArgv(plan.recipe, { + apiKeyHostPath: options.apiKeyHostPath, + containerName: options.containerName, + imageReference: options.imageReference, + model: options.model, + network: { isolation: "docker-internal", name: options.networkName }, + ownerLabel: { name: registryOwnerLabel, value: options.registryOwner }, + runtimeGid: options.runtimeGid, + runtimeUid: options.runtimeUid, + }); } export function validateStartupLog(log: string): { offloadedLayers: number; totalLayers: number } { @@ -560,10 +499,7 @@ function validateCandidateCheckout(invocation: QualificationInvocation): string return root; } -function hashModelFile( - modelHostPath: string, - plan: QualificationPlan, -): { digest: string; sizeBytes: number } { +function hashModelFile(modelHostPath: string, plan: QualificationPlan): VerifiedLocalModelArtifact { if (path.basename(modelHostPath) !== plan.recipe.model.file.path) { throw new Error("model file name does not match the qualification plan"); } @@ -607,7 +543,7 @@ function hashModelFile( if (digest !== plan.recipe.model.file.digest) { throw new Error("model file digest does not match the qualification plan"); } - return { digest, sizeBytes: after.size }; + return { digest, hostPath: modelHostPath, sizeBytes: after.size }; } finally { fs.closeSync(descriptor); } @@ -943,7 +879,7 @@ async function runQualification( apiKeyHostPath, containerName: names.containerName, imageReference: image.reference, - modelHostPath: invocation.modelHostPath, + model, networkName: names.networkName, registryOwner: names.registryOwner, runtimeGid, diff --git a/src/lib/inference/llama-cpp/host-local-runtime.test.ts b/src/lib/inference/llama-cpp/host-local-runtime.test.ts new file mode 100644 index 00000000000..2283902a45d --- /dev/null +++ b/src/lib/inference/llama-cpp/host-local-runtime.test.ts @@ -0,0 +1,264 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildLlamaCppHostLocalDockerArgv, + type LlamaCppHostLocalLaunchContract, + type LlamaCppHostLocalRuntimeBindings, +} from "./host-local-runtime"; + +const MODEL_DIGEST = `sha256:${"a".repeat(64)}`; +const IMAGE = `ghcr.io/nvidia/nemoclaw/llama-cpp-server@sha256:${"b".repeat(64)}`; + +function contract(): LlamaCppHostLocalLaunchContract { + return { + model: { + servedName: "nvidia-nemotron-3-nano-30b-a3b", + file: { + digest: MODEL_DIGEST, + path: "Nemotron-3-Nano-30B-A3B-UD-Q4_K_XL.gguf", + sizeBytes: 22_833_947_424, + }, + }, + policy: { + egress: "disabled", + modelDownloads: "disabled", + modelSource: "verified-local", + }, + runtime: { + gpu: { count: 1, cpuFallback: "reject", offload: "full", vendor: "nvidia" }, + resources: { + memoryBytes: 51_539_607_552, + pidsLimit: 256, + writableStorageBytes: 42_949_672_960, + }, + }, + serve: { + authentication: "bearer", + batchSize: 2_048, + contextSize: 262_144, + flashAttention: "enabled", + idleSleepSeconds: -1, + kvCache: { key: "f16", value: "f16" }, + limits: { + requestTimeoutSeconds: 900, + }, + microBatchSize: 512, + port: 8_081, + protocol: "openai-completions", + slots: 1, + speculativeDecoding: "disabled", + }, + surfaces: { + agentMode: "disabled", + mcpProxy: "disabled", + multimodalProjection: "disabled", + router: "disabled", + serverTools: "disabled", + slotInspection: "disabled", + ui: "disabled", + }, + }; +} + +function bindings(): LlamaCppHostLocalRuntimeBindings { + return { + apiKeyHostPath: "/run/nemoclaw/llama-cpp/api-key", + containerName: "nemoclaw-llama-cpp", + imageReference: IMAGE, + model: { + digest: MODEL_DIGEST, + hostPath: `/home/nvidia/.cache/huggingface/hub/blobs/${"c".repeat(64)}`, + sizeBytes: 22_833_947_424, + }, + network: { isolation: "docker-internal", name: "nemoclaw-llama-cpp-internal" }, + ownerLabel: { name: "io.nvidia.nemoclaw.llama-cpp-owner", value: "gateway.primary" }, + runtimeGid: 1_001, + runtimeUid: 1_001, + }; +} + +function valuesAfter(argv: readonly string[], flag: string): string[] { + return argv.flatMap((value, index) => (value === flag ? [argv[index + 1] ?? ""] : [])); +} + +describe("llama.cpp host-local runtime materializer", () => { + it("binds the matching local GGUF into the declared llama.cpp runtime (#8144)", () => { + const input = contract(); + const runtime = bindings(); + const argv = buildLlamaCppHostLocalDockerArgv(input, runtime); + + expect(valuesAfter(argv, "--mount")).toEqual([ + `type=bind,source=${runtime.model.hostPath},target=/models/${input.model.file.path},readonly`, + `type=bind,source=${runtime.apiKeyHostPath},target=/run/secrets/llama-cpp-api-key,readonly`, + ]); + expect(valuesAfter(argv, "--publish")).toEqual(["127.0.0.1::8081"]); + expect(valuesAfter(argv, "--gpus")).toEqual(["1"]); + expect(valuesAfter(argv, "--gpu-layers")).toEqual(["all"]); + expect(valuesAfter(argv, "--ctx-size")).toEqual([String(input.serve.contextSize)]); + expect(valuesAfter(argv, "--batch-size")).toEqual([String(input.serve.batchSize)]); + expect(valuesAfter(argv, "--ubatch-size")).toEqual([String(input.serve.microBatchSize)]); + expect(valuesAfter(argv, "--timeout")).toEqual([ + String(input.serve.limits.requestTimeoutSeconds), + ]); + expect(argv).toEqual( + expect.arrayContaining([ + "--read-only", + "--cap-drop", + "ALL", + "no-new-privileges=true", + "--no-ui", + "--no-slots", + "--no-mmproj", + "--no-agent", + ]), + ); + expect(argv.join("\n")).not.toContain("HF_TOKEN"); + expect(argv.join("\n")).not.toContain("huggingface.co"); + }); + + it("takes launch settings from the declared contract instead of code defaults (#8144)", () => { + const input = contract(); + const changed = { + ...input, + serve: { + ...input.serve, + batchSize: 1_024, + contextSize: 131_072, + limits: { ...input.serve.limits, requestTimeoutSeconds: 600 }, + microBatchSize: 256, + }, + } satisfies LlamaCppHostLocalLaunchContract; + const argv = buildLlamaCppHostLocalDockerArgv(changed, bindings()); + + expect(valuesAfter(argv, "--ctx-size")).toEqual(["131072"]); + expect(valuesAfter(argv, "--batch-size")).toEqual(["1024"]); + expect(valuesAfter(argv, "--ubatch-size")).toEqual(["256"]); + expect(valuesAfter(argv, "--timeout")).toEqual(["600"]); + }); + + it("materializes the complete Docker argument contract in stable order (#8144)", () => { + const input = contract(); + const runtime = bindings(); + + expect(buildLlamaCppHostLocalDockerArgv(input, runtime)).toEqual([ + "run", + "--detach", + "--name", + runtime.containerName, + "--label", + `${runtime.ownerLabel.name}=${runtime.ownerLabel.value}`, + "--network", + runtime.network.name, + "--user", + "1001:1001", + "--publish", + "127.0.0.1::8081", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges=true", + "--memory", + "51539607552b", + "--memory-swap", + "51539607552b", + "--pids-limit", + "256", + "--gpus", + "1", + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=42949672960,uid=1001,gid=1001,mode=1777", + "--mount", + `type=bind,source=${runtime.model.hostPath},target=/models/${input.model.file.path},readonly`, + "--mount", + `type=bind,source=${runtime.apiKeyHostPath},target=/run/secrets/llama-cpp-api-key,readonly`, + runtime.imageReference, + "--model", + `/models/${input.model.file.path}`, + "--alias", + input.model.servedName, + "--host", + "0.0.0.0", + "--port", + "8081", + "--gpu-layers", + "all", + "--ctx-size", + "262144", + "--parallel", + "1", + "--sleep-idle-seconds", + "-1", + "--batch-size", + "2048", + "--ubatch-size", + "512", + "--cache-type-k", + "f16", + "--cache-type-v", + "f16", + "--flash-attn", + "on", + "--timeout", + "900", + "--api-key-file", + "/run/secrets/llama-cpp-api-key", + "--metrics", + "--no-ui", + "--no-slots", + "--no-mmproj", + "--no-agent", + ]); + }); + + it("rejects an artifact that does not match the declared GGUF identity (#8144)", () => { + expect(() => + buildLlamaCppHostLocalDockerArgv(contract(), { + ...bindings(), + model: { ...bindings().model, digest: `sha256:${"0".repeat(64)}` }, + }), + ).toThrow("verified model artifact does not match"); + expect(() => + buildLlamaCppHostLocalDockerArgv(contract(), { + ...bindings(), + model: { ...bindings().model, sizeBytes: 1 }, + }), + ).toThrow("verified model artifact does not match"); + }); + + it("rejects inputs that violate host-local isolation (#8144)", () => { + expect(() => + buildLlamaCppHostLocalDockerArgv(contract(), { + ...bindings(), + model: { ...bindings().model, hostPath: "/models/../foreign.gguf" }, + }), + ).toThrow("normalized absolute host path"); + expect(() => + buildLlamaCppHostLocalDockerArgv(contract(), { + ...bindings(), + network: { isolation: "docker-internal", name: "host" }, + }), + ).toThrow("runtime binding is invalid"); + expect(() => + buildLlamaCppHostLocalDockerArgv( + { + ...contract(), + policy: { ...contract().policy, egress: "enabled" }, + } as unknown as LlamaCppHostLocalLaunchContract, + bindings(), + ), + ).toThrow("offline policy"); + expect(() => + buildLlamaCppHostLocalDockerArgv( + { + ...contract(), + surfaces: { ...contract().surfaces, ui: "enabled" }, + } as unknown as LlamaCppHostLocalLaunchContract, + bindings(), + ), + ).toThrow("disabled-surface contract"); + }); +}); diff --git a/src/lib/inference/llama-cpp/host-local-runtime.ts b/src/lib/inference/llama-cpp/host-local-runtime.ts new file mode 100644 index 00000000000..8422463fbe8 --- /dev/null +++ b/src/lib/inference/llama-cpp/host-local-runtime.ts @@ -0,0 +1,277 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { isSafeLlamaCppServedModelAlias } from "./contract"; + +export const LLAMA_CPP_HOST_LOCAL_CONTAINER_API_KEY_PATH = + "/run/secrets/llama-cpp-api-key" as const; + +const SAFE_HOST_PATH = /^\/(?:[A-Za-z0-9._-]+\/)*[A-Za-z0-9._-]+$/u; +const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u; +const SAFE_LABEL_NAME = /^[a-z0-9][a-z0-9.-]{0,127}$/u; +const SAFE_LABEL_VALUE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,255}$/u; +const IMAGE_DIGEST = + /^(?:[a-z0-9]+(?:[._-][a-z0-9]+)*(?::[0-9]+)?\/)?(?:[a-z0-9]+(?:[._-][a-z0-9]+)*\/)*[a-z0-9]+(?:[._-][a-z0-9]+)*@sha256:[0-9a-f]{64}$/u; +const GGUF_FILE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,255}\.gguf$/u; +const SHA256_DIGEST = /^sha256:[0-9a-f]{64}$/u; +const DOCKER_BUILTIN_NETWORKS = new Set(["bridge", "host", "none"]); + +export interface LlamaCppHostLocalLaunchContract { + readonly model: { + readonly servedName: string; + readonly file: { + readonly digest: string; + readonly path: string; + readonly sizeBytes: number; + }; + }; + readonly policy: { + readonly egress: "disabled"; + readonly modelDownloads: "disabled"; + readonly modelSource: "verified-local"; + }; + readonly runtime: { + readonly gpu: { + readonly count: 1; + readonly cpuFallback: "reject"; + readonly offload: "full"; + readonly vendor: "nvidia"; + }; + readonly resources: { + readonly memoryBytes: number; + readonly pidsLimit: number; + readonly writableStorageBytes: number; + }; + }; + readonly serve: { + readonly authentication: "bearer"; + readonly batchSize: number; + readonly contextSize: number; + readonly flashAttention: "enabled"; + readonly idleSleepSeconds: -1; + readonly kvCache: { + readonly key: "f16" | "q8_0" | "q4_0"; + readonly value: "f16" | "q8_0" | "q4_0"; + }; + readonly limits: { + readonly requestTimeoutSeconds: number; + }; + readonly microBatchSize: number; + readonly port: number; + readonly protocol: "openai-completions"; + readonly slots: 1; + readonly speculativeDecoding: "disabled"; + }; + readonly surfaces: { + readonly agentMode: "disabled"; + readonly mcpProxy: "disabled"; + readonly multimodalProjection: "disabled"; + readonly router: "disabled"; + readonly serverTools: "disabled"; + readonly slotInspection: "disabled"; + readonly ui: "disabled"; + }; +} + +export interface LlamaCppHostLocalRuntimeBindings { + readonly apiKeyHostPath: string; + readonly containerName: string; + readonly imageReference: string; + readonly model: VerifiedLocalModelArtifact; + /** The caller must create this named Docker network with `--internal` before launch. */ + readonly network: { + readonly isolation: "docker-internal"; + readonly name: string; + }; + readonly ownerLabel: { readonly name: string; readonly value: string }; + readonly runtimeGid: number; + readonly runtimeUid: number; +} + +export interface VerifiedLocalModelArtifact { + readonly digest: string; + readonly hostPath: string; + readonly sizeBytes: number; +} + +function positiveInteger(value: number, label: string, maximum = Number.MAX_SAFE_INTEGER): number { + if (!Number.isSafeInteger(value) || value < 1 || value > maximum) { + throw new Error(`${label} must be a positive safe integer`); + } + return value; +} + +function safeHostPath(value: string, label: string): string { + if (!SAFE_HOST_PATH.test(value) || path.posix.normalize(value) !== value) { + throw new Error(`${label} must be a normalized absolute host path`); + } + return value; +} + +function validateContract(contract: LlamaCppHostLocalLaunchContract): void { + if ( + !GGUF_FILE.test(contract.model.file.path) || + !SHA256_DIGEST.test(contract.model.file.digest) || + !Number.isSafeInteger(contract.model.file.sizeBytes) || + contract.model.file.sizeBytes < 1 + ) { + throw new Error("llama.cpp model file must be one safe GGUF filename"); + } + if ( + !isSafeLlamaCppServedModelAlias(contract.model.servedName) || + contract.policy.egress !== "disabled" || + contract.policy.modelDownloads !== "disabled" || + contract.policy.modelSource !== "verified-local" + ) { + throw new Error("llama.cpp host-local model or offline policy is invalid"); + } + if ( + contract.runtime.gpu.vendor !== "nvidia" || + contract.runtime.gpu.count !== 1 || + contract.runtime.gpu.offload !== "full" || + contract.runtime.gpu.cpuFallback !== "reject" + ) { + throw new Error("llama.cpp host-local runtime must require one fully offloaded NVIDIA GPU"); + } + positiveInteger(contract.runtime.resources.memoryBytes, "llama.cpp memory limit"); + positiveInteger( + contract.runtime.resources.writableStorageBytes, + "llama.cpp writable storage limit", + ); + positiveInteger(contract.runtime.resources.pidsLimit, "llama.cpp PID limit", 4_096); + positiveInteger(contract.serve.port, "llama.cpp server port", 65_535); + positiveInteger(contract.serve.contextSize, "llama.cpp context size"); + positiveInteger(contract.serve.batchSize, "llama.cpp batch size"); + positiveInteger(contract.serve.microBatchSize, "llama.cpp micro-batch size"); + positiveInteger(contract.serve.limits.requestTimeoutSeconds, "llama.cpp request timeout", 86_400); + if (contract.serve.microBatchSize > contract.serve.batchSize) { + throw new Error("llama.cpp micro-batch size exceeds the batch size"); + } + if ( + contract.serve.authentication !== "bearer" || + contract.serve.protocol !== "openai-completions" || + contract.serve.slots !== 1 || + contract.serve.idleSleepSeconds !== -1 || + contract.serve.flashAttention !== "enabled" || + contract.serve.speculativeDecoding !== "disabled" || + Object.values(contract.surfaces).some((state) => state !== "disabled") + ) { + throw new Error("llama.cpp host-local serving or disabled-surface contract is invalid"); + } +} + +function validateBindings( + contract: LlamaCppHostLocalLaunchContract, + bindings: LlamaCppHostLocalRuntimeBindings, +): void { + safeHostPath(bindings.model.hostPath, "llama.cpp model path"); + safeHostPath(bindings.apiKeyHostPath, "llama.cpp API-key path"); + if ( + bindings.model.digest !== contract.model.file.digest || + bindings.model.sizeBytes !== contract.model.file.sizeBytes + ) { + throw new Error( + "llama.cpp verified model artifact does not match the declarative GGUF identity", + ); + } + if ( + !SAFE_NAME.test(bindings.containerName) || + bindings.network.isolation !== "docker-internal" || + !SAFE_NAME.test(bindings.network.name) || + DOCKER_BUILTIN_NETWORKS.has(bindings.network.name) || + !SAFE_LABEL_NAME.test(bindings.ownerLabel.name) || + !SAFE_LABEL_VALUE.test(bindings.ownerLabel.value) || + !IMAGE_DIGEST.test(bindings.imageReference) + ) { + throw new Error("llama.cpp host-local runtime binding is invalid"); + } + positiveInteger(bindings.runtimeUid, "llama.cpp runtime uid", 2_147_483_647); + positiveInteger(bindings.runtimeGid, "llama.cpp runtime gid", 2_147_483_647); +} + +/** + * Materialize the declarative llama.cpp contract around an already verified + * local GGUF. Artifact acquisition is deliberately outside this adapter. + */ +export function buildLlamaCppHostLocalDockerArgv( + contract: LlamaCppHostLocalLaunchContract, + bindings: LlamaCppHostLocalRuntimeBindings, +): string[] { + validateContract(contract); + validateBindings(contract, bindings); + const { resources } = contract.runtime; + const { serve } = contract; + const containerModelPath = `/models/${contract.model.file.path}`; + const runtimeIdentity = `${String(bindings.runtimeUid)}:${String(bindings.runtimeGid)}`; + return [ + "run", + "--detach", + "--name", + bindings.containerName, + "--label", + `${bindings.ownerLabel.name}=${bindings.ownerLabel.value}`, + "--network", + bindings.network.name, + "--user", + runtimeIdentity, + "--publish", + `127.0.0.1::${String(serve.port)}`, + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges=true", + "--memory", + `${String(resources.memoryBytes)}b`, + "--memory-swap", + `${String(resources.memoryBytes)}b`, + "--pids-limit", + String(resources.pidsLimit), + "--gpus", + "1", + "--tmpfs", + `/tmp:rw,noexec,nosuid,nodev,size=${String(resources.writableStorageBytes)},uid=${String(bindings.runtimeUid)},gid=${String(bindings.runtimeGid)},mode=1777`, + "--mount", + `type=bind,source=${bindings.model.hostPath},target=${containerModelPath},readonly`, + "--mount", + `type=bind,source=${bindings.apiKeyHostPath},target=${LLAMA_CPP_HOST_LOCAL_CONTAINER_API_KEY_PATH},readonly`, + bindings.imageReference, + "--model", + containerModelPath, + "--alias", + contract.model.servedName, + "--host", + "0.0.0.0", + "--port", + String(serve.port), + "--gpu-layers", + "all", + "--ctx-size", + String(serve.contextSize), + "--parallel", + String(serve.slots), + "--sleep-idle-seconds", + String(serve.idleSleepSeconds), + "--batch-size", + String(serve.batchSize), + "--ubatch-size", + String(serve.microBatchSize), + "--cache-type-k", + serve.kvCache.key, + "--cache-type-v", + serve.kvCache.value, + "--flash-attn", + "on", + "--timeout", + String(serve.limits.requestTimeoutSeconds), + "--api-key-file", + LLAMA_CPP_HOST_LOCAL_CONTAINER_API_KEY_PATH, + "--metrics", + "--no-ui", + "--no-slots", + "--no-mmproj", + "--no-agent", + ]; +} diff --git a/src/lib/inference/llama-cpp/index.ts b/src/lib/inference/llama-cpp/index.ts index 609c942643c..93081904265 100644 --- a/src/lib/inference/llama-cpp/index.ts +++ b/src/lib/inference/llama-cpp/index.ts @@ -16,6 +16,7 @@ import { export * from "./contract"; export * from "./gguf-cache-plan"; export * from "./gguf-cache-receipt"; +export * from "./host-local-runtime"; type LlamaCppModelEntry = { id?: unknown; diff --git a/test/llama-cpp-dgx-spark-qualification-runner.test.ts b/test/llama-cpp-dgx-spark-qualification-runner.test.ts index 59c3d3de552..0d6f0850d24 100644 --- a/test/llama-cpp-dgx-spark-qualification-runner.test.ts +++ b/test/llama-cpp-dgx-spark-qualification-runner.test.ts @@ -252,7 +252,11 @@ describe("trusted llama.cpp DGX Spark qualification runner", () => { apiKeyHostPath: "/work/tmp/api-key", containerName: "qualified-server", imageReference: `localhost:5000/repo@sha256:${"d".repeat(64)}`, - modelHostPath: MODEL_PATH, + model: { + digest: plan.recipe.model.file.digest, + hostPath: MODEL_PATH, + sizeBytes: plan.recipe.model.file.sizeBytes, + }, networkName: "qualified-internal", registryOwner: expectedRegistryOwner(RUN_ID, RUN_ATTEMPT), runtimeGid: 1001, @@ -313,7 +317,11 @@ describe("trusted llama.cpp DGX Spark qualification runner", () => { apiKeyHostPath: "/work/tmp/api-key", containerName: "qualified-server", imageReference: `localhost:5000/repo@sha256:${"d".repeat(64)}`, - modelHostPath: MODEL_PATH, + model: { + digest: plan.recipe.model.file.digest, + hostPath: MODEL_PATH, + sizeBytes: plan.recipe.model.file.sizeBytes, + }, networkName: "qualified-internal", registryOwner: expectedRegistryOwner(RUN_ID, RUN_ATTEMPT), runtimeGid: 1001,