diff --git a/.cursor/rules/sdk/docs/kv-cache-system.mdc b/.cursor/rules/sdk/docs/kv-cache-system.mdc index df1307131b..5dc9ea9003 100644 --- a/.cursor/rules/sdk/docs/kv-cache-system.mdc +++ b/.cursor/rules/sdk/docs/kv-cache-system.mdc @@ -102,7 +102,7 @@ When a new cache key is used for the first time: ### Cache Initialization (initSystemPromptCache) -Primes a new cache by sending system prompt + tools to the model with `{ cacheKey: cachePath }` in `runOptions`, then cancels generation after the first output token — it only needs to process the cache priming, not produce a response. +Primes a new cache by sending system prompt + tools through `runModel` with `{ cacheKey: cachePath, saveCacheToDisk: true, prefill: true }`. The prompt is ingested into the KV cache and persisted to disk without producing any output tokens, so the call resolves as soon as priming finishes. ### Message Preparation (prepareMessagesForCache) diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 0f74aa7b5c..434cf19376 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -177,7 +177,7 @@ "@qvac/embed-llamacpp": "^0.14.0", "@qvac/error": "^0.1.1", "@qvac/langdetect-text": "^0.1.2", - "@qvac/llm-llamacpp": "^0.17.1", + "@qvac/llm-llamacpp": "^0.17.3", "@qvac/logging": "^0.1.0", "@qvac/ocr-onnx": "^0.4.2", "@qvac/rag": "^0.4.4", diff --git a/packages/sdk/server/bare/plugins/llamacpp-completion/ops/completion-stream.ts b/packages/sdk/server/bare/plugins/llamacpp-completion/ops/completion-stream.ts index efcb0d99c4..517b8025c1 100644 --- a/packages/sdk/server/bare/plugins/llamacpp-completion/ops/completion-stream.ts +++ b/packages/sdk/server/bare/plugins/llamacpp-completion/ops/completion-stream.ts @@ -103,7 +103,10 @@ interface ChatHistory { // dep bump propagates and is harmless once it has. type CompletionGenerationParams = GenerationParams & { json_schema?: string }; -type CompletionRunOptions = Pick & { +type CompletionRunOptions = Pick< + RunOptions, + "cacheKey" | "saveCacheToDisk" | "prefill" +> & { generationParams?: CompletionGenerationParams; }; @@ -244,10 +247,7 @@ async function initSystemPromptCache( const primeResponse = await runModel(model, primeMessages, { cacheKey: cachePathToUse, saveCacheToDisk: true, - }); - - primeResponse.once("output", () => { - void primeResponse.cancel(); + prefill: true, }); await primeResponse.await();